Esempio n. 1
0
def wait(job, _print):
    data = []
    while True:
        if _check_running(job):
            data += _get_data(job, _print)
        else:
            break
    data += _get_data(job, _print)
    if DEBUG or _print:
        screen.print_mutable("", True)
    killall(job)
    del job
    return data
Esempio n. 2
0
def _get_data(job, _print):
    data = []
    while True:
        try:
            chunk = job.out_queue.get_nowait()
        except:
            break
        job.processed_rows +=1
        data.append(chunk)
    if job.processed_rows != 0:
        pct = int((float(job.processed_rows)/job.datalen) * 100)
        if pct != job.pct_complete:
            job.pct_complete = pct
            if DEBUG or _print:
                screen.print_mutable("Processing data... %d%%" % pct)
    return data
Esempio n. 3
0
    def crunchlogs(self):
        global log_regex
        if self.args.format is not None:
            logformat = self.args.format
        else:
            logformat = logformat.TYPES[self.args.type]

        print
        lines = []
        for logfile in self.args.logfile:
            screen.print_mutable("Reading lines from %s:" % logfile.name)
            lines += logfile.readlines()
            screen.print_mutable("Reading lines from %s: %d" % (logfile.name, len(lines)))
            logfile.close()
            screen.print_mutable("", True)

        log_regex = re.compile(parse_format_string(logformat))
        if self.args.lines:
            lines = lines[:self.args.lines]
        st = time.time()
        self.data = parallel.run(log_match, lines, _print=True)
        et = time.time()
        print "%d lines crunched in %0.3f seconds" % (len(lines), (et-st))