def wait(job, _print): data = [] while True: if _check_running(job): data += _get_data(job, _print) else: break data += _get_data(job, _print) if DEBUG or _print: screen.print_mutable("", True) killall(job) del job return data
def _get_data(job, _print): data = [] while True: try: chunk = job.out_queue.get_nowait() except: break job.processed_rows +=1 data.append(chunk) if job.processed_rows != 0: pct = int((float(job.processed_rows)/job.datalen) * 100) if pct != job.pct_complete: job.pct_complete = pct if DEBUG or _print: screen.print_mutable("Processing data... %d%%" % pct) return data
def crunchlogs(self): global log_regex if self.args.format is not None: logformat = self.args.format else: logformat = logformat.TYPES[self.args.type] print lines = [] for logfile in self.args.logfile: screen.print_mutable("Reading lines from %s:" % logfile.name) lines += logfile.readlines() screen.print_mutable("Reading lines from %s: %d" % (logfile.name, len(lines))) logfile.close() screen.print_mutable("", True) log_regex = re.compile(parse_format_string(logformat)) if self.args.lines: lines = lines[:self.args.lines] st = time.time() self.data = parallel.run(log_match, lines, _print=True) et = time.time() print "%d lines crunched in %0.3f seconds" % (len(lines), (et-st))