def merge(self): """Merge all files into a temp file and return it.""" linfos = LineInfo() tmp_file, _ = unique_tmp_path(TMP_PREFIX) filecnt = len(self.files) fileno = 0 pg = ProgressBar('merging', filecnt) with open(tmp_file, 'w') as out: for _file in self.files: pg.animate(fileno) cmd = ['cat', _file.abspath] check_call(cmd, stdout=out) linfos += _file._linfos fileno += 1 pg.done() return TempFile(self._ctx, tmp_file, linfos)
def _to_frame_gen(self, _c, usecols): f = _open_with_codec(self._ctx, self.abspath) if _c.show_prog: pg = ProgressBar('to_frame', _c.linecnt) tfp = None hasna = False smap = {} _c.lineno = 0 fdates = [date._sdate for date in self._linfos.dates] nodes = [node for node in self._linfos.nodes] kinds = [kind for kind in self._linfos.kinds] for line in f: if tfp is None: tfp = self._to_frame_init_tfp() slineno = 0 hasna = False if _c.show_prog: pg.animate(_c.lineno) fdate = fdates[_c.lineno] _date, level, msg, hasna = \ _to_frame_convert_line(tfp, fdate, line, hasna, smap) if _date is not None: tfp.msgs.append(msg) tfp.levels.append(level) tfp.dates.append(_date) tfp.nodes.append(nodes[_c.lineno]) tfp.kinds.append(kinds[_c.lineno]) _c.lineno += 1 slineno += 1 if slineno >= _c.chunk_cnt: df = self._to_frame_build_data_frame(tfp, hasna, usecols) tfp = None yield df df = self._to_frame_build_data_frame(tfp, hasna, usecols) if _c.show_prog: pg.done() yield df
def _load_files_info(ctx, prog_cb): global _load_errors root_list = [] rv, msg = load_files_precalc(ctx, root_list) root_list, filecnt = rv fileno = 0 pg = ProgressBar('collecting file info', filecnt, prog_cb) errors = [] for _root in root_list: fileno, errs = _load_files_root(ctx, _root, filecnt, fileno, pg) if len(errs) > 0: errors += errs pg.done() for _, fobj in ctx.fields.iteritems(): fobj._file_filter() if msg is not None: nprint(msg) _load_errors = errors if len(errors) > 0: nprint(errors[-4:])