def merge(self):
"""Merge all files into a temp file and return it."""
linfos = LineInfo()
tmp_file, _ = unique_tmp_path(TMP_PREFIX)
filecnt = len(self.files)
fileno = 0
pg = ProgressBar('merging', filecnt)
with open(tmp_file, 'w') as out:
for _file in self.files:
pg.animate(fileno)
cmd = ['cat', _file.abspath]
check_call(cmd, stdout=out)
linfos += _file._linfos
fileno += 1
pg.done()
return TempFile(self._ctx, tmp_file, linfos)
def _to_frame_gen(self, _c, usecols):
f = _open_with_codec(self._ctx, self.abspath)
if _c.show_prog:
pg = ProgressBar('to_frame', _c.linecnt)
tfp = None
hasna = False
smap = {}
_c.lineno = 0
fdates = [date._sdate for date in self._linfos.dates]
nodes = [node for node in self._linfos.nodes]
kinds = [kind for kind in self._linfos.kinds]
for line in f:
if tfp is None:
tfp = self._to_frame_init_tfp()
slineno = 0
hasna = False
if _c.show_prog:
pg.animate(_c.lineno)
fdate = fdates[_c.lineno]
_date, level, msg, hasna = \
_to_frame_convert_line(tfp, fdate, line, hasna, smap)
if _date is not None:
tfp.msgs.append(msg)
tfp.levels.append(level)
tfp.dates.append(_date)
tfp.nodes.append(nodes[_c.lineno])
tfp.kinds.append(kinds[_c.lineno])
_c.lineno += 1
slineno += 1
if slineno >= _c.chunk_cnt:
df = self._to_frame_build_data_frame(tfp, hasna, usecols)
tfp = None
yield df
df = self._to_frame_build_data_frame(tfp, hasna, usecols)
if _c.show_prog:
pg.done()
yield df
def _load_files_info(ctx, prog_cb):
global _load_errors
root_list = []
rv, msg = load_files_precalc(ctx, root_list)
root_list, filecnt = rv
fileno = 0
pg = ProgressBar('collecting file info', filecnt, prog_cb)
errors = []
for _root in root_list:
fileno, errs = _load_files_root(ctx, _root, filecnt, fileno, pg)
if len(errs) > 0:
errors += errs
pg.done()
for _, fobj in ctx.fields.iteritems():
fobj._file_filter()
if msg is not None:
nprint(msg)
_load_errors = errors
if len(errors) > 0:
nprint(errors[-4:])
