def _load_files_root(ctx, _root, filecnt, fileno, pg):
root = _root[0]
fields = ctx.fields.values()
fieldcnt = len(fields)
field_getter = [field._value_fn for field in fields]
converted = []
errs = []
for filename in _root[1]:
if FORWARDER_LOG_PREFIX in filename:
continue
pg.animate(fileno)
if 'progress_cb' in ctx.dict:
ctx.dict['progress_cb'](pg.prev_pct)
abspath = os.path.join(root, filename)
abspath = _load_files_root_check_conv(ctx, abspath, converted)
fileo = FileValue(ctx, abspath)
vals, field_errs = _load_files_root_vals(fieldcnt, fields, fileo,
field_getter)
# remove incompatible files
if len(field_errs) > 0:
errs += field_errs
continue
try:
fileo._set_props(vals)
except InvalidProp:
pass
else:
ctx.files.append(fileo)
fileno += 1
if ctx.encoding.startswith('utf-16'):
if len(converted) > 0:
nprint("%d files have been converted." % len(converted))
return fileno, errs
def _load_files_info(ctx, prog_cb):
global _load_errors
root_list = []
rv, msg = load_files_precalc(ctx, root_list)
root_list, filecnt = rv
fileno = 0
pg = ProgressBar('collecting file info', filecnt, prog_cb)
errors = []
for _root in root_list:
fileno, errs = _load_files_root(ctx, _root, filecnt, fileno, pg)
if len(errs) > 0:
errors += errs
pg.done()
for _, fobj in ctx.fields.iteritems():
fobj._file_filter()
if msg is not None:
nprint(msg)
_load_errors = errors
if len(errors) > 0:
nprint(errors[-4:])
def _remove_old():
nprint('deleting old files...')
cfg = make_config()
tmp_dir = get_tmp_dir()
remove_old_tmps(tmp_dir, TMP_PREFIX, cfg["tmp_valid_hour"])
remove_old_tmps(tmp_dir, NAMED_TMP_PREFIX, cfg["named_tmp_valid_hour"])
