def _load_files_root(ctx, _root, filecnt, fileno, pg): root = _root[0] fields = ctx.fields.values() fieldcnt = len(fields) field_getter = [field._value_fn for field in fields] converted = [] errs = [] for filename in _root[1]: if FORWARDER_LOG_PREFIX in filename: continue pg.animate(fileno) if 'progress_cb' in ctx.dict: ctx.dict['progress_cb'](pg.prev_pct) abspath = os.path.join(root, filename) abspath = _load_files_root_check_conv(ctx, abspath, converted) fileo = FileValue(ctx, abspath) vals, field_errs = _load_files_root_vals(fieldcnt, fields, fileo, field_getter) # remove incompatible files if len(field_errs) > 0: errs += field_errs continue try: fileo._set_props(vals) except InvalidProp: pass else: ctx.files.append(fileo) fileno += 1 if ctx.encoding.startswith('utf-16'): if len(converted) > 0: nprint("%d files have been converted." % len(converted)) return fileno, errs
def _load_files_info(ctx, prog_cb): global _load_errors root_list = [] rv, msg = load_files_precalc(ctx, root_list) root_list, filecnt = rv fileno = 0 pg = ProgressBar('collecting file info', filecnt, prog_cb) errors = [] for _root in root_list: fileno, errs = _load_files_root(ctx, _root, filecnt, fileno, pg) if len(errs) > 0: errors += errs pg.done() for _, fobj in ctx.fields.iteritems(): fobj._file_filter() if msg is not None: nprint(msg) _load_errors = errors if len(errors) > 0: nprint(errors[-4:])
def _remove_old(): nprint('deleting old files...') cfg = make_config() tmp_dir = get_tmp_dir() remove_old_tmps(tmp_dir, TMP_PREFIX, cfg["tmp_valid_hour"]) remove_old_tmps(tmp_dir, NAMED_TMP_PREFIX, cfg["named_tmp_valid_hour"])