def load_info(mod, file_type, prog_fn=None): """Load file information. Remove temp files when necessary. """ _remove_old() cfg = make_config() _dir = cfg['data_dir'] encoding = cfg["data_encoding"] if isinstance(encoding, dict): encoding = encoding[file_type] if file_type in encoding else '' if encoding.startswith('utf-16'): _dir = get_conv_dir() ctx = Context(mod, _dir, encoding, file_type) mod['date'] = DateField(ctx) mod['kind'] = KindField(ctx) mod['node'] = NodeField(ctx) # collect files _load_files_info(ctx, prog_fn) if encoding.startswith('utf-16'): ctx.encoding = 'utf-8' ctx.updated = time.time() # make selectors _files = FileSelector(ctx, ctx.mod['all_files'], 'file') mod['files'] = _files mod['kinds'] = Selector(_files, 'kind') mod['dates'] = Selector(_files, 'date') mod['nodes'] = Selector(_files, 'node') mod['slot'] = SlotMap(ctx)
def test_common_config(fxcfg): invalidate_config() cfg = make_config(fxcfg) assert type(cfg) == dict assert cfg['val0'] == 0 assert cfg['val1'] == 1 assert cfg['val2'] == 2 cp = os.environ['WZDAT_CFG'] cfgenv0 = os.path.join(cp, 'test_bcfg') cfgenv1 = os.path.join(cp, 'test_cfg') assert cfg['env0'] == cfgenv0 assert cfg['env1'] == cfgenv1
def test_common_cache(fxlogs, fxdb): log = fxlogs[0] from wzdat.util import get_cache_dir cfg = make_config() d = get_cache_dir() assert os.listdir(d) == [] find_files_and_save(cfg['data_dir'], 'log', True, log.file_filter) assert os.listdir(d) == ['log_found_files.pkl'] os.remove(os.path.join(d, 'log_found_files.pkl')) cache_files() assert len(os.listdir(d)) > 0
def _reset_data(): print 'reset data' from wzdat.util import gen_dummydata, get_var_dir cfg = make_config() ddir = cfg['data_dir'] # remove previous dummy data if os.path.isdir(ddir): print 'remove datadir' shutil.rmtree(ddir) # generate new dummy data gen_dummydata(ddir, 10, False) # make _var_ dir get_var_dir()
import re import os import wzdat from wzdat.make_config import make_config cfg = make_config() localhost = os.environ['WZDAT_HOST'] TEST_IMPORT_ALL = False def test_selector_basic(fxlogs): # Log log = fxlogs[0] update_err = wzdat.selector.get_load_errors() assert len(update_err) == 0 assert len(log.files) == 450 assert set(log.kinds) == set([log.kind.auth, log.kind.community, log.kind.game]) assert set(log.nodes) == set([log.node.jp_node_1, log.node.jp_node_2, log.node.jp_node_3, log.node.kr_node_1, log.node.kr_node_2, log.node.kr_node_3, log.node.us_node_1, log.node.us_node_2, log.node.us_node_3]) assert set(log.dates) == set([log.date.D2014_02_24, log.date.D2014_02_25, log.date.D2014_02_26, log.date.D2014_02_27, log.date.D2014_02_28, log.date.D2014_03_01, log.date.D2014_03_02, log.date.D2014_03_03, log.date.D2014_03_04, log.date.D2014_03_05])
def _remove_old(): nprint('deleting old files...') cfg = make_config() tmp_dir = get_tmp_dir() remove_old_tmps(tmp_dir, TMP_PREFIX, cfg["tmp_valid_hour"]) remove_old_tmps(tmp_dir, NAMED_TMP_PREFIX, cfg["named_tmp_valid_hour"])