def load_info(mod, file_type, prog_fn=None):
"""Load file information.
Remove temp files when necessary.
"""
_remove_old()
cfg = make_config()
_dir = cfg['data_dir']
encoding = cfg["data_encoding"]
if isinstance(encoding, dict):
encoding = encoding[file_type] if file_type in encoding else ''
if encoding.startswith('utf-16'):
_dir = get_conv_dir()
ctx = Context(mod, _dir, encoding, file_type)
mod['date'] = DateField(ctx)
mod['kind'] = KindField(ctx)
mod['node'] = NodeField(ctx)
# collect files
_load_files_info(ctx, prog_fn)
if encoding.startswith('utf-16'):
ctx.encoding = 'utf-8'
ctx.updated = time.time()
# make selectors
_files = FileSelector(ctx, ctx.mod['all_files'], 'file')
mod['files'] = _files
mod['kinds'] = Selector(_files, 'kind')
mod['dates'] = Selector(_files, 'date')
mod['nodes'] = Selector(_files, 'node')
mod['slot'] = SlotMap(ctx)
def test_common_config(fxcfg):
invalidate_config()
cfg = make_config(fxcfg)
assert type(cfg) == dict
assert cfg['val0'] == 0
assert cfg['val1'] == 1
assert cfg['val2'] == 2
cp = os.environ['WZDAT_CFG']
cfgenv0 = os.path.join(cp, 'test_bcfg')
cfgenv1 = os.path.join(cp, 'test_cfg')
assert cfg['env0'] == cfgenv0
assert cfg['env1'] == cfgenv1
def test_common_cache(fxlogs, fxdb):
log = fxlogs[0]
from wzdat.util import get_cache_dir
cfg = make_config()
d = get_cache_dir()
assert os.listdir(d) == []
find_files_and_save(cfg['data_dir'], 'log', True, log.file_filter)
assert os.listdir(d) == ['log_found_files.pkl']
os.remove(os.path.join(d, 'log_found_files.pkl'))
cache_files()
assert len(os.listdir(d)) > 0
def _reset_data():
print 'reset data'
from wzdat.util import gen_dummydata, get_var_dir
cfg = make_config()
ddir = cfg['data_dir']
# remove previous dummy data
if os.path.isdir(ddir):
print 'remove datadir'
shutil.rmtree(ddir)
# generate new dummy data
gen_dummydata(ddir, 10, False)
# make _var_ dir
get_var_dir()
import re
import os
import wzdat
from wzdat.make_config import make_config
cfg = make_config()
localhost = os.environ['WZDAT_HOST']
TEST_IMPORT_ALL = False
def test_selector_basic(fxlogs):
# Log
log = fxlogs[0]
update_err = wzdat.selector.get_load_errors()
assert len(update_err) == 0
assert len(log.files) == 450
assert set(log.kinds) == set([log.kind.auth, log.kind.community,
log.kind.game])
assert set(log.nodes) == set([log.node.jp_node_1, log.node.jp_node_2,
log.node.jp_node_3, log.node.kr_node_1,
log.node.kr_node_2, log.node.kr_node_3,
log.node.us_node_1, log.node.us_node_2,
log.node.us_node_3])
assert set(log.dates) == set([log.date.D2014_02_24, log.date.D2014_02_25,
log.date.D2014_02_26, log.date.D2014_02_27,
log.date.D2014_02_28, log.date.D2014_03_01,
log.date.D2014_03_02, log.date.D2014_03_03,
log.date.D2014_03_04, log.date.D2014_03_05])
def _remove_old():
nprint('deleting old files...')
cfg = make_config()
tmp_dir = get_tmp_dir()
remove_old_tmps(tmp_dir, TMP_PREFIX, cfg["tmp_valid_hour"])
remove_old_tmps(tmp_dir, NAMED_TMP_PREFIX, cfg["named_tmp_valid_hour"])
