def test_manifest_for_merger(): try: c = JsonDictionaryCompiler({"memory_limit_mb": "10"}) c.Add("abc", '{"a" : 2}') c.Compile() c.SetManifest('{"author": "Zapp Brannigan"}') c.WriteToFile('manifest_json_merge1.kv') del c c2 = JsonDictionaryCompiler({"memory_limit_mb": "10"}) c2.Add("abd", '{"a" : 3}') c2.Compile() c2.SetManifest('{"author": "Leela"}') c2.WriteToFile('manifest_json_merge2.kv') del c2 merger = JsonDictionaryMerger({"memory_limit_mb": "10"}) merger.SetManifest('{"author": "Fry"}') merger.Merge('manifest_json_merged.kv') d = Dictionary('manifest_json_merged.kv') m = json.loads(d.GetManifest()) assert m['author'] == "Fry" del d finally: os.remove('manifest_json_merge1.kv') os.remove('manifest_json_merge2.kv') os.remove('manifest_json_merged.kv')
def compile(args): params = {key: value for key, value in args.compiler_params} dict_type = args.dict_type if dict_type == 'json': dictionary = JsonDictionaryCompiler(params) elif dict_type == 'string': dictionary = StringDictionaryCompiler(params) elif dict_type == 'int': dictionary = IntDictionaryCompiler(params) elif dict_type == 'completion': dictionary = CompletionDictionaryCompiler(params) elif dict_type == 'key-only': dictionary = KeyOnlyDictionaryCompiler(params) else: return 'Must never reach here' with open(args.input_file) as file_in: for line in file_in: line = line.rstrip('\n') try: splits = line.split('\t') if dict_type == 'key-only': dictionary.Add(splits[0]) elif dict_type == 'int' or dict_type == 'completion': dictionary.Add(splits[0], int(splits[1])) else: dictionary.Add(splits[0], splits[1]) except: print('Can not parse line: {}'.format(line)) dictionary.Compile() dictionary.WriteToFile(args.output_file)
def test_truncated_file_json(): c = JsonDictionaryCompiler({"memory_limit_mb": "10"}) c.Add('a', '{1:2}') c.Add('b', '{2:4}') c.Add('c', '{4:4}') c.Add('d', '{2:3}') c.Compile() c.WriteToFile(os.path.join(tmp_dir, 'truncation_test.kv')) size = os.path.getsize(os.path.join(tmp_dir, 'truncation_test.kv')) fd_in = open(os.path.join(tmp_dir, 'truncation_test.kv'), 'rb') fd = open(os.path.join(tmp_dir, 'truncation_test1.kv'), 'wb') fd.write(fd_in.read(int(size / 2))) fd.close() fd2 = open(os.path.join(tmp_dir, 'truncation_test2.kv'), 'wb') fd2.write(fd_in.read(int(size - 2))) fd2.close() with pytest.raises(ValueError): d = Dictionary(os.path.join(tmp_dir, 'truncation_test1.kv')) with pytest.raises(ValueError): d = Dictionary(os.path.join(tmp_dir, 'truncation_test2.kv')) os.remove(os.path.join(tmp_dir, 'truncation_test2.kv')) os.remove(os.path.join(tmp_dir, 'truncation_test1.kv')) os.remove(os.path.join(tmp_dir, 'truncation_test.kv'))
def run_compile(tmpdir): c = JsonDictionaryCompiler({ "memory_limit_mb": "10", "temporary_path": tmpdir }) c.Add("abc", "{'a':2}") c.Compile() assert os.listdir(tmpdir) != []
def generate_keyvi(key_values, filename): dictionary_compiler = JsonDictionaryCompiler({"memory_limit_mb": "10"}) for key, value in key_values.items(): dictionary_compiler.Add(key, json.dumps(value)) dictionary_compiler.Compile() dictionary_compiler.WriteToFile(filename)
def test_tmp_dir(): cwd = os.getcwd() os.chdir(tempfile.gettempdir()) try: os.mkdir("tmp_dir_test") os.chdir(os.path.join(tempfile.gettempdir(), "tmp_dir_test")) c = JsonDictionaryCompiler({"memory_limit_mb": "10"}) c.Add("abc", "{'a':2}") assert os.listdir('.') == [] c.Compile() assert os.listdir('.') == [] del c assert os.listdir('.') == [] finally: os.chdir(cwd) os.rmdir(os.path.join(tempfile.gettempdir(), "tmp_dir_test"))