def test_manifest_for_merger():
try:
c = JsonDictionaryCompiler({"memory_limit_mb": "10"})
c.Add("abc", '{"a" : 2}')
c.Compile()
c.SetManifest('{"author": "Zapp Brannigan"}')
c.WriteToFile('manifest_json_merge1.kv')
del c
c2 = JsonDictionaryCompiler({"memory_limit_mb": "10"})
c2.Add("abd", '{"a" : 3}')
c2.Compile()
c2.SetManifest('{"author": "Leela"}')
c2.WriteToFile('manifest_json_merge2.kv')
del c2
merger = JsonDictionaryMerger({"memory_limit_mb": "10"})
merger.SetManifest('{"author": "Fry"}')
merger.Merge('manifest_json_merged.kv')
d = Dictionary('manifest_json_merged.kv')
m = json.loads(d.GetManifest())
assert m['author'] == "Fry"
del d
finally:
os.remove('manifest_json_merge1.kv')
os.remove('manifest_json_merge2.kv')
os.remove('manifest_json_merged.kv')
def compile(args):
params = {key: value for key, value in args.compiler_params}
dict_type = args.dict_type
if dict_type == 'json':
dictionary = JsonDictionaryCompiler(params)
elif dict_type == 'string':
dictionary = StringDictionaryCompiler(params)
elif dict_type == 'int':
dictionary = IntDictionaryCompiler(params)
elif dict_type == 'completion':
dictionary = CompletionDictionaryCompiler(params)
elif dict_type == 'key-only':
dictionary = KeyOnlyDictionaryCompiler(params)
else:
return 'Must never reach here'
with open(args.input_file) as file_in:
for line in file_in:
line = line.rstrip('\n')
try:
splits = line.split('\t')
if dict_type == 'key-only':
dictionary.Add(splits[0])
elif dict_type == 'int' or dict_type == 'completion':
dictionary.Add(splits[0], int(splits[1]))
else:
dictionary.Add(splits[0], splits[1])
except:
print('Can not parse line: {}'.format(line))
dictionary.Compile()
dictionary.WriteToFile(args.output_file)
def test_truncated_file_json():
c = JsonDictionaryCompiler({"memory_limit_mb": "10"})
c.Add('a', '{1:2}')
c.Add('b', '{2:4}')
c.Add('c', '{4:4}')
c.Add('d', '{2:3}')
c.Compile()
c.WriteToFile(os.path.join(tmp_dir, 'truncation_test.kv'))
size = os.path.getsize(os.path.join(tmp_dir, 'truncation_test.kv'))
fd_in = open(os.path.join(tmp_dir, 'truncation_test.kv'), 'rb')
fd = open(os.path.join(tmp_dir, 'truncation_test1.kv'), 'wb')
fd.write(fd_in.read(int(size / 2)))
fd.close()
fd2 = open(os.path.join(tmp_dir, 'truncation_test2.kv'), 'wb')
fd2.write(fd_in.read(int(size - 2)))
fd2.close()
with pytest.raises(ValueError):
d = Dictionary(os.path.join(tmp_dir, 'truncation_test1.kv'))
with pytest.raises(ValueError):
d = Dictionary(os.path.join(tmp_dir, 'truncation_test2.kv'))
os.remove(os.path.join(tmp_dir, 'truncation_test2.kv'))
os.remove(os.path.join(tmp_dir, 'truncation_test1.kv'))
os.remove(os.path.join(tmp_dir, 'truncation_test.kv'))
def generate_keyvi(key_values, filename):
dictionary_compiler = JsonDictionaryCompiler({"memory_limit_mb": "10"})
for key, value in key_values.items():
dictionary_compiler.Add(key, json.dumps(value))
dictionary_compiler.Compile()
dictionary_compiler.WriteToFile(filename)