def generate_keyvi(key_values, filename):
dictionary_compiler = KeyOnlyDictionaryCompiler({"memory_limit_mb": "10"})
for key in key_values:
dictionary_compiler.Add(key)
dictionary_compiler.Compile()
dictionary_compiler.WriteToFile(filename)
def test_manifest():
c = KeyOnlyDictionaryCompiler({"memory_limit_mb": "10"})
c.Add("Leela")
c.Add("Kif")
c.SetManifest('{"author": "Zapp Brannigan"}')
with tmp_dictionary(c, 'brannigan_manifest.kv') as d:
m = json.loads(d.GetManifest())
assert m['author'] == "Zapp Brannigan"
def test_get_value_key_only():
c = KeyOnlyDictionaryCompiler({"memory_limit_mb": "10"})
c.Add("abc")
c.Add("abd")
with tmp_dictionary(c, 'match_object_key_only.kv') as d:
m = d["abc"]
assert m.GetValue() == ''
m = d["abd"]
assert m.GetValue() == ''
def test_statistics():
c = KeyOnlyDictionaryCompiler({"memory_limit_mb": "10"})
c.Add("Leela")
c.Add("Kif")
c.SetManifest('{"author": "Zapp Brannigan"}')
with tmp_dictionary(c, 'brannigan_statistics.kv') as d:
stats = d.GetStatistics()
gen = stats.get('General', {})
man = json.loads(d.GetManifest())
size = int(gen.get('number_of_keys', 0))
assert size == 2
assert man.get('author') == "Zapp Brannigan"
def compile(args):
params = {key: value for key, value in args.compiler_params}
dict_type = args.dict_type
if dict_type == 'json':
dictionary = JsonDictionaryCompiler(params)
elif dict_type == 'string':
dictionary = StringDictionaryCompiler(params)
elif dict_type == 'int':
dictionary = IntDictionaryCompiler(params)
elif dict_type == 'completion':
dictionary = CompletionDictionaryCompiler(params)
elif dict_type == 'key-only':
dictionary = KeyOnlyDictionaryCompiler(params)
else:
return 'Must never reach here'
with open(args.input_file) as file_in:
for line in file_in:
line = line.rstrip('\n')
try:
splits = line.split('\t')
if dict_type == 'key-only':
dictionary.Add(splits[0])
elif dict_type == 'int' or dict_type == 'completion':
dictionary.Add(splits[0], int(splits[1]))
else:
dictionary.Add(splits[0], splits[1])
except:
print('Can not parse line: {}'.format(line))
dictionary.Compile()
dictionary.WriteToFile(args.output_file)
def test_manifest_after_compile():
c = KeyOnlyDictionaryCompiler({"memory_limit_mb": "10"})
c.Add("Leela")
c.Add("Kif")
c.Compile()
c.SetManifest('{"author": "Zapp Brannigan"}')
file_name = os.path.join(tempfile.gettempdir(), 'brannigan_manifest2.kv')
try:
c.WriteToFile(file_name)
d = Dictionary(file_name)
m = json.loads(d.GetManifest())
assert m['author'] == "Zapp Brannigan"
del d
finally:
os.remove(file_name)
def test_compiler_no_compile_edge_case():
c = KeyOnlyDictionaryCompiler({"memory_limit_mb": "10"})
c.Add("abc")
c.Add("abd")
del c
def test_size():
c = KeyOnlyDictionaryCompiler({"memory_limit_mb": "10"})
c.Add("Leela")
c.Add("Kif")
with tmp_dictionary(c, 'brannigan_size.kv') as d:
assert len(d) == 2