def test_0_init_model_store_and_list_online_models(self):
print("\n\33[33m{}\33[0m".format("0. Loading the model store and querying the online database ..."))
from cube.io_utils.model_store import ModelMetadata, ModelStore
model_store_object = ModelStore()
online_models = model_store_object.list_online_models()
print("Found "+str(len(online_models))+ " models online.")
self.assertTrue(len(online_models)>0)
def test_3_2_import_model_in_store(self):
print("\n\33[33m{}\33[0m".format(
"3.2. Import locally created model in store ..."))
command = "python3 " + os.path.join(
self.scripts_path, "import_model.py") + " " + os.path.join(
self.local_model_repo, "my_model-1.0.zip")
print("\n\t\t\33[32m{}\n{}\33[0m".format("Import command:", command))
'''popen = subprocess.Popen(command.split(" ") , stdout=subprocess.PIPE, universal_newlines=True)
output = []
for stdout_line in iter(popen.stdout.readline, ""):
print(stdout_line[:-1])
if stdout_line.strip()!= "":
output.append(stdout_line[:-1])
popen.stdout.close()
return_code = popen.wait()
'''
os.system(command)
# check it is in store
from cube.io_utils.model_store import ModelMetadata, ModelStore
model_store_object = ModelStore()
local_models = model_store_object.list_local_models()
test = False
for model, version in local_models:
if model == "my_model":
test = True
self.assertTrue(test)
def test_get_latest_model_versions(self):
httpretty.register_uri(httpretty.GET,
ModelStore.MODELS_PATH_CLOUD_ALL,
body=XML_CONTENT_FOR_GET_LATEST_VERSION)
model_store = ModelStore()
latest_model_versions = model_store.get_latest_model_versions()
self.assertEqual(latest_model_versions, {'en': '2.0', 'ro': '1.0'})
def test_5_cleanup(self):
print("\n\33[33m{}\33[0m".format("5. Cleanup after myself ..."))
# delete my_model from the store, if it exists
from cube.io_utils.model_store import ModelMetadata, ModelStore
model_store_object = ModelStore()
local_models = model_store_object.list_local_models()
print("\tFound local models:" + str(local_models))
self.assertTrue(len(local_models) > 0)
for model, version in local_models:
if model == "my_model":
# delete local model
print("\tDeleting 'my_model-1.0'...")
model_store_object.delete_model("my_model", "1.0")
local_models_new = model_store_object.list_local_models()
print("\tFound local models:" + str(local_models_new))
self.assertTrue(len(local_models) > len(local_models_new))
break
# delete my_model.zip, if it exists
if os.path.exists(
os.path.join(self.local_model_repo, "my_model-1.0.zip")):
os.remove(os.path.join(self.local_model_repo, "my_model-1.0.zip"))
self.assertFalse(
os.path.exists(
os.path.join(self.local_model_repo, "my_model-1.0.zip")))
def test_4_2_import_model_in_store(self):
print("\n\33[33m{}\33[0m".format(
"4.2. Import locally created model in store (with prior cleanup)..."
))
# first check local models
from cube.io_utils.model_store import ModelMetadata, ModelStore
model_store_object = ModelStore()
local_models = model_store_object.list_local_models()
print("\tFound local models:" + str(local_models))
self.assertTrue(len(local_models) > 0)
# search for my_model
for model, version in local_models:
if model == "my_model":
# delete local model
print("\tDeleting 'my_model-1.0'...")
model_store_object.delete_model("my_model", "1.0")
local_models_new = model_store_object.list_local_models()
print("\tFound local models:" + str(local_models_new))
self.assertTrue(len(local_models) > len(local_models_new))
# import new model
command = "python3 " + os.path.join(
self.scripts_path, "import_model.py") + " " + os.path.join(
self.local_model_repo, "my_model-1.0.zip")
print("\n\t\t\33[32m{}\n{}\33[0m".format("Import command:", command))
'''popen = subprocess.Popen(command.split(" ") , stdout=subprocess.PIPE, universal_newlines=True)
output = []
for stdout_line in iter(popen.stdout.readline, ""):
print(stdout_line[:-1])
if stdout_line.strip()!= "":
output.append(stdout_line[:-1])
popen.stdout.close()
return_code = popen.wait()
'''
os.system(command)
test = os.path.exists(
os.path.join(self.local_model_repo, "my_model-1.0.zip"))
self.assertTrue(test)
# check it is in store
local_models = model_store_object.list_local_models()
test = False
for model, version in local_models:
if model == "my_model":
test = True
self.assertTrue(test)
import sys
import os
# Append parent dir to sys path.
os.sys.path.insert(0,
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from cube.io_utils.model_store import ModelMetadata, ModelStore
from datetime import datetime
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: python3 import_model.py path-to-my-model")
print("Example: 'python3 import_model.py my_model-1.0.zip")
sys.exit(0)
model_path = sys.argv[1]
if not os.path.exists(model_path):
raise Exception("File {} not found!".format(model_path))
if ".zip" not in model_path:
raise Exception(
"File {} must be a zip! See export_model.py and the online tutorial on how to package a locally trained model!"
.format(model_path))
model_store_object = ModelStore()
model_store_object.import_local_model(model_path)
from pathlib import Path
# Append parent dir to sys path.
os.sys.path.insert(0,
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from cube.io_utils.model_store import ModelMetadata, ModelStore
from datetime import datetime
if __name__ == "__main__":
input_models_root_folder = "/work/nlp_cube_models/raw/models-1.1"
output_models_root_folder = "/work/nlp_cube_models/models_zip/1.1"
version = 1.1
model_store = ModelStore()
local_models = [
os.path.basename(os.path.normpath(dI))
for dI in os.listdir(input_models_root_folder)
if os.path.isdir(os.path.join(input_models_root_folder, dI))
]
#print(local_models)
model_tuples = [ #folder, language_code, embedding code
("UD_Afrikaans-AfriBooms", "af", "af"),
("UD_Ancient_Greek-PROIEL", "grc", "grc"),
("UD_Arabic-PADT", "ar", "ar"), ("UD_Armenian-ArmTDP", "hy", "hy"),
("UD_Basque-BDT", "eu", "eu"), ("UD_Bulgarian-BTB", "bg", "bg"),
("UD_Buryat-BDT", "bxr", "bxr"), ("UD_Catalan-AnCora", "ca", "ca"),
("UD_Chinese-GSD", "zh", "zh"), ("UD_Croatian-SET", "hr", "hr"),
("UD_Czech-PDT", "cs", "cs"), ("UD_Danish-DDT", "da", "da"),
print("\n\tModel folder: " + model_folder_path)
print("\tUse tokenizer: {}".format(_tokenizer))
print("\tUse compound word expander: {}".format(_compound_word_expander))
print("\tUse lemmatizer: {}".format(_lemmatizer))
print("\tUse tagger: {}".format(_tagger))
print("\tUse parser: {}\n".format(_parser))
# check if path exists
if not os.path.exists(model_folder_path):
raise Exception("Model folder not found!")
# check if metadata exists
if not os.path.exists(os.path.join(model_folder_path, "metadata.json")):
raise Exception("metadata.json not found in model folder!")
# check if metadata is valid
metadata = ModelMetadata()
metadata.read(os.path.join(model_folder_path, "metadata.json"))
output_folder_path = os.path.dirname(model_folder_path)
model_store_object = ModelStore(disk_path=output_folder_path)
model_store_object.package_model(
model_folder_path,
output_folder_path,
metadata,
should_contain_tokenizer=_tokenizer,
should_contain_compound_word_expander=_compound_word_expander,
should_contain_lemmatizer=_lemmatizer,
should_contain_tagger=_tagger,
should_contain_parser=_parser)
#metrics = ["Tokens", "Sentences", "Words", "UPOS", "XPOS", "UFeats", "AllTags", "Lemmas", "UAS", "LAS", "CLAS", "MLAS", "BLEX"]
#example usage: metrics_test = conll_eval(system,gold)
# test_tok_f1, test_ss_f1 = metrics_test["Tokens"].f1*100., metrics_test["Sentences"].f1*100.
def conll_eval(system_file, gold_file):
gold_ud = load_conllu_file(gold_file)
system_ud = load_conllu_file(system_file)
return evaluate(gold_ud, system_ud)
if len(sys.argv) == 3:
local_model_path = sys.argv[2]
print("Using local model path: " + local_model_path)
else:
local_model_path = None
model_store_object = ModelStore(local_model_path)
local_embeddings_path = model_store_object.embeddings_repository
local_model_path = model_store_object.disk_path
online_models = model_store_object.list_online_models()
print("Found {} online models".format(len(online_models)))
local_models = model_store_object.list_local_models()
print("Found {} local models".format(len(local_models)))
model_count = len(online_models)
# step 1. download all models
for online_model in online_models:
model, version = online_model[0], online_model[1]
if not online_model in local_models: