def test_file(self):
model = GenericModel(source=get_path(self.MODEL_PATH))
self._validate_meta(model)
vendor = configuration.VENDOR
configuration.VENDOR = None
try:
model = GenericModel(source=get_path(self.MODEL_PATH))
self._validate_meta(model)
finally:
configuration.VENDOR = vendor
def test_pickle(self):
docfreq = GenericModel(source=get_path(self.DOCFREQ_PATH))
res = pickle.dumps(docfreq)
docfreq_rec = pickle.loads(res)
for k in docfreq.__dict__:
self.assertEqual(docfreq.__dict__[k], docfreq_rec.__dict__[k])
def publish_model(args: argparse.Namespace, backend: StorageBackend,
log: logging.Logger):
"""
Pushes the model to Google Cloud Storage and updates the index file.
:param args: :class:`argparse.Namespace` with "model", "backend", "args", "force" and \
"update_default".
:return: None if successful, 1 otherwise.
"""
path = os.path.abspath(args.model)
try:
model = GenericModel(source=path, dummy=True)
except ValueError as e:
log.critical('"model" must be a path: %s', e)
return 1
except Exception as e:
log.critical("Failed to load the model: %s: %s" %
(type(e).__name__, e))
return 1
meta = model.meta
with backend.lock():
model_url = backend.upload_model(path, meta, args.force)
log.info("Uploaded as %s", model_url)
log.info("Updating the models index...")
index = backend.fetch_index()
index["models"].setdefault(meta["model"], {})[meta["uuid"]] = \
extract_index_meta(meta, model_url)
if args.update_default:
index["models"][meta["model"]][Model.DEFAULT_NAME] = meta["uuid"]
backend.upload_index(index)
def test_bad_code(self):
def route(url):
self.assertEqual("https://bad_code", url)
return 404
storage_backend.requests = FakeRequests(route)
with self.assertRaises(ValueError):
GenericModel(source="https://bad_code", backend=self.backend)
def test_error(self):
success = True
try:
model = GenericModel(source="f64bacd4-67fb-4c64-8382-399a8e7db52a")
success = False
except ValueError:
pass
self.assertTrue(success)
def test_pickle(self):
docfreq = GenericModel(source=get_path(self.DOCFREQ_PATH))
res = pickle.dumps(docfreq)
docfreq_rec = pickle.loads(res)
for k in docfreq.__dict__:
if k != "tree":
self.assertEqual(getattr(docfreq, k), getattr(docfreq_rec, k), k)
def test_url(self):
def route(url):
self.assertEqual("https://xxx", url)
with open(get_path(self.DOCFREQ_PATH), "rb") as fin:
return fin.read()
modelforge.gcs_backend.requests = FakeRequests(route)
model = GenericModel(source="https://xxx", backend=self.backend)
self._validate_meta(model)
def test_id(self):
def route(url):
self.assertEqual("https://xxx", url)
with open(get_path(self.DOCFREQ_PATH), "rb") as fin:
return fin.read()
back.requests = FakeRequests(route)
model = GenericModel(source="f64bacd4-67fb-4c64-8382-399a8e7db52a",
backend=self.backend)
self._validate_meta(model)
def _load_generic_model(source: str, backend: StorageBackend,
log: logging.Logger) -> Optional[GenericModel]:
try:
return GenericModel(source, backend=backend)
except ValueError as e:
log.critical('"input" must be a path: %s', e)
return None
except Exception as e:
log.critical("Failed to load the model: %s: %s" %
(type(e).__name__, e))
return None
def test_bad_code(self):
def route(url):
self.assertEqual("https://bad_code", url)
return 404
back.requests = FakeRequests(route)
success = True
try:
GenericModel(source="https://bad_code", backend=self.backend)
success = False
except ValueError:
pass
self.assertTrue(success)
def test_id(self):
def route(url):
if GCSBackend.INDEX_FILE in url:
return '{"models": {"docfreq": {' \
'"f64bacd4-67fb-4c64-8382-399a8e7db52a": ' \
'{"url": "https://xxx"}}}}'.encode()
self.assertEqual("https://xxx", url)
with open(get_path(self.DOCFREQ_PATH), "rb") as fin:
return fin.read()
modelforge.gcs_backend.requests = FakeRequests(route)
model = GenericModel(source="f64bacd4-67fb-4c64-8382-399a8e7db52a",
backend=self.backend)
self._validate_meta(model)
def publish_model(args: argparse.Namespace, backend: StorageBackend,
log: logging.Logger):
"""
Push the model to Google Cloud Storage and updates the index file.
:param args: :class:`argparse.Namespace` with "model", "backend", "args", "force", "meta" \
"update_default", "username", "password", "remote_repo", "template_model", \
"template_readme" and "log_level".
:param backend: Backend which is responsible for working with model files.
:param log: Logger supplied by supply_backend
:return: None if successful, 1 otherwise.
"""
path = os.path.abspath(args.model)
try:
model = GenericModel(source=path, dummy=True)
except ValueError as e:
log.critical('"model" must be a path: %s', e)
return 1
except Exception as e:
log.critical("Failed to load the model: %s: %s" %
(type(e).__name__, e))
return 1
base_meta = model.meta
try:
model_url = backend.upload_model(path, base_meta, args.force)
except ModelAlreadyExistsError:
return 1
log.info("Uploaded as %s", model_url)
with open(os.path.join(args.meta), encoding="utf-8") as _in:
extra_meta = json.load(_in)
model_type, model_uuid = base_meta["model"], base_meta["uuid"]
meta = extract_model_meta(base_meta, extra_meta, model_url)
log.info("Updating the models index...")
try:
template_model = backend.index.load_template(args.template_model)
template_readme = backend.index.load_template(args.template_readme)
except ValueError:
return 1
backend.index.add_model(model_type, model_uuid, meta, template_model,
args.update_default)
backend.index.update_readme(template_readme)
try:
backend.index.upload("add", {"model": model_type, "uuid": model_uuid})
except ValueError: # TODO: replace with PorcelainError, see related TODO in index.py:181
return 1
log.info("Successfully published.")
def test_id(self):
def route(url):
self.assertEqual("https://xxx", url)
with open(get_path(self.MODEL_PATH), "rb") as fin:
return fin.read()
storage_backend.requests = FakeRequests(route)
cleaned = False
def fake_rmtree(path):
nonlocal cleaned
cleaned = True
with patch("shutil.rmtree", fake_rmtree):
model = GenericModel(source=UUID, backend=self.backend)
self._validate_meta(model)
self.assertTrue(cleaned)
def dump_model(args: argparse.Namespace, backend: StorageBackend, log: logging.Logger):
"""
Prints the information about the model.
:param args: :class:`argparse.Namespace` with "input", "backend", "args", "username",
"password", "remote_repo" and "log_level".
:param backend: Backend which is responsible for working with model files.
:param log: Logger supplied by supply_backend
:return: None
"""
try:
print(GenericModel(args.input, backend=backend))
except ValueError as e:
log.critical('"input" must be a path: %s', e)
return 1
except Exception as e:
log.critical("Failed to load the model: %s: %s" % (type(e).__name__, e))
return 1
def test_error(self):
with self.assertRaises(ValueError):
GenericModel(source="f64bacd4-67fb-4c64-8382-399a8e7db52a")
def test_file(self):
model = GenericModel(source=get_path(self.DOCFREQ_PATH))
self._validate_meta(model)
def test_error(self):
with self.assertRaises(ValueError):
GenericModel(source=UUID)
