def test_file(self): model = GenericModel(source=get_path(self.MODEL_PATH)) self._validate_meta(model) vendor = configuration.VENDOR configuration.VENDOR = None try: model = GenericModel(source=get_path(self.MODEL_PATH)) self._validate_meta(model) finally: configuration.VENDOR = vendor
def test_pickle(self): docfreq = GenericModel(source=get_path(self.DOCFREQ_PATH)) res = pickle.dumps(docfreq) docfreq_rec = pickle.loads(res) for k in docfreq.__dict__: self.assertEqual(docfreq.__dict__[k], docfreq_rec.__dict__[k])
def publish_model(args: argparse.Namespace, backend: StorageBackend, log: logging.Logger): """ Pushes the model to Google Cloud Storage and updates the index file. :param args: :class:`argparse.Namespace` with "model", "backend", "args", "force" and \ "update_default". :return: None if successful, 1 otherwise. """ path = os.path.abspath(args.model) try: model = GenericModel(source=path, dummy=True) except ValueError as e: log.critical('"model" must be a path: %s', e) return 1 except Exception as e: log.critical("Failed to load the model: %s: %s" % (type(e).__name__, e)) return 1 meta = model.meta with backend.lock(): model_url = backend.upload_model(path, meta, args.force) log.info("Uploaded as %s", model_url) log.info("Updating the models index...") index = backend.fetch_index() index["models"].setdefault(meta["model"], {})[meta["uuid"]] = \ extract_index_meta(meta, model_url) if args.update_default: index["models"][meta["model"]][Model.DEFAULT_NAME] = meta["uuid"] backend.upload_index(index)
def test_bad_code(self): def route(url): self.assertEqual("https://bad_code", url) return 404 storage_backend.requests = FakeRequests(route) with self.assertRaises(ValueError): GenericModel(source="https://bad_code", backend=self.backend)
def test_error(self): success = True try: model = GenericModel(source="f64bacd4-67fb-4c64-8382-399a8e7db52a") success = False except ValueError: pass self.assertTrue(success)
def test_pickle(self): docfreq = GenericModel(source=get_path(self.DOCFREQ_PATH)) res = pickle.dumps(docfreq) docfreq_rec = pickle.loads(res) for k in docfreq.__dict__: if k != "tree": self.assertEqual(getattr(docfreq, k), getattr(docfreq_rec, k), k)
def test_url(self): def route(url): self.assertEqual("https://xxx", url) with open(get_path(self.DOCFREQ_PATH), "rb") as fin: return fin.read() modelforge.gcs_backend.requests = FakeRequests(route) model = GenericModel(source="https://xxx", backend=self.backend) self._validate_meta(model)
def test_id(self): def route(url): self.assertEqual("https://xxx", url) with open(get_path(self.DOCFREQ_PATH), "rb") as fin: return fin.read() back.requests = FakeRequests(route) model = GenericModel(source="f64bacd4-67fb-4c64-8382-399a8e7db52a", backend=self.backend) self._validate_meta(model)
def _load_generic_model(source: str, backend: StorageBackend, log: logging.Logger) -> Optional[GenericModel]: try: return GenericModel(source, backend=backend) except ValueError as e: log.critical('"input" must be a path: %s', e) return None except Exception as e: log.critical("Failed to load the model: %s: %s" % (type(e).__name__, e)) return None
def test_bad_code(self): def route(url): self.assertEqual("https://bad_code", url) return 404 back.requests = FakeRequests(route) success = True try: GenericModel(source="https://bad_code", backend=self.backend) success = False except ValueError: pass self.assertTrue(success)
def test_id(self): def route(url): if GCSBackend.INDEX_FILE in url: return '{"models": {"docfreq": {' \ '"f64bacd4-67fb-4c64-8382-399a8e7db52a": ' \ '{"url": "https://xxx"}}}}'.encode() self.assertEqual("https://xxx", url) with open(get_path(self.DOCFREQ_PATH), "rb") as fin: return fin.read() modelforge.gcs_backend.requests = FakeRequests(route) model = GenericModel(source="f64bacd4-67fb-4c64-8382-399a8e7db52a", backend=self.backend) self._validate_meta(model)
def publish_model(args: argparse.Namespace, backend: StorageBackend, log: logging.Logger): """ Push the model to Google Cloud Storage and updates the index file. :param args: :class:`argparse.Namespace` with "model", "backend", "args", "force", "meta" \ "update_default", "username", "password", "remote_repo", "template_model", \ "template_readme" and "log_level". :param backend: Backend which is responsible for working with model files. :param log: Logger supplied by supply_backend :return: None if successful, 1 otherwise. """ path = os.path.abspath(args.model) try: model = GenericModel(source=path, dummy=True) except ValueError as e: log.critical('"model" must be a path: %s', e) return 1 except Exception as e: log.critical("Failed to load the model: %s: %s" % (type(e).__name__, e)) return 1 base_meta = model.meta try: model_url = backend.upload_model(path, base_meta, args.force) except ModelAlreadyExistsError: return 1 log.info("Uploaded as %s", model_url) with open(os.path.join(args.meta), encoding="utf-8") as _in: extra_meta = json.load(_in) model_type, model_uuid = base_meta["model"], base_meta["uuid"] meta = extract_model_meta(base_meta, extra_meta, model_url) log.info("Updating the models index...") try: template_model = backend.index.load_template(args.template_model) template_readme = backend.index.load_template(args.template_readme) except ValueError: return 1 backend.index.add_model(model_type, model_uuid, meta, template_model, args.update_default) backend.index.update_readme(template_readme) try: backend.index.upload("add", {"model": model_type, "uuid": model_uuid}) except ValueError: # TODO: replace with PorcelainError, see related TODO in index.py:181 return 1 log.info("Successfully published.")
def test_id(self): def route(url): self.assertEqual("https://xxx", url) with open(get_path(self.MODEL_PATH), "rb") as fin: return fin.read() storage_backend.requests = FakeRequests(route) cleaned = False def fake_rmtree(path): nonlocal cleaned cleaned = True with patch("shutil.rmtree", fake_rmtree): model = GenericModel(source=UUID, backend=self.backend) self._validate_meta(model) self.assertTrue(cleaned)
def dump_model(args: argparse.Namespace, backend: StorageBackend, log: logging.Logger): """ Prints the information about the model. :param args: :class:`argparse.Namespace` with "input", "backend", "args", "username", "password", "remote_repo" and "log_level". :param backend: Backend which is responsible for working with model files. :param log: Logger supplied by supply_backend :return: None """ try: print(GenericModel(args.input, backend=backend)) except ValueError as e: log.critical('"input" must be a path: %s', e) return 1 except Exception as e: log.critical("Failed to load the model: %s: %s" % (type(e).__name__, e)) return 1
def test_error(self): with self.assertRaises(ValueError): GenericModel(source="f64bacd4-67fb-4c64-8382-399a8e7db52a")
def test_file(self): model = GenericModel(source=get_path(self.DOCFREQ_PATH)) self._validate_meta(model)
def test_error(self): with self.assertRaises(ValueError): GenericModel(source=UUID)