def test_extract_index_meta(self):
self.maxDiff = None
dt = met.get_datetime_now()
base_meta = {
"created_at": dt,
"model": "docfreq",
"series": "pga-2018",
"uuid": "12345678-9abc-def0-1234-56789abcdef0",
"version": [1, 0, 2],
"parent": "f64bacd4-67fb-4c64-8382-399a8e7db52a",
"dependencies": ["1e3da42a-28b6-4b33-94a2-a5671f4102f4"],
"description": "model_description",
"license": "MIT",
}
extra_meta = {
"code": "model_code %s",
"datasets": [["any", "https://any"]],
"description": "override",
"references": [["any", "ref"]],
"tags": ["one", "two"],
"extra": {
"feature": "value"
},
}
def route(url):
self.assertEqual("https://xxx", url)
return b"content"
met.requests = FakeRequests(route)
model_meta = met.extract_model_meta(base_meta, extra_meta,
"https://xxx")
self.assertIsInstance(model_meta, dict)
self.assertGreater(len(met.format_datetime(dt)), 0)
self.assertDictEqual(
model_meta, {
"default": {
"default": "12345678-9abc-def0-1234-56789abcdef0",
"description": "model_description",
"code": "model_code %s"
},
"model": {
"created_at": met.format_datetime(dt),
"code": "model_code %s",
"description": "model_description",
"dependencies": ["1e3da42a-28b6-4b33-94a2-a5671f4102f4"],
"license": "MIT",
"parent": "f64bacd4-67fb-4c64-8382-399a8e7db52a",
"datasets": [["any", "https://any"]],
"references": [["any", "ref"]],
"size": "7 Bytes",
"series": "pga-2018",
"url": "https://xxx",
"tags": ["one", "two"],
"version": [1, 0, 2],
"extra": {
"feature": "value"
}
}
})
def test_list(self):
def route(url):
return """
{"models": {
"xxx": {
"f64bacd4-67fb-4c64-8382-399a8e7db52a": {
"url": "https://xxx",
"created_at": "12:00"
},
"064bacd4-67fb-4c64-8382-399a8e7db52a": {
"url": "https://xxx2",
"created_at": "13:00"
},
"default": "f64bacd4-67fb-4c64-8382-399a8e7db52a"
},
"yyy": {
"f74bacd4-67fb-4c64-8382-399a8e7db52b": {
"url": "https://yyy",
"created_at": "11:00"
},
"f64bacd4-67fb-4c64-8382-399a8e7db52b": {
"url": "https://yyy",
"created_at": "12:00"
},
"default": "f64bacd4-67fb-4c64-8382-399a8e7db52b"
},
"zzz": {
"f64bacd4-67fb-4c64-8382-399a8e7db52c": {
"url": "https://zzz",
"created_at": "12:00"
},
"default": "f64bacd4-67fb-4c64-8382-399a8e7db52c"
}
}}""".encode()
modelforge.gcs_backend.requests = FakeRequests(route)
args = argparse.Namespace(backend=None, args=None)
with captured_output() as (out, _, _):
list_models(args)
out = out.getvalue().split("\n")
for name, uuids in (("xxx", ("064bacd4-67fb-4c64-8382-399a8e7db52a",
"f64bacd4-67fb-4c64-8382-399a8e7db52a")),
("yyy", ("f64bacd4-67fb-4c64-8382-399a8e7db52b",
"f74bacd4-67fb-4c64-8382-399a8e7db52b")),
("zzz",
("f64bacd4-67fb-4c64-8382-399a8e7db52c", ))):
idx = out.index(name)
self.assertGreaterEqual(idx, 0)
im = -1
while idx < len(out):
idx += 1
im += 1
if out[idx].startswith(" * "):
self.assertIn(uuids[im], out[idx])
break
else:
self.assertEqual(out[idx][:4], " ")
self.assertIn(uuids[im], out[idx])
else:
self.fail("The default model was not found.")
def test_bad_code(self):
def route(url):
self.assertEqual("https://bad_code", url)
return 404
storage_backend.requests = FakeRequests(route)
with self.assertRaises(ValueError):
GenericModel(source="https://bad_code", backend=self.backend)
def test_url(self):
def route(url):
self.assertEqual("https://xxx", url)
with open(get_path(self.DOCFREQ_PATH), "rb") as fin:
return fin.read()
modelforge.gcs_backend.requests = FakeRequests(route)
model = GenericModel(source="https://xxx", backend=self.backend)
self._validate_meta(model)
def test_extract_index_meta(self):
self.maxDiff = None
dt = str(datetime.now())
base_meta = {
"created_at": dt,
"model": "docfreq",
"uuid": "12345678-9abc-def0-1234-56789abcdef0",
"version": [1, 0, 2]
}
extra_meta = {
"code": "readme_code %s",
"description": "readme_description",
"model": {
"code": "model_code %s",
"description": "model_description",
"dependencies": ["1e3da42a-28b6-4b33-94a2-a5671f4102f4"],
"extra": {
"ex": "tra"
},
"license": ["", "undecided"],
"parent": "f64bacd4-67fb-4c64-8382-399a8e7db52a",
"references": [["any", "ref"]]
}
}
def route(url):
self.assertEqual("https://xxx", url)
return b"content"
met.requests = FakeRequests(route)
model_meta = met.extract_model_meta(base_meta, extra_meta,
"https://xxx")
self.assertIsInstance(model_meta, dict)
self.assertDictEqual(
model_meta, {
"default": {
"code": "readme_code %s",
"default": "12345678-9abc-def0-1234-56789abcdef0",
"description": "readme_description"
},
"model": {
"created_at": dt,
"code": "model_code %s",
"description": "model_description",
"dependencies": ["1e3da42a-28b6-4b33-94a2-a5671f4102f4"],
"extra": {
"ex": "tra"
},
"license": ["", "undecided"],
"parent": "f64bacd4-67fb-4c64-8382-399a8e7db52a",
"references": [["any", "ref"]],
"size": "7 Bytes",
"url": "https://xxx",
"version": [1, 0, 2],
}
})
def test_docfreq_url(self):
def route(url):
self.assertEqual("https://xxx", url)
with open(self._get_path(self.DOCFREQ_PATH), "rb") as fin:
return fin.read()
gcs_backend.requests = FakeRequests(route)
with captured_output() as (out, _, _):
dump_model(self._get_args(input="https://xxx"))
self.assertEqual(out.getvalue(), self.DOCFREQ_DUMP)
def test_id(self):
def route(url):
self.assertEqual("https://xxx", url)
with open(get_path(self.DOCFREQ_PATH), "rb") as fin:
return fin.read()
back.requests = FakeRequests(route)
model = GenericModel(source="f64bacd4-67fb-4c64-8382-399a8e7db52a",
backend=self.backend)
self._validate_meta(model)
def test_auto(self):
class FakeModel(GenericModel):
NAME = "docfreq"
def route(url):
self.assertEqual("https://xxx", url)
with open(get_path(self.DOCFREQ_PATH), "rb") as fin:
return fin.read()
back.requests = FakeRequests(route)
model = FakeModel(backend=self.backend)
self._validate_meta(model)
def test_bad_code(self):
def route(url):
self.assertEqual("https://bad_code", url)
return 404
back.requests = FakeRequests(route)
success = True
try:
GenericModel(source="https://bad_code", backend=self.backend)
success = False
except ValueError:
pass
self.assertTrue(success)
def test_id(self):
def route(url):
if GCSBackend.INDEX_FILE in url:
return '{"models": {"docfreq": {' \
'"f64bacd4-67fb-4c64-8382-399a8e7db52a": ' \
'{"url": "https://xxx"}}}}'.encode()
self.assertEqual("https://xxx", url)
with open(get_path(self.DOCFREQ_PATH), "rb") as fin:
return fin.read()
modelforge.gcs_backend.requests = FakeRequests(route)
model = GenericModel(source="f64bacd4-67fb-4c64-8382-399a8e7db52a",
backend=self.backend)
self._validate_meta(model)
def test_docfreq_id(self):
def route(url):
if gcs_backend.GCSBackend.INDEX_FILE in url:
return '{"models": {"docfreq": {' \
'"f64bacd4-67fb-4c64-8382-399a8e7db52a": ' \
'{"url": "https://xxx"}}}}'.encode()
self.assertEqual("https://xxx", url)
with open(self._get_path(self.DOCFREQ_PATH), "rb") as fin:
return fin.read()
gcs_backend.requests = FakeRequests(route)
with captured_output() as (out, err, _):
dump_model(
self._get_args(input="f64bacd4-67fb-4c64-8382-399a8e7db52a"))
self.assertEqual(out.getvalue(), self.DOCFREQ_DUMP)
self.assertFalse(err.getvalue())
def test_id(self):
def route(url):
self.assertEqual("https://xxx", url)
with open(get_path(self.MODEL_PATH), "rb") as fin:
return fin.read()
storage_backend.requests = FakeRequests(route)
cleaned = False
def fake_rmtree(path):
nonlocal cleaned
cleaned = True
with patch("shutil.rmtree", fake_rmtree):
model = GenericModel(source=UUID, backend=self.backend)
self._validate_meta(model)
self.assertTrue(cleaned)
def test_auto(self):
class FakeModel(GenericModel):
NAME = "docfreq"
def route(url):
if GCSBackend.INDEX_FILE in url:
return '{"models": {"docfreq": {' \
'"f64bacd4-67fb-4c64-8382-399a8e7db52a": ' \
'{"url": "https://xxx"}, ' \
'"default": "f64bacd4-67fb-4c64-8382-399a8e7db52a"' \
'}}}'.encode()
self.assertEqual("https://xxx", url)
with open(get_path(self.DOCFREQ_PATH), "rb") as fin:
return fin.read()
modelforge.gcs_backend.requests = FakeRequests(route)
model = FakeModel(backend=create_backend())
self._validate_meta(model)