def test_acl(self):
client = APIClient(self.access_key, self.secret_key, self.api_server)
req = APIRequest(client, self.endpoint, self.version)
r = req.get('/resource')
self.assertEqual(r.status_code, 200)
client = APIClient(self.access_key, self.secret_key, self.api_server)
req = APIRequest(client, self.endpoint, self.version)
r = req.get('/forbidden/')
self.assertEqual(r.status_code, GATEWAY_ERROR_STATUS_CODE)
def test_signature(self):
client = APIClient(self.access_key, 'bad secret key', self.api_server)
req = APIRequest(client, self.endpoint, self.version)
r = req.get('/resource/')
self.assertEqual(r.status_code, GATEWAY_ERROR_STATUS_CODE)
client = APIClient(self.access_key, 'bad secret key', self.api_server)
req = APIRequest(client, self.endpoint, self.version)
r = req.get('/resource/')
self.assertEqual(r.status_code, GATEWAY_ERROR_STATUS_CODE)
def setUp(self):
self.api = APIClient(base_url="http://localhost:5100/v1.0")
self.created_tasks = []
self.task = dict(
name="test task events",
type="training",
)
res, self.task_id = self.api.send('tasks.create',
self.task,
extract="id")
assert (res.meta.result_code == 200)
self.created_tasks.append(self.task_id)
def test_auth(self):
client = APIClient(self.access_key, self.secret_key, self.api_server)
req = APIRequest(client, self.endpoint, self.version)
r = req.get('/resource')
self.assertEqual(r.status_code, 200)
r = req.get('/resource/not_exist')
self.assertEqual(r.status_code, 404)
def test_old_api(self):
legacy_params = {"legacy.1": "val1", "TF_DEFINE/param2": "val2"}
legacy_config = {"design": "hello"}
task_id, _ = self.new_task(execution={
"parameters": legacy_params,
"model_desc": legacy_config
})
config = self._config_dict_from_list(
self._new_config_from_legacy(legacy_config))
params = self._param_dict_from_list(
self._new_params_from_legacy(legacy_params))
old_api = APIClient(base_url="http://localhost:8008/v2.8")
task = old_api.tasks.get_all_ex(id=[task_id]).tasks[0]
self.assertEqual(legacy_params, task.execution.parameters)
self.assertEqual(legacy_config, task.execution.model_desc)
self.assertEqual(params, task.hyperparams)
self.assertEqual(config, task.configuration)
modified_params = {"legacy.2": "val2"}
modified_config = {"design": "by"}
old_api.tasks.edit(task=task_id,
execution=dict(parameters=modified_params,
model_desc=modified_config))
task = old_api.tasks.get_all_ex(id=[task_id]).tasks[0]
self.assertEqual(modified_params, task.execution.parameters)
self.assertEqual(modified_config, task.execution.model_desc)
def test_aes_get(self):
client = APIClient(self.access_key, self.secret_key, self.api_server)
req = APIRequest(client, self.endpoint, self.version, encrypt_type='aes')
r = req.get('/resource/')
self.assertEqual(r.status_code, 200)
self.assertEqual(utf8('get'), utf8(r.content))
def test_post_img(self):
client = APIClient(self.access_key, self.secret_key, self.api_server)
req = APIRequest(client, self.endpoint, self.version)
with open('img.jpg', 'rb') as f:
body = f.read()
r = req.post('/resource/', data=body)
self.assertEqual(r.status_code, 200)
self.assertEqual(utf8(r.content), utf8(body))
def testBackwardsCompatibility(self):
new_api = self.api
self.api = APIClient(base_url="http://localhost:8008/v2.2")
entity_tags = {
"model": "archived",
"project": "public",
"task": "development",
}
for name, system_tag in entity_tags.items():
create_func = getattr(self, f"_temp_{name}")
_id = create_func(tags=[system_tag, "test"])
names = f"{name}s"
# when accessed through the old api all the tags are in the tags field
self.assertGetById(service=names,
entity=name,
_id=_id,
tags=[system_tag, "test"])
entities = self._send(names,
"get_all",
name="Test tags",
tags=[f"-{system_tag}"])[names]
self.assertNotFound(_id, entities)
# when accessed through the new api the tags are in tags and system_tags fields
self.assertGetById(
service=names,
entity=name,
_id=_id,
tags=["test"],
system_tags=[system_tag],
api=new_api,
)
# update operation, remove system tag through the old api
self._send(names, "update", tags=["test"], **{name: _id})
self.assertGetById(service=names,
entity=name,
_id=_id,
tags=["test"])
def test_post_json(self):
client = APIClient(self.access_key, self.secret_key, self.api_server)
req = APIRequest(client, self.endpoint, self.version)
json_data = {
'a': 1,
'b': 'test string',
'c': '中文'
}
body = json.dumps(json_data, ensure_ascii=False)
r = req.post('/resource/', json=json_data)
self.assertEqual(r.status_code, 200)
self.assertEqual(utf8(r.content), utf8(body))
def setUp(self, version="1.7"):
self._api = APIClient(base_url=f"http://localhost:8008/v{version}")
self._deferred = []
header(self.id())
class TestDatasetsService(unittest.TestCase):
def setUp(self):
self.api = APIClient(base_url="http://localhost:5100/v1.0")
self.created_tasks = []
self.task = dict(
name="test task events",
type="training",
)
res, self.task_id = self.api.send('tasks.create',
self.task,
extract="id")
assert (res.meta.result_code == 200)
self.created_tasks.append(self.task_id)
def tearDown(self):
log.info("Cleanup...")
for task_id in self.created_tasks:
try:
self.api.send('tasks.delete', dict(task=task_id, force=True))
except Exception as ex:
log.exception(ex)
def create_task_event(self, type, iteration):
return {
"worker": "test",
"type": type,
"task": self.task_id,
"iter": iteration,
"timestamp": es_factory.get_timestamp_millis()
}
def copy_and_update(self, src_obj, new_data):
obj = src_obj.copy()
obj.update(new_data)
return obj
def test_task_logs(self):
events = []
for iter in range(10):
log_event = self.create_task_event("log", iteration=iter)
events.append(
self.copy_and_update(
log_event, {
"msg":
"This is a log message from test task iter " +
str(iter)
}))
# sleep so timestamp is not the same
import time
time.sleep(0.01)
self.send_batch(events)
data = self.api.events.get_task_log(task=self.task_id)
assert len(data["events"]) == 10
self.api.tasks.reset(task=self.task_id)
data = self.api.events.get_task_log(task=self.task_id)
assert len(data["events"]) == 0
def test_task_plots(self):
event = self.create_task_event("plot", 0)
event["metric"] = "roc"
event.update({
"plot_str":
json.dumps({
"data": [{
"x": [0, 1, 2, 3, 4, 5, 6, 7, 8],
"y": [0, 1, 2, 3, 4, 5, 6, 7, 8],
"text": [
"Th=0.1", "Th=0.2", "Th=0.3", "Th=0.4", "Th=0.5",
"Th=0.6", "Th=0.7", "Th=0.8"
],
"name":
'class1'
}, {
"x": [0, 1, 2, 3, 4, 5, 6, 7, 8],
"y": [2.0, 3.0, 5.0, 8.2, 6.4, 7.5, 9.2, 8.1, 10.0],
"text": [
"Th=0.1", "Th=0.2", "Th=0.3", "Th=0.4", "Th=0.5",
"Th=0.6", "Th=0.7", "Th=0.8"
],
"name":
'class2',
}],
"layout": {
"title": "ROC for iter 0",
"xaxis": {
"title": 'my x axis'
},
"yaxis": {
"title": 'my y axis'
}
}
})
})
self.send(event)
event = self.create_task_event("plot", 100)
event["metric"] = "confusion"
event.update({
"plot_str":
json.dumps({
"data": [{
"y":
["lying", "sitting", "standing", "people", "backgroun"],
"x":
["lying", "sitting", "standing", "people", "backgroun"],
"z": [[758, 163, 0, 0, 23], [63, 858, 3, 0, 0],
[0, 50, 188, 21, 35], [
0,
22,
8,
40,
4,
], [12, 91, 26, 29, 368]],
"type":
"heatmap"
}],
"layout": {
"title": "Confusion Matrix for iter 100",
"xaxis": {
"title": "Predicted value"
},
"yaxis": {
"title": "Real value"
}
}
})
})
self.send(event)
data = self.api.events.get_task_plots(task=self.task_id)
assert len(data["plots"]) == 2
self.api.tasks.reset(task=self.task_id)
data = self.api.events.get_task_plots(task=self.task_id)
assert len(data["plots"]) == 0
def send_batch(self, events):
self.api.send_batch('events.add_batch', events)
def send(self, event):
self.api.send('events.add', event)
class TestTags(TestService):
def setUp(self, version="2.3"):
super().setUp(version)
def testPartition(self):
tags, system_tags = partition_tags("project", ["test"])
self.assertTagsEqual(tags, ["test"])
self.assertTagsEqual(system_tags, [])
tags, system_tags = partition_tags("project", ["test", "archived"])
self.assertTagsEqual(tags, ["test"])
self.assertTagsEqual(system_tags, ["archived"])
tags, system_tags = partition_tags("project", ["test", "archived"],
["custom"])
self.assertTagsEqual(tags, ["test"])
self.assertTagsEqual(system_tags, ["archived", "custom"])
tags, system_tags = partition_tags(
"task", ["test", "development", "annotator20", "Annotation"])
self.assertTagsEqual(tags, ["test"])
self.assertTagsEqual(system_tags,
["development", "annotator20", "Annotation"])
def testBackwardsCompatibility(self):
new_api = self.api
self.api = APIClient(base_url="http://localhost:8008/v2.2")
entity_tags = {
"model": "archived",
"project": "public",
"task": "development",
}
for name, system_tag in entity_tags.items():
create_func = getattr(self, f"_temp_{name}")
_id = create_func(tags=[system_tag, "test"])
names = f"{name}s"
# when accessed through the old api all the tags are in the tags field
self.assertGetById(service=names,
entity=name,
_id=_id,
tags=[system_tag, "test"])
entities = self._send(names,
"get_all",
name="Test tags",
tags=[f"-{system_tag}"])[names]
self.assertNotFound(_id, entities)
# when accessed through the new api the tags are in tags and system_tags fields
self.assertGetById(
service=names,
entity=name,
_id=_id,
tags=["test"],
system_tags=[system_tag],
api=new_api,
)
# update operation, remove system tag through the old api
self._send(names, "update", tags=["test"], **{name: _id})
self.assertGetById(service=names,
entity=name,
_id=_id,
tags=["test"])
def testProjectTags(self):
pr_id = self._temp_project(system_tags=["default"])
# Test getting project with system tags
projects = self.api.projects.get_all(name="Test tags").projects
self.assertFound(pr_id, ["default"], projects)
projects = self.api.projects.get_all(name="Test tags",
system_tags=["default"]).projects
self.assertFound(pr_id, ["default"], projects)
projects = self.api.projects.get_all(name="Test tags",
system_tags=["-default"]).projects
self.assertNotFound(pr_id, projects)
self.api.projects.update(project=pr_id, system_tags=[])
projects = self.api.projects.get_all(name="Test tags",
system_tags=["-default"]).projects
self.assertFound(pr_id, [], projects)
# Test task statistics and delete
task1_id = self._temp_task(name="Tags test1",
project=pr_id,
system_tags=["active"])
self._run_task(task1_id)
task2_id = self._temp_task(name="Tags test2",
project=pr_id,
system_tags=["archived"])
projects = self.api.projects.get_all_ex(name="Test tags").projects
self.assertFound(pr_id, [], projects)
projects = self.api.projects.get_all_ex(name="Test tags",
include_stats=True).projects
project = next(p for p in projects if p.id == pr_id)
self.assertProjectStats(project)
with self.api.raises(bad_request.ProjectHasTasks):
self.api.projects.delete(project=pr_id)
self.api.projects.delete(project=pr_id, force=True)
def testModelTags(self):
model_id = self._temp_model(system_tags=["default"])
models = self.api.models.get_all_ex(name="Test tags",
system_tags=["default"]).models
self.assertFound(model_id, ["default"], models)
models = self.api.models.get_all_ex(name="Test tags",
system_tags=["-default"]).models
self.assertNotFound(model_id, models)
self.api.models.update(model=model_id, system_tags=[])
models = self.api.models.get_all_ex(name="Test tags",
system_tags=["-default"]).models
self.assertFound(model_id, [], models)
def testTaskTags(self):
task_id = self._temp_task(name="Test tags", system_tags=["active"])
tasks = self.api.tasks.get_all_ex(name="Test tags",
system_tags=["active"]).tasks
self.assertFound(task_id, ["active"], tasks)
tasks = self.api.tasks.get_all_ex(name="Test tags",
system_tags=["-active"]).tasks
self.assertNotFound(task_id, tasks)
self.api.tasks.update(task=task_id, system_tags=[])
tasks = self.api.tasks.get_all_ex(name="Test tags",
system_tags=["-active"]).tasks
self.assertFound(task_id, [], tasks)
# test development system tag
self.api.tasks.started(task=task_id)
self.api.tasks.stop(task=task_id)
task = self.api.tasks.get_by_id(task=task_id).task
self.assertEqual(task.status, "in_progress")
self.api.tasks.update(task=task_id, system_tags=["development"])
self.api.tasks.stop(task=task_id)
task = self.api.tasks.get_by_id(task=task_id).task
self.assertEqual(task.status, "stopped")
def assertProjectStats(self, project: AttrDict):
self.assertEqual(set(project.stats.keys()), {"active"})
self.assertAlmostEqual(project.stats.active.total_runtime, 1, places=0)
for status, count in project.stats.active.status_count.items():
self.assertEqual(count, 1 if status == "stopped" else 0)
def _run_task(self, task_id):
"""Imitate 1 second of running"""
self.api.tasks.started(task=task_id)
sleep(1)
self.api.tasks.stopped(task=task_id)
def _temp_project(self, **kwargs):
self._update_missing(kwargs, name="Test tags", description="test")
return self.create_temp("projects", **kwargs)
def _temp_model(self, **kwargs):
self._update_missing(kwargs,
name="Test tags",
uri="file:///a/b",
labels={})
return self.create_temp("models", **kwargs)
def _temp_task(self, **kwargs):
self._update_missing(kwargs,
name="Test tags",
type="testing",
input=dict(view=dict()))
return self.create_temp("tasks", **kwargs)
@staticmethod
def _update_missing(target: dict, **update):
target.update({k: v for k, v in update.items() if k not in target})
def _send(self, service, action, **kwargs):
api = kwargs.pop("api", self.api)
return AttrDict(api.send(f"{service}.{action}", kwargs)[1])
def assertGetById(self,
service,
entity,
_id,
tags,
system_tags=None,
**kwargs):
entity = self._send(service, "get_by_id", **{entity: _id},
**kwargs)[entity]
self.assertEqual(set(entity.tags), set(tags))
if system_tags is not None:
self.assertEqual(set(entity.system_tags), set(system_tags))
def assertFound(self, _id: str, system_tags: Sequence[str],
res: Sequence[AttrDict]):
found = next((r for r in res if _id == r.id), None)
assert found
self.assertTagsEqual(found.system_tags, system_tags)
def assertNotFound(self, _id: str, res: Sequence[AttrDict]):
self.assertFalse(any(r for r in res if r.id == _id))
def assertTagsEqual(self, tags: Sequence[str],
expected_tags: Sequence[str]):
self.assertEqual(set(tags), set(expected_tags))
def test_login_refresh_logout(self):
client = APIClient(self.access_key, self.secret_key, self.api_server)
req = APIRequest(client, self.endpoint, self.version)
r = req.get('/login')
self.assertEqual(r.status_code, 405)
json_data = {
'name': 'name',
'password': '******'
}
r = req.post('/login', json=json_data)
self.assertEqual(r.status_code, 200)
schema = {
'code': {
'type': 'integer',
'required': True,
'allowed': [APIStatusCode.SUCCESS]
},
'msg': {
'type': 'string',
'required': True,
},
'data': {
'type': 'dict',
'required': True,
}
}
v = Validator(schema=schema, allow_unknown=True)
logger.debug(r.json())
logger.debug(v.validate(r.json()))
self.assertEqual(v.validate(r.json()), True)
# refresh_token
json_data = {
'refresh_token': r.json()['data']['refresh_token']
}
logger.debug(json_data)
r = req.post('/token', json=json_data)
self.assertEqual(r.status_code, 200)
v = Validator(schema=schema, allow_unknown=True)
logger.debug(r.json())
self.assertEqual(v.validate(r.json()), True)
# ---------------------
json_data = {
'test': 'test'
}
auth_req = APIRequest(client, 'test_api_login', 'v1')
access_token = r.json()['data']['access_token']
ar = auth_req.post('/protected/?access_token=%s' % access_token,
json=json_data)
self.assertEqual(r.status_code, 200)
v = Validator(schema=schema, allow_unknown=True)
logger.debug(ar.json())
self.assertEqual(v.validate(ar.json()), True)
# ---------------------
# logout
json_data = {
'access_token': r.json()['data']['access_token']
}
r = req.post('/logout', json=json_data)
self.assertEqual(r.status_code, 200)
schema = {
'code': {
'type': 'integer',
'required': True,
'allowed': [APIStatusCode.SUCCESS]
},
'msg': {
'type': 'string',
'required': True,
}
}
v = Validator(schema=schema, allow_unknown=True)
self.assertEqual(v.validate(r.json()), True)