def test_list_experiments(self):
experiments = [
ExperimentConfig(config={}).to_dict() for _ in range(10)
]
httpretty.register_uri(httpretty.GET,
BaseApiHandler.build_url(
self.api_config.base_url, '/',
'experiments'),
body=json.dumps({
'results': experiments,
'count': 10,
'next': None
}),
content_type='application/json',
status=200)
# Schema response
response = self.api_handler.list_experiments()
assert len(response['results']) == 10
assert isinstance(response['results'][0], ExperimentConfig)
# Raw response
self.set_raw_response()
response = self.api_handler.list_experiments()
assert len(response['results']) == 10
assert isinstance(response['results'][0], Mapping)
def test_update_experiment(self):
exp = ExperimentConfig(config={}).to_dict()
httpretty.register_uri(httpretty.PATCH,
BaseApiHandler.build_url(
self.api_config.base_url, '/', 'username',
'project_name', 'experiments', 1),
body=json.dumps(exp),
content_type='application/json',
status=200)
# Schema response
result = self.api_handler.update_experiment('username', 'project_name',
1, {'name': 'new'})
assert result.to_dict() == exp
# Raw response
self.set_raw_response()
result = self.api_handler.update_experiment('username', 'project_name',
1, {'name': 'new'})
assert result == exp
# Async
self.assert_async_call(api_handler_call=lambda: self.api_handler.
update_experiment('username',
'project_name',
1, {'name': 'new'},
background=True),
method='patch')
def test_resume_experiment_with_config(self):
exp = ExperimentConfig(config={}).to_dict()
config = {'config': {'declarations': {'lr': 0.1}}}
httpretty.register_uri(httpretty.POST,
BaseApiHandler.build_url(
self.api_config.base_url, '/', 'username',
'project_name', 'experiments', 1, 'resume'),
body=json.dumps(exp),
content_type='application/json',
status=200)
# Schema response
result = self.api_handler.resume('username', 'project_name', 1, config)
assert result.to_dict() == exp
# Raw response
self.set_raw_response()
result = self.api_handler.resume('username', 'project_name', 1, config)
assert result == exp
# Async
self.assert_async_call(
api_handler_call=lambda: self.api_handler.resume(
'username', 'project_name', 1, config, background=True),
method='post')
def test_get_bookmarked_experiments(self):
project_uuid = uuid.uuid4().hex
obj_uuid = uuid.uuid4().hex
objs = [{
'content_object':
ExperimentConfig(config={}, uuid=obj_uuid,
project=project_uuid).to_dict()
} for _ in range(10)]
httpretty.register_uri(httpretty.GET,
BaseApiHandler.build_url(
self.api_config.base_url, '/bookmarks',
'user', 'experiments'),
body=json.dumps({
'results': objs,
'count': 10,
'next': None
}),
content_type='application/json',
status=200)
# Schema response
result = self.api_handler.experiments('user')
assert len(result['results']) == 10
assert isinstance(result['results'][0], ExperimentConfig)
# Raw response
self.set_raw_response()
result = self.api_handler.experiments('user')
assert len(result['results']) == 10
assert isinstance(result['results'][0], Mapping)
def test_validate_config(self):
assert isinstance(
self.api_handler.validate_config(config={},
config_schema=BuildJobConfig),
BuildJobConfig)
with self.assertRaises(PolyaxonClientException):
self.api_handler.validate_config(config=ExperimentConfig(),
config_schema=BuildJobConfig)
def test_get_experiment(self):
exp = ExperimentConfig(config={}).to_dict()
httpretty.register_uri(httpretty.GET,
BaseApiHandler.build_url(
self.api_config.base_url, '/', 'username',
'project_name', 'experiments', 1),
body=json.dumps(exp),
content_type='application/json',
status=200)
# Schema response
result = self.api_handler.get_experiment('username', 'project_name', 1)
assert result.to_dict() == exp
# Raw response
self.set_raw_response()
result = self.api_handler.get_experiment('username', 'project_name', 1)
assert result == exp
def test_create_experiment(self):
project_uuid = uuid.uuid4().hex
obj = ExperimentConfig(project=project_uuid, config={})
obj_dict = obj.to_dict()
httpretty.register_uri(httpretty.POST,
BaseApiHandler.build_url(
self.api_config.base_url, '/', 'user',
'project', 'experiments'),
body=json.dumps(obj_dict),
content_type='application/json',
status=200)
# Schema response
result = self.api_handler.create_experiment('user', 'project', obj)
assert result.to_dict() == obj_dict
# Raw response
self.set_raw_response()
result = self.api_handler.create_experiment('user', 'project', obj)
assert result == obj_dict
# Async
self.assert_async_call(
api_handler_call=lambda: self.api_handler.create_experiment(
'user', 'project', obj, background=True),
method='post')
# Test create experiment with dict
httpretty.register_uri(httpretty.POST,
BaseApiHandler.build_url(
self.api_config.base_url, '/', 'user',
'project', 'experiments'),
body=json.dumps(obj_dict),
content_type='application/json',
status=200)
# Schema response
self.set_schema_response()
result = self.api_handler.create_experiment('user', 'project',
obj_dict)
assert result.to_dict() == obj_dict
# Raw response
self.set_raw_response()
result = self.api_handler.create_experiment('user', 'project',
obj_dict)
assert result == obj_dict
# Async
self.assert_async_call(
api_handler_call=lambda: self.api_handler.create_experiment(
'user', 'project', obj_dict, background=True),
method='post')
# Test create experiment with group
group_id = 1
httpretty.register_uri(httpretty.POST,
BaseApiHandler.build_url(
self.api_config.base_url, '/', 'user',
'project', 'experiments'),
body=json.dumps(obj_dict),
content_type='application/json',
status=200)
# Schema response
self.set_schema_response()
result = self.api_handler.create_experiment('user',
'project',
obj_dict,
group=group_id)
assert result.to_dict() == obj_dict
# Raw response
self.set_raw_response()
result = self.api_handler.create_experiment('user', 'project',
obj_dict)
assert result == obj.to_dict()
# Async
self.assert_async_call(
api_handler_call=lambda: self.api_handler.create_experiment(
'user', 'project', obj_dict, group=group_id, background=True),
method='post')
def test_list_experiments(self):
project_uuid = uuid.uuid4().hex
xp_uuid = uuid.uuid4().hex
xps = [
ExperimentConfig(config={}, uuid=xp_uuid,
project=project_uuid).to_dict() for _ in range(10)
]
httpretty.register_uri(httpretty.GET,
BaseApiHandler.build_url(
self.api_config.base_url, '/', 'user',
'project', 'experiments'),
body=json.dumps({
'results': xps,
'count': 10,
'next': None
}),
content_type='application/json',
status=200)
# Schema response
response = self.api_handler.list_experiments('user', 'project')
assert len(response['results']) == 10
assert isinstance(response['results'][0], ExperimentConfig)
# Raw response
self.set_raw_response()
response = self.api_handler.list_experiments('user', 'project')
assert len(response['results']) == 10
assert isinstance(response['results'][0], Mapping)
# Pagination
httpretty.register_uri(
httpretty.GET,
BaseApiHandler.build_url(self.api_config.base_url, '/', 'user',
'project', 'experiments') + '?offset=2',
body=json.dumps({
'results': xps,
'count': 10,
'next': None
}),
content_type='application/json',
status=200)
# Schema response
self.set_schema_response()
response = self.api_handler.list_experiments('user', 'project', page=2)
assert len(response['results']) == 10
assert isinstance(response['results'][0], ExperimentConfig)
# Raw response
self.set_raw_response()
response = self.api_handler.list_experiments('user', 'project', page=2)
assert len(response['results']) == 10
assert isinstance(response['results'][0], Mapping)
# Metrics & Declarations
for xp in xps:
xp['last_metric'] = {'loss': 0.1}
xp['declarations'] = {'foo': 'bar'}
httpretty.register_uri(
httpretty.GET,
BaseApiHandler.build_url(self.api_config.base_url, '/', 'user',
'project', 'experiments') +
'?metrics=true',
body=json.dumps({
'results': xps,
'count': 10,
'next': None
}),
content_type='application/json',
status=200)
response = self.api_handler.list_experiments('user',
'project',
metrics=True,
page=2)
assert len(response['results']) == 10
httpretty.register_uri(
httpretty.GET,
BaseApiHandler.build_url(self.api_config.base_url, '/', 'user',
'project', 'experiments') +
'?declarations=true',
body=json.dumps({
'results': xps,
'count': 10,
'next': None
}),
content_type='application/json',
status=200)
response = self.api_handler.list_experiments('user',
'project',
declarations=True,
page=2)
assert len(response['results']) == 10
# Query, Sort
httpretty.register_uri(
httpretty.GET,
BaseApiHandler.build_url(self.api_config.base_url, '/', 'username',
'project_name', 'experiments') +
'?independent=true&query=started_at:>=2010-10-10,sort=created_at',
body=json.dumps({
'results': xps,
'count': 10,
'next': None
}),
content_type='application/json',
status=200)
# Schema response
self.set_schema_response()
response = self.api_handler.list_experiments(
'user',
'project',
True,
query='started_at:>=2010-10-10',
sort='created_at')
assert len(response['results']) == 10
assert isinstance(response['results'][0], ExperimentConfig)
# Raw response
self.set_raw_response()
response = self.api_handler.list_experiments(
'user',
'project',
True,
query='started_at:>=2010-10-10',
sort='created_at')
assert len(response['results']) == 10
assert isinstance(response['results'][0], Mapping)
def test_list_experiments(self):
group_uuid = uuid.uuid4().hex
project_uuid = uuid.uuid4().hex
xp_uuid = uuid.uuid4().hex
xps = [
ExperimentConfig(uuid=xp_uuid,
config={},
project=project_uuid,
experiment_group=group_uuid).to_dict()
for _ in range(10)
]
httpretty.register_uri(
httpretty.GET,
BaseApiHandler.build_url(self.api_config.base_url, '/', 'username',
'project_name', 'experiments') +
'?group=1',
body=json.dumps({
'results': xps,
'count': 10,
'next': None
}),
content_type='application/json',
status=200)
# Schema response
response = self.api_handler.list_experiments('username',
'project_name', 1)
assert len(response['results']) == 10
assert isinstance(response['results'][0], ExperimentConfig)
assert response['count'] == 10
assert response['next'] is None
assert response['previous'] is None
# Raw response
self.set_raw_response()
response = self.api_handler.list_experiments('username',
'project_name', 1)
assert len(response['results']) == 10
assert isinstance(response['results'][0], Mapping)
assert response['count'] == 10
assert response['next'] is None
assert response['previous'] is None
# Pagination
httpretty.register_uri(
httpretty.GET,
BaseApiHandler.build_url(self.api_config.base_url, '/', 'username',
'project_name', 'experiments') +
'?group=1&offset=2',
body=json.dumps({
'results': xps,
'count': 10,
'next': None
}),
content_type='application/json',
status=200)
# Schema response
self.set_schema_response()
response = self.api_handler.list_experiments('username',
'project_name',
1,
page=2)
assert len(response['results']) == 10
assert isinstance(response['results'][0], ExperimentConfig)
# Raw response
self.set_raw_response()
response = self.api_handler.list_experiments('username',
'project_name',
1,
page=2)
assert len(response['results']) == 10
assert isinstance(response['results'][0], Mapping)
# Query, Sort
httpretty.register_uri(
httpretty.GET,
BaseApiHandler.build_url(self.api_config.base_url, '/', 'username',
'project_name', 'experiments') +
'?group=1&query=started_at:>=2010-10-10,sort=created_at',
body=json.dumps({
'results': xps,
'count': 10,
'next': None
}),
content_type='application/json',
status=200)
# Schema response
self.set_schema_response()
response = self.api_handler.list_experiments(
'username',
'project_name',
1,
query='started_at:>=2010-10-10',
sort='created_at')
assert len(response['results']) == 10
assert isinstance(response['results'][0], ExperimentConfig)
# Raw response
self.set_raw_response()
response = self.api_handler.list_experiments(
'username',
'project_name',
1,
query='started_at:>=2010-10-10',
sort='created_at')
assert len(response['results']) == 10
assert isinstance(response['results'][0], Mapping)