def test_mixed_search_custom_index_range(self):
client = self.client
response = client.get('/api/runs/search/images/',
params={
'q': '',
'index_range': '3:5'
})
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(
decode_encoded_tree_stream(
response.iter_content(chunk_size=512 * 1024)))
self.assertEqual(1, len(decoded_response))
run_data = decoded_response[self.run_hash]
self.assertEqual([0, 5], run_data['ranges']['record_range'])
self.assertEqual([0, 5, 1], run_data['ranges']['record_slice'])
self.assertEqual([3, 5], run_data['ranges']['index_range'])
self.assertEqual([3, 5, 1], run_data['ranges']['index_slice'])
self.assertEqual(2, len(run_data['traces']))
trace_data = run_data['traces'][0]
self.assertEqual('image_lists', trace_data['name'])
self.assertEqual(5, len(trace_data['values']))
img_list = trace_data['values'][0]
self.assertEqual(2, len(img_list))
trace_data = run_data['traces'][1]
self.assertEqual('single_images', trace_data['name'])
self.assertEqual(5, len(trace_data['values']))
img_list = trace_data['values'][0]
self.assertListEqual([], img_list)
def test_search_aligned_metrics_api(self):
client = self.client
hash_names = []
for run, _ in zip(self.repo.iter_runs(), range(2)):
hash_names.append(run.hashname)
response = client.post('/api/runs/search/metric/align/', json={
'align_by': 'accuracy',
'runs': [{
'run_id': hash_names[0],
'traces': [{'metric_name': 'loss', 'slice': [0, 100, 1], 'context': {'is_training': False}}]
}, {
'run_id': hash_names[1],
'traces': [{'metric_name': 'loss', 'slice': [0, 100, 1], 'context': {'is_training': True, 'subset': 'train'}}]
}]
})
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024*1024)))
self.assertEqual(2, len(decoded_response))
self.assertListEqual(hash_names, list(decoded_response.keys()))
self.assertEqual([], decoded_response[hash_names[1]])
traces = decoded_response[hash_names[0]][0]
self.assertEqual('loss', traces['metric_name'])
self.assertDictEqual({'is_training': False}, traces['context'])
self.assertEqual(99, traces['x_axis_values']['shape'])
def test_query_images_api_custom_densities_sparse(self):
client = self.client
response = client.get('/api/runs/search/images/',
params={
'record_density': 10,
'index_density': 4
})
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(
decode_encoded_tree_stream(
response.iter_content(chunk_size=512 * 1024)))
self.assertEqual(1, len(decoded_response))
run_data = decoded_response[self.run_hash]
trace_data = run_data['traces'][0]
self.assertEqual(10, len(trace_data['values']))
self.assertEqual(10, len(trace_data['iters']))
self.assertEqual(4, len(trace_data['values'][0]))
self.assertEqual(4, len(trace_data['values'][9]))
image_indices = [img['index'] for img in trace_data['values'][0]]
self.assertListEqual([0, 4, 8, 12], image_indices)
image_names = [img['caption'] for img in trace_data['values'][9]]
self.assertListEqual(
['Image 90 0', 'Image 90 4', 'Image 90 8', 'Image 90 12'],
image_names)
def test_query_images_api_defaults(self):
client = self.client
response = client.get('/api/runs/search/images/', params={'q': ''})
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(
decode_encoded_tree_stream(
response.iter_content(chunk_size=512 * 1024)))
self.assertEqual(1, len(decoded_response))
run_data = decoded_response[self.run_hash]
self.assertEqual([0, 100], run_data['ranges']['record_range'])
self.assertEqual([0, 100, 2], run_data['ranges']['record_slice'])
self.assertEqual([0, 16], run_data['ranges']['index_range'])
self.assertEqual([0, 16, 3], run_data['ranges']['index_slice'])
self.assertEqual(16, run_data['params']['images_per_step'])
trace_data = run_data['traces'][0]
self.assertEqual('random_images', trace_data['name'])
self.assertEqual(50, len(trace_data['iters']))
self.assertEqual(4, trace_data['iters'][2])
self.assertEqual(50, len(trace_data['values']))
img_list = trace_data['values'][2]
self.assertEqual(6, len(img_list))
img = img_list[3]
self.assertEqual('Image 4 9', img['caption'])
self.assertEqual(9, img['index'])
self.assertEqual(16, img['width'])
self.assertEqual(16, img['height'])
self.assertEqual('png', img['format'])
self.assertIn('blob_uri', img)
def test_run_images_bulk_load_api(self):
client = self.client
requested_traces = [
{
'name': 'random_images',
'context': {}
},
]
response = client.post(f'/api/runs/{self.run_hash}/images/get-batch/',
json=requested_traces)
self.assertEqual(200, response.status_code)
trace_data = decode_tree(
decode_encoded_tree_stream(
response.iter_content(chunk_size=512 * 1024)))
self.assertEqual('random_images', trace_data['name'])
self.assertDictEqual({}, trace_data['context'])
self.assertEqual(50, len(trace_data['values']))
self.assertListEqual(list(range(0, 100, 2)), trace_data['iters'])
img_list = trace_data['values'][2]
self.assertEqual(6, len(img_list))
img = img_list[3]
self.assertEqual('Image 4 9', img['caption'])
self.assertEqual(9, img['index'])
self.assertEqual(16, img['width'])
self.assertEqual(16, img['height'])
self.assertEqual('png', img['format'])
self.assertIn('blob_uri', img)
def test_query_images_api_calculate_ranges(self):
client = self.client
response = client.get('/api/runs/search/images/',
params={
'record_range': '10:20',
'index_range': '3:6',
'calc_ranges': True
})
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(
decode_encoded_tree_stream(
response.iter_content(chunk_size=512 * 1024)))
self.assertEqual(1, len(decoded_response))
run_data = decoded_response[self.run_hash]
self.assertEqual([0, 100], run_data['ranges']['record_range'])
self.assertEqual([10, 20, 1], run_data['ranges']['record_slice'])
self.assertEqual([0, 16], run_data['ranges']['index_range'])
self.assertEqual([3, 6, 1], run_data['ranges']['index_slice'])
trace_data = run_data['traces'][0]
self.assertEqual(10, len(trace_data['values']))
self.assertEqual(10, len(trace_data['iters']))
self.assertEqual(10, trace_data['iters'][0])
self.assertEqual(19, trace_data['iters'][-1])
img_list = trace_data['values'][2]
self.assertEqual(3, len(img_list))
img_indices = [img['index'] for img in img_list]
self.assertListEqual([3, 4, 5], img_indices)
def test_query_images_api_custom_record_ranges(self, input_range, rstart,
rstop, start, stop, step,
count):
client = self.client
response = client.get('/api/runs/search/images/',
params={
'record_range': input_range,
'record_density': 100
})
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(
decode_encoded_tree_stream(
response.iter_content(chunk_size=512 * 1024)))
self.assertEqual(1, len(decoded_response))
run_data = decoded_response[self.run_hash]
self.assertEqual([rstart, rstop], run_data['ranges']['record_range'])
self.assertEqual([start, stop, step],
run_data['ranges']['record_slice'])
trace_data = run_data['traces'][0]
self.assertEqual(count, len(trace_data['values']))
self.assertEqual(count, len(trace_data['iters']))
self.assertEqual(start, trace_data['iters'][0])
self.assertEqual(stop - 1, trace_data['iters'][-1])
def test_search_runs_api(self):
client = self.client
response = client.get('/api/runs/search/run/', params={'q': 'run["name"] == "Run # 3"'})
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024*1024)))
self.assertEqual(1, len(decoded_response))
for hashname, run in decoded_response.items():
self.assertEqual(4, len(run['traces']))
for trace in run['traces']:
self.assertAlmostEqual(0.99, trace['last_value']['last'])
def test_images_uri_bulk_load_api(self, uri_count):
# take random N URIs
uris = random.sample(self.uri_map.keys(), uri_count)
client = self.client
response = client.post('/api/runs/images/get-batch', json=uris)
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(
decode_encoded_tree_stream(
response.iter_content(chunk_size=512 * 1024)))
self.assertEqual(uri_count, len(decoded_response))
for uri, blob in decoded_response.items():
expected_blob = self.image_blobs[self.uri_map[uri]]
self.assertEqual(expected_blob, blob)
def test_search_runs_api_paginated(self):
client = self.client
response = client.get('/api/runs/search/run/', params={'q': 'run["name"] in ["Run # 2","Run # 3"]',
'limit': 1})
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024 * 1024)))
self.assertEqual(1, len(decoded_response))
offset = ''
for run_hash, run in decoded_response.items():
offset = run_hash
self.assertEqual('Run # 3', run['props']['name'])
response = client.get('/api/runs/search/run/', params={'q': 'run["name"] in ["Run # 2","Run # 3"]',
'limit': 5,
'offset': offset})
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024 * 1024)))
self.assertEqual(1, len(decoded_response))
for run_hash, run in decoded_response.items():
self.assertEqual('Run # 2', run['props']['name'])
def collect(
self,
path: Union[AimObjectKey, AimObjectPath] = (),
strict: bool = True
) -> AimObject:
if path == Ellipsis:
path = ()
if isinstance(path, (int, str)):
path = (path,)
prefix = E.encode_path(path)
it = self.container.view(prefix).items()
try:
return treeutils.decode_tree(it, strict=strict)
except KeyError:
raise KeyError('No key {} is present.'.format(path))
def setUp(self) -> None:
super().setUp()
self.uri_map = {}
client = self.client
response = client.get('/api/runs/search/images/',
params={
'record_range': '0:10',
'index_range': '0:5',
})
decoded_response = decode_tree(
decode_encoded_tree_stream(
response.iter_content(chunk_size=512 * 1024)))
run_data = decoded_response[self.run_hash]
trace_data = run_data['traces'][0]
for img_list in trace_data['values']:
for img_data in img_list:
self.uri_map[img_data['blob_uri']] = img_data['caption']
def test_search_metrics_api_default_step(self):
client = self.client
response = client.get('/api/runs/search/metric/', params={'q': 'run["name"] == "Run # 3"'})
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024*1024)))
for run in decoded_response.values():
for trace in run['traces']:
self.assertEqual([0, 100, 2], trace['slice'])
values = trace['values']
dtype = values['dtype']
shape = values['shape']
data = values['blob']
array = np.frombuffer(data, dtype=dtype).reshape(shape)
self.assertEqual(51, shape) # default 50 steps
self.assertAlmostEqual(0.99, array.max())
self.assertAlmostEqual(0.99, array[50])
self.assertEqual(51, len(array))
def test_search_aligned_metrics_api_with_wrong_context(self):
client = self.client
hash_names = []
for run, _ in zip(self.repo.iter_runs(), range(2)):
hash_names.append(run.hashname)
response = client.post('/api/runs/search/metric/align/', json={
'align_by': 'accuracy',
'runs': [{
'run_id': hash_names[0],
'traces': [{'metric_name': 'loss', 'slice': [0, 20, 1], 'context': {'is_training': True, 'subset': 'training'}}]
}, {
'run_id': hash_names[1],
'traces': [{'metric_name': 'loss', 'slice': [0, 10, 1], 'context': {'is_training': True, 'subset': 'val'}}]
}]
})
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024 * 1024)))
self.assertEqual(2, len(decoded_response))
for run in decoded_response.values():
self.assertEqual([], run)
def test_query_images_api_custom_densities_dense(self):
client = self.client
response = client.get('/api/runs/search/images/',
params={
'record_density': 200,
'index_density': 10
})
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(
decode_encoded_tree_stream(response.iter_content(chunk_size=512 *
1024),
concat_chunks=True))
self.assertEqual(1, len(decoded_response))
run_data = decoded_response[self.run_hash]
trace_data = run_data['traces'][0]
self.assertEqual(100, len(trace_data['values']))
self.assertEqual(100, len(trace_data['iters']))
self.assertEqual(16, len(trace_data['values'][0]))
self.assertEqual(16, len(trace_data['values'][99]))
def run_instruction(self, resource, method, args=()):
args = deepcopy(args)
def message_stream_generator():
header = rpc_messages.InstructionRequest(
header=rpc_messages.RequestHeader(version='0.1',
handler=resource,
client_uri=self.uri,
method_name=method))
yield header
stream = pack_stream(encode_tree(args))
for chunk in stream:
yield rpc_messages.InstructionRequest(message=chunk)
resp = self.remote.run_instruction(message_stream_generator())
status_msg = next(resp)
assert status_msg.WhichOneof('instruction') == 'header'
if status_msg.header.status == rpc_messages.ResponseHeader.Status.ERROR:
raise_exception(status_msg.header.exception)
return decode_tree(unpack_stream(resp))
def test_search_metrics_api_custom_step(self, step_count, expected_step_count):
client = self.client
response = client.get('/api/runs/search/metric/', params={'q': 'run["name"] == "Run # 3"', 'p': step_count})
self.assertEqual(200, response.status_code)
decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024*1024)))
if (100 // step_count) <= 1:
array_last_idx = 99
else:
array_last_idx = step_count
for run in decoded_response.values():
for trace in run['traces']:
self.assertEqual([0, 100, (100 // step_count) or 1], trace['slice'])
values = trace['values']
dtype = values['dtype']
shape = values['shape']
data = values['blob']
array = np.frombuffer(data, dtype=dtype).reshape(shape)
self.assertEqual(expected_step_count, shape)
self.assertAlmostEqual(0.99, array.max())
self.assertAlmostEqual(0.99, array[array_last_idx])
self.assertEqual(0.0, array[0])
self.assertEqual(expected_step_count, len(array))
