def test_mixed_search_custom_index_range(self): client = self.client response = client.get('/api/runs/search/images/', params={ 'q': '', 'index_range': '3:5' }) self.assertEqual(200, response.status_code) decoded_response = decode_tree( decode_encoded_tree_stream( response.iter_content(chunk_size=512 * 1024))) self.assertEqual(1, len(decoded_response)) run_data = decoded_response[self.run_hash] self.assertEqual([0, 5], run_data['ranges']['record_range']) self.assertEqual([0, 5, 1], run_data['ranges']['record_slice']) self.assertEqual([3, 5], run_data['ranges']['index_range']) self.assertEqual([3, 5, 1], run_data['ranges']['index_slice']) self.assertEqual(2, len(run_data['traces'])) trace_data = run_data['traces'][0] self.assertEqual('image_lists', trace_data['name']) self.assertEqual(5, len(trace_data['values'])) img_list = trace_data['values'][0] self.assertEqual(2, len(img_list)) trace_data = run_data['traces'][1] self.assertEqual('single_images', trace_data['name']) self.assertEqual(5, len(trace_data['values'])) img_list = trace_data['values'][0] self.assertListEqual([], img_list)
def test_search_aligned_metrics_api(self): client = self.client hash_names = [] for run, _ in zip(self.repo.iter_runs(), range(2)): hash_names.append(run.hashname) response = client.post('/api/runs/search/metric/align/', json={ 'align_by': 'accuracy', 'runs': [{ 'run_id': hash_names[0], 'traces': [{'metric_name': 'loss', 'slice': [0, 100, 1], 'context': {'is_training': False}}] }, { 'run_id': hash_names[1], 'traces': [{'metric_name': 'loss', 'slice': [0, 100, 1], 'context': {'is_training': True, 'subset': 'train'}}] }] }) self.assertEqual(200, response.status_code) decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024*1024))) self.assertEqual(2, len(decoded_response)) self.assertListEqual(hash_names, list(decoded_response.keys())) self.assertEqual([], decoded_response[hash_names[1]]) traces = decoded_response[hash_names[0]][0] self.assertEqual('loss', traces['metric_name']) self.assertDictEqual({'is_training': False}, traces['context']) self.assertEqual(99, traces['x_axis_values']['shape'])
def test_query_images_api_custom_densities_sparse(self): client = self.client response = client.get('/api/runs/search/images/', params={ 'record_density': 10, 'index_density': 4 }) self.assertEqual(200, response.status_code) decoded_response = decode_tree( decode_encoded_tree_stream( response.iter_content(chunk_size=512 * 1024))) self.assertEqual(1, len(decoded_response)) run_data = decoded_response[self.run_hash] trace_data = run_data['traces'][0] self.assertEqual(10, len(trace_data['values'])) self.assertEqual(10, len(trace_data['iters'])) self.assertEqual(4, len(trace_data['values'][0])) self.assertEqual(4, len(trace_data['values'][9])) image_indices = [img['index'] for img in trace_data['values'][0]] self.assertListEqual([0, 4, 8, 12], image_indices) image_names = [img['caption'] for img in trace_data['values'][9]] self.assertListEqual( ['Image 90 0', 'Image 90 4', 'Image 90 8', 'Image 90 12'], image_names)
def test_query_images_api_defaults(self): client = self.client response = client.get('/api/runs/search/images/', params={'q': ''}) self.assertEqual(200, response.status_code) decoded_response = decode_tree( decode_encoded_tree_stream( response.iter_content(chunk_size=512 * 1024))) self.assertEqual(1, len(decoded_response)) run_data = decoded_response[self.run_hash] self.assertEqual([0, 100], run_data['ranges']['record_range']) self.assertEqual([0, 100, 2], run_data['ranges']['record_slice']) self.assertEqual([0, 16], run_data['ranges']['index_range']) self.assertEqual([0, 16, 3], run_data['ranges']['index_slice']) self.assertEqual(16, run_data['params']['images_per_step']) trace_data = run_data['traces'][0] self.assertEqual('random_images', trace_data['name']) self.assertEqual(50, len(trace_data['iters'])) self.assertEqual(4, trace_data['iters'][2]) self.assertEqual(50, len(trace_data['values'])) img_list = trace_data['values'][2] self.assertEqual(6, len(img_list)) img = img_list[3] self.assertEqual('Image 4 9', img['caption']) self.assertEqual(9, img['index']) self.assertEqual(16, img['width']) self.assertEqual(16, img['height']) self.assertEqual('png', img['format']) self.assertIn('blob_uri', img)
def test_run_images_bulk_load_api(self): client = self.client requested_traces = [ { 'name': 'random_images', 'context': {} }, ] response = client.post(f'/api/runs/{self.run_hash}/images/get-batch/', json=requested_traces) self.assertEqual(200, response.status_code) trace_data = decode_tree( decode_encoded_tree_stream( response.iter_content(chunk_size=512 * 1024))) self.assertEqual('random_images', trace_data['name']) self.assertDictEqual({}, trace_data['context']) self.assertEqual(50, len(trace_data['values'])) self.assertListEqual(list(range(0, 100, 2)), trace_data['iters']) img_list = trace_data['values'][2] self.assertEqual(6, len(img_list)) img = img_list[3] self.assertEqual('Image 4 9', img['caption']) self.assertEqual(9, img['index']) self.assertEqual(16, img['width']) self.assertEqual(16, img['height']) self.assertEqual('png', img['format']) self.assertIn('blob_uri', img)
def test_query_images_api_calculate_ranges(self): client = self.client response = client.get('/api/runs/search/images/', params={ 'record_range': '10:20', 'index_range': '3:6', 'calc_ranges': True }) self.assertEqual(200, response.status_code) decoded_response = decode_tree( decode_encoded_tree_stream( response.iter_content(chunk_size=512 * 1024))) self.assertEqual(1, len(decoded_response)) run_data = decoded_response[self.run_hash] self.assertEqual([0, 100], run_data['ranges']['record_range']) self.assertEqual([10, 20, 1], run_data['ranges']['record_slice']) self.assertEqual([0, 16], run_data['ranges']['index_range']) self.assertEqual([3, 6, 1], run_data['ranges']['index_slice']) trace_data = run_data['traces'][0] self.assertEqual(10, len(trace_data['values'])) self.assertEqual(10, len(trace_data['iters'])) self.assertEqual(10, trace_data['iters'][0]) self.assertEqual(19, trace_data['iters'][-1]) img_list = trace_data['values'][2] self.assertEqual(3, len(img_list)) img_indices = [img['index'] for img in img_list] self.assertListEqual([3, 4, 5], img_indices)
def test_query_images_api_custom_record_ranges(self, input_range, rstart, rstop, start, stop, step, count): client = self.client response = client.get('/api/runs/search/images/', params={ 'record_range': input_range, 'record_density': 100 }) self.assertEqual(200, response.status_code) decoded_response = decode_tree( decode_encoded_tree_stream( response.iter_content(chunk_size=512 * 1024))) self.assertEqual(1, len(decoded_response)) run_data = decoded_response[self.run_hash] self.assertEqual([rstart, rstop], run_data['ranges']['record_range']) self.assertEqual([start, stop, step], run_data['ranges']['record_slice']) trace_data = run_data['traces'][0] self.assertEqual(count, len(trace_data['values'])) self.assertEqual(count, len(trace_data['iters'])) self.assertEqual(start, trace_data['iters'][0]) self.assertEqual(stop - 1, trace_data['iters'][-1])
def test_search_runs_api(self): client = self.client response = client.get('/api/runs/search/run/', params={'q': 'run["name"] == "Run # 3"'}) self.assertEqual(200, response.status_code) decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024*1024))) self.assertEqual(1, len(decoded_response)) for hashname, run in decoded_response.items(): self.assertEqual(4, len(run['traces'])) for trace in run['traces']: self.assertAlmostEqual(0.99, trace['last_value']['last'])
def test_images_uri_bulk_load_api(self, uri_count): # take random N URIs uris = random.sample(self.uri_map.keys(), uri_count) client = self.client response = client.post('/api/runs/images/get-batch', json=uris) self.assertEqual(200, response.status_code) decoded_response = decode_tree( decode_encoded_tree_stream( response.iter_content(chunk_size=512 * 1024))) self.assertEqual(uri_count, len(decoded_response)) for uri, blob in decoded_response.items(): expected_blob = self.image_blobs[self.uri_map[uri]] self.assertEqual(expected_blob, blob)
def test_search_runs_api_paginated(self): client = self.client response = client.get('/api/runs/search/run/', params={'q': 'run["name"] in ["Run # 2","Run # 3"]', 'limit': 1}) self.assertEqual(200, response.status_code) decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024 * 1024))) self.assertEqual(1, len(decoded_response)) offset = '' for run_hash, run in decoded_response.items(): offset = run_hash self.assertEqual('Run # 3', run['props']['name']) response = client.get('/api/runs/search/run/', params={'q': 'run["name"] in ["Run # 2","Run # 3"]', 'limit': 5, 'offset': offset}) self.assertEqual(200, response.status_code) decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024 * 1024))) self.assertEqual(1, len(decoded_response)) for run_hash, run in decoded_response.items(): self.assertEqual('Run # 2', run['props']['name'])
def collect( self, path: Union[AimObjectKey, AimObjectPath] = (), strict: bool = True ) -> AimObject: if path == Ellipsis: path = () if isinstance(path, (int, str)): path = (path,) prefix = E.encode_path(path) it = self.container.view(prefix).items() try: return treeutils.decode_tree(it, strict=strict) except KeyError: raise KeyError('No key {} is present.'.format(path))
def setUp(self) -> None: super().setUp() self.uri_map = {} client = self.client response = client.get('/api/runs/search/images/', params={ 'record_range': '0:10', 'index_range': '0:5', }) decoded_response = decode_tree( decode_encoded_tree_stream( response.iter_content(chunk_size=512 * 1024))) run_data = decoded_response[self.run_hash] trace_data = run_data['traces'][0] for img_list in trace_data['values']: for img_data in img_list: self.uri_map[img_data['blob_uri']] = img_data['caption']
def test_search_metrics_api_default_step(self): client = self.client response = client.get('/api/runs/search/metric/', params={'q': 'run["name"] == "Run # 3"'}) self.assertEqual(200, response.status_code) decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024*1024))) for run in decoded_response.values(): for trace in run['traces']: self.assertEqual([0, 100, 2], trace['slice']) values = trace['values'] dtype = values['dtype'] shape = values['shape'] data = values['blob'] array = np.frombuffer(data, dtype=dtype).reshape(shape) self.assertEqual(51, shape) # default 50 steps self.assertAlmostEqual(0.99, array.max()) self.assertAlmostEqual(0.99, array[50]) self.assertEqual(51, len(array))
def test_search_aligned_metrics_api_with_wrong_context(self): client = self.client hash_names = [] for run, _ in zip(self.repo.iter_runs(), range(2)): hash_names.append(run.hashname) response = client.post('/api/runs/search/metric/align/', json={ 'align_by': 'accuracy', 'runs': [{ 'run_id': hash_names[0], 'traces': [{'metric_name': 'loss', 'slice': [0, 20, 1], 'context': {'is_training': True, 'subset': 'training'}}] }, { 'run_id': hash_names[1], 'traces': [{'metric_name': 'loss', 'slice': [0, 10, 1], 'context': {'is_training': True, 'subset': 'val'}}] }] }) self.assertEqual(200, response.status_code) decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024 * 1024))) self.assertEqual(2, len(decoded_response)) for run in decoded_response.values(): self.assertEqual([], run)
def test_query_images_api_custom_densities_dense(self): client = self.client response = client.get('/api/runs/search/images/', params={ 'record_density': 200, 'index_density': 10 }) self.assertEqual(200, response.status_code) decoded_response = decode_tree( decode_encoded_tree_stream(response.iter_content(chunk_size=512 * 1024), concat_chunks=True)) self.assertEqual(1, len(decoded_response)) run_data = decoded_response[self.run_hash] trace_data = run_data['traces'][0] self.assertEqual(100, len(trace_data['values'])) self.assertEqual(100, len(trace_data['iters'])) self.assertEqual(16, len(trace_data['values'][0])) self.assertEqual(16, len(trace_data['values'][99]))
def run_instruction(self, resource, method, args=()): args = deepcopy(args) def message_stream_generator(): header = rpc_messages.InstructionRequest( header=rpc_messages.RequestHeader(version='0.1', handler=resource, client_uri=self.uri, method_name=method)) yield header stream = pack_stream(encode_tree(args)) for chunk in stream: yield rpc_messages.InstructionRequest(message=chunk) resp = self.remote.run_instruction(message_stream_generator()) status_msg = next(resp) assert status_msg.WhichOneof('instruction') == 'header' if status_msg.header.status == rpc_messages.ResponseHeader.Status.ERROR: raise_exception(status_msg.header.exception) return decode_tree(unpack_stream(resp))
def test_search_metrics_api_custom_step(self, step_count, expected_step_count): client = self.client response = client.get('/api/runs/search/metric/', params={'q': 'run["name"] == "Run # 3"', 'p': step_count}) self.assertEqual(200, response.status_code) decoded_response = decode_tree(decode_encoded_tree_stream(response.iter_content(chunk_size=1024*1024))) if (100 // step_count) <= 1: array_last_idx = 99 else: array_last_idx = step_count for run in decoded_response.values(): for trace in run['traces']: self.assertEqual([0, 100, (100 // step_count) or 1], trace['slice']) values = trace['values'] dtype = values['dtype'] shape = values['shape'] data = values['blob'] array = np.frombuffer(data, dtype=dtype).reshape(shape) self.assertEqual(expected_step_count, shape) self.assertAlmostEqual(0.99, array.max()) self.assertAlmostEqual(0.99, array[array_last_idx]) self.assertEqual(0.0, array[0]) self.assertEqual(expected_step_count, len(array))