def test_meta_run_tree_contexts_and_names(self): train_context = {'subset': 'train'} val_context = {'subset': 'val'} empty_context = {} run = Run(repo=self.repo, system_tracking_interval=None) run.track(1, name='metric 1', context=train_context) run.track(1, name='metric 2', context=train_context) run.track(1, name='metric 1', context=val_context) run.track(1, name='metric 2', context=val_context) run.track(1, name='metric 3', context=val_context) run.track(0, name='metric') meta_container_path = os.path.join(self.repo.path, 'meta', 'chunks', run.hash) rc = RocksContainer(meta_container_path, read_only=True) tree = ContainerTreeView(rc) contexts = tree.view(('meta', 'chunks', run.hash, 'contexts')).collect() for ctx in [train_context, val_context, empty_context]: self.assertIn(Context(ctx).idx, contexts) self.assertDictEqual(contexts[Context(ctx).idx], ctx) traces = tree.view(('meta', 'chunks', run.hash, 'traces', Context(train_context).idx)).collect() self.assertSetEqual({'metric 1', 'metric 2'}, set(traces.keys())) traces = tree.view(('meta', 'chunks', run.hash, 'traces', Context(val_context).idx)).collect() self.assertSetEqual({'metric 1', 'metric 2', 'metric 3'}, set(traces.keys())) traces = tree.view(('meta', 'chunks', run.hash, 'traces', Context(empty_context).idx)).collect() self.assertSetEqual({'metric'}, set(traces.keys()))
def custom_aligned_metrics_streamer(requested_runs: List[AlignedRunIn], x_axis: str, repo: 'Repo') -> bytes: for run_data in requested_runs: run_hash = run_data.run_id requested_traces = run_data.traces run = Run(run_hash, repo=repo, read_only=True) traces_list = [] for trace_data in requested_traces: context = Context(trace_data.context) trace = run.get_metric(name=trace_data.name, context=context) x_axis_trace = run.get_metric(name=x_axis, context=context) if not (trace and x_axis_trace): continue _slice = slice(*trace_data.slice) iters = trace.values.sparse_numpy()[0] sliced_iters = sliced_np_array(iters, _slice) x_axis_iters, x_axis_values = collect_x_axis_data(x_axis_trace, sliced_iters) traces_list.append({ 'name': trace.name, 'context': trace.context.to_dict(), 'x_axis_values': x_axis_values, 'x_axis_iters': x_axis_iters, }) run_dict = { run_hash: traces_list } encoded_tree = encode_tree(run_dict) yield collect_run_streamable_data(encoded_tree)
def test_disable_resource_tracking(self): run = Run(system_tracking_interval=None) run_hash = run.hash run.track(1, name='metric') time.sleep(.1) # allow tracker to add resource usage metrics del run metrics = list( self.repo.query_metrics( f'run.hash == "{run_hash}" and metric.name.startswith("__")')) self.assertListEqual([], metrics)
def requested_distribution_traces_streamer(run: Run, requested_traces: List[TraceBase], rec_range, rec_num: int = 50) -> List[dict]: for requested_trace in requested_traces: trace_name = requested_trace.name context = Context(requested_trace.context) trace = run.get_distribution_sequence(name=trace_name, context=context) if not trace: continue record_range_missing = rec_range.start is None or rec_range.stop is None if record_range_missing: rec_range = IndexRange(trace.first_step(), trace.last_step() + 1) steps_vals = trace.values.items_in_range(rec_range.start, rec_range.stop, rec_num) steps = [] values = [] for step, val in steps_vals: steps.append(step) values.append(dist_record_to_encodable(val)) trace_dict = { 'record_range': (trace.first_step(), trace.last_step() + 1), 'name': trace.name, 'context': trace.context.to_dict(), 'values': values, 'iters': steps, } encoded_tree = encode_tree(trace_dict) yield collect_run_streamable_data(encoded_tree)
def collect_requested_metric_traces(run: Run, requested_traces: List[TraceBase], steps_num: int = 200) -> List[dict]: processed_traces_list = [] for requested_trace in requested_traces: metric_name = requested_trace.name context = Context(requested_trace.context) trace = run.get_metric(name=metric_name, context=context) if not trace: continue iters, values = trace.values.sparse_list() values = list(map(lambda x: x if float('-inf') < x < float('inf') and x == x else None, values)) num_records = len(values) step = (num_records // steps_num) or 1 _slice = slice(0, num_records, step) processed_traces_list.append({ 'name': trace.name, 'context': trace.context.to_dict(), 'values': sliced_array(values, _slice), 'iters': sliced_array(iters, _slice), }) return processed_traces_list
def test_default_tracking_interval(self): run = Run() # default tracking interval 10s run_hash = run.hash run.track(1, name='metric') time.sleep(.1) # allow tracker to add resource usage metrics del run metrics = list( self.repo.query_metrics( f'run.hash == "{run_hash}" and metric.name.startswith("__")')) self.assertEqual(4, len(metrics)) expected_metrics = { '__system__cpu', '__system__disk_percent', '__system__memory_percent', '__system__p_memory_percent' } self.assertSetEqual(expected_metrics, set(m.name for m in metrics))
def test_custom_tracking_interval(self): run = Run(system_tracking_interval=1) run_hash = run.hash run.track(1, name='metric') time.sleep(3) # allow tracker to add resource usage metrics del run metrics = list( self.repo.query_metrics( f'run.hash == "{run_hash}" and metric.name.startswith("__")')) self.assertEqual(4, len(metrics)) expected_metrics = { '__system__cpu', '__system__disk_percent', '__system__memory_percent', '__system__p_memory_percent' } self.assertSetEqual(expected_metrics, set(m.name for m in metrics)) for metric in metrics: # 3 sec. runtime, 1 sec. interval self.assertGreaterEqual(len(metric.values), 3) self.assertLessEqual(len(metric.values), 4)
def test_numpy_scalar_types_track(self): """covers https://github.com/aimhubio/aim/issues/1206""" run = Run(system_tracking_interval=None) run.track(np.array([1.0]), name='single_item_array') run.track(np.array([[[1.0]]]), name='single_item_3d_array') run.track(np.float64(1.0), name='numpy_float64') run.track(np.float32(1.0), name='numpy_float32') repo = Repo.default_repo() metric_names = {metric.name for metric in repo.query_metrics()} expected_metric_names = { 'single_item_array', 'single_item_3d_array', 'numpy_float64', 'numpy_float32' } self.assertSetEqual(expected_metric_names, metric_names)
def test_run_trace_dtype_and_last_value(self): run = Run() run.track(1.0, name='metric 1', context={}) run.track(2.0, name='metric 1', context={}) run.track(3.0, name='metric 1', context={}) run.track(1.0, name='metric 1', context={'subset': 'train'}) meta_container_path = os.path.join(self.repo.path, 'meta', 'chunks', run.hash) rc = RocksContainer(meta_container_path, read_only=True) tree = ContainerTreeView(rc) metric_1_dict = tree.view(('meta', 'chunks', run.hash, 'traces', Context({}).idx, 'metric 1')).collect() self.assertEqual(3.0, metric_1_dict['last']) self.assertEqual('float', metric_1_dict['dtype']) metric_1_dict = tree.view(('meta', 'chunks', run.hash, 'traces', Context({'subset': 'train'}).idx, 'metric 1')).collect() self.assertEqual(1.0, metric_1_dict['last'])
def requested_text_traces_streamer(run: Run, requested_traces: List[TraceBase], rec_range, idx_range, rec_num: int = 50, idx_num: int = 5) -> List[dict]: for requested_trace in requested_traces: trace_name = requested_trace.name context = Context(requested_trace.context) trace = run.get_text_sequence(name=trace_name, context=context) if not trace: continue record_range_missing = rec_range.start is None or rec_range.stop is None if record_range_missing: rec_range = IndexRange(trace.first_step(), trace.last_step() + 1) index_range_missing = idx_range.start is None or idx_range.stop is None if index_range_missing: idx_range = IndexRange(0, trace.record_length() or 1) rec_length = trace.record_length() or 1 idx_step = rec_length // idx_num or 1 idx_slice = slice(idx_range.start, idx_range.stop, idx_step) steps_vals = trace.values.items_in_range(rec_range.start, rec_range.stop, rec_num) steps = [] values = [] for step, val in steps_vals: steps.append(step) if isinstance(val, list): values.append(text_collection_to_encodable(sliced_custom_object_record(val, idx_slice))) elif idx_slice.start == 0: values.append(text_record_to_encodable(val)) else: values.append([]) trace_dict = { 'record_range': (trace.first_step(), trace.last_step() + 1), 'index_range': (0, rec_length), 'name': trace.name, 'context': trace.context.to_dict(), 'values': values, 'iters': steps, } encoded_tree = encode_tree(trace_dict) yield collect_run_streamable_data(encoded_tree)
def test_reject_non_scalar_arrays_track(self): run = Run(system_tracking_interval=None) with self.assertRaises(ValueError): run.track(np.array([1.0, 2.0]), name='fail')
def test_series_tree_values(self): # sequential steps run = Run() run.track(1.0, name='metric 1', context={}) run.track(2.0, name='metric 1', context={}) run.track(3.0, name='metric 1', context={}) series_container_path = os.path.join(self.repo.path, 'seqs', 'chunks', run.hash) rc = RocksContainer(series_container_path, read_only=True) tree = ContainerTreeView(rc) traces_dict = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).collect() self.assertSetEqual({'val', 'epoch', 'time'}, set(traces_dict.keys())) self.assertEqual(3, len(traces_dict['val'])) self.assertEqual(3, len(traces_dict['epoch'])) self.assertEqual(3, len(traces_dict['time'])) self.assertEqual(1.0, traces_dict['val'][0]) self.assertEqual(2.0, traces_dict['val'][1]) self.assertEqual(3.0, traces_dict['val'][2]) # user-specified steps run = Run() run.track(1.0, name='metric 1', step=10, context={}) run.track(2.0, name='metric 1', step=20, context={}) run.track(3.0, name='metric 1', step=30, context={}) series_container_path = os.path.join(self.repo.path, 'seqs', 'chunks', run.hash) rc = RocksContainer(series_container_path, read_only=True) tree = ContainerTreeView(rc) traces_dict = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).collect() self.assertEqual(31, len(traces_dict['val'])) # last index is 30 # sparse array self.assertTrue(all(x is None for x in traces_dict['val'][0:10])) self.assertEqual(1.0, traces_dict['val'][10]) self.assertTrue(all(x is None for x in traces_dict['val'][11:20])) self.assertEqual(2.0, traces_dict['val'][20]) self.assertTrue(all(x is None for x in traces_dict['val'][21:30])) self.assertEqual(3.0, traces_dict['val'][30]) val_array_view = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).array('val') self.assertEqual(31, len(val_array_view)) self.assertEqual(3, len(list(val_array_view))) self.assertEqual(1.0, val_array_view[10]) self.assertEqual(2.0, val_array_view[20]) self.assertEqual(3.0, val_array_view[30]) # user-specified steps, unordered run = Run() run.track(3.0, name='metric 1', step=30, context={}) run.track(1.0, name='metric 1', step=10, context={}) run.track(2.0, name='metric 1', step=20, context={}) series_container_path = os.path.join(self.repo.path, 'seqs', 'chunks', run.hash) rc = RocksContainer(series_container_path, read_only=True) tree = ContainerTreeView(rc) traces_dict = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).collect() self.assertEqual(31, len(traces_dict['val'])) # last index is 30 # sparse array self.assertTrue(all(x is None for x in traces_dict['val'][0:10])) self.assertEqual(1.0, traces_dict['val'][10]) self.assertTrue(all(x is None for x in traces_dict['val'][11:20])) self.assertEqual(2.0, traces_dict['val'][20]) self.assertTrue(all(x is None for x in traces_dict['val'][21:30])) self.assertEqual(3.0, traces_dict['val'][30])
def test_run_set_param_meta_tree(self): run = Run() run['p1'] = 1 run['p2'] = {'foo': 'bar'} run['p3'] = 0.0 run['p4'] = True run['p5'] = ['x', 1, 2] run['p6'] = None run['p7'] = 'bazե\x00a-a' run['p8'] = b'blob' meta_container_path = os.path.join(self.repo.path, 'meta', 'chunks', run.hash) rc = RocksContainer(meta_container_path, read_only=True) tree = ContainerTreeView(rc) meta_attrs_tree = tree.view(('meta', 'attrs')) meta_run_attrs_tree = tree.view(('meta', 'chunks', run.hash, 'attrs')) val = meta_attrs_tree['p1'] self.assertEqual(int, type(val)) self.assertEqual(1, val) val = meta_run_attrs_tree['p1'] self.assertEqual(int, type(val)) self.assertEqual(1, val) val = meta_attrs_tree['p3'] self.assertEqual(float, type(val)) self.assertEqual(0.0, val) val = meta_run_attrs_tree['p3'] self.assertEqual(float, type(val)) self.assertEqual(0.0, val) val = meta_attrs_tree['p4'] self.assertEqual(bool, type(val)) self.assertEqual(True, val) val = meta_run_attrs_tree['p4'] self.assertEqual(bool, type(val)) self.assertEqual(True, val) val = meta_attrs_tree['p2'] self.assertEqual(dict, type(val)) self.assertEqual({'foo': 'bar'}, val) val = meta_run_attrs_tree['p2'] self.assertEqual(dict, type(val)) self.assertEqual({'foo': 'bar'}, val) val = meta_attrs_tree['p5'] self.assertEqual(list, type(val)) self.assertEqual(['x', 1, 2], val) val = meta_run_attrs_tree['p5'] self.assertEqual(list, type(val)) self.assertEqual(['x', 1, 2], val) val = meta_attrs_tree['p6'] self.assertEqual(None, val) val = meta_run_attrs_tree['p6'] self.assertEqual(None, val) val = meta_attrs_tree['p7'] self.assertEqual(str, type(val)) self.assertEqual('bazե\x00a-a', val) val = meta_run_attrs_tree['p7'] self.assertEqual(str, type(val)) self.assertEqual('bazե\x00a-a', val) val = meta_attrs_tree['p8'] self.assertEqual(bytes, type(val)) self.assertEqual(b'blob', val) val = meta_run_attrs_tree['p8'] self.assertEqual(bytes, type(val)) self.assertEqual(b'blob', val)
def test_resource_tracking_interval_limits(self): run = Run(system_tracking_interval=0) self.assertIsNone(run._system_resource_tracker) run = Run(system_tracking_interval=2 * 24 * 3600) # two days self.assertIsNone(run._system_resource_tracker)