def test_meta_run_tree_contexts_and_names(self):
train_context = {'subset': 'train'}
val_context = {'subset': 'val'}
empty_context = {}
run = Run(repo=self.repo, system_tracking_interval=None)
run.track(1, name='metric 1', context=train_context)
run.track(1, name='metric 2', context=train_context)
run.track(1, name='metric 1', context=val_context)
run.track(1, name='metric 2', context=val_context)
run.track(1, name='metric 3', context=val_context)
run.track(0, name='metric')
meta_container_path = os.path.join(self.repo.path, 'meta', 'chunks', run.hash)
rc = RocksContainer(meta_container_path, read_only=True)
tree = ContainerTreeView(rc)
contexts = tree.view(('meta', 'chunks', run.hash, 'contexts')).collect()
for ctx in [train_context, val_context, empty_context]:
self.assertIn(Context(ctx).idx, contexts)
self.assertDictEqual(contexts[Context(ctx).idx], ctx)
traces = tree.view(('meta', 'chunks', run.hash, 'traces', Context(train_context).idx)).collect()
self.assertSetEqual({'metric 1', 'metric 2'}, set(traces.keys()))
traces = tree.view(('meta', 'chunks', run.hash, 'traces', Context(val_context).idx)).collect()
self.assertSetEqual({'metric 1', 'metric 2', 'metric 3'}, set(traces.keys()))
traces = tree.view(('meta', 'chunks', run.hash, 'traces', Context(empty_context).idx)).collect()
self.assertSetEqual({'metric'}, set(traces.keys()))
def custom_aligned_metrics_streamer(requested_runs: List[AlignedRunIn], x_axis: str, repo: 'Repo') -> bytes:
for run_data in requested_runs:
run_hash = run_data.run_id
requested_traces = run_data.traces
run = Run(run_hash, repo=repo, read_only=True)
traces_list = []
for trace_data in requested_traces:
context = Context(trace_data.context)
trace = run.get_metric(name=trace_data.name,
context=context)
x_axis_trace = run.get_metric(name=x_axis,
context=context)
if not (trace and x_axis_trace):
continue
_slice = slice(*trace_data.slice)
iters = trace.values.sparse_numpy()[0]
sliced_iters = sliced_np_array(iters, _slice)
x_axis_iters, x_axis_values = collect_x_axis_data(x_axis_trace, sliced_iters)
traces_list.append({
'name': trace.name,
'context': trace.context.to_dict(),
'x_axis_values': x_axis_values,
'x_axis_iters': x_axis_iters,
})
run_dict = {
run_hash: traces_list
}
encoded_tree = encode_tree(run_dict)
yield collect_run_streamable_data(encoded_tree)
def test_disable_resource_tracking(self):
run = Run(system_tracking_interval=None)
run_hash = run.hash
run.track(1, name='metric')
time.sleep(.1) # allow tracker to add resource usage metrics
del run
metrics = list(
self.repo.query_metrics(
f'run.hash == "{run_hash}" and metric.name.startswith("__")'))
self.assertListEqual([], metrics)
def requested_distribution_traces_streamer(run: Run,
requested_traces: List[TraceBase],
rec_range,
rec_num: int = 50) -> List[dict]:
for requested_trace in requested_traces:
trace_name = requested_trace.name
context = Context(requested_trace.context)
trace = run.get_distribution_sequence(name=trace_name, context=context)
if not trace:
continue
record_range_missing = rec_range.start is None or rec_range.stop is None
if record_range_missing:
rec_range = IndexRange(trace.first_step(), trace.last_step() + 1)
steps_vals = trace.values.items_in_range(rec_range.start, rec_range.stop, rec_num)
steps = []
values = []
for step, val in steps_vals:
steps.append(step)
values.append(dist_record_to_encodable(val))
trace_dict = {
'record_range': (trace.first_step(), trace.last_step() + 1),
'name': trace.name,
'context': trace.context.to_dict(),
'values': values,
'iters': steps,
}
encoded_tree = encode_tree(trace_dict)
yield collect_run_streamable_data(encoded_tree)
def collect_requested_metric_traces(run: Run, requested_traces: List[TraceBase], steps_num: int = 200) -> List[dict]:
processed_traces_list = []
for requested_trace in requested_traces:
metric_name = requested_trace.name
context = Context(requested_trace.context)
trace = run.get_metric(name=metric_name, context=context)
if not trace:
continue
iters, values = trace.values.sparse_list()
values = list(map(lambda x: x if float('-inf') < x < float('inf') and x == x else None, values))
num_records = len(values)
step = (num_records // steps_num) or 1
_slice = slice(0, num_records, step)
processed_traces_list.append({
'name': trace.name,
'context': trace.context.to_dict(),
'values': sliced_array(values, _slice),
'iters': sliced_array(iters, _slice),
})
return processed_traces_list
def test_default_tracking_interval(self):
run = Run() # default tracking interval 10s
run_hash = run.hash
run.track(1, name='metric')
time.sleep(.1) # allow tracker to add resource usage metrics
del run
metrics = list(
self.repo.query_metrics(
f'run.hash == "{run_hash}" and metric.name.startswith("__")'))
self.assertEqual(4, len(metrics))
expected_metrics = {
'__system__cpu', '__system__disk_percent',
'__system__memory_percent', '__system__p_memory_percent'
}
self.assertSetEqual(expected_metrics, set(m.name for m in metrics))
def test_custom_tracking_interval(self):
run = Run(system_tracking_interval=1)
run_hash = run.hash
run.track(1, name='metric')
time.sleep(3) # allow tracker to add resource usage metrics
del run
metrics = list(
self.repo.query_metrics(
f'run.hash == "{run_hash}" and metric.name.startswith("__")'))
self.assertEqual(4, len(metrics))
expected_metrics = {
'__system__cpu', '__system__disk_percent',
'__system__memory_percent', '__system__p_memory_percent'
}
self.assertSetEqual(expected_metrics, set(m.name for m in metrics))
for metric in metrics:
# 3 sec. runtime, 1 sec. interval
self.assertGreaterEqual(len(metric.values), 3)
self.assertLessEqual(len(metric.values), 4)
def test_numpy_scalar_types_track(self):
"""covers https://github.com/aimhubio/aim/issues/1206"""
run = Run(system_tracking_interval=None)
run.track(np.array([1.0]), name='single_item_array')
run.track(np.array([[[1.0]]]), name='single_item_3d_array')
run.track(np.float64(1.0), name='numpy_float64')
run.track(np.float32(1.0), name='numpy_float32')
repo = Repo.default_repo()
metric_names = {metric.name for metric in repo.query_metrics()}
expected_metric_names = {
'single_item_array', 'single_item_3d_array', 'numpy_float64',
'numpy_float32'
}
self.assertSetEqual(expected_metric_names, metric_names)
def test_run_trace_dtype_and_last_value(self):
run = Run()
run.track(1.0, name='metric 1', context={})
run.track(2.0, name='metric 1', context={})
run.track(3.0, name='metric 1', context={})
run.track(1.0, name='metric 1', context={'subset': 'train'})
meta_container_path = os.path.join(self.repo.path, 'meta', 'chunks', run.hash)
rc = RocksContainer(meta_container_path, read_only=True)
tree = ContainerTreeView(rc)
metric_1_dict = tree.view(('meta', 'chunks', run.hash, 'traces', Context({}).idx, 'metric 1')).collect()
self.assertEqual(3.0, metric_1_dict['last'])
self.assertEqual('float', metric_1_dict['dtype'])
metric_1_dict = tree.view(('meta', 'chunks', run.hash, 'traces',
Context({'subset': 'train'}).idx, 'metric 1')).collect()
self.assertEqual(1.0, metric_1_dict['last'])
def requested_text_traces_streamer(run: Run,
requested_traces: List[TraceBase],
rec_range, idx_range,
rec_num: int = 50, idx_num: int = 5) -> List[dict]:
for requested_trace in requested_traces:
trace_name = requested_trace.name
context = Context(requested_trace.context)
trace = run.get_text_sequence(name=trace_name, context=context)
if not trace:
continue
record_range_missing = rec_range.start is None or rec_range.stop is None
if record_range_missing:
rec_range = IndexRange(trace.first_step(), trace.last_step() + 1)
index_range_missing = idx_range.start is None or idx_range.stop is None
if index_range_missing:
idx_range = IndexRange(0, trace.record_length() or 1)
rec_length = trace.record_length() or 1
idx_step = rec_length // idx_num or 1
idx_slice = slice(idx_range.start, idx_range.stop, idx_step)
steps_vals = trace.values.items_in_range(rec_range.start, rec_range.stop, rec_num)
steps = []
values = []
for step, val in steps_vals:
steps.append(step)
if isinstance(val, list):
values.append(text_collection_to_encodable(sliced_custom_object_record(val, idx_slice)))
elif idx_slice.start == 0:
values.append(text_record_to_encodable(val))
else:
values.append([])
trace_dict = {
'record_range': (trace.first_step(), trace.last_step() + 1),
'index_range': (0, rec_length),
'name': trace.name,
'context': trace.context.to_dict(),
'values': values,
'iters': steps,
}
encoded_tree = encode_tree(trace_dict)
yield collect_run_streamable_data(encoded_tree)
def test_reject_non_scalar_arrays_track(self):
run = Run(system_tracking_interval=None)
with self.assertRaises(ValueError):
run.track(np.array([1.0, 2.0]), name='fail')
def test_series_tree_values(self):
# sequential steps
run = Run()
run.track(1.0, name='metric 1', context={})
run.track(2.0, name='metric 1', context={})
run.track(3.0, name='metric 1', context={})
series_container_path = os.path.join(self.repo.path, 'seqs', 'chunks', run.hash)
rc = RocksContainer(series_container_path, read_only=True)
tree = ContainerTreeView(rc)
traces_dict = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).collect()
self.assertSetEqual({'val', 'epoch', 'time'}, set(traces_dict.keys()))
self.assertEqual(3, len(traces_dict['val']))
self.assertEqual(3, len(traces_dict['epoch']))
self.assertEqual(3, len(traces_dict['time']))
self.assertEqual(1.0, traces_dict['val'][0])
self.assertEqual(2.0, traces_dict['val'][1])
self.assertEqual(3.0, traces_dict['val'][2])
# user-specified steps
run = Run()
run.track(1.0, name='metric 1', step=10, context={})
run.track(2.0, name='metric 1', step=20, context={})
run.track(3.0, name='metric 1', step=30, context={})
series_container_path = os.path.join(self.repo.path, 'seqs', 'chunks', run.hash)
rc = RocksContainer(series_container_path, read_only=True)
tree = ContainerTreeView(rc)
traces_dict = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).collect()
self.assertEqual(31, len(traces_dict['val'])) # last index is 30
# sparse array
self.assertTrue(all(x is None for x in traces_dict['val'][0:10]))
self.assertEqual(1.0, traces_dict['val'][10])
self.assertTrue(all(x is None for x in traces_dict['val'][11:20]))
self.assertEqual(2.0, traces_dict['val'][20])
self.assertTrue(all(x is None for x in traces_dict['val'][21:30]))
self.assertEqual(3.0, traces_dict['val'][30])
val_array_view = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).array('val')
self.assertEqual(31, len(val_array_view))
self.assertEqual(3, len(list(val_array_view)))
self.assertEqual(1.0, val_array_view[10])
self.assertEqual(2.0, val_array_view[20])
self.assertEqual(3.0, val_array_view[30])
# user-specified steps, unordered
run = Run()
run.track(3.0, name='metric 1', step=30, context={})
run.track(1.0, name='metric 1', step=10, context={})
run.track(2.0, name='metric 1', step=20, context={})
series_container_path = os.path.join(self.repo.path, 'seqs', 'chunks', run.hash)
rc = RocksContainer(series_container_path, read_only=True)
tree = ContainerTreeView(rc)
traces_dict = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).collect()
self.assertEqual(31, len(traces_dict['val'])) # last index is 30
# sparse array
self.assertTrue(all(x is None for x in traces_dict['val'][0:10]))
self.assertEqual(1.0, traces_dict['val'][10])
self.assertTrue(all(x is None for x in traces_dict['val'][11:20]))
self.assertEqual(2.0, traces_dict['val'][20])
self.assertTrue(all(x is None for x in traces_dict['val'][21:30]))
self.assertEqual(3.0, traces_dict['val'][30])
def test_run_set_param_meta_tree(self):
run = Run()
run['p1'] = 1
run['p2'] = {'foo': 'bar'}
run['p3'] = 0.0
run['p4'] = True
run['p5'] = ['x', 1, 2]
run['p6'] = None
run['p7'] = 'bazե\x00a-a'
run['p8'] = b'blob'
meta_container_path = os.path.join(self.repo.path, 'meta', 'chunks', run.hash)
rc = RocksContainer(meta_container_path, read_only=True)
tree = ContainerTreeView(rc)
meta_attrs_tree = tree.view(('meta', 'attrs'))
meta_run_attrs_tree = tree.view(('meta', 'chunks', run.hash, 'attrs'))
val = meta_attrs_tree['p1']
self.assertEqual(int, type(val))
self.assertEqual(1, val)
val = meta_run_attrs_tree['p1']
self.assertEqual(int, type(val))
self.assertEqual(1, val)
val = meta_attrs_tree['p3']
self.assertEqual(float, type(val))
self.assertEqual(0.0, val)
val = meta_run_attrs_tree['p3']
self.assertEqual(float, type(val))
self.assertEqual(0.0, val)
val = meta_attrs_tree['p4']
self.assertEqual(bool, type(val))
self.assertEqual(True, val)
val = meta_run_attrs_tree['p4']
self.assertEqual(bool, type(val))
self.assertEqual(True, val)
val = meta_attrs_tree['p2']
self.assertEqual(dict, type(val))
self.assertEqual({'foo': 'bar'}, val)
val = meta_run_attrs_tree['p2']
self.assertEqual(dict, type(val))
self.assertEqual({'foo': 'bar'}, val)
val = meta_attrs_tree['p5']
self.assertEqual(list, type(val))
self.assertEqual(['x', 1, 2], val)
val = meta_run_attrs_tree['p5']
self.assertEqual(list, type(val))
self.assertEqual(['x', 1, 2], val)
val = meta_attrs_tree['p6']
self.assertEqual(None, val)
val = meta_run_attrs_tree['p6']
self.assertEqual(None, val)
val = meta_attrs_tree['p7']
self.assertEqual(str, type(val))
self.assertEqual('bazե\x00a-a', val)
val = meta_run_attrs_tree['p7']
self.assertEqual(str, type(val))
self.assertEqual('bazե\x00a-a', val)
val = meta_attrs_tree['p8']
self.assertEqual(bytes, type(val))
self.assertEqual(b'blob', val)
val = meta_run_attrs_tree['p8']
self.assertEqual(bytes, type(val))
self.assertEqual(b'blob', val)
def test_resource_tracking_interval_limits(self):
run = Run(system_tracking_interval=0)
self.assertIsNone(run._system_resource_tracker)
run = Run(system_tracking_interval=2 * 24 * 3600) # two days
self.assertIsNone(run._system_resource_tracker)