def test_disable_resource_tracking(self):
run = Run(system_tracking_interval=None)
run_hash = run.hash
run.track(1, name='metric')
time.sleep(.1) # allow tracker to add resource usage metrics
del run
metrics = list(
self.repo.query_metrics(
f'run.hash == "{run_hash}" and metric.name.startswith("__")'))
self.assertListEqual([], metrics)
def test_meta_run_tree_contexts_and_names(self):
train_context = {'subset': 'train'}
val_context = {'subset': 'val'}
empty_context = {}
run = Run(repo=self.repo, system_tracking_interval=None)
run.track(1, name='metric 1', context=train_context)
run.track(1, name='metric 2', context=train_context)
run.track(1, name='metric 1', context=val_context)
run.track(1, name='metric 2', context=val_context)
run.track(1, name='metric 3', context=val_context)
run.track(0, name='metric')
meta_container_path = os.path.join(self.repo.path, 'meta', 'chunks', run.hash)
rc = RocksContainer(meta_container_path, read_only=True)
tree = ContainerTreeView(rc)
contexts = tree.view(('meta', 'chunks', run.hash, 'contexts')).collect()
for ctx in [train_context, val_context, empty_context]:
self.assertIn(Context(ctx).idx, contexts)
self.assertDictEqual(contexts[Context(ctx).idx], ctx)
traces = tree.view(('meta', 'chunks', run.hash, 'traces', Context(train_context).idx)).collect()
self.assertSetEqual({'metric 1', 'metric 2'}, set(traces.keys()))
traces = tree.view(('meta', 'chunks', run.hash, 'traces', Context(val_context).idx)).collect()
self.assertSetEqual({'metric 1', 'metric 2', 'metric 3'}, set(traces.keys()))
traces = tree.view(('meta', 'chunks', run.hash, 'traces', Context(empty_context).idx)).collect()
self.assertSetEqual({'metric'}, set(traces.keys()))
def test_default_tracking_interval(self):
run = Run() # default tracking interval 10s
run_hash = run.hash
run.track(1, name='metric')
time.sleep(.1) # allow tracker to add resource usage metrics
del run
metrics = list(
self.repo.query_metrics(
f'run.hash == "{run_hash}" and metric.name.startswith("__")'))
self.assertEqual(4, len(metrics))
expected_metrics = {
'__system__cpu', '__system__disk_percent',
'__system__memory_percent', '__system__p_memory_percent'
}
self.assertSetEqual(expected_metrics, set(m.name for m in metrics))
def test_custom_tracking_interval(self):
run = Run(system_tracking_interval=1)
run_hash = run.hash
run.track(1, name='metric')
time.sleep(3) # allow tracker to add resource usage metrics
del run
metrics = list(
self.repo.query_metrics(
f'run.hash == "{run_hash}" and metric.name.startswith("__")'))
self.assertEqual(4, len(metrics))
expected_metrics = {
'__system__cpu', '__system__disk_percent',
'__system__memory_percent', '__system__p_memory_percent'
}
self.assertSetEqual(expected_metrics, set(m.name for m in metrics))
for metric in metrics:
# 3 sec. runtime, 1 sec. interval
self.assertGreaterEqual(len(metric.values), 3)
self.assertLessEqual(len(metric.values), 4)
def test_numpy_scalar_types_track(self):
"""covers https://github.com/aimhubio/aim/issues/1206"""
run = Run(system_tracking_interval=None)
run.track(np.array([1.0]), name='single_item_array')
run.track(np.array([[[1.0]]]), name='single_item_3d_array')
run.track(np.float64(1.0), name='numpy_float64')
run.track(np.float32(1.0), name='numpy_float32')
repo = Repo.default_repo()
metric_names = {metric.name for metric in repo.query_metrics()}
expected_metric_names = {
'single_item_array', 'single_item_3d_array', 'numpy_float64',
'numpy_float32'
}
self.assertSetEqual(expected_metric_names, metric_names)
def test_run_trace_dtype_and_last_value(self):
run = Run()
run.track(1.0, name='metric 1', context={})
run.track(2.0, name='metric 1', context={})
run.track(3.0, name='metric 1', context={})
run.track(1.0, name='metric 1', context={'subset': 'train'})
meta_container_path = os.path.join(self.repo.path, 'meta', 'chunks', run.hash)
rc = RocksContainer(meta_container_path, read_only=True)
tree = ContainerTreeView(rc)
metric_1_dict = tree.view(('meta', 'chunks', run.hash, 'traces', Context({}).idx, 'metric 1')).collect()
self.assertEqual(3.0, metric_1_dict['last'])
self.assertEqual('float', metric_1_dict['dtype'])
metric_1_dict = tree.view(('meta', 'chunks', run.hash, 'traces',
Context({'subset': 'train'}).idx, 'metric 1')).collect()
self.assertEqual(1.0, metric_1_dict['last'])
def test_reject_non_scalar_arrays_track(self):
run = Run(system_tracking_interval=None)
with self.assertRaises(ValueError):
run.track(np.array([1.0, 2.0]), name='fail')
def test_series_tree_values(self):
# sequential steps
run = Run()
run.track(1.0, name='metric 1', context={})
run.track(2.0, name='metric 1', context={})
run.track(3.0, name='metric 1', context={})
series_container_path = os.path.join(self.repo.path, 'seqs', 'chunks', run.hash)
rc = RocksContainer(series_container_path, read_only=True)
tree = ContainerTreeView(rc)
traces_dict = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).collect()
self.assertSetEqual({'val', 'epoch', 'time'}, set(traces_dict.keys()))
self.assertEqual(3, len(traces_dict['val']))
self.assertEqual(3, len(traces_dict['epoch']))
self.assertEqual(3, len(traces_dict['time']))
self.assertEqual(1.0, traces_dict['val'][0])
self.assertEqual(2.0, traces_dict['val'][1])
self.assertEqual(3.0, traces_dict['val'][2])
# user-specified steps
run = Run()
run.track(1.0, name='metric 1', step=10, context={})
run.track(2.0, name='metric 1', step=20, context={})
run.track(3.0, name='metric 1', step=30, context={})
series_container_path = os.path.join(self.repo.path, 'seqs', 'chunks', run.hash)
rc = RocksContainer(series_container_path, read_only=True)
tree = ContainerTreeView(rc)
traces_dict = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).collect()
self.assertEqual(31, len(traces_dict['val'])) # last index is 30
# sparse array
self.assertTrue(all(x is None for x in traces_dict['val'][0:10]))
self.assertEqual(1.0, traces_dict['val'][10])
self.assertTrue(all(x is None for x in traces_dict['val'][11:20]))
self.assertEqual(2.0, traces_dict['val'][20])
self.assertTrue(all(x is None for x in traces_dict['val'][21:30]))
self.assertEqual(3.0, traces_dict['val'][30])
val_array_view = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).array('val')
self.assertEqual(31, len(val_array_view))
self.assertEqual(3, len(list(val_array_view)))
self.assertEqual(1.0, val_array_view[10])
self.assertEqual(2.0, val_array_view[20])
self.assertEqual(3.0, val_array_view[30])
# user-specified steps, unordered
run = Run()
run.track(3.0, name='metric 1', step=30, context={})
run.track(1.0, name='metric 1', step=10, context={})
run.track(2.0, name='metric 1', step=20, context={})
series_container_path = os.path.join(self.repo.path, 'seqs', 'chunks', run.hash)
rc = RocksContainer(series_container_path, read_only=True)
tree = ContainerTreeView(rc)
traces_dict = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).collect()
self.assertEqual(31, len(traces_dict['val'])) # last index is 30
# sparse array
self.assertTrue(all(x is None for x in traces_dict['val'][0:10]))
self.assertEqual(1.0, traces_dict['val'][10])
self.assertTrue(all(x is None for x in traces_dict['val'][11:20]))
self.assertEqual(2.0, traces_dict['val'][20])
self.assertTrue(all(x is None for x in traces_dict['val'][21:30]))
self.assertEqual(3.0, traces_dict['val'][30])