def test_meta_run_tree_contexts_and_names(self):
train_context = {'subset': 'train'}
val_context = {'subset': 'val'}
empty_context = {}
run = Run(repo=self.repo, system_tracking_interval=None)
run.track(1, name='metric 1', context=train_context)
run.track(1, name='metric 2', context=train_context)
run.track(1, name='metric 1', context=val_context)
run.track(1, name='metric 2', context=val_context)
run.track(1, name='metric 3', context=val_context)
run.track(0, name='metric')
meta_container_path = os.path.join(self.repo.path, 'meta', 'chunks', run.hash)
rc = RocksContainer(meta_container_path, read_only=True)
tree = ContainerTreeView(rc)
contexts = tree.view(('meta', 'chunks', run.hash, 'contexts')).collect()
for ctx in [train_context, val_context, empty_context]:
self.assertIn(Context(ctx).idx, contexts)
self.assertDictEqual(contexts[Context(ctx).idx], ctx)
traces = tree.view(('meta', 'chunks', run.hash, 'traces', Context(train_context).idx)).collect()
self.assertSetEqual({'metric 1', 'metric 2'}, set(traces.keys()))
traces = tree.view(('meta', 'chunks', run.hash, 'traces', Context(val_context).idx)).collect()
self.assertSetEqual({'metric 1', 'metric 2', 'metric 3'}, set(traces.keys()))
traces = tree.view(('meta', 'chunks', run.hash, 'traces', Context(empty_context).idx)).collect()
self.assertSetEqual({'metric'}, set(traces.keys()))
def _track_legacy_run_step(run: Run, metric_name: str, context: dict, val):
(value, step, epoch, timestamp) = val
from aim.storage.context import Context, Metric
if context is None:
context = {}
ctx = Context(context)
metric = Metric(metric_name, ctx)
if ctx not in run.contexts:
run.meta_tree['contexts', ctx.idx] = ctx.to_dict()
run.meta_run_tree['contexts', ctx.idx] = ctx.to_dict()
run.contexts[ctx] = ctx.idx
run._idx_to_ctx[ctx.idx] = ctx
time_view = run.series_run_tree.view(metric.selector).array('time').allocate()
val_view = run.series_run_tree.view(metric.selector).array('val').allocate()
epoch_view = run.series_run_tree.view(metric.selector).array('epoch').allocate()
max_idx = run.series_counters.get((ctx, metric_name), None)
if max_idx == None:
max_idx = len(val_view)
if max_idx == 0:
run.meta_tree['traces', ctx.idx, metric_name] = 1
run.meta_run_tree['traces', ctx.idx, metric_name, "last"] = value
run.series_counters[ctx, metric_name] = max_idx + 1
time_view[step] = timestamp
val_view[step] = value
epoch_view[step] = epoch
def track(
self,
value,
name: str,
step: int = None,
epoch: int = None,
*,
context: AimObject = None,
):
track_time = time()
# TODO move to Trace
if context is None:
context = {}
value = convert_to_py_number(value)
ctx = Context(context)
metric = Metric(name, ctx)
if ctx not in self.contexts:
self.meta_tree['contexts', ctx.idx] = ctx.to_dict()
self.meta_run_tree['contexts', ctx.idx] = ctx.to_dict()
self.contexts[ctx] = ctx.idx
self._idx_to_ctx[ctx.idx] = ctx
val_view = self.series_run_tree.view(
metric.selector).array('val').allocate()
epoch_view = self.series_run_tree.view(
metric.selector).array('epoch').allocate()
time_view = self.series_run_tree.view(
metric.selector).array('time').allocate()
max_idx = self.series_counters.get((ctx, name), None)
if max_idx == None:
max_idx = len(val_view)
if max_idx == 0:
self.meta_tree['traces', ctx.idx, name] = 1
self.meta_run_tree['traces', ctx.idx, name, "last"] = value
self.series_counters[ctx, name] = max_idx + 1
# TODO perform assignments in an atomic way
if step is None:
step = max_idx
val_view[step] = value
epoch_view[step] = epoch
time_view[step] = track_time
def requested_figure_object_traces_streamer(
run: Run, requested_traces: List[TraceBase], rec_range, rec_num: int = 50
) -> List[dict]:
for requested_trace in requested_traces:
trace_name = requested_trace.name
context = Context(requested_trace.context)
trace = run.get_figure_sequence(name=trace_name, context=context)
if not trace:
continue
record_range_missing = rec_range.start is None or rec_range.stop is None
if record_range_missing:
rec_range = IndexRange(trace.first_step(), trace.last_step() + 1)
steps = []
values = []
steps_vals = trace.values.items_in_range(
rec_range.start, rec_range.stop, rec_num
)
for step, val in steps_vals:
steps.append(step)
values.append(preparer(val, trace, step, decode=True))
trace_dict = {
'name': trace.name,
'context': trace.context.to_dict(),
'values': values,
'iters': steps,
'record_range': (trace.first_step(), trace.last_step() + 1),
}
encoded_tree = encode_tree(trace_dict)
yield collect_run_streamable_data(encoded_tree)
def collect_requested_metric_traces(run: Run, requested_traces: List[TraceBase], steps_num: int = 200) -> List[dict]:
processed_traces_list = []
for requested_trace in requested_traces:
metric_name = requested_trace.name
context = Context(requested_trace.context)
trace = run.get_metric(name=metric_name, context=context)
if not trace:
continue
iters, values = trace.values.sparse_list()
values = list(map(lambda x: x if float('-inf') < x < float('inf') and x == x else None, values))
num_records = len(values)
step = (num_records // steps_num) or 1
_slice = slice(0, num_records, step)
processed_traces_list.append({
'name': trace.name,
'context': trace.context.to_dict(),
'values': sliced_array(values, _slice),
'iters': sliced_array(iters, _slice),
})
return processed_traces_list
def custom_aligned_metrics_streamer(requested_runs: List[AlignedRunIn], x_axis: str, repo: 'Repo') -> bytes:
for run_data in requested_runs:
run_hash = run_data.run_id
requested_traces = run_data.traces
run = Run(run_hash, repo=repo, read_only=True)
traces_list = []
for trace_data in requested_traces:
context = Context(trace_data.context)
trace = run.get_metric(name=trace_data.name,
context=context)
x_axis_trace = run.get_metric(name=x_axis,
context=context)
if not (trace and x_axis_trace):
continue
_slice = slice(*trace_data.slice)
iters = trace.values.sparse_numpy()[0]
sliced_iters = sliced_np_array(iters, _slice)
x_axis_iters, x_axis_values = collect_x_axis_data(x_axis_trace, sliced_iters)
traces_list.append({
'name': trace.name,
'context': trace.context.to_dict(),
'x_axis_values': x_axis_values,
'x_axis_iters': x_axis_iters,
})
run_dict = {
run_hash: traces_list
}
encoded_tree = encode_tree(run_dict)
yield collect_run_streamable_data(encoded_tree)
def test_type_compatibility_for_empty_list(self):
run = Run(system_tracking_interval=None)
context = {}
ctx = Context(context)
seq_name = 'obj_list'
sequence_info = run.meta_run_tree.subtree(
('traces', ctx.idx, seq_name))
typed_sequences_info = run.meta_tree.subtree('traces_types')
run.track([], name=seq_name, context=context)
self.assertEqual('list', sequence_info['dtype'])
self.assertEqual(1, typed_sequences_info['list', ctx.idx, seq_name])
self.assertIsNone(
typed_sequences_info.get(('list(float)', ctx.idx, seq_name), None))
run.track([], name=seq_name, context=context)
self.assertEqual('list', sequence_info['dtype'])
self.assertIsNone(
typed_sequences_info.get(('list(float)', ctx.idx, seq_name), None))
run.track([1.], name=seq_name, context=context)
self.assertEqual('list(float)', sequence_info['dtype'])
self.assertEqual(
1, typed_sequences_info['list(float)', ctx.idx, seq_name])
run.track([], name=seq_name, context=context)
self.assertEqual('list(float)', sequence_info['dtype'])
with self.assertRaises(ValueError) as cm:
run.track([5], name=seq_name, context=context)
exception = cm.exception
self.assertEqual(
f'Cannot log value \'{[5]}\' on sequence \'{seq_name}\'. Incompatible data types.',
exception.args[0])
def test_run_trace_dtype_and_last_value(self):
run = Run()
run.track(1.0, name='metric 1', context={})
run.track(2.0, name='metric 1', context={})
run.track(3.0, name='metric 1', context={})
run.track(1.0, name='metric 1', context={'subset': 'train'})
meta_container_path = os.path.join(self.repo.path, 'meta', 'chunks', run.hash)
rc = RocksContainer(meta_container_path, read_only=True)
tree = ContainerTreeView(rc)
metric_1_dict = tree.view(('meta', 'chunks', run.hash, 'traces', Context({}).idx, 'metric 1')).collect()
self.assertEqual(3.0, metric_1_dict['last'])
self.assertEqual('float', metric_1_dict['dtype'])
metric_1_dict = tree.view(('meta', 'chunks', run.hash, 'traces',
Context({'subset': 'train'}).idx, 'metric 1')).collect()
self.assertEqual(1.0, metric_1_dict['last'])
def idx_to_ctx(self, idx: int) -> Context:
ctx = Run._idx_to_ctx.get(idx)
if ctx is not None:
return ctx
ctx = Context(self.meta_tree['contexts', idx])
Run._idx_to_ctx[idx] = ctx
self.contexts[ctx] = idx
return ctx
def setUpClass(cls) -> None:
super().setUpClass()
cls.image_blobs = {}
run = Run(run_hash=cls.run_hash, read_only=True)
empty_context = Context({})
for step in range(10):
for idx in range(5):
img_view = run.series_run_tree.subtree(
(empty_context.idx, 'random_images', 'val', step, idx))
cls.image_blobs[img_view['caption']] = img_view['data'].load()
def requested_image_traces_streamer(run: Run,
requested_traces: List[TraceBase],
rec_range, idx_range,
rec_num: int = 50, idx_num: int = 5) -> List[dict]:
for requested_trace in requested_traces:
trace_name = requested_trace.name
context = Context(requested_trace.context)
trace = run.get_image_sequence(name=trace_name, context=context)
if not trace:
continue
record_range_missing = rec_range.start is None or rec_range.stop is None
if record_range_missing:
rec_range = IndexRange(trace.first_step(), trace.last_step() + 1)
index_range_missing = idx_range.start is None or idx_range.stop is None
if index_range_missing:
idx_range = IndexRange(0, trace.record_length() or 1)
rec_length = trace.record_length() or 1
idx_step = rec_length // idx_num or 1
idx_slice = slice(idx_range.start, idx_range.stop, idx_step)
steps_vals = trace.values.items_in_range(rec_range.start, rec_range.stop, rec_num)
steps = []
values = []
for step, val in steps_vals:
steps.append(step)
if isinstance(val, list):
values.append(
img_collection_record_to_encodable(sliced_custom_object_record(val, idx_slice), trace, step)
)
elif idx_slice.start == 0:
values.append(img_record_to_encodable(val, trace, step))
else:
values.append([])
trace_dict = {
'record_range': (trace.first_step(), trace.last_step() + 1),
'index_range': (0, rec_length),
'name': trace.name,
'context': trace.context.to_dict(),
'values': values,
'iters': steps,
}
encoded_tree = encode_tree(trace_dict)
yield collect_run_streamable_data(encoded_tree)
def test_series_tree_values(self):
# sequential steps
run = Run()
run.track(1.0, name='metric 1', context={})
run.track(2.0, name='metric 1', context={})
run.track(3.0, name='metric 1', context={})
series_container_path = os.path.join(self.repo.path, 'seqs', 'chunks', run.hash)
rc = RocksContainer(series_container_path, read_only=True)
tree = ContainerTreeView(rc)
traces_dict = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).collect()
self.assertSetEqual({'val', 'epoch', 'time'}, set(traces_dict.keys()))
self.assertEqual(3, len(traces_dict['val']))
self.assertEqual(3, len(traces_dict['epoch']))
self.assertEqual(3, len(traces_dict['time']))
self.assertEqual(1.0, traces_dict['val'][0])
self.assertEqual(2.0, traces_dict['val'][1])
self.assertEqual(3.0, traces_dict['val'][2])
# user-specified steps
run = Run()
run.track(1.0, name='metric 1', step=10, context={})
run.track(2.0, name='metric 1', step=20, context={})
run.track(3.0, name='metric 1', step=30, context={})
series_container_path = os.path.join(self.repo.path, 'seqs', 'chunks', run.hash)
rc = RocksContainer(series_container_path, read_only=True)
tree = ContainerTreeView(rc)
traces_dict = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).collect()
self.assertEqual(31, len(traces_dict['val'])) # last index is 30
# sparse array
self.assertTrue(all(x is None for x in traces_dict['val'][0:10]))
self.assertEqual(1.0, traces_dict['val'][10])
self.assertTrue(all(x is None for x in traces_dict['val'][11:20]))
self.assertEqual(2.0, traces_dict['val'][20])
self.assertTrue(all(x is None for x in traces_dict['val'][21:30]))
self.assertEqual(3.0, traces_dict['val'][30])
val_array_view = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).array('val')
self.assertEqual(31, len(val_array_view))
self.assertEqual(3, len(list(val_array_view)))
self.assertEqual(1.0, val_array_view[10])
self.assertEqual(2.0, val_array_view[20])
self.assertEqual(3.0, val_array_view[30])
# user-specified steps, unordered
run = Run()
run.track(3.0, name='metric 1', step=30, context={})
run.track(1.0, name='metric 1', step=10, context={})
run.track(2.0, name='metric 1', step=20, context={})
series_container_path = os.path.join(self.repo.path, 'seqs', 'chunks', run.hash)
rc = RocksContainer(series_container_path, read_only=True)
tree = ContainerTreeView(rc)
traces_dict = tree.view(('seqs', 'chunks', run.hash, Context({}).idx, 'metric 1')).collect()
self.assertEqual(31, len(traces_dict['val'])) # last index is 30
# sparse array
self.assertTrue(all(x is None for x in traces_dict['val'][0:10]))
self.assertEqual(1.0, traces_dict['val'][10])
self.assertTrue(all(x is None for x in traces_dict['val'][11:20]))
self.assertEqual(2.0, traces_dict['val'][20])
self.assertTrue(all(x is None for x in traces_dict['val'][21:30]))
self.assertEqual(3.0, traces_dict['val'][30])
def _track_impl(
self,
value,
track_time: float,
name: str,
step: int = None,
epoch: int = None,
*,
context: AimObject = None,
):
if context is None:
context = {}
if is_number(value):
val = convert_to_py_number(value)
elif isinstance(value, (CustomObject, list, tuple)):
val = value
else:
raise ValueError(
f'Input metric of type {type(value)} is neither python number nor AimObject'
)
dtype = get_object_typename(val)
ctx = Context(context)
sequence = SequenceDescriptor(name, ctx)
if ctx not in self.contexts:
self.meta_tree['contexts', ctx.idx] = context
self.meta_run_tree['contexts', ctx.idx] = context
self.contexts[ctx] = ctx.idx
self._idx_to_ctx[ctx.idx] = ctx
seq_info = self.sequence_info[sequence.selector]
if not seq_info.initialized:
seq_info.val_view = self.series_run_tree.subtree(
sequence.selector).array('val').allocate()
seq_info.epoch_view = self.series_run_tree.subtree(
sequence.selector).array('epoch').allocate()
seq_info.time_view = self.series_run_tree.subtree(
sequence.selector).array('time').allocate()
seq_info.count = len(seq_info.val_view)
seq_info.sequence_dtype = self.meta_run_tree.get(
('traces', ctx.idx, name, 'dtype'), None)
if seq_info.count != 0 and seq_info.sequence_dtype is None: # continue tracking on old sequence
seq_info.sequence_dtype = 'float'
seq_info.initialized = True
if seq_info.sequence_dtype is not None:
def update_trace_dtype(new_dtype):
self.meta_tree['traces_types', new_dtype, ctx.idx, name] = 1
seq_info.sequence_dtype = self.meta_run_tree[
'traces', ctx.idx, name, 'dtype'] = new_dtype
compatible = check_types_compatibility(dtype,
seq_info.sequence_dtype,
update_trace_dtype)
if not compatible:
raise ValueError(
f'Cannot log value \'{value}\' on sequence \'{name}\'. Incompatible data types.'
)
step = step or seq_info.count
if seq_info.count == 0:
self.meta_tree['traces_types', dtype, ctx.idx, name] = 1
seq_info.sequence_dtype = self.meta_run_tree['traces', ctx.idx,
name, 'dtype'] = dtype
self.meta_run_tree['traces', ctx.idx, name, 'first_step'] = step
self.meta_run_tree['traces', ctx.idx, name, 'last'] = val
self.meta_run_tree['traces', ctx.idx, name, 'last_step'] = step
if isinstance(val, (tuple, list)):
record_max_length = self.meta_run_tree.get(
('traces', ctx.idx, name, 'record_max_length'), 0)
self.meta_run_tree['traces', ctx.idx, name,
'record_max_length'] = max(
record_max_length, len(val))
# TODO perform assignments in an atomic way
seq_info.val_view[step] = val
seq_info.epoch_view[step] = epoch
seq_info.time_view[step] = track_time
seq_info.count = seq_info.count + 1