def custom_aligned_metrics_streamer(requested_runs: List[AlignedRunIn], x_axis: str, repo: 'Repo') -> bytes:
for run_data in requested_runs:
run_hash = run_data.run_id
requested_traces = run_data.traces
run = Run(run_hash, repo=repo, read_only=True)
traces_list = []
for trace_data in requested_traces:
context = Context(trace_data.context)
trace = run.get_metric(name=trace_data.name,
context=context)
x_axis_trace = run.get_metric(name=x_axis,
context=context)
if not (trace and x_axis_trace):
continue
_slice = slice(*trace_data.slice)
iters = trace.values.sparse_numpy()[0]
sliced_iters = sliced_np_array(iters, _slice)
x_axis_iters, x_axis_values = collect_x_axis_data(x_axis_trace, sliced_iters)
traces_list.append({
'name': trace.name,
'context': trace.context.to_dict(),
'x_axis_values': x_axis_values,
'x_axis_iters': x_axis_iters,
})
run_dict = {
run_hash: traces_list
}
encoded_tree = encode_tree(run_dict)
yield collect_run_streamable_data(encoded_tree)
def requested_figure_object_traces_streamer(
run: Run, requested_traces: List[TraceBase], rec_range, rec_num: int = 50
) -> List[dict]:
for requested_trace in requested_traces:
trace_name = requested_trace.name
context = Context(requested_trace.context)
trace = run.get_figure_sequence(name=trace_name, context=context)
if not trace:
continue
record_range_missing = rec_range.start is None or rec_range.stop is None
if record_range_missing:
rec_range = IndexRange(trace.first_step(), trace.last_step() + 1)
steps = []
values = []
steps_vals = trace.values.items_in_range(
rec_range.start, rec_range.stop, rec_num
)
for step, val in steps_vals:
steps.append(step)
values.append(preparer(val, trace, step, decode=True))
trace_dict = {
'name': trace.name,
'context': trace.context.to_dict(),
'values': values,
'iters': steps,
'record_range': (trace.first_step(), trace.last_step() + 1),
}
encoded_tree = encode_tree(trace_dict)
yield collect_run_streamable_data(encoded_tree)
def message_stream_generator():
header = rpc_messages.InstructionRequest(
header=rpc_messages.RequestHeader(version='0.1',
handler=resource,
client_uri=self.uri,
method_name=method))
yield header
stream = pack_stream(encode_tree(args))
for chunk in stream:
yield rpc_messages.InstructionRequest(message=chunk)
async def metric_search_result_streamer(traces: QueryTraceCollection,
steps_num: int,
x_axis: Optional[str]) -> bytes:
for run_trace_collection in traces.iter_runs():
run = None
traces_list = []
for trace in run_trace_collection.iter():
if not run:
run = run_trace_collection.run
iters, values = trace.values.sparse_numpy()
num_records = len(values)
step = (num_records // steps_num) or 1
_slice = slice(0, num_records, step)
sliced_iters = sliced_np_array(iters, _slice)
x_axis_trace = run.get_trace(x_axis,
trace.context) if x_axis else None
x_axis_iters, x_axis_values = collect_x_axis_data(
x_axis_trace, sliced_iters)
traces_list.append({
'metric_name':
trace.name,
'context':
trace.context.to_dict(),
'slice': [0, num_records, step],
'values':
numpy_to_encodable(sliced_np_array(values, _slice)),
'iters':
numpy_to_encodable(sliced_iters),
'epochs':
numpy_to_encodable(
sliced_np_array(trace.epochs.values_numpy(), _slice)),
'timestamps':
numpy_to_encodable(
sliced_np_array(trace.timestamps.values_numpy(), _slice)),
'x_axis_values':
x_axis_values,
'x_axis_iters':
x_axis_iters,
})
if run:
run_dict = {
run.hashname: {
'params': run[...],
'traces': traces_list,
'props': get_run_props(run)
}
}
encoded_tree = encode_tree(run_dict)
yield collect_run_streamable_data(encoded_tree)
async def run_search_result_streamer(runs: QueryRunTraceCollection) -> bytes:
for run_trace_collection in runs.iter_runs():
run = run_trace_collection.run
run_dict = {
run.hashname: {
'params': run[...],
'traces': run.get_traces_overview(),
'props': get_run_props(run)
}
}
encoded_tree = encode_tree(run_dict)
yield collect_run_streamable_data(encoded_tree)
def requested_image_traces_streamer(run: Run,
requested_traces: List[TraceBase],
rec_range, idx_range,
rec_num: int = 50, idx_num: int = 5) -> List[dict]:
for requested_trace in requested_traces:
trace_name = requested_trace.name
context = Context(requested_trace.context)
trace = run.get_image_sequence(name=trace_name, context=context)
if not trace:
continue
record_range_missing = rec_range.start is None or rec_range.stop is None
if record_range_missing:
rec_range = IndexRange(trace.first_step(), trace.last_step() + 1)
index_range_missing = idx_range.start is None or idx_range.stop is None
if index_range_missing:
idx_range = IndexRange(0, trace.record_length() or 1)
rec_length = trace.record_length() or 1
idx_step = rec_length // idx_num or 1
idx_slice = slice(idx_range.start, idx_range.stop, idx_step)
steps_vals = trace.values.items_in_range(rec_range.start, rec_range.stop, rec_num)
steps = []
values = []
for step, val in steps_vals:
steps.append(step)
if isinstance(val, list):
values.append(
img_collection_record_to_encodable(sliced_custom_object_record(val, idx_slice), trace, step)
)
elif idx_slice.start == 0:
values.append(img_record_to_encodable(val, trace, step))
else:
values.append([])
trace_dict = {
'record_range': (trace.first_step(), trace.last_step() + 1),
'index_range': (0, rec_length),
'name': trace.name,
'context': trace.context.to_dict(),
'values': values,
'iters': steps,
}
encoded_tree = encode_tree(trace_dict)
yield collect_run_streamable_data(encoded_tree)
def _pack_run_data(run_: Run, traces_: list):
_rec_range = (
trcs_rec_range if record_range_missing or calc_total_ranges else rec_range
)
run_dict = {
run_.hash: {
'ranges': {
'record_range': [_rec_range.start, _rec_range.stop],
'record_slice': [rec_slice.start, rec_slice.stop, rec_slice.step],
},
'params': run_.get(...),
'traces': traces_,
'props': get_run_props(run_),
}
}
encoded_tree = encode_tree(run_dict)
return collect_run_streamable_data(encoded_tree)
def run_search_result_streamer(runs: SequenceCollection, limit: int) -> bytes:
run_count = 0
for run_trace_collection in runs.iter_runs():
run = run_trace_collection.run
run_dict = {
run.hash: {
'params': run.get(...),
'traces': run.collect_sequence_info(sequence_types='metric'),
'props': get_run_props(run)
}
}
encoded_tree = encode_tree(run_dict)
yield collect_run_streamable_data(encoded_tree)
run_count += 1
if limit and run_count >= limit:
break
def set(
self,
path: Union[AimObjectKey, AimObjectPath],
value: AimObject,
strict: bool = True
):
if path == Ellipsis:
path = ()
if not isinstance(path, (tuple, list)):
path = (path,)
batch = self.container.batch()
encoded_path = E.encode_path(path)
self.container.delete_range(encoded_path, encoded_path + b'\xff',
store_batch=batch)
for key, val in treeutils.encode_tree(value, strict=strict):
self.container.set(encoded_path + key, val,
store_batch=batch)
self.container.commit(batch)
def audios_batch_result_streamer(uri_batch: List[str], repo: 'Repo'):
uri_service = URIService(repo=repo)
batch_iterator = uri_service.request_batch(uri_batch=uri_batch)
for it in batch_iterator:
yield collect_run_streamable_data(encode_tree(it))