def select_run_metrics(self, experiment_name: str, run_hash: str,
select_metrics: Union[str, List[str], Tuple[str]]
) -> Optional[Run]:
if not self.run_exists(experiment_name, run_hash):
return None
if isinstance(select_metrics, str):
select_metrics = [select_metrics]
run = Run(self, experiment_name, run_hash)
for metric_name, metric in run.get_all_metrics().items():
if metric_name in select_metrics:
for trace in metric.get_all_traces():
metric.append(trace)
run.add(metric)
return run
def select_runs(
self,
expression: Optional[Union[str, Expression,
BinaryExpressionTree]] = None,
default_expression: Optional[Union[str, Expression,
BinaryExpressionTree]] = None,
) -> List[Run]:
runs = {
exp_name: [
Run(self, exp_name, run_hash)
for run_hash in self.list_branch_commits(exp_name)
]
for exp_name in self.list_branches()
}
matched_runs = [] # type: List[Run]
# Build expression tree
if expression:
expression = build_bet(expression)
expression.strict = True
if default_expression:
default_expression = build_bet(default_expression)
default_expression.strict = True
if expression:
expression.concat(default_expression)
else:
expression = default_expression
for experiment_runs in runs.values():
for run in experiment_runs:
# Add metrics path modifier
expression.dump_path_modifiers()
if AIM_MAP_METRICS_KEYWORD in run.params.keys():
expression.add_path_modifier(
lambda path_token: self.metrics_path_checker(
path_token, run.config.keys()),
lambda path_token: self.metrics_path_modifier(
path_token, run.params[AIM_MAP_METRICS_KEYWORD]))
# Dictionary representing all search fields
fields = {
'experiment': run.experiment_name,
'run':
run.config, # Run configs (date, name, archived etc)
'params': run.params, # Run parameters (`NestedMap`)
}
# Default parameters - ones passed without namespace
default_params = run.params.get(AIM_NESTED_MAP_DEFAULT) or {}
if not expression:
res = True
else:
res = expression.match(fields, run.params, default_params)
if res is True:
matched_runs.append(run)
return matched_runs
def select_metrics(self, select_metrics: Union[str, List[str], Tuple[str]],
expression: Optional[
Union[str,
Expression,
BinaryExpressionTree]] = None,
default_expression: Optional[
Union[str,
Expression,
BinaryExpressionTree]] = None,
) -> List[Run]:
"""
Searches repo and returns matching metrics
"""
if isinstance(select_metrics, str):
select_metrics = [select_metrics]
runs = {
exp_name: [
Run(self, exp_name, run_hash)
for run_hash in self.list_branch_commits(exp_name)
]
for exp_name in self.list_branches()
}
matched_runs: List[Run] = []
expression = build_bet(expression)
expression.strict = True
if default_expression:
default_expression = build_bet(default_expression)
expression.concat(default_expression)
for experiment_runs in runs.values():
for run in experiment_runs:
# Add metrics path modifier
expression.dump_path_modifiers()
if AIM_MAP_METRICS_KEYWORD in run.params.keys():
expression.add_path_modifier(
lambda path_token: self.metrics_path_checker(
path_token,
run.config.keys()),
lambda path_token: self.metrics_path_modifier(
path_token,
run.params[AIM_MAP_METRICS_KEYWORD])
)
# Dictionary representing all search fields
fields = {
'experiment': run.experiment_name,
'run': run.config, # Run configs (date, name, archived etc)
'params': run.params, # Run parameters (`NestedMap`)
}
# Default parameters - ones passed without namespace
default_params = run.params.get(AIM_NESTED_MAP_DEFAULT) or {}
# Search metrics
for metric_name, metric in run.get_all_metrics().items():
if metric_name not in select_metrics:
continue
fields['metric'] = metric_name
for trace in metric.get_all_traces():
fields['context'] = trace.context
# Pass fields in descending order by priority
if expression is None:
res = True
else:
res = expression.match(fields,
run.params,
default_params)
if res is True:
metric.append(trace)
run.add(metric)
if run not in matched_runs:
matched_runs.append(run)
return matched_runs
def select(
self,
select_fields: List[str] = [],
expression: Optional[Union[str, Expression,
BinaryExpressionTree]] = None,
default_expression: Optional[Union[str, Expression,
BinaryExpressionTree]] = None,
):
select_result = SelectResult(select_fields)
runs = {
exp_name: [
Run(self, exp_name, run_hash)
for run_hash in self.list_branch_commits(exp_name)
]
for exp_name in self.list_branches()
}
# Build expression tree
if expression:
expression = build_bet(expression)
expression.strict = True
if default_expression:
default_expression = build_bet(default_expression)
default_expression.strict = True
if expression:
expression.concat(default_expression)
else:
expression = default_expression
for experiment_runs in runs.values():
for run in experiment_runs:
# Dictionary representing all search fields
fields = {
'experiment': run.experiment_name,
'run':
run.config, # Run configs (date, name, archived etc)
'params': run.params, # Run parameters (`NestedMap`)
}
# Default parameters - those passed without namespace
default_params = {
'params': (run.params.get(AIM_NESTED_MAP_DEFAULT) or {}),
}
# Search metrics
for metric_name, metric in run.get_all_metrics().items():
fields['metric'] = metric_name
for trace in metric.get_all_traces():
fields['context'] = trace.context
# Pass fields in descending order by priority
if expression is None:
res = True
else:
res = expression.match(fields, run.params,
default_params)
if res is not True:
continue
# Append trace data if metric is selected
for select_field in select_fields:
if select_field == metric_name:
metric.append(trace)
run.add(metric)
break
# Append run if either metric or param is selected
for select_field in select_fields:
if select_field == metric_name:
select_result.append_run(run)
break
field_val = get_dict_item_by_path(
run.params, select_field)
if field_val is not None:
select_result.append_run(run)
break
return select_result