def get_data_structure_info(result_struct):
all_settings_ = result_struct.keys()
first_result = result_struct.values()[0]
all_measurements_ = remove_duplicates([
k for v in first_result.values() for k in v.keys()
]) # ['flops', 'error', ...]
all_datasets_, all_subsets_, all_nets_ = [
remove_duplicates(all_category_entries)
for all_category_entries in zip(
*[categories for categories in first_result.keys()])
]
return all_settings_, all_measurements_, all_datasets_, all_subsets_, all_nets_
def test_remove_duplicates():
assert remove_duplicates(['a', 'b', 'a', 'c', 'c']) == ['a', 'b', 'c']
assert remove_duplicates(['a', 'b', 'a', 'c', 'c'],
keep_last=True) == ['b', 'a', 'c']
assert remove_duplicates(
['Alfred', 'Bob', 'Cindy', 'Alina', 'Karol', 'Betty'],
key=lambda x: x[0]) == ['Alfred', 'Bob', 'Cindy', 'Karol']
assert remove_duplicates(
['Alfred', 'Bob', 'Cindy', 'Alina', 'Karol', 'Betty'],
key=lambda x: x[0],
keep_last=True) == ['Cindy', 'Alina', 'Karol', 'Betty']
assert remove_duplicates(
['Alfred', 'Bob', 'Cindy', 'Alina', 'Karol', 'Betty'],
key=lambda x: x[0],
keep_last=True,
hashable=False) == ['Cindy', 'Alina', 'Karol', 'Betty']
def get_best_score(score_info_pairs, subset = 'test', prediction_function = None, score_measure = None, lower_is_better = False):
"""
DEPRECATED!!!
Given a list of (info, score) pairs which represet the progress over training, find the best score and return it.
:param score_info_pairs: A list<(IterationInfo, dict)> of the type returned in train_and_test_online_predictor
:param subset: 'train' or 'test' ... which subset to use to look for the best score
:param prediction_function: Which prediction function (if there are multiple prediction functions, otherwise leave blank)
:param score_measure: Which score measure (if there are multiple score measures, otherwise leave blank)
:param lower_is_better: True if a lower score is better for the chosen score_measure
:return: best_info, best_score
best_info is an InterationInfo object
best_score is a dict<(subset, prediction_function, score_measure) -> score>
"""
assert len(score_info_pairs)>0, "You need to have at least one score to determine the best one."
_, first_score = score_info_pairs[0]
all_subsets, all_functions, all_measures = [remove_duplicates(s) for s in zip(*first_score.keys())]
if prediction_function is None:
assert len(all_functions)==1, "You did not specify prediction_function... options are: {}".format(all_functions)
prediction_function = all_functions[0]
if score_measure is None:
assert len(all_measures)==1, "You did not specify a score_measure... options are: {}".format(all_measures)
score_measure = all_measures[0]
best_info = None
best_score = None
for info, score in score_info_pairs:
this_is_the_best = best_score is None or \
(score[subset, prediction_function, score_measure]<best_score[subset, prediction_function, score_measure]) == lower_is_better
if this_is_the_best:
best_score = score
best_info = info
return best_info, best_score
def get_record_invalid_arg_string(record, recursive=True, ignore_valid_keys=(), note_version = 'full'):
"""
Return a string identifying ig the arguments for this experiment are still valid.
:return:
"""
assert note_version in ('full', 'short')
experiment_id = record.get_experiment_id()
if is_experiment_loadable(experiment_id):
if record.info.has_field(ExpInfoFields.ARGS):
last_run_args = OrderedDict([(k,v) for k,v in record.get_args().items() if k not in ignore_valid_keys])
current_args = OrderedDict([(k,v) for k,v in record.get_experiment().get_args().items() if k not in ignore_valid_keys])
if recursive:
last_run_args = OrderedDict(flatten_struct(last_run_args, first_dict_is_namespace=True, primatives = PRIMATIVE_TYPES + (UnPicklableArg, )))
last_run_args = OrderedDict([(k, v) for k, v in last_run_args.items() if k not in ignore_valid_keys])
current_args = OrderedDict(flatten_struct(current_args, first_dict_is_namespace=True))
current_args = OrderedDict([(k, v) for k, v in current_args.items() if k not in ignore_valid_keys])
validity = record.args_valid(last_run_args=last_run_args, current_args=current_args)
if validity is False:
common, (old_args, new_args) = separate_common_items([list(last_run_args.items()), list(current_args.items())])
if len(old_args)+len(new_args)==0:
raise Exception('Error displaying different args. Bug Peter.')
all_changed_arg_names = remove_duplicates(list(name for name, _ in old_args)+list(name for name, _ in new_args))
changestr = ', '.join("{}:{}->{}".format(k, last_run_args[k] if k in last_run_args else '<N/A>', current_args[k] if k in current_args else '<N/A>') for k in all_changed_arg_names)
notes = ("Change: " if note_version=='full' else "") + changestr
elif validity is None:
notes = "Cannot Determine: Unhashable Args" if note_version=='full' else '<Unhashable Args>'
else:
notes = "<No Change>"
else:
notes = "Cannot Determine: Inconsistent Experiment Record" if note_version == 'full' else '<Inconsistent Record>'
else:
notes = "Cannot Determine: Experiment Not Imported" if note_version=='full' else '<Not Imported>'
return notes
def get_best_score(score_info_pairs, subset = 'test', prediction_function = None, score_measure = None, lower_is_better = False):
"""
DEPRECATED!!!
Given a list of (info, score) pairs which represet the progress over training, find the best score and return it.
:param score_info_pairs: A list<(IterationInfo, dict)> of the type returned in train_and_test_online_predictor
:param subset: 'train' or 'test' ... which subset to use to look for the best score
:param prediction_function: Which prediction function (if there are multiple prediction functions, otherwise leave blank)
:param score_measure: Which score measure (if there are multiple score measures, otherwise leave blank)
:param lower_is_better: True if a lower score is better for the chosen score_measure
:return: best_info, best_score
best_info is an InterationInfo object
best_score is a dict<(subset, prediction_function, score_measure) -> score>
"""
assert len(score_info_pairs)>0, "You need to have at least one score to determine the best one."
_, first_score = score_info_pairs[0]
all_subsets, all_functions, all_measures = [remove_duplicates(s) for s in zip(*first_score.keys())]
if prediction_function is None:
assert len(all_functions)==1, "You did not specify prediction_function... options are: {}".format(all_functions)
prediction_function = all_functions[0]
if score_measure is None:
assert len(all_measures)==1, "You did not specify a score_measure... options are: {}".format(all_measures)
score_measure = all_measures[0]
best_info = None
best_score = None
for info, score in score_info_pairs:
this_is_the_best = best_score is None or \
(score[subset, prediction_function, score_measure]<best_score[subset, prediction_function, score_measure]) == lower_is_better
if this_is_the_best:
best_score = score
best_info = info
return best_info, best_score
def print_score_results(score, info=None):
"""
:param results: An OrderedDict in the format returned by assess_prediction_functions_on_generator.
:return:
"""
if info is not None:
print('Epoch {} (after {:.3g}s)'.format(info.epoch, info.time))
test_pair_names, function_names, cost_names = [remove_duplicates(k) for k in zip(*score.keys())]
rows = build_table(
lookup_fcn=lambda test_pair_name_and_function_name_, cost_name_: score[test_pair_name_and_function_name_[0], test_pair_name_and_function_name_[1], cost_name_],
row_categories = [[test_pair_name for test_pair_name in test_pair_names], [function_name for function_name in function_names]],
column_categories = [cost_name for cost_name in cost_names],
row_header_labels=['Subset', 'Function'],
clear_repeated_headers = False,
remove_unchanging_cols=True
)
import tabulate
print(tabulate.tabulate(rows))
def print_score_results(score, info=None):
"""
:param results: An OrderedDict in the format returned by assess_prediction_functions_on_generator.
:return:
"""
if info is not None:
print('Epoch {} (after {:.3g}s)'.format(info.epoch, info.time))
test_pair_names, function_names, cost_names = [remove_duplicates(k) for k in zip(*score.keys())]
rows = build_table(
lookup_fcn=lambda test_pair_name_and_function_name_, cost_name_: score[test_pair_name_and_function_name_[0], test_pair_name_and_function_name_[1], cost_name_],
row_categories = [[test_pair_name for test_pair_name in test_pair_names], [function_name for function_name in function_names]],
column_categories = [cost_name for cost_name in cost_names],
row_header_labels=['Subset', 'Function'],
clear_repeated_headers = False,
remove_unchanging_cols=True
)
import tabulate
print(tabulate.tabulate(rows))
def get_costs(self):
return remove_duplicates([c for _, c, c in self.scores.keys()])
def get_prediction_functions(self):
return remove_duplicates([f for _, f, _ in self.scores.keys()])
def get_data_subsets(self):
return remove_duplicates([s for s, _, _ in self.scores.keys()])
def get_record_invalid_arg_string(record,
recursive=True,
ignore_valid_keys=(),
note_version='full'):
"""
Return a string identifying ig the arguments for this experiment are still valid.
:return:
"""
assert note_version in ('full', 'short')
experiment_id = record.get_experiment_id()
if is_experiment_loadable(experiment_id):
if record.info.has_field(ExpInfoFields.ARGS):
last_run_args = OrderedDict([(k, v)
for k, v in record.get_args().items()
if k not in ignore_valid_keys])
current_args = OrderedDict([
(k, v) for k, v in record.get_experiment().get_args().items()
if k not in ignore_valid_keys
])
if recursive:
last_run_args = OrderedDict(
flatten_struct(last_run_args,
first_dict_is_namespace=True,
primatives=PRIMATIVE_TYPES +
(UnPicklableArg, )))
last_run_args = OrderedDict([(k, v)
for k, v in last_run_args.items()
if k not in ignore_valid_keys])
current_args = OrderedDict(
flatten_struct(current_args, first_dict_is_namespace=True))
current_args = OrderedDict([(k, v)
for k, v in current_args.items()
if k not in ignore_valid_keys])
validity = record.args_valid(last_run_args=last_run_args,
current_args=current_args)
if validity is False:
common, (old_args, new_args) = separate_common_items(
[list(last_run_args.items()),
list(current_args.items())])
if len(old_args) + len(new_args) == 0:
raise Exception(
'Error displaying different args. Bug Peter.')
all_changed_arg_names = remove_duplicates(
list(name for name, _ in old_args) +
list(name for name, _ in new_args))
changestr = ', '.join("{}:{}->{}".format(
k, last_run_args[k] if k in last_run_args else '<N/A>',
current_args[k] if k in current_args else '<N/A>')
for k in all_changed_arg_names)
notes = ("Change: "
if note_version == 'full' else "") + changestr
elif validity is None:
notes = "Cannot Determine: Unhashable Args" if note_version == 'full' else '<Unhashable Args>'
else:
notes = "<No Change>"
else:
notes = "Cannot Determine: Inconsistent Experiment Record" if note_version == 'full' else '<Inconsistent Record>'
else:
notes = "Cannot Determine: Experiment Not Imported" if note_version == 'full' else '<Not Imported>'
return notes
def test_remove_duplicates():
assert remove_duplicates(['a', 'b', 'a', 'c', 'c'])==['a', 'b', 'c']
assert remove_duplicates(['a', 'b', 'a', 'c', 'c'], keep_last=True)==['b', 'a', 'c']
assert remove_duplicates(['Alfred', 'Bob', 'Cindy', 'Alina', 'Karol', 'Betty'], key=lambda x: x[0])==['Alfred', 'Bob', 'Cindy', 'Karol']
assert remove_duplicates(['Alfred', 'Bob', 'Cindy', 'Alina', 'Karol', 'Betty'], key=lambda x: x[0], keep_last=True)==['Cindy', 'Alina', 'Karol', 'Betty']
assert remove_duplicates(['Alfred', 'Bob', 'Cindy', 'Alina', 'Karol', 'Betty'], key=lambda x: x[0], keep_last=True, hashable=False)==['Cindy', 'Alina', 'Karol', 'Betty']
def get_costs(self):
return remove_duplicates([c for _, c, c in self.scores.keys()])
def get_prediction_functions(self):
return remove_duplicates([f for _, f, _ in self.scores.keys()])
def get_data_subsets(self):
return remove_duplicates([s for s, _, _ in self.scores.keys()])
