def print_experiment_record_argtable(records):
"""
Print a table comparing experiment arguments and their results.
"""
funtion_names = [record.info.get_field(ExpInfoFields.FUNCTION) for record in records]
args = [record.get_args() for record in records]
common_args, different_args = separate_common_items(args)
record_ids = [record.get_id() for record in records]
def lookup_fcn(record_id, column):
index = record_ids.index(record_id)
if column=='Function':
return funtion_names[index]
elif column=='Run Time':
return records[index].info.get_field_text(ExpInfoFields.RUNTIME)
elif column=='Common Args':
return ', '.join('{}={}'.format(k, v) for k, v in common_args.items())
elif column=='Different Args':
return ', '.join('{}={}'.format(k, v) for k, v in different_args[index].items())
elif column=='Result':
return get_oneline_result_string(records[index])
else:
bad_value(column)
rows = build_table(lookup_fcn,
row_categories=record_ids,
column_categories=['Function', 'Run Time', 'Common Args', 'Different Args', 'Result'],
prettify_labels=False
)
print(tabulate(rows))
def print_experiment_record_argtable(records):
"""
Print a table comparing experiment arguments and their results.
"""
funtion_names = [record.info.get_field(ExpInfoFields.FUNCTION) for record in records]
args = [record.info.get_field(ExpInfoFields.ARGS) for record in records]
results = [record.get_result(err_if_none=False) for record in records]
common_args, different_args = separate_common_items(args)
record_ids = [record.get_id() for record in records]
def lookup_fcn(record_id, column):
index = record_ids.index(record_id)
if column=='Function':
return funtion_names[index]
elif column=='Run Time':
return records[index].info.get_field_text(ExpInfoFields.RUNTIME)
elif column=='Common Args':
return ', '.join('{}={}'.format(k, v) for k, v in common_args)
elif column=='Different Args':
return ', '.join('{}={}'.format(k, v) for k, v in different_args[index])
elif column=='Result':
return results[index]
else:
bad_value(column)
rows = build_table(lookup_fcn,
row_categories=record_ids,
column_categories=['Function', 'Run Time', 'Common Args', 'Different Args', 'Result'],
prettify_labels=False
)
print tabulate(rows)
def test_build_table(show_table=True):
def lookup_function(prisoner_a_choice, prisoner_b_choice):
total_utility = \
2 if prisoner_a_choice=='cooperate' and prisoner_b_choice=='cooperate' else \
3 if prisoner_a_choice != prisoner_b_choice else \
4 if prisoner_b_choice=='betray' and prisoner_a_choice=='betray' \
else bad_value((prisoner_a_choice, prisoner_b_choice))
return total_utility
rows = build_table(lookup_function, row_categories=['cooperate', 'betray'], column_categories=['cooperate', 'betray'])
assert rows == [[' ', 'Cooperate', 'Betray'], ['Cooperate', 2, 3], ['Betray', 3, 4]]
if show_table:
import tabulate
print(tabulate.tabulate(rows))
def get_table(self):
# test_pair_names, function_names, cost_names = [remove_duplicates(k) for k in zip(*self.scores.keys())]
def lookup( test_pair_name_and_function_name_, cost_name_):
test_pair_name_, function_name_ = test_pair_name_and_function_name_
return self[test_pair_name_, function_name_, cost_name_]
rows = build_table(
lookup_fcn=lookup,
row_categories=[[test_pair_name for test_pair_name in self.get_data_subsets()], [function_name for function_name in self.get_prediction_functions()]],
column_categories=[cost_name for cost_name in self.get_costs()],
row_header_labels=['Subset', 'Function'],
clear_repeated_headers=False,
remove_unchanging_cols=False
)
import tabulate
return tabulate.tabulate(rows)
def get_table(self):
# test_pair_names, function_names, cost_names = [remove_duplicates(k) for k in zip(*self.scores.keys())]
def lookup( test_pair_name_and_function_name_, cost_name_):
test_pair_name_, function_name_ = test_pair_name_and_function_name_
return self[test_pair_name_, function_name_, cost_name_]
rows = build_table(
lookup_fcn=lookup,
row_categories=[[test_pair_name for test_pair_name in self.get_data_subsets()], [function_name for function_name in self.get_prediction_functions()]],
column_categories=[cost_name for cost_name in self.get_costs()],
row_header_labels=['Subset', 'Function'],
clear_repeated_headers=False,
remove_unchanging_cols=False
)
import tabulate
return tabulate.tabulate(rows)
def print_score_results(score, info=None):
"""
:param results: An OrderedDict in the format returned by assess_prediction_functions_on_generator.
:return:
"""
if info is not None:
print('Epoch {} (after {:.3g}s)'.format(info.epoch, info.time))
test_pair_names, function_names, cost_names = [remove_duplicates(k) for k in zip(*score.keys())]
rows = build_table(
lookup_fcn=lambda test_pair_name_and_function_name_, cost_name_: score[test_pair_name_and_function_name_[0], test_pair_name_and_function_name_[1], cost_name_],
row_categories = [[test_pair_name for test_pair_name in test_pair_names], [function_name for function_name in function_names]],
column_categories = [cost_name for cost_name in cost_names],
row_header_labels=['Subset', 'Function'],
clear_repeated_headers = False,
remove_unchanging_cols=True
)
import tabulate
print(tabulate.tabulate(rows))
def print_score_results(score, info=None):
"""
:param results: An OrderedDict in the format returned by assess_prediction_functions_on_generator.
:return:
"""
if info is not None:
print('Epoch {} (after {:.3g}s)'.format(info.epoch, info.time))
test_pair_names, function_names, cost_names = [remove_duplicates(k) for k in zip(*score.keys())]
rows = build_table(
lookup_fcn=lambda test_pair_name_and_function_name_, cost_name_: score[test_pair_name_and_function_name_[0], test_pair_name_and_function_name_[1], cost_name_],
row_categories = [[test_pair_name for test_pair_name in test_pair_names], [function_name for function_name in function_names]],
column_categories = [cost_name for cost_name in cost_names],
row_header_labels=['Subset', 'Function'],
clear_repeated_headers = False,
remove_unchanging_cols=True
)
import tabulate
print(tabulate.tabulate(rows))
def test_build_table(show_table=True):
def lookup_function(prisoner_a_choice, prisoner_b_choice):
total_utility = \
2 if prisoner_a_choice=='cooperate' and prisoner_b_choice=='cooperate' else \
3 if prisoner_a_choice != prisoner_b_choice else \
4 if prisoner_b_choice=='betray' and prisoner_a_choice=='betray' \
else bad_value((prisoner_a_choice, prisoner_b_choice))
return total_utility
rows = build_table(lookup_function,
row_categories=['cooperate', 'betray'],
column_categories=['cooperate', 'betray'])
assert rows == [[' ', 'Cooperate', 'Betray'], ['Cooperate', 2, 3],
['Betray', 3, 4]]
if show_table:
import tabulate
print tabulate.tabulate(rows)
def compare_experiment_records(record_identifiers):
records = [load_experiment_record(ident) for ident in record_identifiers]
# info_results = OrderedDict([(identifier, record.get_info()) for identifier, record in zip(record_identifiers, records)]])
funtion_names = [
record.info.get_field(ExpInfoFields.FUNCTION) for record in records
]
args = [record.info.get_field(ExpInfoFields.ARGS) for record in records]
results = [record.get_result() for record in records]
common_args, different_args = separate_common_items(args)
def lookup_fcn(identifier, column):
index = record_identifiers.index(identifier)
if column == 'Function':
return funtion_names[index]
elif column == 'Run Time':
return records[index].info.get_field_text(ExpInfoFields.RUNTIME)
elif column == 'Common Args':
return ', '.join('{}={}'.format(k, v) for k, v in common_args)
elif column == 'Different Args':
return ', '.join('{}={}'.format(k, v)
for k, v in different_args[index])
elif column == 'Result':
return results[index]
else:
bad_value(column)
rows = build_table(lookup_fcn,
row_categories=record_identifiers,
column_categories=[
'Function', 'Run Time', 'Common Args',
'Different Args', 'Result'
],
prettify_labels=False)
print tabulate(rows)
','+str(cost_fcn),
self.get_score(subset, pred_fcn, cost_fcn))
for subset, pred_fcn, cost_fcn in self.keys()
]
return '{}:{}'.format(self.__class__.__name__, ','.join(sections))
def get_table(self):
# test_pair_names, function_names, cost_names = [remove_duplicates(k) for k in zip(*self.scores.keys())]
def lookup( (test_pair_name_, function_name_), cost_name_):
return self[test_pair_name_, function_name_, cost_name_]
rows = build_table(
lookup_fcn=lookup,
row_categories=[[test_pair_name for test_pair_name in self.get_data_subsets()], [function_name for function_name in self.get_prediction_functions()]],
column_categories=[cost_name for cost_name in self.get_costs()],
row_header_labels=['Subset', 'Function'],
clear_repeated_headers=False,
remove_unchanging_cols=False
)
import tabulate
return tabulate.tabulate(rows)
class InfoScorePair(object):
def __init__(self, info, score):
"""
:param info: An IterationInfo Object
:param v: A ModelTestScore object
"""
assert isinstance(info, IterationInfo)
