def print_experiment_record_argtable(records):
"""
Print a table comparing experiment arguments and their results.
"""
funtion_names = [record.info.get_field(ExpInfoFields.FUNCTION) for record in records]
args = [record.info.get_field(ExpInfoFields.ARGS) for record in records]
results = [record.get_result(err_if_none=False) for record in records]
common_args, different_args = separate_common_items(args)
record_ids = [record.get_id() for record in records]
def lookup_fcn(record_id, column):
index = record_ids.index(record_id)
if column=='Function':
return funtion_names[index]
elif column=='Run Time':
return records[index].info.get_field_text(ExpInfoFields.RUNTIME)
elif column=='Common Args':
return ', '.join('{}={}'.format(k, v) for k, v in common_args)
elif column=='Different Args':
return ', '.join('{}={}'.format(k, v) for k, v in different_args[index])
elif column=='Result':
return results[index]
else:
bad_value(column)
rows = build_table(lookup_fcn,
row_categories=record_ids,
column_categories=['Function', 'Run Time', 'Common Args', 'Different Args', 'Result'],
prettify_labels=False
)
print tabulate(rows)
def print_experiment_record_argtable(records):
"""
Print a table comparing experiment arguments and their results.
"""
funtion_names = [record.info.get_field(ExpInfoFields.FUNCTION) for record in records]
args = [record.get_args() for record in records]
common_args, different_args = separate_common_items(args)
record_ids = [record.get_id() for record in records]
def lookup_fcn(record_id, column):
index = record_ids.index(record_id)
if column=='Function':
return funtion_names[index]
elif column=='Run Time':
return records[index].info.get_field_text(ExpInfoFields.RUNTIME)
elif column=='Common Args':
return ', '.join('{}={}'.format(k, v) for k, v in common_args.items())
elif column=='Different Args':
return ', '.join('{}={}'.format(k, v) for k, v in different_args[index].items())
elif column=='Result':
return get_oneline_result_string(records[index])
else:
bad_value(column)
rows = build_table(lookup_fcn,
row_categories=record_ids,
column_categories=['Function', 'Run Time', 'Common Args', 'Different Args', 'Result'],
prettify_labels=False
)
print(tabulate(rows))
def get_invalid_arg_note(self):
"""
Return a string identifying ig the arguments for this experiment are still valid.
:return:
"""
experiment_id = record_id_to_experiment_id(self.get_identifier())
if is_experiment_loadable(experiment_id):
last_run_args = dict(self.info.get_field(ExpInfoFields.ARGS))
current_args = dict(
load_experiment(
record_id_to_experiment_id(
self.get_identifier())).get_args())
if not self.is_valid(last_run_args=last_run_args,
current_args=current_args):
last_arg_str, this_arg_str = [
['{}:{}'.format(k, v)
for k, v in argdict.iteritems()] if isinstance(
argdict,
dict) else ['{}:{}'.format(k, v) for k, v in argdict]
for argdict in (last_run_args, current_args)
]
common, (old_args, new_args) = separate_common_items(
[last_arg_str, this_arg_str])
notes = "Args changed!: {{{}}}->{{{}}}".format(
','.join(old_args), ','.join(new_args))
else:
notes = "Yes"
else:
notes = "<Experiment Not Currently Imported>"
return notes
def separate_common_args(records,
as_dicts=False,
return_dict=False,
only_shared_argdiffs=False):
"""
:param records: A List of records
:param return_dict: Return the different args as a dict<ExperimentRecord: args>
:return: (common, different)
Where common is an OrderedDict of common args
different is a list (the same lengths of records) of OrderedDicts containing args that are not the same in all records.
"""
common, argdiff = separate_common_items(
[list(rec.get_args().items()) for rec in records])
if only_shared_argdiffs:
args_that_they_all_have = set.intersection(*(set(k
for k, v in different)
for different in argdiff))
argdiff = [[(k, v) for k, v in ad if k in args_that_they_all_have]
for ad in argdiff]
if as_dicts:
common = OrderedDict(common)
argdiff = [OrderedDict(ad) for ad in argdiff]
if return_dict:
argdiff = {rec.get_id(): args for rec, args in zip(records, argdiff)}
return common, argdiff
def infer_arg_values(f, args=(), kwargs={}):
"""
DEPRECATED! Use advanced_getargspec instead.
Get the full list of arguments to a function f called with args, kwargs, or throw an error if the function cannot be
called by the given arguments (e.g. if the provided args, kwargs do not provide all required arguments to the function).
:param f: A function
:param args: A list of args
:param kwargs: A dict of keyword args
:return: An OrderedDict(arg_name->arg_value)
"""
all_arg_names, varargs_name, kwargs_name, defaults = inspect.getargspec(f)
assert varargs_name is None, "This function doesn't work with unnamed args"
default_args = {k: v for k, v in zip(all_arg_names[len(all_arg_names)-(len(defaults) if defaults is not None else 0):], defaults if defaults is not None else [])}
args_with_values = set(all_arg_names[:len(args)]+list(default_args.keys())+list(kwargs.keys()))
assert set(all_arg_names).issubset(args_with_values), "Arguments {} require values but are not given any. ".format(tuple(set(all_arg_names).difference(args_with_values)))
assert len(args) <= len(all_arg_names), "You provided {} arguments, but the function only takes {}".format(len(args), len(all_arg_names))
full_args = tuple(
list(zip(all_arg_names, args)) # Handle unnamed args f(1, 2)
+ [(name, kwargs[name] if name in kwargs else default_args[name]) for name in all_arg_names[len(args):]] # Handle named keyworkd args f(a=1, b=2)
+ [(name, kwargs[name]) for name in kwargs if name not in all_arg_names[len(args):]] # Need to handle case if f takes **kwargs
)
duplicates = tuple(item for item, count in collections.Counter([a for a, _ in full_args]).items() if count > 1)
assert len(duplicates)==0, 'Arguments {} have been defined multiple times: {}'.format(duplicates, full_args)
common_args, (different_args, different_given_args) = separate_common_items([tuple(all_arg_names), tuple(n for n, _ in full_args)])
if kwargs_name is None: # There is no **kwargs
assert len(different_given_args)==0, "Function {} was given args {} but didn't ask for them".format(f, different_given_args)
assert len(different_args)==0, "Function {} needs values for args {} but didn't get them".format(f, different_args)
return OrderedDict(full_args)
def make_record_comparison_table(records, args_to_show=None, results_extractor = None, print_table = False, tablefmt='simple', reorder_by_args=False):
"""
Make a table comparing the arguments and results of different experiment records. You can use the output
of this function with the tabulate package to make a nice readable table.
:param records: A list of records whose results to compare
:param args_to_show: A list of arguments to show. If none, it will just show all arguments
that differ between experiments.
:param results_extractor: A dict<str->callable> where the callables take the result of the
experiment as an argument and return an entry in the table.
:param print_table: Optionally, import tabulate and print the table here and now.
:return: headers, rows
headers is a list of of headers for the top of the table
rows is a list of lists filling in the information.
example usage:
headers, rows = make_record_comparison_table(
record_ids = [experiment_id_to_latest_record_id(eid) for eid in [
'demo_fast_weight_mlp.multilayer_baseline.1epoch.version=mlp',
'demo_fast_weight_mlp.multilayer_baseline.1epoch.full-gd.n_steps=1',
'demo_fast_weight_mlp.multilayer_baseline.1epoch.full-gd.n_steps=20',
]],
results_extractor={
'Test': lambda result: result.get_best('test').score.get_score('test'),
'Train': lambda result: result.get_best('test').score.get_score('train'),
}
)
import tabulate
print tabulate.tabulate(rows, headers=headers, tablefmt=tablefmt)
"""
args = [rec.get_args(ExpInfoFields.ARGS) for rec in records]
if args_to_show is None:
common, separate = separate_common_items(args)
args_to_show = [k for k, v in separate[0]]
if results_extractor is None:
results_extractor = {'Result': str}
elif callable(results_extractor):
results_extractor = {'Result': results_extractor}
else:
assert isinstance(results_extractor, dict)
headers = args_to_show + results_extractor.keys()
rows = []
for record, record_args in izip_equal(records, args):
arg_dict = dict(record_args)
args_vals = [arg_dict[k] for k in args_to_show]
results = record.get_result()
rows.append(args_vals+[f(results) for f in results_extractor.values()])
if reorder_by_args:
rows = sorted(rows)
if print_table:
import tabulate
print(tabulate.tabulate(rows, headers=headers, tablefmt=tablefmt))
return headers, rows
def get_record_invalid_arg_string(record, recursive=True, ignore_valid_keys=(), note_version = 'full'):
"""
Return a string identifying ig the arguments for this experiment are still valid.
:return:
"""
assert note_version in ('full', 'short')
experiment_id = record.get_experiment_id()
if is_experiment_loadable(experiment_id):
if record.info.has_field(ExpInfoFields.ARGS):
last_run_args = OrderedDict([(k,v) for k,v in record.get_args().items() if k not in ignore_valid_keys])
current_args = OrderedDict([(k,v) for k,v in record.get_experiment().get_args().items() if k not in ignore_valid_keys])
if recursive:
last_run_args = OrderedDict(flatten_struct(last_run_args, first_dict_is_namespace=True, primatives = PRIMATIVE_TYPES + (UnPicklableArg, )))
last_run_args = OrderedDict([(k, v) for k, v in last_run_args.items() if k not in ignore_valid_keys])
current_args = OrderedDict(flatten_struct(current_args, first_dict_is_namespace=True))
current_args = OrderedDict([(k, v) for k, v in current_args.items() if k not in ignore_valid_keys])
validity = record.args_valid(last_run_args=last_run_args, current_args=current_args)
if validity is False:
common, (old_args, new_args) = separate_common_items([list(last_run_args.items()), list(current_args.items())])
if len(old_args)+len(new_args)==0:
raise Exception('Error displaying different args. Bug Peter.')
all_changed_arg_names = remove_duplicates(list(name for name, _ in old_args)+list(name for name, _ in new_args))
changestr = ', '.join("{}:{}->{}".format(k, last_run_args[k] if k in last_run_args else '<N/A>', current_args[k] if k in current_args else '<N/A>') for k in all_changed_arg_names)
notes = ("Change: " if note_version=='full' else "") + changestr
elif validity is None:
notes = "Cannot Determine: Unhashable Args" if note_version=='full' else '<Unhashable Args>'
else:
notes = "<No Change>"
else:
notes = "Cannot Determine: Inconsistent Experiment Record" if note_version == 'full' else '<Inconsistent Record>'
else:
notes = "Cannot Determine: Experiment Not Imported" if note_version=='full' else '<Not Imported>'
return notes
def get_record_invalid_arg_string(record, recursive=True):
"""
Return a string identifying ig the arguments for this experiment are still valid.
:return:
"""
experiment_id = record.get_experiment_id()
if is_experiment_loadable(experiment_id):
if record.info.has_field(ExpInfoFields.ARGS):
last_run_args = record.get_args()
current_args = record.get_experiment().get_args()
validity = record.args_valid(last_run_args=last_run_args, current_args=current_args)
if validity is False:
if recursive:
last_run_args = OrderedDict(flatten_struct(last_run_args, first_dict_is_namespace=True))
current_args = OrderedDict(flatten_struct(current_args, first_dict_is_namespace=True))
last_arg_str, this_arg_str = [['{}:{}'.format(k, v) for k, v in argdict.iteritems()] for argdict in (last_run_args, current_args)]
common, (old_args, new_args) = separate_common_items([last_arg_str, this_arg_str])
notes = "No: Args changed!: {{{}}}->{{{}}}".format(','.join(old_args), ','.join(new_args))
elif validity is None:
notes = "Cannot Determine: Unhashable Args"
else:
notes = "Yes"
else:
notes = "Cannot Determine: Inconsistent Experiment Record"
else:
notes = "Cannot Determine: Experiment Not Imported"
return notes
def test_separate_common_items():
x = {'a': 1, 'b': 2, 'c': 3}
y = {'a': 1, 'b': 4, 'c': 8}
z = {'a': 1, 'b': 2, 'c': 9}
common, separate = separate_common_items([x, y, z])
assert common == {'a': 1}
assert separate == [{'b': 2, 'c': 3}, {'b': 4, 'c': 8}, {'b': 2, 'c': 9}]
def test_separate_common_items():
x={'a': 1, 'b':2, 'c':3}
y={'a': 1, 'b':4, 'c':8}
z={'a': 1, 'b':2, 'c':9}
common, separate = separate_common_items([x, y, z])
assert common == {'a': 1}
assert separate == [{'b':2, 'c':3}, {'b':4, 'c':8}, {'b':2, 'c':9}]
def infer_arg_values(f, *args, **kwargs):
"""
:param f: Get the full list of arguments to a function, or throw an error if the function cannot be called by the
given arguments.
:param args: A list of args
:param kwargs: A dict of keyword args
:return: An OrderedDict(arg_name->arg_value)
"""
all_arg_names, varargs_name, kwargs_name, defaults = inspect.getargspec(f)
assert varargs_name is None, "This function doesn't work with unnamed args"
default_args = {
k: v
for k, v in zip(
all_arg_names[len(all_arg_names) -
(len(defaults) if defaults is not None else 0):],
defaults if defaults is not None else [])
}
args_with_values = set(all_arg_names[:len(args)] + default_args.keys() +
kwargs.keys())
assert set(all_arg_names).issubset(
args_with_values
), "Arguments {} require values but are not given any. ".format(
tuple(set(all_arg_names).difference(args_with_values)))
assert len(args) <= len(
all_arg_names
), "You provided {} arguments, but the function only takes {}".format(
len(args), len(all_arg_names))
full_args = tuple(
zip(all_arg_names, args) # Handle unnamed args f(1, 2)
+ [(name, kwargs[name] if name in kwargs else default_args[name])
for name in all_arg_names[len(args):]
] # Handle named keyworkd args f(a=1, b=2)
+ [(name, kwargs[name])
for name in kwargs if name not in all_arg_names[len(args):]
] # Need to handle case if f takes **kwargs
)
duplicates = tuple(item for item, count in collections.Counter(
[a for a, _ in full_args]).iteritems() if count > 1)
assert len(
duplicates
) == 0, 'Arguments {} have been defined multiple times: {}'.format(
duplicates, full_args)
common_args, (different_args,
different_given_args) = separate_common_items(
[tuple(all_arg_names),
tuple(n for n, _ in full_args)])
if kwargs_name is None: # There is no **kwargs
assert len(
different_given_args
) == 0, "Function {} was given args {} but didn't ask for them".format(
f, different_given_args)
assert len(
different_args
) == 0, "Function {} needs values for args {} but didn't get them".format(
f, different_args)
return OrderedDict(full_args)
def make_record_comparison_table(records, args_to_show=None, results_extractor = None, print_table = False):
"""
Make a table comparing the arguments and results of different experiment records. You can use the output
of this function with the tabulate package to make a nice readable table.
:param records: A list of records whose results to compare
:param args_to_show: A list of arguments to show. If none, it will just show all arguments
that differ between experiments.
:param results_extractor: A dict<str->callable> where the callables take the result of the
experiment as an argument and return an entry in the table.
:param print_table: Optionally, import tabulate and print the table here and now.
:return: headers, rows
headers is a list of of headers for the top of the table
rows is a list of lists filling in the information.
example usage:
headers, rows = make_record_comparison_table(
record_ids = [experiment_id_to_latest_record_id(eid) for eid in [
'demo_fast_weight_mlp.multilayer_baseline.1epoch.version=mlp',
'demo_fast_weight_mlp.multilayer_baseline.1epoch.full-gd.n_steps=1',
'demo_fast_weight_mlp.multilayer_baseline.1epoch.full-gd.n_steps=20',
]],
results_extractor={
'Test': lambda result: result.get_best('test').score.get_score('test'),
'Train': lambda result: result.get_best('test').score.get_score('train'),
}
)
import tabulate
print tabulate.tabulate(rows, headers=headers, tablefmt=tablefmt)
"""
args = [rec.info.get_field(ExpInfoFields.ARGS) for rec in records]
if args_to_show is None:
common, separate = separate_common_items(args)
args_to_show = [k for k, v in separate[0]]
if results_extractor is None:
results_extractor = {'Result': str}
elif callable(results_extractor):
results_extractor = {'Result': results_extractor}
else:
assert isinstance(results_extractor, dict)
headers = args_to_show + results_extractor.keys()
rows = []
for record, record_args in izip_equal(records, args):
arg_dict = dict(record_args)
args_vals = [arg_dict[k] for k in args_to_show]
results = record.get_result()
rows.append(args_vals+[f(results) for f in results_extractor.values()])
if print_table:
import tabulate
print tabulate.tabulate(rows, headers=headers, tablefmt='simple')
return headers, rows
def get_record_invalid_arg_string(record,
recursive=True,
ignore_valid_keys=(),
note_version='full'):
"""
Return a string identifying ig the arguments for this experiment are still valid.
:return:
"""
assert note_version in ('full', 'short')
experiment_id = record.get_experiment_id()
if is_experiment_loadable(experiment_id):
if record.info.has_field(ExpInfoFields.ARGS):
last_run_args = OrderedDict([(k, v)
for k, v in record.get_args().items()
if k not in ignore_valid_keys])
current_args = OrderedDict([
(k, v) for k, v in record.get_experiment().get_args().items()
if k not in ignore_valid_keys
])
if recursive:
last_run_args = OrderedDict(
flatten_struct(last_run_args,
first_dict_is_namespace=True))
last_run_args = OrderedDict([(k, v)
for k, v in last_run_args.items()
if k not in ignore_valid_keys])
current_args = OrderedDict(
flatten_struct(current_args, first_dict_is_namespace=True))
current_args = OrderedDict([(k, v)
for k, v in current_args.items()
if k not in ignore_valid_keys])
validity = record.args_valid(last_run_args=last_run_args,
current_args=current_args)
if validity is False:
last_arg_str, this_arg_str = [[
'{}:{}'.format(k, v) for k, v in argdict.items()
] for argdict in (last_run_args, current_args)]
common, (old_args, new_args) = separate_common_items(
[last_arg_str, this_arg_str])
if len(old_args) + len(new_args) == 0:
raise Exception(
'Error displaying different args. Bug Peter.')
changestr = "{{{}}}->{{{}}}".format(','.join(old_args),
','.join(new_args))
notes = ("Change: "
if note_version == 'full' else "") + changestr
elif validity is None:
notes = "Cannot Determine: Unhashable Args" if note_version == 'full' else '<Unhashable Args>'
else:
notes = "<No Change>"
else:
notes = "Cannot Determine: Inconsistent Experiment Record" if note_version == 'full' else '<Inconsistent Record>'
else:
notes = "Cannot Determine: Experiment Not Imported" if note_version == 'full' else '<Not Imported>'
return notes
def separate_common_args(records, return_dict=False):
"""
:param records: A List of records
:param return_dict: Return the different args as a dict<ExperimentRecord: args>
:return: (common, different)
Where common is an OrderedDict of common args
different is a list (the same lengths of records) of OrderedDicts containing args that are not the same in all records.
"""
common, argdiff = separate_common_items(
[list(rec.get_args().items()) for rec in records])
if return_dict:
argdiff = {rec.get_id(): args for rec, args in zip(records, argdiff)}
return common, argdiff
def separate_common_args(records, as_dicts=False, return_dict = False, only_shared_argdiffs = False):
"""
:param records: A List of records
:param return_dict: Return the different args as a dict<ExperimentRecord: args>
:return: (common, different)
Where common is an OrderedDict of common args
different is a list (the same lengths of records) of OrderedDicts containing args that are not the same in all records.
"""
common, argdiff = separate_common_items([list(rec.get_args().items()) for rec in records])
if only_shared_argdiffs:
args_that_they_all_have = set.intersection(*(set(k for k, v in different) for different in argdiff))
argdiff = [[(k, v) for k, v in ad if k in args_that_they_all_have] for ad in argdiff]
if as_dicts:
common = OrderedDict(common)
argdiff = [OrderedDict(ad) for ad in argdiff]
if return_dict:
argdiff = {rec.get_id(): args for rec, args in zip(records, argdiff)}
return common, argdiff
def compare_experiment_records(record_identifiers):
records = [load_experiment_record(ident) for ident in record_identifiers]
# info_results = OrderedDict([(identifier, record.get_info()) for identifier, record in zip(record_identifiers, records)]])
funtion_names = [
record.info.get_field(ExpInfoFields.FUNCTION) for record in records
]
args = [record.info.get_field(ExpInfoFields.ARGS) for record in records]
results = [record.get_result() for record in records]
common_args, different_args = separate_common_items(args)
def lookup_fcn(identifier, column):
index = record_identifiers.index(identifier)
if column == 'Function':
return funtion_names[index]
elif column == 'Run Time':
return records[index].info.get_field_text(ExpInfoFields.RUNTIME)
elif column == 'Common Args':
return ', '.join('{}={}'.format(k, v) for k, v in common_args)
elif column == 'Different Args':
return ', '.join('{}={}'.format(k, v)
for k, v in different_args[index])
elif column == 'Result':
return results[index]
else:
bad_value(column)
rows = build_table(lookup_fcn,
row_categories=record_identifiers,
column_categories=[
'Function', 'Run Time', 'Common Args',
'Different Args', 'Result'
],
prettify_labels=False)
print tabulate(rows)
def get_record_invalid_arg_string(record,
recursive=True,
ignore_valid_keys=(),
note_version='full'):
"""
Return a string identifying ig the arguments for this experiment are still valid.
:return:
"""
assert note_version in ('full', 'short')
experiment_id = record.get_experiment_id()
if is_experiment_loadable(experiment_id):
if record.info.has_field(ExpInfoFields.ARGS):
last_run_args = OrderedDict([(k, v)
for k, v in record.get_args().items()
if k not in ignore_valid_keys])
current_args = OrderedDict([
(k, v) for k, v in record.get_experiment().get_args().items()
if k not in ignore_valid_keys
])
if recursive:
last_run_args = OrderedDict(
flatten_struct(last_run_args,
first_dict_is_namespace=True,
primatives=PRIMATIVE_TYPES +
(UnPicklableArg, )))
last_run_args = OrderedDict([(k, v)
for k, v in last_run_args.items()
if k not in ignore_valid_keys])
current_args = OrderedDict(
flatten_struct(current_args, first_dict_is_namespace=True))
current_args = OrderedDict([(k, v)
for k, v in current_args.items()
if k not in ignore_valid_keys])
validity = record.args_valid(last_run_args=last_run_args,
current_args=current_args)
if validity is False:
common, (old_args, new_args) = separate_common_items(
[list(last_run_args.items()),
list(current_args.items())])
if len(old_args) + len(new_args) == 0:
raise Exception(
'Error displaying different args. Bug Peter.')
all_changed_arg_names = remove_duplicates(
list(name for name, _ in old_args) +
list(name for name, _ in new_args))
changestr = ', '.join("{}:{}->{}".format(
k, last_run_args[k] if k in last_run_args else '<N/A>',
current_args[k] if k in current_args else '<N/A>')
for k in all_changed_arg_names)
notes = ("Change: "
if note_version == 'full' else "") + changestr
elif validity is None:
notes = "Cannot Determine: Unhashable Args" if note_version == 'full' else '<Unhashable Args>'
else:
notes = "<No Change>"
else:
notes = "Cannot Determine: Inconsistent Experiment Record" if note_version == 'full' else '<Inconsistent Record>'
else:
notes = "Cannot Determine: Experiment Not Imported" if note_version == 'full' else '<Not Imported>'
return notes
def get_experiment_list_str(self, exp_record_dict):
headers = {
'full': [ExpRecordDisplayFields.RUNS, ExpRecordDisplayFields.DURATION, ExpRecordDisplayFields.STATUS, ExpRecordDisplayFields.ARGS_CHANGED, ExpRecordDisplayFields.RESULT_STR, ExpRecordDisplayFields.NOTES],
'results': [ExpRecordDisplayFields.RESULT_STR]
}[self.view_mode]
if self.remove_prefix:
deprefixed_ids = deprefix_experiment_ids(exp_record_dict.keys())
exp_record_dict = OrderedDict((k, v) for k, v in zip(deprefixed_ids, exp_record_dict.values()))
row_func = _get_record_rows_cached if self.cache_result_string else _get_record_rows
header_names = [h.value for h in headers]
def remove_notes_if_no_notes(_record_rows, _record_headers):
notes_column_index = _record_headers.index(ExpRecordDisplayFields.NOTES.value) if ExpRecordDisplayFields.NOTES.value in _record_headers else None
# Remove the notes column if there are no notes!
if notes_column_index is not None and all(row[notes_column_index]=='' or row[notes_column_index]=='-' for row in _record_rows):
new_rows = []
for row in _record_rows:
new_rows.append(row[:notes_column_index]+row[notes_column_index+1:])
new_headers = _record_headers[:notes_column_index]+_record_headers[notes_column_index+1:]
else:
new_rows = _record_rows
new_headers = _record_headers
return new_rows, new_headers
if self.display_format=='nested': # New Display mode
if self.show_args:
_, argdiff = separate_common_items([load_experiment(ex).get_args().items() for ex in exp_record_dict])
argdiff = {k: args for k, args in izip_equal(exp_record_dict.keys(), argdiff)}
# Build a list of experiments and a list of records.
full_headers = ['#']+header_names
record_rows = []
experiment_rows = []
experiment_row_ixs = []
counter = 1 # Start at 2 because record table has the headers.
for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()):
experiment_row_ixs.append(counter)
exp_identifier = exp_id if not self.show_args else ','.join('{}={}'.format(k, v) for k, v in argdiff[exp_id])
experiment_rows.append([i, exp_identifier])
for j, record_id in enumerate(record_ids):
record_rows.append([j]+row_func(record_id, headers, raise_display_errors=self.raise_display_errors, truncate_to=self.truncate_result_to, ignore_valid_keys=self.ignore_valid_keys))
counter+=1
record_rows, full_headers = remove_notes_if_no_notes(record_rows, full_headers)
# Merge the experiments table and record table.
if self.table_package=='tabulate':
record_table_rows = tabulate(record_rows, headers=full_headers, tablefmt="pipe").split('\n')
del record_table_rows[1] # Get rid of that silly line.
experiment_table_rows = tabulate(experiment_rows, numalign='left').split('\n')[1:-1] # First and last are just borders
longest_row = max(max(len(r) for r in record_table_rows), max(len(r) for r in experiment_table_rows)+4) if len(experiment_table_rows)>0 and len(record_table_rows)>0 else 0
record_table_rows = [r if len(r)==longest_row else r[:-1] + ' '*(longest_row-len(r)) + r[-1] for r in record_table_rows]
experiment_table_rows = [('=' if i==0 else '-')*longest_row+'\n'+r + ' '*(longest_row-len(r)-1)+'|' for i, r in enumerate(experiment_table_rows)]
all_rows = [surround_with_header('Experiments', width=longest_row, char='=')] + (insert_at(record_table_rows, experiment_table_rows, indices=experiment_row_ixs) if len(experiment_table_rows)>0 else ['<No non-root Experiments>']) + ['='*longest_row]
table = '\n'.join(all_rows)
else:
raise NotImplementedError(self.table_package)
elif self.display_format=='flat': # Display First record on same row
full_headers = ['E#', 'R#', 'Experiment']+header_names
rows = []
for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()):
if len(record_ids)==0:
rows.append([str(i), '', exp_id, '<No Records>'] + ['-']*(len(headers)-1))
else:
for j, record_id in enumerate(record_ids):
rows.append([str(i) if j==0 else '', j, exp_id if j==0 else '']+row_func(record_id, headers, raise_display_errors=self.raise_display_errors, truncate_to=self.truncate_result_to, ignore_valid_keys=self.ignore_valid_keys))
assert len(rows[0])==len(full_headers)
rows, full_headers = remove_notes_if_no_notes(rows, full_headers)
if self.table_package == 'pretty_table':
from prettytable.prettytable import PrettyTable
table = str(PrettyTable(rows, field_names=full_headers, align='l', max_table_width=self.max_width))
elif self.table_package == 'tabulate':
table = tabulate(rows, headers=full_headers)
else:
raise NotImplementedError(self.table_package)
else:
raise NotImplementedError(self.display_format)
return table
def get_experiment_list_str(self, exp_record_dict):
headers = {
'full': [
ExpRecordDisplayFields.RUNS, ExpRecordDisplayFields.DURATION,
ExpRecordDisplayFields.STATUS,
ExpRecordDisplayFields.ARGS_CHANGED,
ExpRecordDisplayFields.RESULT_STR, ExpRecordDisplayFields.NOTES
],
'results': [ExpRecordDisplayFields.RESULT_STR]
}[self.view_mode]
if self.remove_prefix:
deprefixed_ids = deprefix_experiment_ids(exp_record_dict.keys())
exp_record_dict = OrderedDict(
(k, v)
for k, v in zip(deprefixed_ids, exp_record_dict.values()))
row_func = _get_record_rows_cached if self.cache_result_string else _get_record_rows
header_names = [h.value for h in headers]
def remove_notes_if_no_notes(_record_rows):
notes_column_index = headers.index(
ExpRecordDisplayFields.NOTES
) if ExpRecordDisplayFields.NOTES in headers else None
# Remove the notes column if there are no notes!
if notes_column_index is not None and all(
row[notes_column_index] == '' for row in _record_rows):
for row in _record_rows:
del row[notes_column_index]
if self.display_format == 'nested': # New Display mode
if self.show_args:
_, argdiff = separate_common_items([
load_experiment(ex).get_args().items()
for ex in exp_record_dict
])
argdiff = {
k: args
for k, args in izip_equal(exp_record_dict.keys(), argdiff)
}
# Build a list of experiments and a list of records.
full_headers = ['#'] + header_names
record_rows = []
experiment_rows = []
experiment_row_ixs = []
counter = 1 # Start at 2 because record table has the headers.
for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()):
experiment_row_ixs.append(counter)
exp_identifier = exp_id if not self.show_args else ','.join(
'{}={}'.format(k, v) for k, v in argdiff[exp_id])
experiment_rows.append([i, exp_identifier])
for j, record_id in enumerate(record_ids):
record_rows.append([j] + row_func(
record_id,
headers,
raise_display_errors=self.raise_display_errors,
truncate_to=self.truncate_result_to,
ignore_valid_keys=self.ignore_valid_keys))
counter += 1
remove_notes_if_no_notes(record_rows)
# Merge the experiments table and record table.
record_table_rows = tabulate(record_rows,
headers=full_headers,
tablefmt="pipe").split('\n')
del record_table_rows[1] # Get rid of that silly line.
experiment_table_rows = tabulate(
experiment_rows, numalign='left').split('\n')[
1:-1] # First and last are just borders
longest_row = max(max(len(r) for r in record_table_rows),
max(len(r) for r in experiment_table_rows) +
4) if len(record_table_rows) > 0 else 0
record_table_rows = [
r if len(r) == longest_row else r[:-1] + ' ' *
(longest_row - len(r)) + r[-1] for r in record_table_rows
]
experiment_table_rows = [
('=' if i == 0 else '-') * longest_row + '\n' + r + ' ' *
(longest_row - len(r) - 1) + '|'
for i, r in enumerate(experiment_table_rows)
]
all_rows = [
surround_with_header(
'Experiments', width=longest_row, char='=')
] + insert_at(record_table_rows,
experiment_table_rows,
indices=experiment_row_ixs) + ['=' * longest_row]
table = '\n'.join(all_rows)
elif self.display_format == 'flat': # Display First record on same row
full_headers = ['E#', 'R#', 'Experiment'] + header_names
rows = []
for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()):
if len(record_ids) == 0:
rows.append([str(i), '', exp_id, '<No Records>'] + ['-'] *
(len(headers) - 1))
else:
for j, record_id in enumerate(record_ids):
rows.append([
str(i) if j == 0 else '', j, exp_id if j ==
0 else ''
] + row_func(
record_id,
headers,
raise_display_errors=self.raise_display_errors,
truncate_to=self.truncate_result_to,
ignore_valid_keys=self.ignore_valid_keys))
remove_notes_if_no_notes(rows)
table = tabulate(rows, headers=full_headers)
else:
raise NotImplementedError(self.display_format)
return table
