def _get_record_rows(record_id,
headers,
raise_display_errors,
truncate_to,
ignore_valid_keys=()):
rec = load_experiment_record(record_id)
with _DisplaySettings(dict(ignore_valid_keys=ignore_valid_keys)):
if not raise_display_errors:
values = []
for h in headers:
try:
values.append(_exp_record_field_getters[h](rec))
except:
values.append('<Display Error>')
else:
values = [_exp_record_field_getters[h](rec) for h in headers]
if truncate_to is not None:
values = [
truncate_string(val, truncation=truncate_to, message='...')
for val in values
]
return values
def _filter_experiment_record_list(user_range, experiment_record_ids):
if user_range=='all':
return [True]*len(experiment_record_ids)
elif user_range=='new':
return detect_duplicates(experiment_record_ids, key=record_id_to_experiment_id, keep_last=True)
# return [n for n, is_old in izip_equal(get_record_ids(), old) if not old]
elif user_range=='old':
return [not x for x in _filter_records(user_range, 'new')]
elif user_range=='orphans':
orphans = []
global_lib = get_global_experiment_library()
for i, record_id in enumerate(experiment_record_ids):
info = load_experiment_record(record_id).info
if 'Module' in info:
try:
import_module(info['Module'])
if not record_id_to_experiment_id(record_id) in global_lib:
orphans.append(True)
else:
orphans.append(False)
except ImportError:
orphans.append(True)
else: # They must be old... lets kill them!
orphans.append(True)
return orphans
# elif user_range
else:
which_ones = interpret_numbers(user_range)
if which_ones is None:
raise Exception('Could not interpret user range: "{}"'.format(user_range))
filters = [False]*len(experiment_record_ids)
for i in which_ones:
filters[i] = True
return filters
def get_oneline_result_string(record,
truncate_to=None,
array_float_format='.3g',
array_print_threshold=8):
"""
Get a string that describes the result of the record in one line. This can optionally be specified by
experiment.one_liner_function.
:param record: An ExperimentRecord.
:param truncate_to:
:param array_float_format:
:param array_print_threshold:
:return: A string with no newlines briefly describing the result of the record.
"""
if isinstance(record, string_types):
record = load_experiment_record(record)
if not is_experiment_loadable(record.get_experiment_id()):
one_liner_function = str
else:
one_liner_function = record.get_experiment().one_liner_function
if one_liner_function is None:
one_liner_function = str
return get_record_result_string(
record,
func=one_liner_function,
truncate_to=truncate_to,
array_print_threshold=array_print_threshold,
array_float_format=array_float_format,
oneline=True)
def _filter_experiment_record_list(user_range, experiment_record_ids):
if user_range=='all':
return [True]*len(experiment_record_ids)
elif user_range=='new':
return detect_duplicates(experiment_record_ids, key=record_id_to_experiment_id, keep_last=True)
# return [n for n, is_old in izip_equal(get_record_ids(), old) if not old]
elif user_range=='old':
return [not x for x in _filter_records(user_range, 'new')]
elif user_range=='orphans':
orphans = []
global_lib = get_global_experiment_library()
for i, record_id in enumerate(experiment_record_ids):
info = load_experiment_record(record_id).info
if 'Module' in info:
try:
import_module(info['Module'])
if not record_id_to_experiment_id(record_id) in global_lib:
orphans.append(True)
else:
orphans.append(False)
except ImportError:
orphans.append(True)
else: # They must be old... lets kill them!
orphans.append(True)
return orphans
else:
which_ones = interpret_numbers(user_range)
if which_ones is None:
raise Exception('Could not interpret user range: "{}"'.format(user_range))
filters = [False]*len(experiment_record_ids)
for i in which_ones:
filters[i] = True
return filters
def errortrace(self, user_range):
record_ids = select_experiment_records(user_range,
self.exp_record_dict,
flat=True)
with IndentPrint("Error Traces:", show_line=True, show_end=True):
for erid in record_ids:
with IndentPrint(erid, show_line=True):
record = load_experiment_record(erid)
print record.get_error_trace()
_warn_with_prompt()
def test_get_latest_identifier():
with experiment_testing_context():
exp_rec = experiment_test_function.run()
print(get_experiment_info('experiment_test_function'))
assert_experiment_record_is_correct(exp_rec)
last_experiment_identifier = load_experiment('experiment_test_function').get_latest_record().get_id()
assert last_experiment_identifier is not None, 'Experiment was run, this should not be none'
same_exp_rec = load_experiment_record(last_experiment_identifier)
assert_experiment_record_is_correct(same_exp_rec)
def get_records(self, only_completed=False):
"""
Get all records associated with this experiment.
:param only_completed: Only include records that have run to completion.
:return: A list of ExperimentRecord objects.
"""
records = [load_experiment_record(rid) for rid in experiment_id_to_record_ids(self.name)]
if only_completed:
records = [record for record in records if record.get_status()==ExpStatusOptions.FINISHED]
return records
def get_records(self, only_completed=False):
records = [
load_experiment_record(rid)
for rid in experiment_id_to_record_ids(self.name)
]
if only_completed:
records = [
record for record in records
if record.get_status() == ExpStatusOptions.FINISHED
]
return records
def test_get_latest_identifier():
with experiment_testing_context():
exp_rec = experiment_test_function.run()
print(get_experiment_info('experiment_test_function'))
assert_experiment_record_is_correct(exp_rec)
last_experiment_identifier = load_experiment(
'experiment_test_function').get_latest_record().get_id()
assert last_experiment_identifier is not None, 'Experiment was run, this should not be none'
same_exp_rec = load_experiment_record(last_experiment_identifier)
assert_experiment_record_is_correct(same_exp_rec)
def get_records(self, only_completed=False):
"""
Get all records associated with this experiment.
:param only_completed: Only include records that have run to completion.
:return: A list of ExperimentRecord objects.
"""
records = [load_experiment_record(rid) for rid in experiment_id_to_record_ids(self.name)]
if only_completed:
records = [record for record in records if record.get_status()==ExpStatusOptions.FINISHED]
return records
def launch(self):
func_lookup = {
'q': self.quit,
'h': self.help,
'filter': self.filter,
'showall': self.showall,
'args': self.args,
'rmfilters': self.rmfilters,
'viewfilters': self.viewfilters,
'side_by_side': self.compare,
'show': self.show,
'search': self.search,
'delete': self.delete,
}
while True:
print("=============== Experiment Records ==================")
# print tabulate([[i]+e.get_row() for i, e in enumerate(record_infos)], headers=['#']+_ExperimentInfo.get_headers())
print(
self.get_record_table(
[load_experiment_record(rid) for rid in self.record_ids],
raise_display_errors=self.raise_display_errors))
print('-----------------------------------------------------')
if self.experiment_names is not None or len(self.filters) != 0:
print(
'Not showing all experiments. Type "showall" to see all experiments, or "viewfilters" to view current filters.'
)
user_input = input(
'Enter Command (show # to show and experiment, or h for help) >>'
).strip()
parts = shlex.split(user_input)
if len(parts) == 0:
print("You need to specify an command. Press h for help.")
continue
cmd = parts[0]
args = parts[1:]
try:
if cmd not in func_lookup:
raise _warn_with_prompt('Unknown Command: {}'.format(cmd))
else:
return_val = func_lookup[cmd](*args)
if return_val == self.QUIT:
break
except Exception as e:
res = input(
'%s: %s\nEnter "e" to view the stacktrace, or anything else to continue.'
% (e.__class__.__name__, e.message))
if res.strip().lower() == 'e':
raise
def get_record_table(record_ids=None,
headers=('#', 'Identifier', 'Start Time', 'Duration',
'Status', 'Notes', 'Result'),
raise_display_errors=False):
d = {
'#':
lambda: i,
'Identifier':
lambda: record_id,
'Start Time':
lambda: experiment_record.info.get_field_text(
ExpInfoFields.TIMESTAMP, replacement_if_none='?'),
'Duration':
lambda: experiment_record.info.get_field_text(
ExpInfoFields.RUNTIME, replacement_if_none='?'),
'Status':
lambda: experiment_record.info.get_field_text(
ExpInfoFields.STATUS, replacement_if_none='?'),
'Args':
lambda: experiment_record.info.get_field_text(
ExpInfoFields.ARGS, replacement_if_none='?'),
'Notes':
lambda: experiment_record.info.get_field_text(
ExpInfoFields.NOTES, replacement_if_none='?'),
'Result':
lambda: experiment_record.get_one_liner(),
}
def get_col_info(headers):
info = []
for h in headers:
try:
info.append(d[h]())
except:
info.append('<Error displaying info>')
if raise_display_errors:
raise
return info
rows = []
for i, record_id in enumerate(record_ids):
experiment_record = load_experiment_record(record_id)
rows.append(get_col_info(headers))
assert all_equal(
[len(headers)] +
[len(row)
for row in rows]), 'Header length: {}, Row Lengths: \n {}'.format(
len(headers), [len(row) for row in rows])
return tabulate(rows, headers=headers)
def results(self, user_range='all'):
record_ids = select_experiment_records(user_range,
self.exp_record_dict)
with IndentPrint("Results:", show_line=True, show_end=True):
for erid in record_ids:
record = load_experiment_record(erid)
with IndentPrint(erid, show_line=True, show_end=True):
try:
result = record.get_result()
record.get_experiment().display_last(result,
err_if_none=False)
except NoSavedResultError:
print '<No result was saved>'
except Exception as err:
print err
_warn_with_prompt()
def get_experient_to_record_dict(experiment_ids = None):
"""
Given a list of experiment ids, return an OrderedDict whose keys are the experiment ids and whose values
are lists of experiment record ids.
:param experiment_ids: A list of experiment ids. (Defaults to all imported experiments)
:return: A dict<experiment_id -> list<experiment_record_id>
"""
if experiment_ids is None:
experiment_ids = get_global_experiment_library().keys()
record_ids = get_all_record_ids(experiment_ids)
exp_rec_dict = OrderedDict((exp_id, []) for exp_id in experiment_ids)
for rid in record_ids:
rec = load_experiment_record(rid)
exp_id = rec.get_experiment_id()
exp_rec_dict[exp_id].append(rid)
return exp_rec_dict
def get_experient_to_record_dict(experiment_ids=None):
"""
Given a list of experiment ids, return an OrderedDict whose keys are the experiment ids and whose values
are lists of experiment record ids.
:param experiment_ids: A list of experiment ids. (Defaults to all imported experiments)
:return: A dict<experiment_id -> list<experiment_record_id>
"""
if experiment_ids is None:
experiment_ids = get_global_experiment_library().keys()
record_ids = get_all_record_ids(experiment_ids)
exp_rec_dict = OrderedDict((exp_id, []) for exp_id in experiment_ids)
for rid in record_ids:
rec = load_experiment_record(rid)
exp_id = rec.get_experiment_id()
exp_rec_dict[exp_id].append(rid)
return exp_rec_dict
def _get_record_rows(record_id, headers, raise_display_errors, truncate_to, ignore_valid_keys = ()):
rec = load_experiment_record(record_id)
with _DisplaySettings(dict(ignore_valid_keys=ignore_valid_keys)):
if not raise_display_errors:
values = []
for h in headers:
try:
values.append(_exp_record_field_getters[h](rec))
except:
values.append('<Display Error>')
else:
values = [_exp_record_field_getters[h](rec) for h in headers]
if truncate_to is not None:
values = [truncate_string(val, truncation=truncate_to, message='...') for val in values]
return values
def launch(self):
func_lookup = {
'q': self.quit,
'h': self.help,
'filter': self.filter,
'showall': self.showall,
'args': self.args,
'rmfilters': self.rmfilters,
'viewfilters': self.viewfilters,
'side_by_side': self.compare,
'show': self.show,
'search': self.search,
'delete': self.delete,
}
while True:
print("=============== Experiment Records ==================")
# print tabulate([[i]+e.get_row() for i, e in enumerate(record_infos)], headers=['#']+_ExperimentInfo.get_headers())
print(self.get_record_table([load_experiment_record(rid) for rid in self.record_ids], raise_display_errors = self.raise_display_errors))
print('-----------------------------------------------------')
if self.experiment_names is not None or len(self.filters) != 0:
print('Not showing all experiments. Type "showall" to see all experiments, or "viewfilters" to view current filters.')
user_input = input('Enter Command (show # to show and experiment, or h for help) >>').strip()
parts = shlex.split(user_input)
if len(parts)==0:
print("You need to specify an command. Press h for help.")
continue
cmd = parts[0]
args = parts[1:]
try:
if cmd not in func_lookup:
raise _warn_with_prompt('Unknown Command: {}'.format(cmd))
else:
return_val = func_lookup[cmd](*args)
if return_val==self.QUIT:
break
except Exception as e:
res = input('%s: %s\nEnter "e" to view the stacktrace, or anything else to continue.' % (e.__class__.__name__, e.message))
if res.strip().lower() == 'e':
raise
def select_experiment_records(user_range, exp_record_dict, flat=True):
"""
:param user_range:
:param exp_record_dict: An OrderedDict<experiment_name: list<experiment_record_name>>
:param flat: Return a list of experiment records, instead of an OrderedDict
:return: if not flat, an An OrderedDict<experiment_name: list<experiment_record_name>>
otherwise a list<experiment_record_name>
"""
filters = _filter_records(user_range, exp_record_dict)
filtered_dict = OrderedDict((k, [
load_experiment_record(record_id)
for record_id, f in izip_equal(exp_record_dict[k], filters[k]) if f
]) for k in exp_record_dict.keys())
if flat:
return [
record_id for records in filtered_dict.values()
for record_id in records
]
else:
return filtered_dict
def get_oneline_result_string(record, truncate_to=None, array_float_format='.3g', array_print_threshold=8):
"""
Get a string that describes the result of the record in one line. This can optionally be specified by
experiment.one_liner_function.
:param record: An ExperimentRecord.
:param truncate_to:
:param array_float_format:
:param array_print_threshold:
:return: A string with no newlines briefly describing the result of the record.
"""
if isinstance(record, string_types):
record = load_experiment_record(record)
if not is_experiment_loadable(record.get_experiment_id()):
one_liner_function=str
else:
one_liner_function = record.get_experiment().one_liner_function
if one_liner_function is None:
one_liner_function = str
return get_record_result_string(record, func=one_liner_function, truncate_to=truncate_to, array_print_threshold=array_print_threshold,
array_float_format=array_float_format, oneline=True)
def compare_experiment_records(record_identifiers):
records = [load_experiment_record(ident) for ident in record_identifiers]
# info_results = OrderedDict([(identifier, record.get_info()) for identifier, record in zip(record_identifiers, records)]])
funtion_names = [
record.info.get_field(ExpInfoFields.FUNCTION) for record in records
]
args = [record.info.get_field(ExpInfoFields.ARGS) for record in records]
results = [record.get_result() for record in records]
common_args, different_args = separate_common_items(args)
def lookup_fcn(identifier, column):
index = record_identifiers.index(identifier)
if column == 'Function':
return funtion_names[index]
elif column == 'Run Time':
return records[index].info.get_field_text(ExpInfoFields.RUNTIME)
elif column == 'Common Args':
return ', '.join('{}={}'.format(k, v) for k, v in common_args)
elif column == 'Different Args':
return ', '.join('{}={}'.format(k, v)
for k, v in different_args[index])
elif column == 'Result':
return results[index]
else:
bad_value(column)
rows = build_table(lookup_fcn,
row_categories=record_identifiers,
column_categories=[
'Function', 'Run Time', 'Common Args',
'Different Args', 'Result'
],
prettify_labels=False)
print tabulate(rows)
def show(self, user_range):
record_ids = self._select_records(user_range)
show_experiment_records(
[load_experiment_record(rid) for rid in record_ids])
_warn_with_prompt('')
def get_experiment_list_str(exp_record_dict,
just_last_record,
view_mode='full',
raise_display_errors=False,
truncate_result_to=100,
cache_result_string=True):
headers = {
'full': [
'E#', 'R#', 'Name',
'Last Run' if just_last_record else 'All Runs', 'Duration',
'Status', 'Valid', 'Result'
],
'results': ['E#', 'R#', 'Name', 'Result']
}[view_mode]
rows = []
oneliner_func = memoize_to_disk_with_settings(suppress_info=True)(
get_oneline_result_string
) if cache_result_string else get_oneline_result_string
def get_field(header):
try:
return \
index if header=='#' else \
(str(i) if j==0 else '') if header == 'E#' else \
j if header == 'R#' else \
(name if j==0 else '') if header=='Name' else \
experiment_record.info.get_field_text(ExpInfoFields.TIMESTAMP) if header in ('Last Run', 'All Runs') else \
experiment_record.info.get_field_text(ExpInfoFields.RUNTIME) if header=='Duration' else \
experiment_record.info.get_field_text(ExpInfoFields.STATUS) if header=='Status' else \
get_record_invalid_arg_string(experiment_record) if header=='Valid' else \
oneliner_func(experiment_record.get_id(), truncate_to=truncate_result_to) if header=='Result' else \
'???'
except:
if raise_display_errors:
raise
return '<Display Error>'
for i, (exp_id, record_ids) in enumerate(exp_record_dict.iteritems()):
if len(record_ids) == 0:
if exp_id in exp_record_dict:
rows.append([
str(i), '', exp_id, '<No Records>', '-', '-', '-', '-'
])
else:
for j, record_id in enumerate(record_ids):
index, name = ['{}.{}'.format(
i, j), exp_id] if j == 0 else [
'{}.{}'.format('`' * len(str(i)), j), exp_id
]
try:
experiment_record = load_experiment_record(record_id)
except:
experiment_record = None
rows.append([get_field(h) for h in headers])
assert all_equal([len(headers)] + [len(row) for row in rows]
), 'Header length: {}, Row Lengths: \n {}'.format(
len(headers),
'\n'.join([len(row) for row in rows]))
table = tabulate(rows, headers=headers)
return table
def _select_record_from_filters(filters, exp_record_dict):
return OrderedDict((k, [
load_experiment_record(record_id)
for record_id, f in izip_equal(exp_record_dict[k], filters[k]) if f
]) for k in exp_record_dict.keys())
def _filter_records(user_range, exp_record_dict):
"""
:param user_range:
:param exp_record_dict:
:return: An OrderedDict<experiment_id -> list<True or False>> indicating whether each record from the given experiment passed the filter
"""
def _bitwise(op, filter_set_1, filter_set_2):
assert op in ('and', 'or')
filter_set_3 = filter_set_1.copy()
for k in filter_set_1.keys():
filter_set_3[k] = [
(a or b) if op == 'or' else (a and b)
for a, b in izip_equal(filter_set_1[k], filter_set_2[k])
]
return filter_set_3
base = OrderedDict(
(k, [False] * len(v)) for k, v in exp_record_dict.iteritems())
if user_range in exp_record_dict: # User just lists an experiment
base[user_range] = [True] * len(base[user_range])
return base
if '|' in user_range:
return reduce(lambda a, b: _bitwise('or', a, b), [
_filter_records(subrange, exp_record_dict)
for subrange in user_range.split('|')
])
if '&' in user_range:
return reduce(lambda a, b: _bitwise('and', a, b), [
_filter_records(subrange, exp_record_dict)
for subrange in user_range.split('&')
])
number_range = interpret_numbers(user_range)
keys = exp_record_dict.keys()
if number_range is not None:
for i in number_range:
base[keys[i]] = [True] * len(base[keys[i]])
elif '.' in user_range:
exp_rec_pairs = interpret_record_identifier(user_range)
for exp_number, rec_number in exp_rec_pairs:
base[keys[exp_number]][rec_number] = True
elif user_range == 'old':
for k, v in base.iteritems():
base[k] = ([True] * (len(v) - 1) + [False]) if len(v) > 0 else []
elif user_range == 'unfinished':
for k, v in base.iteritems():
base[k] = [
load_experiment_record(rec_id).info.get_field(
ExpInfoFields.STATUS) != ExpStatusOptions.FINISHED
for rec_id in exp_record_dict[k]
]
# filtered_dict = OrderedDict((exp_id, [rec_id for rec_id in records if load_experiment_record(rec_id).info.get_field(ExpInfoFields.STATUS) != ExpStatusOptions.FINISHED]) for exp_id, records in exp_record_dict.iteritems())
elif user_range == 'invalid':
for k, v in base.iteritems():
base[k] = [
load_experiment_record(rec_id).args_valid() is False
for rec_id in exp_record_dict[k]
]
elif user_range == 'all':
for k, v in base.iteritems():
base[k] = [True] * len(v)
elif user_range == 'errors':
for k, v in base.iteritems():
base[k] = [
load_experiment_record(rec_id).info.get_field(
ExpInfoFields.STATUS) == ExpStatusOptions.ERROR
for rec_id in exp_record_dict[k]
]
else:
raise Exception(
"Don't know how to interpret subset '{}'".format(user_range))
return base
def _filter_records(user_range, exp_record_dict):
"""
:param user_range:
:param exp_record_dict:
:return: An OrderedDict<experiment_id -> list<True or False>> indicating whether each record from the given experiment passed the filter
"""
if user_range == 'unfinished':
return _filter_records('~finished', exp_record_dict)
elif user_range == 'last':
return _filter_records('~old', exp_record_dict)
elif '|' in user_range:
return _bitwise_filter_op(
'or', *[
_filter_records(subrange, exp_record_dict)
for subrange in user_range.split('|')
])
elif '&' in user_range:
return _bitwise_filter_op(
'and', *[
_filter_records(subrange, exp_record_dict)
for subrange in user_range.split('&')
])
elif '@' in user_range:
ix = user_range.index('@')
first_part, second_part = user_range[:ix], user_range[ix + 1:]
_first_stage_filters = _filter_records(first_part, exp_record_dict)
_new_dict = _select_record_ids_from_filters(_first_stage_filters,
exp_record_dict)
_second_stage_filters = _filter_records(second_part, _new_dict)
return _bitwise_filter_op('andcascade', _first_stage_filters,
_second_stage_filters)
elif user_range.startswith('~'):
return _bitwise_filter_op(
'not', _filter_records(user_range[1:], exp_record_dict))
base = OrderedDict(
(k, [False] * len(v)) for k, v in exp_record_dict.items())
if user_range in exp_record_dict: # User just lists an experiment
base[user_range] = [True] * len(base[user_range])
return base
number_range = interpret_numbers(user_range)
keys = list(exp_record_dict.keys())
if user_range in _named_record_filters: # e.g. 'finished'
for exp_id, _ in base.items():
base[exp_id] = _named_record_filters[user_range](
exp_record_dict[exp_id])
elif number_range is not None: # e.g. '6-12'
for i in number_range:
if i > len(keys):
raise RecordSelectionError(
'Experiment {} does not exist (they go from 0 to {})'.
format(i,
len(keys) - 1))
base[keys[i]] = [True] * len(base[keys[i]])
elif '.' in user_range: # e.b. 6.3-4
exp_rec_pairs = interpret_record_identifier(user_range)
for exp_number, rec_number in exp_rec_pairs:
if rec_number >= len(base[keys[exp_number]]):
raise RecordSelectionError(
'Selection {}.{} does not exist.'.format(
exp_number, rec_number))
base[keys[exp_number]][rec_number] = True
elif user_range.startswith('since:'): # eg. 'since:24'
time_id = user_range[len('since:'):]
try:
seconds_ago = int(time_id) * 3600
except:
raise RecordSelectionError(
'Cannot interpret "{}" as a time. Currently, it should be an integer, which means "within the last X hours"'
.format(time_id))
current_time = time()
for exp_id, _ in base.items():
base[exp_id] = [
(current_time - load_experiment_record(rec_id).get_timestamp())
< seconds_ago for rec_id in exp_record_dict[exp_id]
]
elif user_range.startswith(
'dur'): # Eg dur<25 Means "All records that ran less than 25s"
try:
sign = user_range[3]
assert sign in ('<', '>')
seconds = int(user_range[4:])
except:
raise RecordSelectionError(
'Could not interpret "{}" as a duration. Example is dur<25 to select all experiments that ran less than 25s.'
.format(user_range))
for exp_id, _ in base.items():
durations = [
load_experiment_record(rec_id).info.get_field(
ExpInfoFields.RUNTIME, default=None)
for rec_id in exp_record_dict[exp_id]
]
base[exp_id] = [
False if dur is None else
dur < seconds if sign == '<' else dur > seconds
for dur in durations
]
else:
raise RecordSelectionError(
"Don't know how to interpret subset '{}'. Possible subsets: {}".
format(user_range, list(_named_record_filters.keys())))
return base
def show(self, user_range):
for rid in self._select_records(user_range):
load_experiment_record(rid).show()
_warn_with_prompt('')
for k in output_set.keys():
output_set[k] = _bitwise_not(filter_sets[0][k])
elif op in ('and', 'or'):
for k in output_set.keys():
output_set[k] = reduce(_bitwise_and if op=='and' else _bitwise_or, [fs[k] for fs in filter_sets])
elif op=='andcascade':
for k in output_set.keys():
output_set[k] = reduce(_bitwise_andcascade, [fs[k] for fs in filter_sets[::-1]])
else:
raise AssertionError('op should be one of {}'.format(('and', 'or', 'andcascade', 'not')))
return output_set
_named_record_filters = {}
_named_record_filters['old'] = lambda rec_ids: ([True]*(len(rec_ids)-1)+[False]) if len(rec_ids)>0 else []
_named_record_filters['corrupt'] = lambda rec_ids: [load_experiment_record(rec_id).info.get_status_field()==ExpStatusOptions.CORRUPT for rec_id in rec_ids]
_named_record_filters['finished'] = lambda rec_ids: [load_experiment_record(rec_id).info.get_field(ExpInfoFields.STATUS) == ExpStatusOptions.FINISHED for rec_id in rec_ids]
_named_record_filters['invalid'] = lambda rec_ids: [load_experiment_record(rec_id).args_valid() is False for rec_id in rec_ids]
_named_record_filters['all'] = lambda rec_ids: [True]*len(rec_ids)
_named_record_filters['errors'] = lambda rec_ids: [load_experiment_record(rec_id).info.get_field(ExpInfoFields.STATUS)==ExpStatusOptions.ERROR for rec_id in rec_ids]
_named_record_filters['result'] = lambda rec_ids: [load_experiment_record(rec_id).has_result() for rec_id in rec_ids]
_named_record_filters['running'] = lambda rec_ids: [load_experiment_record(rec_id).info.get_field(ExpInfoFields.STATUS)==ExpStatusOptions.STARTED for rec_id in rec_ids]
def _filter_records(user_range, exp_record_dict):
"""
:param user_range: A string specifying which records to select. Examples:
4.2 Select record 2 for experiment 4
4 Select all records for experiment 4
4-6 Select all records for experiments 4, 5, 6
4.2-5 Select records 2, 3, 4, 5 for experiment 4
[fs[k] for fs in filter_sets])
elif op == 'andcascade':
for k in output_set.keys():
output_set[k] = reduce(_bitwise_andcascade,
[fs[k] for fs in filter_sets[::-1]])
else:
raise AssertionError('op should be one of {}'.format(
('and', 'or', 'andcascade', 'not')))
return output_set
_named_record_filters = {}
_named_record_filters['old'] = lambda rec_ids: ([True] * (len(
rec_ids) - 1) + [False]) if len(rec_ids) > 0 else []
_named_record_filters['corrupt'] = lambda rec_ids: [
load_experiment_record(rec_id).info.get_status_field() == ExpStatusOptions.
CORRUPT for rec_id in rec_ids
]
_named_record_filters['finished'] = lambda rec_ids: [
load_experiment_record(rec_id).info.get_field(ExpInfoFields.STATUS) ==
ExpStatusOptions.FINISHED for rec_id in rec_ids
]
_named_record_filters['invalid'] = lambda rec_ids: [
load_experiment_record(rec_id).args_valid() is False for rec_id in rec_ids
]
_named_record_filters['all'] = lambda rec_ids: [True] * len(rec_ids)
_named_record_filters['errors'] = lambda rec_ids: [
load_experiment_record(rec_id).info.get_field(ExpInfoFields.STATUS) ==
ExpStatusOptions.ERROR for rec_id in rec_ids
]
_named_record_filters['result'] = lambda rec_ids: [
def show(self, user_range):
record_ids = self._select_records(user_range)
show_multiple_records([load_experiment_record(rid) for rid in record_ids])
_warn_with_prompt('')
def _filter_records(user_range, exp_record_dict):
"""
:param user_range: A string specifying which records to select. Examples:
4.2 Select record 2 for experiment 4
4 Select all records for experiment 4
4-6 Select all records for experiments 4, 5, 6
4.2-5 Select records 2, 3, 4, 5 for experiment 4
4.3,4.4 Select records 4.3, 4.4
all Select all records
old Select all records that are not the the most recent run for that experiment
finished Select all records that have not run to completion
invalid Select all records for which the arguments to their experiments have changed since they were run
errors Select all records that ended in error
~invalid Select all records that are not invalid (the '~' can be used to negate any of the above)
invalid|errors Select all records that are invalid or ended in error (the '|' can be used to "or" any of the above)
invalid&errors Select all records that are invalid and ended in error (the '&' can be used to "and" any of the above)
finished@last Select the last finished record of each experiment (the '@' can be used to cascade any of the above)
dur>4m Select records which ran (ie had a duration) of more than 4 minutes
age<24h Select records which are less than 24h old.
:param exp_record_dict:
:return: An OrderedDict<experiment_id -> list<True or False>> indicating whether each record from the given experiment passed the filter
"""
if user_range == 'unfinished':
return _filter_records('~finished', exp_record_dict)
elif user_range == 'last':
return _filter_records('~old', exp_record_dict)
elif '|' in user_range:
return _bitwise_filter_op(
'or', *[
_filter_records(subrange, exp_record_dict)
for subrange in user_range.split('|')
])
elif '&' in user_range:
return _bitwise_filter_op(
'and', *[
_filter_records(subrange, exp_record_dict)
for subrange in user_range.split('&')
])
elif '@' in user_range:
ix = user_range.index('@')
first_part, second_part = user_range[:ix], user_range[ix + 1:]
_first_stage_filters = _filter_records(first_part, exp_record_dict)
_new_dict = _select_record_ids_from_filters(_first_stage_filters,
exp_record_dict)
_second_stage_filters = _filter_records(second_part, _new_dict)
return _bitwise_filter_op('andcascade', _first_stage_filters,
_second_stage_filters)
elif user_range.startswith('~'):
return _bitwise_filter_op(
'not', _filter_records(user_range[1:], exp_record_dict))
base = OrderedDict(
(k, [False] * len(v)) for k, v in exp_record_dict.items())
if user_range in exp_record_dict: # User just lists an experiment
base[user_range] = [True] * len(base[user_range])
return base
number_range = interpret_numbers(user_range)
keys = list(exp_record_dict.keys())
if user_range in _named_record_filters: # e.g. 'finished'
for exp_id, _ in base.items():
base[exp_id] = _named_record_filters[user_range](
exp_record_dict[exp_id])
elif number_range is not None: # e.g. '6-12'
for i in number_range:
if i > len(keys):
raise RecordSelectionError(
'Experiment {} does not exist (they go from 0 to {})'.
format(i,
len(keys) - 1))
base[keys[i]] = [True] * len(base[keys[i]])
elif '.' in user_range: # e.b. 6.3-4
exp_rec_pairs = interpret_record_identifier(user_range)
for exp_number, rec_number in exp_rec_pairs:
if rec_number >= len(base[keys[exp_number]]):
raise RecordSelectionError(
'Selection {}.{} does not exist.'.format(
exp_number, rec_number))
base[keys[exp_number]][rec_number] = True
elif user_range.startswith('dur') or user_range.startswith(
'age'): # Eg dur<25 Means "All records that ran less than 25s"
try:
sign = user_range[3]
assert sign in ('<', '>')
filter_func = (
lambda a, b: (a is not None and b is not None) and a < b
) if sign == '<' else (lambda a, b:
(a is not None and b is not None) and a > b)
time_delta = parse_time(user_range[4:])
except:
if user_range.startswith('dur'):
raise RecordSelectionError(
'Could not interpret "{}" as duration. Example is dur<25s to select all experiments that ran less than 25s.'
.format(user_range))
else:
raise RecordSelectionError(
'Could not interpret "{}" as age. Example is age<24h to select all experiments were started in the last 24h.'
.format(user_range))
if user_range.startswith('dur'):
for exp_id, _ in base.items():
base[exp_id] = [
filter_func(
load_experiment_record(rec_id).get_runtime(),
time_delta) for rec_id in exp_record_dict[exp_id]
]
else:
current_time = datetime.now()
for exp_id, _ in base.items():
base[exp_id] = [
filter_func(
current_time -
load_experiment_record(rec_id).get_datetime(),
time_delta) for rec_id in exp_record_dict[exp_id]
]
elif user_range.startswith('has:'):
phrase = user_range[len('has:'):]
for exp_id, records in base.items():
base[exp_id] = [True] * len(
records) if phrase in exp_id else [False] * len(records)
else:
raise RecordSelectionError(
"Don't know how to interpret subset '{}'. Possible subsets: {}".
format(user_range, list(_named_record_filters.keys())))
return base
def _select_record_from_filters(filters, exp_record_dict):
return OrderedDict((k, [load_experiment_record(record_id) for record_id, f in izip_equal(exp_record_dict[k], filters[k]) if f]) for k in exp_record_dict.keys())
def get_experiment_list_str(exp_record_dict,
just_last_record,
view_mode='full',
raise_display_errors=False,
exp_filter=None):
headers = {
'full': [
'E#', 'R#', 'Name',
'Last Run' if just_last_record else 'All Runs', 'Duration',
'Status', 'Valid', 'Result'
],
'results': ['E#', 'R#', 'Name', 'Result']
}[view_mode]
rows = []
def get_field(header):
try:
return \
index if header=='#' else \
(str(i) if j==0 else '') if header == 'E#' else \
j if header == 'R#' else \
(name if j==0 else '') if header=='Name' else \
experiment_record.info.get_field_text(ExpInfoFields.TIMESTAMP) if header in ('Last Run', 'All Runs') else \
experiment_record.info.get_field_text(ExpInfoFields.RUNTIME) if header=='Duration' else \
experiment_record.info.get_field_text(ExpInfoFields.STATUS) if header=='Status' else \
experiment_record.get_invalid_arg_note() if header=='Valid' else \
experiment_record.get_one_liner() if header=='Result' else \
'???'
except:
if raise_display_errors:
raise
return '<Error>'
exps_to_show = set(
exp_record_dict.keys()) if exp_filter is None else set(
select_experiments(exp_filter, exp_record_dict))
for i, (exp_id, record_ids) in enumerate(exp_record_dict.iteritems()):
if len(record_ids) == 0:
rows.append(
[str(i), '', exp_id, '<No Records>', '-', '-', '-', '-'])
else:
for j, record_id in enumerate(record_ids):
index, name = ['{}.{}'.format(
i, j), exp_id] if j == 0 else [
'{}.{}'.format('`' * len(str(i)), j), exp_id
]
try:
experiment_record = load_experiment_record(record_id)
except:
experiment_record = None
if exp_id in exps_to_show:
rows.append([get_field(h) for h in headers])
assert all_equal([len(headers)] + [len(row) for row in rows]
), 'Header length: {}, Row Lengths: \n {}'.format(
len(headers),
'\n'.join([len(row) for row in rows]))
table = tabulate(rows, headers=headers)
if exp_filter:
table += '\n[Filtered with "{}" to show {}/{} experiments]'.format(
exp_filter, len(exps_to_show), len(exp_record_dict))
return table
def _filter_records(user_range, exp_record_dict):
"""
:param user_range:
:param exp_record_dict:
:return: An OrderedDict<experiment_id -> list<True or False>> indicating whether each record from the given experiment passed the filter
"""
if user_range=='unfinished':
return _filter_records('~finished', exp_record_dict)
elif user_range=='last':
return _filter_records('~old', exp_record_dict)
elif '|' in user_range:
return _bitwise_filter_op('or', *[_filter_records(subrange, exp_record_dict) for subrange in user_range.split('|')])
elif '&' in user_range:
return _bitwise_filter_op('and', *[_filter_records(subrange, exp_record_dict) for subrange in user_range.split('&')])
elif '>' in user_range:
ix = user_range.index('>')
first_part, second_part = user_range[:ix], user_range[ix+1:]
_first_stage_filters = _filter_records(first_part, exp_record_dict)
_new_dict = _select_record_ids_from_filters(_first_stage_filters, exp_record_dict)
_second_stage_filters = _filter_records(second_part, _new_dict)
return _bitwise_filter_op('andcascade', _first_stage_filters, _second_stage_filters)
elif user_range.startswith('~'):
return _bitwise_filter_op('not', _filter_records(user_range[1:], exp_record_dict))
base = OrderedDict((k, [False]*len(v)) for k, v in exp_record_dict.items())
if user_range in exp_record_dict: # User just lists an experiment
base[user_range] = [True]*len(base[user_range])
return base
number_range = interpret_numbers(user_range)
keys = list(exp_record_dict.keys())
if number_range is not None:
for i in number_range:
if i>len(keys):
raise RecordSelectionError('Experiment {} does not exist (they go from 0 to {})'.format(i, len(keys)-1))
base[keys[i]] = [True]*len(base[keys[i]])
elif '.' in user_range:
exp_rec_pairs = interpret_record_identifier(user_range)
for exp_number, rec_number in exp_rec_pairs:
if rec_number>=len(base[keys[exp_number]]):
raise RecordSelectionError('Selection {}.{} does not exist.'.format(exp_number, rec_number))
base[keys[exp_number]][rec_number] = True
elif user_range == 'old':
for k, v in base.items():
base[k] = ([True]*(len(v)-1)+[False]) if len(v)>0 else []
elif user_range == 'corrupt':
for k, v in base.items():
base[k] = [load_experiment_record(rec_id).info.get_status_field()==ExpStatusOptions.CORRUPT for rec_id in exp_record_dict[k]]
elif user_range == 'finished':
for k, v in base.iteritems():
base[k] = [load_experiment_record(rec_id).info.get_field(ExpInfoFields.STATUS) == ExpStatusOptions.FINISHED for rec_id in exp_record_dict[k]]
elif user_range == 'invalid':
for k, v in base.items():
base[k] = [load_experiment_record(rec_id).args_valid() is False for rec_id in exp_record_dict[k]]
elif user_range == 'all':
for k, v in base.items():
base[k] = [True]*len(v)
elif user_range == 'errors':
for k, v in base.items():
base[k] = [load_experiment_record(rec_id).info.get_field(ExpInfoFields.STATUS)==ExpStatusOptions.ERROR for rec_id in exp_record_dict[k]]
elif user_range == 'result':
for k, v in base.items():
base[k] = [load_experiment_record(rec_id).has_result() for rec_id in exp_record_dict[k]]
else:
raise RecordSelectionError("Don't know how to interpret subset '{}'".format(user_range))
return base
def _filter_records(user_range, exp_record_dict):
"""
:param user_range: A string specifying which records to select. Examples:
4.2 Select record 2 for experiment 4
4 Select all records for experiment 4
4-6 Select all records for experiments 4, 5, 6
4.2-5 Select records 2, 3, 4, 5 for experiment 4
4.3,4.4 Select records 4.3, 4.4
all Select all records
old Select all records that are not the the most recent run for that experiment
finished Select all records that have not run to completion
invalid Select all records for which the arguments to their experiments have changed since they were run
errors Select all records that ended in error
~invalid Select all records that are not invalid (the '~' can be used to negate any of the above)
invalid|errors Select all records that are invalid or ended in error (the '|' can be used to "or" any of the above)
invalid&errors Select all records that are invalid and ended in error (the '&' can be used to "and" any of the above)
finished@last Select the last finished record of each experiment (the '@' can be used to cascade any of the above)
dur>4m Select records which ran (ie had a duration) of more than 4 minutes
age<24h Select records which are less than 24h old.
:param exp_record_dict:
:return: An OrderedDict<experiment_id -> list<True or False>> indicating whether each record from the given experiment passed the filter
"""
if user_range=='unfinished':
return _filter_records('~finished', exp_record_dict)
elif user_range=='last':
return _filter_records('~old', exp_record_dict)
elif '|' in user_range:
return _bitwise_filter_op('or', *[_filter_records(subrange, exp_record_dict) for subrange in user_range.split('|')])
elif '&' in user_range:
return _bitwise_filter_op('and', *[_filter_records(subrange, exp_record_dict) for subrange in user_range.split('&')])
elif '@' in user_range:
ix = user_range.index('@')
first_part, second_part = user_range[:ix], user_range[ix+1:]
_first_stage_filters = _filter_records(first_part, exp_record_dict)
_new_dict = _select_record_ids_from_filters(_first_stage_filters, exp_record_dict)
_second_stage_filters = _filter_records(second_part, _new_dict)
return _bitwise_filter_op('andcascade', _first_stage_filters, _second_stage_filters)
elif user_range.startswith('~'):
return _bitwise_filter_op('not', _filter_records(user_range[1:], exp_record_dict))
base = OrderedDict((k, [False]*len(v)) for k, v in exp_record_dict.items())
if user_range in exp_record_dict: # User just lists an experiment
base[user_range] = [True]*len(base[user_range])
return base
number_range = interpret_numbers(user_range)
keys = list(exp_record_dict.keys())
if user_range in _named_record_filters: # e.g. 'finished'
for exp_id, _ in base.items():
base[exp_id] = _named_record_filters[user_range](exp_record_dict[exp_id])
elif number_range is not None: # e.g. '6-12'
for i in number_range:
if i>len(keys):
raise RecordSelectionError('Experiment {} does not exist (they go from 0 to {})'.format(i, len(keys)-1))
base[keys[i]] = [True]*len(base[keys[i]])
elif '.' in user_range: # e.b. 6.3-4
exp_rec_pairs = interpret_record_identifier(user_range)
for exp_number, rec_number in exp_rec_pairs:
if rec_number>=len(base[keys[exp_number]]):
raise RecordSelectionError('Selection {}.{} does not exist.'.format(exp_number, rec_number))
base[keys[exp_number]][rec_number] = True
elif user_range.startswith('dur') or user_range.startswith('age'): # Eg dur<25 Means "All records that ran less than 25s"
try:
sign = user_range[3]
assert sign in ('<', '>')
filter_func = (lambda a, b: a<b) if sign == '<' else (lambda a, b: a>b)
time_delta = parse_time(user_range[4:])
except:
if user_range.startswith('dur'):
raise RecordSelectionError('Could not interpret "{}" as duration. Example is dur<25s to select all experiments that ran less than 25s.'.format(user_range))
else:
raise RecordSelectionError('Could not interpret "{}" as age. Example is age<24h to select all experiments were started in the last 24h.'.format(user_range))
if user_range.startswith('dur'):
for exp_id, _ in base.items():
base[exp_id] = [filter_func(load_experiment_record(rec_id).get_runtime(), time_delta) for rec_id in exp_record_dict[exp_id]]
else:
current_time = datetime.now()
for exp_id, _ in base.items():
base[exp_id] = [filter_func(current_time - load_experiment_record(rec_id).get_datetime(), time_delta) for rec_id in exp_record_dict[exp_id]]
else:
raise RecordSelectionError("Don't know how to interpret subset '{}'. Possible subsets: {}".format(user_range, list(_named_record_filters.keys())))
return base
def show(self, user_range):
for rid in select_experiment_records(user_range,
self.exp_record_dict,
flat=True):
load_experiment_record(rid).show()
_warn_with_prompt()
