def run(self, *args):
parser = argparse.ArgumentParser()
parser.add_argument(
'user_range',
action='store',
help=
'A selection of experiments to run. Examples: "3" or "3-5", or "3,4,5"'
)
parser.add_argument('-p',
'--parallel',
default=False,
action="store_true")
parser.add_argument('-np', '--n_processes', default=1, type=int)
parser.add_argument('-n', '--note')
parser.add_argument('-e',
'--raise_errors',
default=False,
action="store_true")
parser.add_argument('-d',
'--display_results',
default=False,
action="store_true")
parser.add_argument('-s',
'--slurm',
default=False,
action="store_true",
help='Run with slurm')
args = parser.parse_args(args)
ids = select_experiments(args.user_range, self.exp_record_dict)
if args.slurm:
run_multiple_experiments_with_slurm(
experiments=[load_experiment(eid) for eid in ids],
n_parallel=args.n_processes,
raise_exceptions=args.raise_errors,
run_args=self.run_args,
slurm_kwargs=self.slurm_kwargs)
else:
run_multiple_experiments(
experiments=[load_experiment(eid) for eid in ids],
parallel=args.parallel,
cpu_count=args.n_processes,
raise_exceptions=args.raise_errors,
run_args=self.run_args,
notes=(args.note, ) if args.note is not None else (),
display_results=args.display_results)
result = _warn_with_prompt(
'Finished running {} experiment{}.'.format(
len(ids), '' if len(ids) == 1 else 's'),
use_prompt=not self.close_after,
prompt='Press Enter to Continue, or "q" then Enter to Quit')
if result == 'q':
quit()
def run(self, *args):
parser = argparse.ArgumentParser()
parser.add_argument('user_range', action='store', help='A selection of experiments to run. Examples: "3" or "3-5", or "3,4,5"')
parser.add_argument('-p', '--parallel', default=False, nargs='*')
parser.add_argument('-n', '--note')
parser.add_argument('-e', '--raise_errors', default='single', nargs='*', help='By default, error are only raised if a single experiment is run. Set "-e" to always rays errors. "-e 0" to never raise errors.')
parser.add_argument('-d', '--display_results', default=False, action = "store_true")
parser.add_argument('-s', '--slurm', default=False, action = "store_true", help='Run with slurm')
args = parser.parse_args(args)
n_processes = \
None if args.parallel is False else \
'all' if len(args.parallel)==0 else \
int(args.parallel[0]) if len(args.parallel)==1 else \
bad_value(args.parallel, '-p can have 0 or 1 arguments. Got: {}'.format(args.parallel))
ids = select_experiments(args.user_range, self.exp_record_dict)
# Raise errors if:
# -e
# -e 1
# No arg, and only 1 experiment running
raise_errors = (len(args.raise_errors)==0 or (len(args.raise_errors)==1 and args.raise_errors[0]=='1') or (args.raise_errors == 'single' and len(ids)==1))
if args.slurm:
run_multiple_experiments_with_slurm(
experiments=[load_experiment(eid) for eid in ids],
n_parallel = n_processes,
raise_exceptions = raise_errors,
run_args=self.run_args,
slurm_kwargs=self.slurm_kwargs
)
else:
exp_names = list(self.exp_record_dict.keys())
run_multiple_experiments(
experiments=[load_experiment(eid) for eid in ids],
prefixes=[exp_names.index(eid) for eid in ids],
parallel=n_processes,
raise_exceptions = raise_errors,
run_args=self.run_args,
notes=(args.note, ) if args.note is not None else (),
display_results=args.display_results
)
result = _warn_with_prompt('Finished running {} experiment{}.'.format(len(ids), '' if len(ids)==1 else 's'),
use_prompt=not self.close_after,
prompt='Press Enter to Continue, or "q" then Enter to Quit')
if result=='q':
quit()
def run(self, *args):
parser = argparse.ArgumentParser()
parser.add_argument('user_range', action='store', help='A selection of experiments to run. Examples: "3" or "3-5", or "3,4,5"')
parser.add_argument('-p', '--parallel', default=False, nargs='*')
parser.add_argument('-n', '--note')
parser.add_argument('-e', '--raise_errors', default='single', nargs='*', help='By default, error are only raised if a single experiment is run. Set "-e" to always rays errors. "-e 0" to never raise errors.')
parser.add_argument('-d', '--display_results', default=False, action = "store_true")
parser.add_argument('-s', '--slurm', default=False, action = "store_true", help='Run with slurm')
args = parser.parse_args(args)
n_processes = \
None if args.parallel is False else \
'all' if len(args.parallel)==0 else \
int(args.parallel[0]) if len(args.parallel)==1 else \
bad_value(args.parallel, '-p can have 0 or 1 arguments. Got: {}'.format(args.parallel))
ids = select_experiments(args.user_range, self.exp_record_dict)
# Raise errors if:
# -e
# -e 1
# No arg, and only 1 experiment running
raise_errors = (len(args.raise_errors)==0 or (len(args.raise_errors)==1 and args.raise_errors[0]=='1') or (args.raise_errors == 'single' and len(ids)==1))
if args.slurm:
run_multiple_experiments_with_slurm(
experiments=[load_experiment(eid) for eid in ids],
n_parallel = n_processes,
raise_exceptions = raise_errors,
run_args=self.run_args,
slurm_kwargs=self.slurm_kwargs
)
else:
exp_names = list(self.exp_record_dict.keys())
run_multiple_experiments(
experiments=[load_experiment(eid) for eid in ids],
prefixes=[exp_names.index(eid) for eid in ids],
parallel=n_processes,
raise_exceptions = raise_errors,
run_args=self.run_args,
notes=(args.note, ) if args.note is not None else (),
display_results=args.display_results
)
result = _warn_with_prompt('Finished running {} experiment{}.'.format(len(ids), '' if len(ids)==1 else 's'),
use_prompt=not self.close_after,
prompt='Press Enter to Continue, or "q" then Enter to Quit')
if result=='q':
quit()
def load_lastest_experiment_results(experiments, error_if_no_result=True):
"""
:param experiments:
:param error_if_no_result:
:return:
"""
results = OrderedDict()
for ex in experiments:
ex = load_experiment(ex) if isinstance(ex, basestring) else ex
name = experiments[ex.get_id()] if isinstance(
experiments,
dict) else ex if isinstance(ex, basestring) else ex.get_id()
record = ex.get_latest_record(err_if_none=error_if_no_result,
only_completed=True)
if record is None:
if error_if_no_result:
raise Exception(
"Experiment {} had no result. Run this experiment to completion before trying to compare its results."
.format(ex.get_id()))
else:
ARTEMIS_LOGGER.warn(
'Experiment {} had no records. Not including this in results'
.format(ex.get_id()))
else:
results[name] = record.get_result()
if len(results) == 0:
ARTEMIS_LOGGER.warn(
'None of your experiments had any results. Your comparison function will probably show no meaningful result.'
)
return results
def _filter_experiments(user_range, exp_record_dict):
if user_range in exp_record_dict:
is_in = [k==user_range for k in exp_record_dict]
else:
number_range = interpret_numbers(user_range)
if number_range is not None:
# experiment_ids = [experiment_list[i] for i in number_range]
is_in = [i in number_range for i in xrange(len(exp_record_dict))]
elif user_range == 'all':
# experiment_ids = experiment_list
is_in = [True]*len(exp_record_dict)
elif user_range.startswith('has:'):
phrase = user_range[len('has:'):]
# experiment_ids = [exp_id for exp_id in experiment_list if phrase in exp_id]
is_in = [phrase in exp_id for exp_id in exp_record_dict]
elif user_range.startswith('1diff:'):
# select experiments whose arguments differ by one element from the selected experiments
base_range = user_range[len('1diff:'):]
base_range_exps = select_experiments(base_range, exp_record_dict) # list<experiment_id>
all_exp_args_hashes = {eid: set(compute_fixed_hash(a) for a in load_experiment(eid).get_args().items()) for eid in exp_record_dict} # dict<experiment_id : set<arg_hashes>>
# assert all_equal_length(all_exp_args_hashes.values()), 'All variants must have the same number of arguments' # Note: we diable this because we may have lists of experiments with different root functions.
is_in = [any(len(all_exp_args_hashes[eid].difference(all_exp_args_hashes[other_eid]))<=1 for other_eid in base_range_exps) for eid in exp_record_dict]
elif user_range.startswith('hasnot:'):
phrase = user_range[len('hasnot:'):]
# experiment_ids = [exp_id for exp_id in experiment_list if phrase not in exp_id]
is_in = [phrase not in exp_id for exp_id in exp_record_dict]
elif user_range in ('unfinished', 'invalid', 'corrupt'): # Return all experiments where all records are unfinished/invalid/corrupt
record_filters = _filter_records(user_range, exp_record_dict)
# experiment_ids = [exp_id for exp_id in experiment_list if len(record_filters[exp_id])]
is_in = [all(record_filters[exp_id]) for exp_id in exp_record_dict]
else:
raise Exception("Don't know how to use input '{}' to select experiments".format(user_range))
return OrderedDict((exp_id, exp_is_in) for exp_id, exp_is_in in izip_equal(exp_record_dict, is_in))
def get_multiple_records(experiment,
n,
only_completed=True,
if_not_enough='run'):
"""
Get n records from a single experiment.
:param Experiment experiment: The experiment
:param int n: Number of records to get
:param only_completed: True if you only want completed records
:param if_not_enough: What to do if there are not enough records ready.
'run': Run more
'cut': Just return the number that are already calculated
'err': Raise an excepetion
:return:
"""
if isinstance(experiment, str):
experiment = load_experiment(experiment)
assert if_not_enough in ('run', 'cut', 'err')
records = experiment.get_records(only_completed=only_completed)
if if_not_enough == 'err':
assert len(
records
) >= n, "You asked for {} records, but only {} were available".format(
n, len(records))
return records[-n:]
elif if_not_enough == 'run':
for k in range(n - len(records)):
record = experiment.run()
records.append(record)
return records[-n:]
else:
return records
def test_get_latest():
with experiment_testing_context():
record_1 = experiment_test_function.run()
time.sleep(0.01)
record_2 = experiment_test_function.run()
identifier = load_experiment('experiment_test_function').get_latest_record().get_id()
assert identifier == record_2.get_id()
def get_experiment(self):
"""
Load the experiment associated with this record.
Note that this will raise an ExperimentNotFoundError if the experiment has not been imported.
:return: An Experiment object
"""
from artemis.experiments.experiments import load_experiment
return load_experiment(self.get_experiment_id())
def test_get_latest():
with experiment_testing_context():
record_1 = experiment_test_function.run()
time.sleep(0.01)
record_2 = experiment_test_function.run()
identifier = load_experiment(
'experiment_test_function').get_latest_record().get_id()
assert identifier == record_2.get_id()
def get_experiment(self):
"""
Load the experiment associated with this record.
Note that this will raise an ExperimentNotFoundError if the experiment has not been imported.
:return: An Experiment object
"""
from artemis.experiments.experiments import load_experiment
return load_experiment(self.get_experiment_id())
def test_get_latest_identifier():
with experiment_testing_context():
exp_rec = experiment_test_function.run()
print(get_experiment_info('experiment_test_function'))
assert_experiment_record_is_correct(exp_rec)
last_experiment_identifier = load_experiment('experiment_test_function').get_latest_record().get_id()
assert last_experiment_identifier is not None, 'Experiment was run, this should not be none'
same_exp_rec = load_experiment_record(last_experiment_identifier)
assert_experiment_record_is_correct(same_exp_rec)
def test_get_latest_identifier():
with experiment_testing_context():
exp_rec = experiment_test_function.run()
print(get_experiment_info('experiment_test_function'))
assert_experiment_record_is_correct(exp_rec)
last_experiment_identifier = load_experiment(
'experiment_test_function').get_latest_record().get_id()
assert last_experiment_identifier is not None, 'Experiment was run, this should not be none'
same_exp_rec = load_experiment_record(last_experiment_identifier)
assert_experiment_record_is_correct(same_exp_rec)
def select_last_record_of_experiments(user_range, exp_record_dict):
experiments = select_experiments(user_range=user_range,
exp_record_dict=exp_record_dict)
records = [
load_experiment(ex).get_latest_record(only_completed=True,
err_if_none=False)
for ex in experiments
]
if None in records:
print('WARNING: Experiments {} have no completed records.',
[e for e, r in izip_equal(experiments, records) if r is None])
return records
def load_lastest_experiment_results(experiments, error_if_no_result = True):
"""
Given a list of experiments (or experiment ids), return an OrderedDict<record_id: result>
:param experiments: A list of Experiment objects (or strings identifying experiment ID is ok too)
:param error_if_no_result: Raise an error if an experiment has no completed results.
:return: OrderedDict<record_id: result>
"""
experiments = [load_experiment(ex) if isinstance(ex, string_types) else ex for ex in experiments]
records = [ex.get_latest_record(err_if_none=error_if_no_result, only_completed=True) for ex in experiments]
record_results = load_record_results([r for r in records if r is not None], err_if_no_result=error_if_no_result)
experiment_latest_results = OrderedDict((rec.get_experiment_id(), val) for rec, val in record_results.items())
return experiment_latest_results
def load_lastest_experiment_results(experiments, error_if_no_result = True):
"""
Given a list of experiments (or experiment ids), return an OrderedDict<record_id: result>
:param experiments: A list of Experiment objects (or strings identifying experiment ID is ok too)
:param error_if_no_result: Raise an error if an experiment has no completed results.
:return: OrderedDict<record_id: result>
"""
experiments = [load_experiment(ex) if isinstance(ex, string_types) else ex for ex in experiments]
records = [ex.get_latest_record(if_none='err' if error_if_no_result else 'skip', only_completed=True) for ex in experiments]
record_results = load_record_results([r for r in records if r is not None], err_if_no_result=error_if_no_result)
experiment_latest_results = OrderedDict((rec.get_experiment_id(), val) for rec, val in record_results.items())
return experiment_latest_results
def run(self, user_range, mode='-s', raise_exceptions=''):
assert mode in ('-s', '-e') or mode.startswith('-p')
ids = select_experiments(user_range, self.exp_record_dict)
run_multiple_experiments(
experiments=[load_experiment(eid) for eid in ids],
parallel=len(ids) > 1 and mode.startswith('-p'),
raise_exceptions=raise_exceptions == '-e',
run_args=self.run_args)
result = _warn_with_prompt(
'Finished running {} experiment{}.'.format(
len(ids), '' if len(ids) == 1 else 's'),
use_prompt=not self.close_after,
prompt='Press Enter to Continue, or "q" then Enter to Quit')
if result == 'q':
quit()
def _filter_experiments(user_range, exp_record_dict, return_is_in = False):
if user_range.startswith('~'):
is_in = _filter_experiments(user_range=user_range[1:], exp_record_dict=exp_record_dict, return_is_in=True)
is_in = [not r for r in is_in]
else:
if user_range in exp_record_dict:
is_in = [k==user_range for k in exp_record_dict]
else:
number_range = interpret_numbers(user_range)
if number_range is not None:
is_in = [i in number_range for i in xrange(len(exp_record_dict))]
elif user_range == 'all':
is_in = [True]*len(exp_record_dict)
elif user_range.startswith('has:'):
phrase = user_range[len('has:'):]
is_in = [phrase in exp_id for exp_id in exp_record_dict]
elif user_range.startswith('1diff:'):
base_range = user_range[len('1diff:'):]
base_range_exps = select_experiments(base_range, exp_record_dict) # list<experiment_id>
all_exp_args_hashes = {eid: set(compute_fixed_hash(a) for a in load_experiment(eid).get_args().items()) for eid in exp_record_dict} # dict<experiment_id : set<arg_hashes>>
is_in = [any(len(all_exp_args_hashes[eid].difference(all_exp_args_hashes[other_eid]))<=1 for other_eid in base_range_exps) for eid in exp_record_dict]
elif user_range.startswith('hasnot:'):
phrase = user_range[len('hasnot:'):]
is_in = [phrase not in exp_id for exp_id in exp_record_dict]
elif user_range in ('unfinished', 'invalid', 'corrupt'): # Return all experiments where all records are unfinished/invalid/corrupt
record_filters = _filter_records(user_range, exp_record_dict)
is_in = [all(record_filters[exp_id]) for exp_id in exp_record_dict]
elif user_range == 'started':
is_in = [has_experiment_record(exp_id) for exp_id in exp_record_dict]
else:
raise RecordSelectionError("Don't know how to use input '{}' to select experiments".format(user_range))
if return_is_in:
return is_in
else:
return OrderedDict((exp_id, exp_is_in) for exp_id, exp_is_in in izip_equal(exp_record_dict, is_in))
def get_experiment_list_str(self, exp_record_dict):
headers = {
'full': [
ExpRecordDisplayFields.RUNS, ExpRecordDisplayFields.DURATION,
ExpRecordDisplayFields.STATUS,
ExpRecordDisplayFields.ARGS_CHANGED,
ExpRecordDisplayFields.RESULT_STR, ExpRecordDisplayFields.NOTES
],
'results': [ExpRecordDisplayFields.RESULT_STR]
}[self.view_mode]
if self.remove_prefix:
deprefixed_ids = deprefix_experiment_ids(exp_record_dict.keys())
exp_record_dict = OrderedDict(
(k, v)
for k, v in zip(deprefixed_ids, exp_record_dict.values()))
row_func = _get_record_rows_cached if self.cache_result_string else _get_record_rows
header_names = [h.value for h in headers]
def remove_notes_if_no_notes(_record_rows):
notes_column_index = headers.index(
ExpRecordDisplayFields.NOTES
) if ExpRecordDisplayFields.NOTES in headers else None
# Remove the notes column if there are no notes!
if notes_column_index is not None and all(
row[notes_column_index] == '' for row in _record_rows):
for row in _record_rows:
del row[notes_column_index]
if self.display_format == 'nested': # New Display mode
if self.show_args:
_, argdiff = separate_common_items([
load_experiment(ex).get_args().items()
for ex in exp_record_dict
])
argdiff = {
k: args
for k, args in izip_equal(exp_record_dict.keys(), argdiff)
}
# Build a list of experiments and a list of records.
full_headers = ['#'] + header_names
record_rows = []
experiment_rows = []
experiment_row_ixs = []
counter = 1 # Start at 2 because record table has the headers.
for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()):
experiment_row_ixs.append(counter)
exp_identifier = exp_id if not self.show_args else ','.join(
'{}={}'.format(k, v) for k, v in argdiff[exp_id])
experiment_rows.append([i, exp_identifier])
for j, record_id in enumerate(record_ids):
record_rows.append([j] + row_func(
record_id,
headers,
raise_display_errors=self.raise_display_errors,
truncate_to=self.truncate_result_to,
ignore_valid_keys=self.ignore_valid_keys))
counter += 1
remove_notes_if_no_notes(record_rows)
# Merge the experiments table and record table.
record_table_rows = tabulate(record_rows,
headers=full_headers,
tablefmt="pipe").split('\n')
del record_table_rows[1] # Get rid of that silly line.
experiment_table_rows = tabulate(
experiment_rows, numalign='left').split('\n')[
1:-1] # First and last are just borders
longest_row = max(max(len(r) for r in record_table_rows),
max(len(r) for r in experiment_table_rows) +
4) if len(record_table_rows) > 0 else 0
record_table_rows = [
r if len(r) == longest_row else r[:-1] + ' ' *
(longest_row - len(r)) + r[-1] for r in record_table_rows
]
experiment_table_rows = [
('=' if i == 0 else '-') * longest_row + '\n' + r + ' ' *
(longest_row - len(r) - 1) + '|'
for i, r in enumerate(experiment_table_rows)
]
all_rows = [
surround_with_header(
'Experiments', width=longest_row, char='=')
] + insert_at(record_table_rows,
experiment_table_rows,
indices=experiment_row_ixs) + ['=' * longest_row]
table = '\n'.join(all_rows)
elif self.display_format == 'flat': # Display First record on same row
full_headers = ['E#', 'R#', 'Experiment'] + header_names
rows = []
for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()):
if len(record_ids) == 0:
rows.append([str(i), '', exp_id, '<No Records>'] + ['-'] *
(len(headers) - 1))
else:
for j, record_id in enumerate(record_ids):
rows.append([
str(i) if j == 0 else '', j, exp_id if j ==
0 else ''
] + row_func(
record_id,
headers,
raise_display_errors=self.raise_display_errors,
truncate_to=self.truncate_result_to,
ignore_valid_keys=self.ignore_valid_keys))
remove_notes_if_no_notes(rows)
table = tabulate(rows, headers=full_headers)
else:
raise NotImplementedError(self.display_format)
return table
def get_experiment_list_str(self, exp_record_dict):
headers = {
'full': [ExpRecordDisplayFields.RUNS, ExpRecordDisplayFields.DURATION, ExpRecordDisplayFields.STATUS, ExpRecordDisplayFields.ARGS_CHANGED, ExpRecordDisplayFields.RESULT_STR, ExpRecordDisplayFields.NOTES],
'results': [ExpRecordDisplayFields.RESULT_STR]
}[self.view_mode]
if self.remove_prefix:
deprefixed_ids = deprefix_experiment_ids(exp_record_dict.keys())
exp_record_dict = OrderedDict((k, v) for k, v in zip(deprefixed_ids, exp_record_dict.values()))
row_func = _get_record_rows_cached if self.cache_result_string else _get_record_rows
header_names = [h.value for h in headers]
def remove_notes_if_no_notes(_record_rows, _record_headers):
notes_column_index = _record_headers.index(ExpRecordDisplayFields.NOTES.value) if ExpRecordDisplayFields.NOTES.value in _record_headers else None
# Remove the notes column if there are no notes!
if notes_column_index is not None and all(row[notes_column_index]=='' or row[notes_column_index]=='-' for row in _record_rows):
new_rows = []
for row in _record_rows:
new_rows.append(row[:notes_column_index]+row[notes_column_index+1:])
new_headers = _record_headers[:notes_column_index]+_record_headers[notes_column_index+1:]
else:
new_rows = _record_rows
new_headers = _record_headers
return new_rows, new_headers
if self.display_format=='nested': # New Display mode
if self.show_args:
_, argdiff = separate_common_items([load_experiment(ex).get_args().items() for ex in exp_record_dict])
argdiff = {k: args for k, args in izip_equal(exp_record_dict.keys(), argdiff)}
# Build a list of experiments and a list of records.
full_headers = ['#']+header_names
record_rows = []
experiment_rows = []
experiment_row_ixs = []
counter = 1 # Start at 2 because record table has the headers.
for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()):
experiment_row_ixs.append(counter)
exp_identifier = exp_id if not self.show_args else ','.join('{}={}'.format(k, v) for k, v in argdiff[exp_id])
experiment_rows.append([i, exp_identifier])
for j, record_id in enumerate(record_ids):
record_rows.append([j]+row_func(record_id, headers, raise_display_errors=self.raise_display_errors, truncate_to=self.truncate_result_to, ignore_valid_keys=self.ignore_valid_keys))
counter+=1
record_rows, full_headers = remove_notes_if_no_notes(record_rows, full_headers)
# Merge the experiments table and record table.
if self.table_package=='tabulate':
record_table_rows = tabulate(record_rows, headers=full_headers, tablefmt="pipe").split('\n')
del record_table_rows[1] # Get rid of that silly line.
experiment_table_rows = tabulate(experiment_rows, numalign='left').split('\n')[1:-1] # First and last are just borders
longest_row = max(max(len(r) for r in record_table_rows), max(len(r) for r in experiment_table_rows)+4) if len(experiment_table_rows)>0 and len(record_table_rows)>0 else 0
record_table_rows = [r if len(r)==longest_row else r[:-1] + ' '*(longest_row-len(r)) + r[-1] for r in record_table_rows]
experiment_table_rows = [('=' if i==0 else '-')*longest_row+'\n'+r + ' '*(longest_row-len(r)-1)+'|' for i, r in enumerate(experiment_table_rows)]
all_rows = [surround_with_header('Experiments', width=longest_row, char='=')] + (insert_at(record_table_rows, experiment_table_rows, indices=experiment_row_ixs) if len(experiment_table_rows)>0 else ['<No non-root Experiments>']) + ['='*longest_row]
table = '\n'.join(all_rows)
else:
raise NotImplementedError(self.table_package)
elif self.display_format=='flat': # Display First record on same row
full_headers = ['E#', 'R#', 'Experiment']+header_names
rows = []
for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()):
if len(record_ids)==0:
rows.append([str(i), '', exp_id, '<No Records>'] + ['-']*(len(headers)-1))
else:
for j, record_id in enumerate(record_ids):
rows.append([str(i) if j==0 else '', j, exp_id if j==0 else '']+row_func(record_id, headers, raise_display_errors=self.raise_display_errors, truncate_to=self.truncate_result_to, ignore_valid_keys=self.ignore_valid_keys))
assert len(rows[0])==len(full_headers)
rows, full_headers = remove_notes_if_no_notes(rows, full_headers)
if self.table_package == 'pretty_table':
from prettytable.prettytable import PrettyTable
table = str(PrettyTable(rows, field_names=full_headers, align='l', max_table_width=self.max_width))
elif self.table_package == 'tabulate':
table = tabulate(rows, headers=full_headers)
else:
raise NotImplementedError(self.table_package)
else:
raise NotImplementedError(self.display_format)
return table
def call(self, user_range):
ids = select_experiments(user_range, self.exp_record_dict)
for experiment_identifier in ids:
load_experiment(experiment_identifier).call()
def _filter_experiments(user_range, exp_record_dict, return_is_in=False):
if '|' in user_range:
is_in = [
any(xs) for xs in zip(*(_filter_experiments(
subrange, exp_record_dict, return_is_in=True)
for subrange in user_range.split('|')))
]
elif '&' in user_range:
is_in = [
all(xs) for xs in zip(*(_filter_experiments(
subrange, exp_record_dict, return_is_in=True)
for subrange in user_range.split('&')))
]
elif user_range.startswith('~'):
is_in = _filter_experiments(user_range=user_range[1:],
exp_record_dict=exp_record_dict,
return_is_in=True)
is_in = [not r for r in is_in]
else:
if user_range in exp_record_dict:
is_in = [k == user_range for k in exp_record_dict]
else:
number_range = interpret_numbers(user_range)
if number_range is not None:
is_in = [
i in number_range for i in xrange(len(exp_record_dict))
]
elif user_range == 'all':
is_in = [True] * len(exp_record_dict)
elif user_range.startswith('has:'):
phrase = user_range[len('has:'):]
is_in = [phrase in exp_id for exp_id in exp_record_dict]
elif user_range.startswith('tag:'):
tag = user_range[len('tag:'):]
is_in = [
tag in load_experiment(exp_id).get_tags()
for exp_id in exp_record_dict
]
elif user_range.startswith('1diff:'):
base_range = user_range[len('1diff:'):]
base_range_exps = select_experiments(
base_range, exp_record_dict) # list<experiment_id>
all_exp_args_hashes = {
eid: set(
compute_fixed_hash(a)
for a in load_experiment(eid).get_args().items())
for eid in exp_record_dict
} # dict<experiment_id : set<arg_hashes>>
is_in = [
any(
len(all_exp_args_hashes[eid].difference(
all_exp_args_hashes[other_eid])) <= 1
for other_eid in base_range_exps)
for eid in exp_record_dict
]
elif user_range.startswith('hasnot:'):
phrase = user_range[len('hasnot:'):]
is_in = [phrase not in exp_id for exp_id in exp_record_dict]
elif user_range in (
'unfinished', 'invalid', 'corrupt'
): # Return all experiments where all records are unfinished/invalid/corrupt
record_filters = _filter_records(user_range, exp_record_dict)
is_in = [
all(record_filters[exp_id]) for exp_id in exp_record_dict
]
elif user_range == 'started':
is_in = [
has_experiment_record(exp_id) for exp_id in exp_record_dict
]
else:
raise RecordSelectionError(
"Don't know how to use input '{}' to select experiments".
format(user_range))
if return_is_in:
return is_in
else:
return OrderedDict(
(exp_id, exp_is_in)
for exp_id, exp_is_in in izip_equal(exp_record_dict, is_in))
def compare(self, user_range):
experiment_ids = select_experiments(user_range, self.exp_record_dict)
experiments = [load_experiment(eid) for eid in experiment_ids]
compare_experiment_results(experiments, error_if_no_result=False)
_warn_with_prompt(use_prompt=not self.close_after)
def call(self, user_range):
ids = select_experiments(user_range, self.exp_record_dict)
for experiment_identifier in ids:
load_experiment(experiment_identifier)()
