def errortrace(self, user_range):
records = select_experiment_records(user_range, self.exp_record_dict, flat=True)
with IndentPrint("Error Traces:", show_line=True, show_end=True):
for record in records:
with IndentPrint(record.get_id(), show_line=True):
print(record.get_error_trace())
_warn_with_prompt(use_prompt=False)
def errortrace(self, user_range):
record_ids = select_experiment_records(user_range,
self.exp_record_dict,
flat=True)
with IndentPrint("Error Traces:", show_line=True, show_end=True):
for erid in record_ids:
with IndentPrint(erid, show_line=True):
record = load_experiment_record(erid)
print record.get_error_trace()
_warn_with_prompt()
def display(self, user_range='all', skip_if_single=True):
records = select_experiment_records(user_range,
self.exp_record_dict,
flat=True)
if len(records) == 1 and skip_if_single:
display_experiment_record(records[0])
else:
with IndentPrint("Results:", show_line=True, show_end=True):
for record in records:
with IndentPrint(record.get_id(),
show_line=True,
show_end=True):
display_experiment_record(record)
_warn_with_prompt(use_prompt=not self.close_after)
def kill(self, *args):
parser = argparse.ArgumentParser()
parser.add_argument('user_range', action='store', help='A selection of experiments whose records to pull. Examples: "3" or "3-5", or "3,4,5"')
parser.add_argument('-s', '--skip', action='store_true', default=True, help='Skip the check that all selected records are currently running (just filter running ones)')
args = parser.parse_args(args)
records = select_experiment_records(args.user_range, self.exp_record_dict, flat=True)
if not args.skip and (not all(record.get_status() is ExpStatusOptions.STARTED for record in records)):
raise RecordSelectionError('Not all records you selected to kill were running: \n- {}'.format('\n- '.join('{}: {}'.format(rec.get_id(), rec.get_status()) for rec in records)))
elif args.skip:
records = [rec for rec in records if rec.get_status() is ExpStatusOptions.STARTED]
if len(records)==0:
raise RecordSelectionError('Selection "{}" selected no active processes to kill'.format(args.user_range))
print('{} Running Experiments will be killed.'.format(len(records)))
with IndentPrint():
print(ExperimentRecordBrowser.get_record_table(records, ))
response = input('Type "yes" to continue. >').strip().lower()
if response == 'yes':
for rec in records:
rec.kill()
print('Experiments killed.')
return ExperimentBrowser.REFRESH
else:
_warn_with_prompt('Experiments were not killed.', use_prompt=False)
def test_indent_print():
with CaptureStdOut() as cap:
print 'aaa'
print 'bbb'
with IndentPrint():
print 'ccc'
print 'ddd'
with IndentPrint():
print 'eee'
print 'fff'
print 'ggg'
print 'hhh'
print 'iii'
print 'jjj'
assert '\n' + cap.read() == _desired
def results(self, user_range='all'):
record_ids = select_experiment_records(user_range,
self.exp_record_dict)
with IndentPrint("Results:", show_line=True, show_end=True):
for erid in record_ids:
record = load_experiment_record(erid)
with IndentPrint(erid, show_line=True, show_end=True):
try:
result = record.get_result()
record.get_experiment().display_last(result,
err_if_none=False)
except NoSavedResultError:
print '<No result was saved>'
except Exception as err:
print err
_warn_with_prompt()
def test_indent_print():
with CaptureStdOut() as cap:
print('aaa')
print('bbb')
with IndentPrint():
print('ccc')
print('ddd')
with IndentPrint():
print('eee')
print('fff')
print('ggg')
print('hhh')
print('iii')
print('jjj')
assert '\n'+cap.read() == _desired
def debug_section(name):
parent_frame = inspect.currentframe().f_back.f_back
parent_loc = parent_frame.f_code.co_filename
_LOGGER.info('Launching {name} @ File "{loc}", line {line}'.format(name=name, loc=parent_loc, line=parent_frame.f_lineno))
start_time = time.time()
with IndentPrint():
yield
end_time = time.time()
_LOGGER.info('... Finished {} after {:.4g}s'.format(name, end_time-start_time))
def select(self, user_range):
record_ids = select_experiment_records(user_range,
self.exp_record_dict,
flat=True)
with IndentPrint():
print ExperimentRecordBrowser.get_record_table(record_ids)
_warn_with_prompt(
'Selection "{}" includes {} out of {} records.'.format(
user_range, len(record_ids),
sum(len(recs) for recs in self.exp_record_dict.values())))
def selectexp(self, user_range):
exps_to_records = select_experiments(user_range,
self.exp_record_dict,
return_dict=True)
with IndentPrint():
print(self.get_experiment_list_str(exps_to_records))
_warn_with_prompt(
'Experiment Selection "{}" includes {} out of {} experiments.'.
format(user_range, len(exps_to_records),
len(self.exp_record_dict)),
use_prompt=not self.close_after)
def test_callback(info, score):
if plot:
dbplot(net.layers[0].w.get_value().T.reshape(-1, 28, 28),
'w0',
cornertext='Epoch {}'.format(info.epoch))
if swap_mlp:
all_layer_sizes = [dataset.input_size
] + hidden_sizes + [dataset.target_size]
fwd_ops = [
info.sample * d1 * d2
for d1, d2 in zip(all_layer_sizes[:-1], all_layer_sizes[1:])
]
back_ops = [
info.sample * d1 * d2
for d1, d2 in zip(all_layer_sizes[:-1], all_layer_sizes[1:])
]
update_ops = [
info.sample * d1 * d2
for d1, d2 in zip(all_layer_sizes[:-1], all_layer_sizes[1:])
]
else:
fwd_ops = [
layer_.fwd_op_count.get_value() for layer_ in net.layers
]
back_ops = [
layer_.back_op_count.get_value() for layer_ in net.layers
]
update_ops = [
layer_.update_op_count.get_value() for layer_ in net.layers
]
if info.epoch != 0:
with IndentPrint('Mean Ops by epoch {}'.format(info.epoch)):
print 'Fwd: {}'.format([
si_format(ops / info.epoch,
format_str='{value} {prefix}Ops')
for ops in fwd_ops
])
print 'Back: {}'.format([
si_format(ops / info.epoch,
format_str='{value} {prefix}Ops')
for ops in back_ops
])
print 'Update: {}'.format([
si_format(ops / info.epoch,
format_str='{value} {prefix}Ops')
for ops in update_ops
])
if info.epoch > max(
0.5, 2 * test_period) and not swap_mlp and score.get_score(
'train', 'noise_free') < 20:
raise Exception("This horse ain't goin' nowhere.")
op_count_info.append((info, (fwd_ops, back_ops, update_ops)))
def delete(self, user_range):
records = select_experiment_records(user_range, self.exp_record_dict, flat=True)
print('{} out of {} Records will be deleted.'.format(len(records), sum(len(recs) for recs in self.exp_record_dict.values())))
with IndentPrint():
print(ExperimentRecordBrowser.get_record_table(records, ))
response = input('Type "yes" to continue. >').strip().lower()
if response == 'yes':
clear_experiment_records([record.get_id() for record in records])
print('Records deleted.')
return ExperimentBrowser.REFRESH
else:
_warn_with_prompt('Records were not deleted.', use_prompt=False)
def delete(self, user_range):
record_ids = select_experiment_records(user_range,
self.exp_record_dict,
flat=True)
print '{} out of {} Records will be deleted.'.format(
len(record_ids),
sum(len(recs) for recs in self.exp_record_dict.values()))
with IndentPrint():
print ExperimentRecordBrowser.get_record_table(record_ids, )
response = raw_input('Type "yes" to continue. >')
if response.lower() == 'yes':
clear_experiment_records(record_ids)
print 'Records deleted.'
else:
_warn_with_prompt('Records were not deleted.')
def selectexp(self, user_range):
exps_to_records = select_experiments(user_range,
self.exp_record_dict,
return_dict=True)
with IndentPrint():
print self.get_experiment_list_str(
exps_to_records,
just_last_record=self.just_last_record,
view_mode=self.view_mode,
raise_display_errors=self.raise_display_errors)
# print ExperimentRecordBrowser.get_record_table(record_ids)
_warn_with_prompt(
'Experiment Selection "{}" includes {} out of {} experiments.'.
format(user_range, len(exps_to_records),
len(self.exp_record_dict)),
use_prompt=not self.close_after)
def launch(self, command=None):
func_dict = {
'run': self.run,
'test': self.test,
'show': self.show,
'call': self.call,
'selectexp': self.selectexp,
'selectrec': self.selectrec,
'allruns': self.allruns,
'view': self.view,
'h': self.help,
'filter': self.filter,
'explist': self.explist,
'sidebyside': self.side_by_side,
'argtable': self.argtable,
'compare': self.compare,
'display': self.display,
'delete': self.delete,
'errortrace': self.errortrace,
'q': self.quit,
'records': self.records,
'pull': self.pull
}
while True:
all_experiments = self.reload_record_dict()
print "==================== Experiments ===================="
self.exp_record_dict = all_experiments if self._filter is None else \
OrderedDict((exp_name, all_experiments[exp_name]) for exp_name in select_experiments(self._filter, all_experiments))
print self.get_experiment_list_str(
self.exp_record_dict,
just_last_record=self.just_last_record,
view_mode=self.view_mode,
raise_display_errors=self.raise_display_errors,
truncate_result_to=self.truncate_result_to,
cache_result_string=self.cache_result_string)
if self._filter is not None:
print '[Filtered with "{}" to show {}/{} experiments]'.format(
self._filter, len(self.exp_record_dict),
len(all_experiments))
print '-----------------------------------------------------'
if command is None:
user_input = raw_input(
'Enter command or experiment # to run (h for help) >> '
).lstrip(' ').rstrip(' ')
else:
user_input = command
command = None
with IndentPrint():
try:
split = user_input.split(' ')
if len(split) == 0:
continue
cmd = split[0]
args = split[1:]
if cmd == '':
continue
elif cmd in func_dict:
out = func_dict[cmd](*args)
elif interpret_numbers(cmd) is not None:
if not any(x in args for x in ('-s', '-e', '-p')):
args = args + ['-e']
out = self.run(cmd, *args)
elif cmd == 'r': # Refresh
continue
else:
edit_distances = [
levenshtein_distance(cmd, other_cmd)
for other_cmd in func_dict.keys()
]
min_distance = min(edit_distances)
closest = func_dict.keys()[edit_distances.index(
min_distance)]
suggestion = ' Did you mean "{}"?. '.format(
closest) if min_distance <= 2 else ''
response = raw_input(
'Unrecognised command: "{}". {}Type "h" for help or Enter to continue. >'
.format(cmd, suggestion, closest))
if response.lower() == 'h':
self.help()
out = None
if out is self.QUIT or self.close_after:
break
except Exception as name:
if self.catch_errors:
res = raw_input(
'%s: %s\nEnter "e" to view the stacktrace, or anything else to continue.'
% (name.__class__.__name__, name.message))
if res == 'e':
raise
else:
raise
def launch(self):
func_dict = {
'run': self.run,
'test': self.test,
'show': self.show,
'call': self.call,
'select': self.select,
'allruns': self.allruns,
'view': self.view,
'h': self.help,
'results': self.results,
'filter': self.filter,
'explist': self.explist,
'sidebyside': self.side_by_side,
'compare': self.compare,
'delete': self.delete,
'errortrace': self.errortrace,
'q': self.quit,
'records': self.records
}
while True:
self.exp_record_dict = self.reload_record_dict()
print "==================== Experiments ===================="
print self.get_experiment_list_str(
self.exp_record_dict,
just_last_record=self.just_last_record,
view_mode=self.view_mode,
raise_display_errors=self.raise_display_errors,
exp_filter=self._filter)
print '-----------------------------------------------------'
user_input = raw_input(
'Enter command or experiment # to run (h for help) >> '
).lstrip(' ').rstrip(' ')
with IndentPrint():
try:
split = user_input.split(' ')
if len(split) == 0:
continue
cmd = split[0]
args = split[1:]
if cmd == '':
continue
elif cmd in func_dict:
out = func_dict[cmd](*args)
elif interpret_numbers(cmd) is not None:
if not any(x in args for x in ('-s', '-e', '-p')):
args = args + ['-e']
out = self.run(cmd, *args)
elif cmd == 'r': # Refresh
continue
else:
response = raw_input(
'Unrecognised command: "{}". Type "h" for help or Enter to continue. >'
.format(cmd))
if response.lower() == 'h':
self.help()
out = None
if out is self.QUIT:
break
except Exception as name:
if self.catch_errors:
res = raw_input(
'%s: %s\nEnter "e" to view the stacktrace, or anything else to continue.'
% (name.__class__.__name__, name.message))
if res == 'e':
raise
else:
raise
def launch(self, command=None):
func_dict = {
'run': self.run,
'test': self.test,
'show': self.show,
'call': self.call,
'selectexp': self.selectexp,
'selectrec': self.selectrec,
'view': self.view,
'h': self.help,
'filter': self.filter,
'filterrec': self.filterrec,
'displayformat': self.displayformat,
'explist': self.explist,
'sidebyside': self.side_by_side,
'argtable': self.argtable,
'compare': self.compare,
'delete': self.delete,
'errortrace': self.errortrace,
'q': self.quit,
'records': self.records,
'pull': self.pull,
'clearcache': clear_ui_cache,
}
while True:
all_experiments = self.reload_record_dict()
# Display Table:
self.exp_record_dict = all_experiments if self._filter is None else \
OrderedDict((exp_name, all_experiments[exp_name]) for exp_name in select_experiments(self._filter, all_experiments))
if self._filterrec is not None:
self.exp_record_dict = select_experiment_records(
self._filterrec,
self.exp_record_dict,
load_records=False,
flat=False)
print(self.get_experiment_list_str(self.exp_record_dict))
if self._filter is not None or self._filterrec is not None:
print(
'[Filtered out {}/{} experiments with "{}" and {}/{} records with "{}"]'
.format(len(self.exp_record_dict), len(all_experiments),
self._filter,
sum(len(v) for v in self.exp_record_dict.values()),
sum(len(v) for v in all_experiments.values()),
self._filterrec))
# Get command from user
if command is None:
user_input = input(
'Enter command or experiment # to run (h for help) >> '
).strip()
else:
user_input = command
command = None
# Respond to user input
with IndentPrint():
try:
split = user_input.split(' ')
if len(split) == 0:
continue
cmd = split[0]
args = split[1:]
if cmd == '':
continue
elif cmd in func_dict:
out = func_dict[cmd](*args)
elif interpret_numbers(cmd) is not None:
if not any(x in args for x in ('-s', '-e', '-p')):
args = args + ['-e']
out = self.run(cmd, *args)
elif cmd == 'r': # Refresh
continue
else:
edit_distances = [
levenshtein_distance(cmd, other_cmd)
for other_cmd in func_dict.keys()
]
min_distance = min(edit_distances)
closest = func_dict.keys()[edit_distances.index(
min_distance)]
suggestion = ' Did you mean "{}"?. '.format(
closest) if min_distance <= 2 else ''
if self.close_after:
raise Exception(
'Unrecognised command: "{}"'.format(cmd))
else:
response = input(
'Unrecognised command: "{}". {}Type "h" for help or Enter to continue. >'
.format(cmd, suggestion, closest))
if response.strip().lower() == 'h':
self.help()
out = None
if out is self.QUIT or self.close_after:
break
except Exception as name:
if self.catch_errors:
res = input(
'%s: %s\nEnter "e" to view the stacktrace, or anything else to continue.'
% (name.__class__.__name__, name.message))
if res.strip().lower() == 'e':
raise
else:
raise
def launch(self, command=None):
func_dict = {
'run': self.run,
'test': self.test,
'show': self.show,
'call': self.call,
'kill': self.kill,
'selectexp': self.selectexp,
'selectrec': self.selectrec,
'view': self.view,
'h': self.help,
'filter': self.filter,
'filterrec': self.filterrec,
'displayformat': self.displayformat,
'explist': self.explist,
'sidebyside': self.side_by_side,
'argtable': self.argtable,
'compare': self.compare,
'delete': self.delete,
'errortrace': self.errortrace,
'q': self.quit,
'records': self.records,
'pull': self.pull,
'clearcache': clear_ui_cache,
}
display_again = True
while True:
if display_again:
all_experiments = self._reload_record_dict()
try:
self.exp_record_dict = self._filter_record_dict(all_experiments)
except RecordSelectionError as err:
_warn_with_prompt(str(err), use_prompt=self.catch_selection_errors)
if not self.catch_selection_errors:
raise
else:
continue
print(self.get_experiment_list_str(self.exp_record_dict))
if self._filter is not None or self._filterrec is not None:
print('[Showing {}/{} experiments and {}/{} records after Experiment Filter: "{}" and Record Filter: "{}"]'.format(
len(self.exp_record_dict), len(all_experiments),
sum(len(v) for v in self.exp_record_dict.values()), sum(len(v) for v in all_experiments.values()),
self._filter, self._filterrec
))
# Get command from user
if command is None:
user_input = input('Enter command or experiment # to run (h for help) >> ').strip()
else:
user_input = command
command = None
display_again = True
out = None
# Respond to user input
with IndentPrint():
try:
split = user_input.split(' ')
if len(split)==0:
continue
cmd = split[0]
args = split[1:]
if cmd in ('', 'r'): # Refresh
continue
elif interpret_numbers(cmd) is not None: # If you've listed a number, you implicitely call run
args = [cmd]+args
cmd = 'run'
if cmd in func_dict:
out = func_dict[cmd](*args)
display_again = False
else:
edit_distances = [levenshtein_distance(cmd, other_cmd) for other_cmd in func_dict.keys()]
min_distance = min(edit_distances)
closest = func_dict.keys()[edit_distances.index(min_distance)]
suggestion = ' Did you mean "{}"?. '.format(closest) if min_distance<=2 else ''
if self.close_after:
raise Exception('Unrecognised command: "{}"'.format(cmd))
else:
print('Unrecognised command: "{}". {}'.format(cmd, suggestion))
display_again = False
if out is self.QUIT or self.close_after:
break
elif out is self.REFRESH:
display_again = True
except Exception as err:
print ("CAUGHT: {}".format(err))
if self.catch_errors or (isinstance(err, RecordSelectionError) and self.catch_selection_errors):
res = input('Enter "e" to view the stacktrace, or anything else to continue.')
if res.strip().lower() == 'e':
raise
display_again = True
else:
raise