def _start_experiment(self, name):
print_headline('Start experiment', style='=')
if not self._directory:
print('Dry run; no results will be stored!')
return None
timestamp = time.strftime('%Y-%m-%dT%H-%M-%S', time.gmtime())
name = '{}-{}'.format(timestamp, name)
experiment = os.path.join(self._directory, name)
print('Result will be stored in', experiment)
return experiment
def __call__(self, lock):
with lock:
print_headline(self._prefix, 'Start job')
try:
self._execute()
except Exception as e:
self._handle_error(lock, e)
finally:
for env in self._envs:
env.close()
def __call__(self, lock):
with lock:
print_headline(self._prefix, 'Start job')
self._task.directory and dump_yaml(
self._algo_def, self._task.directory, 'algorithm.yaml')
try:
algorithm = self._create_algorithm()
training = self._create_training(algorithm)
testing = self._create_testing(algorithm)
for _ in range(self._epochs):
self._epoch(algorithm, training, testing)
except Exception as e:
message = '{} ({})'.format(e, type(e).__name__)
filepath = os.path.join(self._task.directory, 'errors.txt')
with open(filepath, 'a') as log:
log.write(message + ':\n')
log.write(traceback.format_exc() + '\n\n')
with lock:
print(self._prefix, message)
finally:
for env in self._envs:
env.close()
def _log_finish(self, experiment, duration):
message = 'Benchmark finished after {} hours'
print_headline(message.format(duration), style='=')
if experiment:
print('Find results in', experiment)