def leaderboard(self, dataset, run_id=None):
name = self.ctx.config.get('experiment/name', None)
if name is None:
raise AugerException('Please specify Experiment name...')
if run_id is None:
run_id = self.ctx.config.get('experiment/experiment_session_id',
None)
leaderboard, status, run_id = Experiment(self.ctx, dataset,
name).leaderboard(run_id)
if leaderboard is None:
raise AugerException('No leaderboard was found...')
self.ctx.log('Leaderboard for Run %s' % run_id)
print_table(self.ctx.log, leaderboard[::-1])
messages = {
'preprocess': 'Search is preprocessing data for training...',
'started': 'Search is in progress...',
'completed': 'Search is completed.',
'interrupted': 'Search was interrupted.'
}
message = messages.get(status, None)
if message:
self.ctx.log(message)
else:
self.ctx.log('Search status is %s' % status)
return {'run_id': run_id, 'leaderboard': leaderboard, 'status': status}
def evaluate(self):
# verify avalability of auger credentials
self.credentials.verify()
experiment_session_id = self.ctx.config['auger'].get(
'experiment/experiment_session_id')
if experiment_session_id is None:
raise AugerException(
'Can\'t find previously run experiments'
' (auger.yaml/experiment/experiment_session_id option).')
experiment_session_api = AugerExperimentSessionApi(
self.ctx, None, None, experiment_session_id)
print_table(self.ctx.log, experiment_session_api.get_leaderboard())
status = experiment_session_api.properties().get('status')
messages = {
'preprocess': 'Search is preprocessing data for traing...',
'started': 'Search is in progress...',
'completed': 'Search is completed.'
}
message = messages.get(status, None)
if message:
self.ctx.log(message)
else:
self.ctx.log('Search status is %s' % status)
def leaderboard(self, run_id=None):
ws = AzureProject(self.ctx)._get_ws()
experiment_name = self.ctx.config.get('experiment/name', None)
if experiment_name is None:
raise AzureException('Please specify Experiment name...')
if run_id is None:
run_id = self.ctx.config.get('experiment/run_id', None)
if run_id is None:
raise AzureException(
'Pleae provide Run ID (experiment/run_id) to evaluate')
experiment = Experiment(ws, experiment_name)
run = AutoMLRun(experiment=experiment, run_id=run_id)
leaderboard = self._get_leaderboard(run).to_dict('records')
self.ctx.log('Leaderboard for Run %s' % run_id)
print_table(self.ctx.log, leaderboard)
status = run.get_status()
self.ctx.log('Status: %s' % status)
return {'run_id': run_id, 'leaderboard': leaderboard, 'status': status}
def history(self):
ws = AzureProject(self.ctx)._get_ws()
experiment_name = self.ctx.config.get('experiment/name', None)
if experiment_name is None:
raise AzureException('Please specify Experiment name...')
experiment = Experiment(ws, experiment_name)
runs = Run.list(experiment)
result = []
for run in runs:
details = run.get_details()
st = dt.datetime.strptime(
details['startTimeUtc'], '%Y-%m-%dT%H:%M:%S.%fZ')
et = dt.datetime.strptime(
details['endTimeUtc'], '%Y-%m-%dT%H:%M:%S.%fZ')
duratin = str(et-st)
result.append({
'id': run.id,
'start time': details['startTimeUtc'],
'duratin': duratin,
'status': details['status']
})
print_table(self.ctx.log, result)
return {'history': result}
def leaderboard(self, run_id=None):
ws = AzureProject(self.ctx)._get_ws()
experiment_name = self.ctx.config.get('experiment/name', None)
if experiment_name is None:
raise AzureException('Please specify Experiment name...')
if run_id is None:
run_id = self.ctx.config.get('experiment/run_id', None)
if run_id is None:
raise AzureException(
'Pleae provide Run ID (experiment/run_id) to evaluate')
experiment = Experiment(ws, experiment_name)
run = AutoMLRun(experiment=experiment, run_id=run_id)
leaderboard = self._get_leaderboard(run).to_dict('records')
self.ctx.log('Leaderboard for Run %s' % run_id)
print_table(self.ctx.log, leaderboard)
provider_status = run.get_status()
status = self._map_provider_status(provider_status)
result = {
'run_id': run_id,
'leaderboard': leaderboard,
'status': status,
'provider_status': provider_status,
}
if status == 'error':
result['error'] = run.properties.get('errors')
result['error_details'] = run.get_details().get('error', {}).get(
'error', {}).get('message')
self.ctx.log('Status: %s, Error: %s, Details: %s' %
(status, error, error_details))
self.ctx.log_debug(run.get_details().get('error'))
else:
self.ctx.log('Status: %s' % status)
return result