def leaderboard(self, dataset, run_id=None):
name = self.ctx.config.get('experiment/name', None)
if name is None:
raise AugerException('Please specify Experiment name...')
if run_id is None:
run_id = self.ctx.config.get('experiment/experiment_session_id',
None)
leaderboard, status, run_id = Experiment(self.ctx, dataset,
name).leaderboard(run_id)
if leaderboard is None:
raise AugerException('No leaderboard was found...')
self.ctx.log('Leaderboard for Run %s' % run_id)
print_table(self.ctx.log, leaderboard[::-1])
messages = {
'preprocess': 'Search is preprocessing data for traing...',
'started': 'Search is in progress...',
'completed': 'Search is completed.',
'interrupted': 'Search was interrupted.'
}
message = messages.get(status, None)
if message:
self.ctx.log(message)
else:
self.ctx.log('Search status is %s' % status)
return {'run_id': run_id, 'leaderboard': leaderboard, 'status': status}
def _setup_op(self, name, verify_project=True):
old_name = self.ctx.config.get('name', None)
if name is None:
name = old_name
if name is None:
raise AugerException('Please specify Project name...')
project = Project(self.ctx, name)
if verify_project and not project.is_exists:
raise AugerException('Project %s doesn\'t exists...' % name)
return old_name, name, project
def _get_project(self, autocreate):
project_name = self.ctx.config.get('name', None)
if project_name is None:
raise AugerException(
'Please specify project name in auger.yaml/name...')
project = Project(self.ctx, project_name)
project_properties = project.properties()
if project_properties is None:
if autocreate:
self.ctx.log('Can\'t find project %s on the Auger Cloud. '
'Creating...' % project_name)
project.create()
else:
raise AugerException('Can\'t find project %s' % project_name)
return project
def mk_project_folder(self):
if os.path.exists(os.path.join(os.getcwd(), 'auger.yaml')):
raise AugerException("Can't create '%s' inside a project."
" './auger.yaml' already exists" %
self.project_name)
project_path = os.path.join(os.getcwd(), self.project_name)
try:
os.makedirs(project_path)
except OSError as e:
if e.errno == errno.EEXIST:
raise AugerException(
"Can't create '%s'. Folder already exists." %
self.project_name)
self.ctx.log('Created project folder %s', self.project_name)
return project_path
def _docker_run_predict(self, filename, threshold, model_path):
cluster_settings = AugerClusterApi.get_cluster_settings(self.ctx)
docker_tag = cluster_settings.get('kubernetes_stack')
result_file = os.path.basename(filename)
data_path = os.path.dirname(filename)
call_args = "--path_to_predict=./model_data/%s %s" % \
(result_file, "--threshold=%s" % str(threshold) if threshold else '')
command = (r"docker run "
"-v {model_path}:/var/src/auger-ml-worker/exported_model "
"-v {data_path}:/var/src/auger-ml-worker/model_data "
"deeplearninc/auger-ml-worker:{docker_tag} "
"python ./exported_model/client.py {call_args}").format(
model_path=model_path, data_path=data_path,
docker_tag=docker_tag, call_args=call_args)
try:
self.ctx.log(
'Running model in deeplearninc/'
'auger-ml-worker:%s' % docker_tag)
subprocess.check_call(
command, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError as e:
raise AugerException('Error running Docker container...')
return os.path.join(data_path,
os.path.splitext(result_file)[0] + "_predicted.csv")
def stop(self, dataset):
name = self.ctx.get_config('auger').get('experiment/name', None)
if name is None:
raise AugerException('Please specify Experiment name...')
if Experiment(self.ctx, dataset, name).stop():
self.ctx.log('Search is stopped...')
else:
self.ctx.log('Search is not running. Stop is ignored.')
def wrapper(self, *args, **kwargs):
project = _get_project(self, False)
data_set_name = self.ctx.config.get('dataset', None)
if data_set_name is None:
raise AugerException(
'Please specify dataset name in auger.yaml/dataset...')
dataset = DataSet(self.ctx, project, data_set_name)
return decorated(self, dataset, *args, **kwargs)
def history(self, dataset):
name = self.ctx.get_config('auger').get('experiment/name', None)
if name is None:
raise AugerException('Please specify Experiment name...')
for exp_run in iter(Experiment(self.ctx, dataset, name).history()):
self.ctx.log("run id: {}, start time: {}, status: {}".format(
exp_run.get('id'),
exp_run.get('model_settings').get('start_time'),
exp_run.get('status')))
def leaderboard(self, dataset, run_id):
name = self.ctx.get_config('auger').get('experiment/name', None)
if name is None:
raise AugerException('Please specify Experiment name...')
leaderboard, status = Experiment(
self.ctx, dataset, name).leaderboard(run_id)
if leaderboard is None:
raise AugerException('No leaderboard was found...')
print_table(self.ctx.log, leaderboard)
messages = {
'preprocess': 'Search is preprocessing data for traing...',
'started': 'Search is in progress...',
'completed': 'Search is completed.',
'interrupted': 'Search was interrupted.'
}
message = messages.get(status, None)
if message:
self.ctx.log(message)
else:
self.ctx.log('Search status is %s' % status)
def _docker_pull_image(self):
cluster_settings = AugerClusterApi.get_cluster_settings(self.ctx)
docker_tag = cluster_settings.get('kubernetes_stack')
try:
subprocess.check_call(
'docker pull deeplearninc/auger-ml-worker:%s' % \
docker_tag, shell=True)
except subprocess.CalledProcessError as e:
raise AugerException('Can\'t pull Docker container...')
return docker_tag
def _predict_locally(self, filename, model_id, threshold):
is_model_loaded, model_path, model_name = \
AugerDeploy.verify_local_model(model_id)
if not is_model_loaded:
raise AugerException('Model isn\'t loaded locally. '
'Please use a2ml depoly command to download model.')
model_path, model_existed = self._exstract_model(model_name)
try:
predicted = \
self._docker_run_predict(filename, threshold, model_path)
finally:
# clean up unzipped model
# if it wasn't unzipped before
if not model_existed:
shutil.rmtree(model_path, ignore_errors=True)
return predicted
def train(self):
# verify avalability of auger credentials
self.credentials.verify()
self.start_project()
data_set_name = self.ctx.config['auger'].get('dataset')
if data_set_name is None:
raise AugerException(
'Plese specify DataSet name in auger.yaml/dataset')
experiment_api = AugerExperimentApi(self.ctx, self.project_api)
experiment_api.create(data_set_name)
self.ctx.log('Created Experiment %s ' % experiment_api.object_name)
experiment_session_id = experiment_api.run()
self.ctx.log('Started Experiment %s training.' %
experiment_api.object_name)
AugerConfig(self.ctx).set_experiment(experiment_api.object_name,
experiment_session_id)
def mock(*args, **kwargs):
raise AugerException('Email or password incorrect')
