def post(self):
"""
Executes a prep job to create an image corpus for training.
Use this method to start a prep job.
"""
job_def = request.json
job_def['process_json'] = True # Hardcode to process json file from project folder
job = Job(job_def['name'],job_def)
job.type = 'preprocess'
dt = newdt.now()
job.start_time = int(dt.timestamp()*1000)
job.request = {'full_path': request.full_path,'remote_addr':request.remote_addr,'method':request.method}
jb = aug_queue.enqueue(
preprocess, job,job_timeout=-1,result_ttl=86400,ttl=-1)
jb.meta['job_def'] = job_def
dt = newdt.now()
jb.meta['job_init_time'] = str(int(dt.timestamp()*1000))
jb.status = 'Running'
jb.save_meta()
json_str = job.to_json_string()
st = {
'BUCKET' : job.bucket,
'USE_GCS' : job.use_gcs,
'ACCESS_KEY' : access_key,
'SECRET_KEY' : secret_key,
'S3_URL' : s3_url
}
storage = Storage(st)
storage.upload_data(json_str,'jobs/running/{}_0_preprocess_r_{}.json'.format(str(job.start_time),jb.id),contentType='application/json')
storage.upload_data(json_str,'jobs/all/{}_0_preprocess_r_{}.json'.format(str(job.start_time),jb.id),contentType='application/json')
return {
"status": jb.status,
'job_id': jb.id,
'meta':jb.meta},201
def post(self):
"""
Executes a training.
Use this method to start a training.
"""
job_def = request.json
job = Job(job_def['name'], job_def)
job.type = 'train'
dt = newdt.now()
job.start_time = int(dt.timestamp() * 1000)
job.request = {
'full_path': request.full_path,
'remote_addr': request.remote_addr,
'method': request.method
}
if hasattr(job, 'ml_engine') and job.ml_engine:
jb = train_queue.enqueue(train_mlengine,
job,
job_timeout=-1,
result_ttl=-1)
else:
jb = train_queue.enqueue(train_job_method, job, job_timeout=-1)
jb.meta['job_init_time'] = str(int(dt.timestamp() * 1000))
jb.meta['job_def'] = job_def
jb.save_meta()
json_str = job.to_json_string()
st = {
'BUCKET': job.bucket,
'USE_GCS': job.use_gcs,
'ACCESS_KEY': access_key,
'SECRET_KEY': secret_key,
'S3_URL': s3_url
}
storage = Storage(st)
storage.upload_data(json_str,
'jobs/running/{}_0_train_r_{}.json'.format(
str(job.start_time), jb.id),
contentType='application/json')
storage.upload_data(json_str,
'jobs/all/{}_0_train_r_{}.json'.format(
str(job.start_time), jb.id),
contentType='application/json')
return {
"status": jb.get_status(),
'job_id': jb.id,
'meta': jb.meta
}, 201
