def job_status(job_id):
job = db.get_job(job_id)
if not job:
return jsonify(success=False, error_code="INVALID_JOB_ID"), 400
status_code = db.Job.STATUS_CODES[job.status]
return jsonify(success=True, status_code=status_code)
def job_listener(event):
'''Listens to completed job'''
job_id = event.job.args[0]
if event.code == events.EVENT_JOB_MISSED:
db.mark_job_as_missed(job_id)
elif event.exception:
if isinstance(event.exception, util.JobError):
error_object = event.exception.as_dict()
else:
error_object = "\n".join(
traceback.format_tb(event.traceback) + [repr(event.exception)])
db.mark_job_as_errored(job_id, error_object)
else:
db.mark_job_as_completed(job_id, event.retval)
api_key = db.get_job(job_id)["api_key"]
result_ok = send_result(job_id, api_key)
if not result_ok:
db.mark_job_as_failed_to_post_result(job_id)
# Optionally notify tests that job_listener() has finished.
if "_TEST_CALLBACK_URL" in app.config:
requests.get(app.config["_TEST_CALLBACK_URL"])
def job_delete(job_id):
'''Deletes the job together with its logs and metadata.
:param job_id: An identifier for the job
:type job_id: string
:statuscode 200: no error
:statuscode 403: not authorized to delete the job
:statuscode 404: the job could not be found
:statuscode 409: an error occurred
'''
conn = db.ENGINE.connect()
job = db.get_job(job_id)
if not job:
return json.dumps({'error': 'job_id not found'}), 404, headers
if not is_authorized(job):
return json.dumps({'error': 'not authorized'}), 403, headers
trans = conn.begin()
try:
conn.execute(db.JOBS_TABLE.delete().where(
db.JOBS_TABLE.c.job_id == job_id))
trans.commit()
return json.dumps({'success': True}), 200, headers
except Exception, e:
trans.rollback()
return json.dumps({'error': str(e)}), 409, headers
def job_delete(job_id):
'''Deletes the job together with its logs and metadata.
:param job_id: An identifier for the job
:type job_id: string
:statuscode 200: no error
:statuscode 403: not authorized to delete the job
:statuscode 404: the job could not be found
:statuscode 409: an error occurred
'''
conn = db.ENGINE.connect()
job = db.get_job(job_id)
if not job:
return json.dumps({'error': 'job_id not found'}), 404, headers
if not is_authorized(job):
return json.dumps({'error': 'not authorized'}), 403, headers
trans = conn.begin()
try:
conn.execute(
db.JOBS_TABLE.delete().where(db.JOBS_TABLE.c.job_id == job_id))
trans.commit()
return json.dumps({'success': True}), 200, headers
except Exception, e:
trans.rollback()
return json.dumps({'error': str(e)}), 409, headers
def job_listener(event):
'''Listens to completed job'''
job_id = event.job.args[0]
if event.code == events.EVENT_JOB_MISSED:
db.mark_job_as_missed(job_id)
elif event.exception:
if isinstance(event.exception, util.JobError):
error_object = event.exception.as_dict()
else:
error_object = "\n".join(traceback.format_tb(event.traceback) +
[repr(event.exception)])
db.mark_job_as_errored(job_id, error_object)
else:
db.mark_job_as_completed(job_id, event.retval)
api_key = db.get_job(job_id)["api_key"]
result_ok = send_result(job_id, api_key)
if not result_ok:
db.mark_job_as_failed_to_post_result(job_id)
# Optionally notify tests that job_listener() has finished.
if "_TEST_CALLBACK_URL" in app.config:
requests.get(app.config["_TEST_CALLBACK_URL"])
def work_and_save(job_id):
db.set_job_status(job_id, 'running')
job = db.get_job(job_id)
result = work(job.input)
db.set_job_status_and_output(job_id, 'done', result)
def xloader_status(context, data_dict):
''' Get the status of a ckanext-xloader job for a certain resource.
:param resource_id: The resource id of the resource that you want the
status for.
:type resource_id: string
'''
p.toolkit.check_access('xloader_status', context, data_dict)
if 'id' in data_dict:
data_dict['resource_id'] = data_dict['id']
res_id = _get_or_bust(data_dict, 'resource_id')
task = p.toolkit.get_action('task_status_show')(context, {
'entity_id': res_id,
'task_type': 'xloader',
'key': 'xloader'
})
datapusher_url = config.get('ckan.datapusher.url')
if not datapusher_url:
raise p.toolkit.ValidationError(
{'configuration': ['ckan.datapusher.url not in config file']})
value = json.loads(task['value'])
job_id = value.get('job_id')
url = None
job_detail = None
if job_id:
# get logs from the xloader db
db.init(config)
job_detail = db.get_job(job_id)
# timestamp is a date, so not sure why this code was there
# for log in job_detail['logs']:
# if 'timestamp' in log:
# date = time.strptime(
# log['timestamp'], "%Y-%m-%dT%H:%M:%S.%f")
# date = datetime.datetime.utcfromtimestamp(
# time.mktime(date))
# log['timestamp'] = date
try:
error = json.loads(task['error'])
except ValueError:
# this happens occasionally, such as when the job times out
error = task['error']
return {
'status': task['state'],
'job_id': job_id,
'job_url': url,
'last_updated': task['last_updated'],
'task_info': job_detail,
'error': error,
}
def handle_phone_response(data):
device_id = data['device_id']
job_id = data['job_id']
job = db.get_job(job_id=job_id)
job.status = db.Job.ASSIGNED
job.assigned_device = device_id
job.num_attempts += 1
job.save()
checker.remove_pending_acknowledgement(job.id)
print(f"Device id={device_id} has acknowledged job id={job_id}")
def cancel_and_reschedule_job(self, job_id):
job = db.get_job(job_id)
job_json = job.to_json()
# cancel the job
job.cancel()
print(f"[JOB {job.id}] cancelled and rescheduled.")
# Reschedule the job for another device, if and only if the number of retries don't go past 3.
if job.can_be_retried:
if (schedule_job(job)):
print(f"[JOB {job.id}] Rescheduled")
else:
print(f"[JOB {job.id}] Reschedule Limit Hit")
def handle_phone_cancel_job_response(data):
success = data['success']
device_id = data['device_id']
job_id = data['job_id']
if success:
# the phone was able to stop this job
job = db.get_job(job_id=job_id)
job.status = db.Job.FAILED
job.assigned_device = None
job.save()
print(f"Device id={device_id} was able to cancel job id={job_id}")
else:
# the phone could not cancel this job
# in this case, we don't do anything on this part of the SERVER side, since we should have a separate server cron process that handles failed jobs
# (e.g., picks such failed jobs up and retry them if needed)
# the device, however, would need to somehow end this task.
print(f"Device id={device_id} was NOT able to cancel job id={job_id}")
def job_status(job_id, show_job_key=False, ignore_auth=False):
'''Show a specific job.
:param limit: Limit the number of logs
:type limit: integer
**Results:**
:rtype: A dictionary with the following keys
:param status: Status of job (complete, error)
:type status: string
:param sent_data: Input data for job
:type sent_data: json encodable data
:param job_id: An identifier for the job
:type job_id: string
:param result_url: Callback url
:type result_url: url string
:param data: Results from job.
:type data: json encodable data
:param error: Error raised during job execution
:type error: string
:param metadata: Metadata provided when submitting job.
:type metadata: list of key - value pairs
:param requested_timestamp: Time the job started
:type requested_timestamp: timestamp
:param finished_timestamp: Time the job finished
:type finished_timestamp: timestamp
:statuscode 200: no error
:statuscode 403: not authorized to view the job's data
:statuscode 404: job id not found
:statuscode 409: an error occurred
'''
limit = flask.request.args.get('limit')
job_dict = db.get_job(job_id, limit=limit)
if not job_dict:
return json.dumps({'error': 'job_id not found'}), 404, headers
if not ignore_auth and not is_authorized(job_dict):
return json.dumps({'error': 'not authorized'}), 403, headers
job_dict.pop('api_key', None)
if not show_job_key:
job_dict.pop('job_key', None)
return flask.Response(json.dumps(job_dict, cls=DatetimeJsonEncoder),
mimetype='application/json')
def send_result(job_id, api_key=None):
''' Send results to where requested.
If api_key is provided, it is used, otherwiese
the key from the job will be used.
'''
job_dict = db.get_job(job_id)
result_url = job_dict.get('result_url')
if not result_url:
# A job with an API key (for using when posting to the callback URL)
# but no callback URL is weird, but it can happen.
db.delete_api_key(job_id)
return True
api_key_from_job = job_dict.pop('api_key', None)
if not api_key:
api_key = api_key_from_job
headers = {'Content-Type': 'application/json'}
if api_key:
if ':' in api_key:
header, key = api_key.split(':')
else:
header, key = 'Authorization', api_key
headers[header] = key
try:
result = requests.post(result_url,
data=json.dumps(job_dict,
cls=DatetimeJsonEncoder),
headers=headers,
verify=SSL_VERIFY)
db.delete_api_key(job_id)
except requests.ConnectionError:
return False
return result.status_code == requests.codes.ok
def job_status(job_id, show_job_key=False, ignore_auth=False):
'''Show a specific job.
**Results:**
:rtype: A dictionary with the following keys
:param status: Status of job (complete, error)
:type status: string
:param sent_data: Input data for job
:type sent_data: json encodable data
:param job_id: An identifier for the job
:type job_id: string
:param result_url: Callback url
:type result_url: url string
:param data: Results from job.
:type data: json encodable data
:param error: Error raised during job execution
:type error: string
:param metadata: Metadata provided when submitting job.
:type metadata: list of key - value pairs
:param requested_timestamp: Time the job started
:type requested_timestamp: timestamp
:param finished_timestamp: Time the job finished
:type finished_timestamp: timestamp
:statuscode 200: no error
:statuscode 403: not authorized to view the job's data
:statuscode 404: job id not found
:statuscode 409: an error occurred
'''
job_dict = db.get_job(job_id)
if not job_dict:
return json.dumps({'error': 'job_id not found'}), 404, headers
if not ignore_auth and not is_authorized(job_dict):
return json.dumps({'error': 'not authorized'}), 403, headers
job_dict.pop('api_key', None)
if not show_job_key:
job_dict.pop('job_key', None)
return flask.Response(json.dumps(job_dict, cls=DatetimeJsonEncoder),
mimetype='application/json')
def send_result(job_id, api_key=None):
''' Send results to where requested.
If api_key is provided, it is used, otherwiese
the key from the job will be used.
'''
job_dict = db.get_job(job_id)
result_url = job_dict.get('result_url')
if not result_url:
# A job with an API key (for using when posting to the callback URL)
# but no callback URL is weird, but it can happen.
db.delete_api_key(job_id)
return True
api_key_from_job = job_dict.pop('api_key', None)
if not api_key:
api_key = api_key_from_job
headers = {'Content-Type': 'application/json'}
if api_key:
if ':' in api_key:
header, key = api_key.split(':')
else:
header, key = 'Authorization', api_key
headers[header] = key
try:
result = requests.post(
result_url,
data=json.dumps(job_dict, cls=DatetimeJsonEncoder),
headers=headers, verify=SSL_VERIFY)
db.delete_api_key(job_id)
except requests.ConnectionError:
return False
return result.status_code == requests.codes.ok
def job_update_status(job_id):
body = request.get_json()
assert "device_id" in body
assert "status" in body
device_id = body['device_id']
status = body['status']
result = body.get("result")
device = db.get_device(device_id)
if not device:
return jsonify(success=False, error_code="INVALID_DEVICE_ID"), 400
job = db.get_job(job_id)
if not job:
return jsonify(success=False, error_code="INVALID_JOB_ID"), 400
if not job.assigned_device:
return jsonify(success=False,
error_code="CANNOT_UPDATE_UNASSIGNED_JOB")
if job.assigned_device.id != device_id:
return jsonify(success=False,
error_code="JOB_ASSIGNED_TO_ANOTHER_DEVICE"), 400
job.status = status
if status == db.Job.SUCCEEDED or status == db.Job.FAILED:
# This job has finished, so it's no longer assigned to a device
job.assigned_device = None
if result:
print(f"\n\n[JOB RESULT] Received output for job id {job_id}:")
print(result, "\n\n")
job.save()
return jsonify(success=True)
def job_data(job_id):
'''Get the raw data that the job returned. The mimetype
will be the value provided in the metdata for the key ``mimetype``.
**Results:**
:rtype: string
:statuscode 200: no error
:statuscode 403: not authorized to view the job's data
:statuscode 404: job id not found
:statuscode 409: an error occurred
'''
job_dict = db.get_job(job_id)
if not job_dict:
return json.dumps({'error': 'job_id not found'}), 404, headers
if not is_authorized(job_dict):
return json.dumps({'error': 'not authorized'}), 403, headers
if job_dict['error']:
return json.dumps({'error': job_dict['error']}), 409, headers
content_type = job_dict['metadata'].get('mimetype')
return flask.Response(job_dict['data'], mimetype=content_type)
def job_data(job_id):
'''Get the raw data that the job returned. The mimetype
will be the value provided in the metdata for the key ``mimetype``.
**Results:**
:rtype: string
:statuscode 200: no error
:statuscode 403: not authorized to view the job's data
:statuscode 404: job id not found
:statuscode 409: an error occurred
'''
job_dict = db.get_job(job_id)
if not job_dict:
return json.dumps({'error': 'job_id not found'}), 404, headers
if not is_authorized(job_dict):
return json.dumps({'error': 'not authorized'}), 403, headers
if job_dict['error']:
return json.dumps({'error': job_dict['error']}), 409, headers
content_type = job_dict['metadata'].get('mimetype')
return flask.Response(job_dict['data'], mimetype=content_type)
def undermind_client():
for job_id, generation_id, strain1_id, model1_data, strain2_id, model2_data in db.get_job(
):
try:
print(
f"[Undermind client] Generation {generation_id} job {job_id} started"
)
(result1, result2) = openbw(model1_data, model2_data, strain1_id,
strain2_id, generation_id)
print(
f"[Undermind client] Generation {generation_id} job {job_id} done"
)
mr1 = db.create_model(strain1_id, result1)
mr2 = db.create_model(strain2_id, result2)
print(f"[Undermind client] Models created")
db.create_result(job_id, generation_id, mr1)
db.create_result(job_id, generation_id, mr2)
db.finish_job(job_id)
print(f"[Undermind client] Results created")
except Exception as e:
print(f"[Undermind client] Job {job_id} has no pants!")
traceback.print_exc()
db.reset_job(job_id)
result = job(job_id, input)
if hasattr(result, "__call__"):
db.mark_job_as_completed(job_id)
return flask.Response(result(), mimetype='application/json')
else:
db.mark_job_as_completed(job_id, result)
except util.JobError, e:
db.mark_job_as_errored(job_id, e.as_dict())
except Exception, e:
db.mark_job_as_errored(
job_id,
traceback.format_tb(sys.exc_traceback)[-1] + repr(e))
api_key = db.get_job(job_id)['api_key']
result_ok = send_result(job_id, api_key)
if not result_ok:
db.mark_job_as_failed_to_post_result(job_id)
return job_status(job_id=job_id, show_job_key=True, ignore_auth=True)
def run_asynchronous_job(job, job_id, job_key, input):
if not scheduler.running:
scheduler.start()
try:
db.add_pending_job(job_id, job_key, **input)
except sa.exc.IntegrityError:
error_string = 'job_id {} already exists'.format(job_id)
try:
result = job(job_id, input)
if hasattr(result, "__call__"):
db.mark_job_as_completed(job_id)
return flask.Response(result(), mimetype='application/json')
else:
db.mark_job_as_completed(job_id, result)
except util.JobError, e:
db.mark_job_as_errored(job_id, e.as_dict())
except Exception, e:
db.mark_job_as_errored(
job_id, traceback.format_tb(sys.exc_traceback)[-1] + repr(e))
api_key = db.get_job(job_id)['api_key']
result_ok = send_result(job_id, api_key)
if not result_ok:
db.mark_job_as_failed_to_post_result(job_id)
return job_status(job_id=job_id, show_job_key=True, ignore_auth=True)
def run_asynchronous_job(job, job_id, job_key, input):
if not scheduler.running:
scheduler.start()
try:
db.add_pending_job(job_id, job_key, **input)
except sa.exc.IntegrityError:
error_string = 'job_id {} already exists'.format(job_id)
