def job_status(job_id): job = db.get_job(job_id) if not job: return jsonify(success=False, error_code="INVALID_JOB_ID"), 400 status_code = db.Job.STATUS_CODES[job.status] return jsonify(success=True, status_code=status_code)
def job_listener(event): '''Listens to completed job''' job_id = event.job.args[0] if event.code == events.EVENT_JOB_MISSED: db.mark_job_as_missed(job_id) elif event.exception: if isinstance(event.exception, util.JobError): error_object = event.exception.as_dict() else: error_object = "\n".join( traceback.format_tb(event.traceback) + [repr(event.exception)]) db.mark_job_as_errored(job_id, error_object) else: db.mark_job_as_completed(job_id, event.retval) api_key = db.get_job(job_id)["api_key"] result_ok = send_result(job_id, api_key) if not result_ok: db.mark_job_as_failed_to_post_result(job_id) # Optionally notify tests that job_listener() has finished. if "_TEST_CALLBACK_URL" in app.config: requests.get(app.config["_TEST_CALLBACK_URL"])
def job_delete(job_id): '''Deletes the job together with its logs and metadata. :param job_id: An identifier for the job :type job_id: string :statuscode 200: no error :statuscode 403: not authorized to delete the job :statuscode 404: the job could not be found :statuscode 409: an error occurred ''' conn = db.ENGINE.connect() job = db.get_job(job_id) if not job: return json.dumps({'error': 'job_id not found'}), 404, headers if not is_authorized(job): return json.dumps({'error': 'not authorized'}), 403, headers trans = conn.begin() try: conn.execute(db.JOBS_TABLE.delete().where( db.JOBS_TABLE.c.job_id == job_id)) trans.commit() return json.dumps({'success': True}), 200, headers except Exception, e: trans.rollback() return json.dumps({'error': str(e)}), 409, headers
def job_delete(job_id): '''Deletes the job together with its logs and metadata. :param job_id: An identifier for the job :type job_id: string :statuscode 200: no error :statuscode 403: not authorized to delete the job :statuscode 404: the job could not be found :statuscode 409: an error occurred ''' conn = db.ENGINE.connect() job = db.get_job(job_id) if not job: return json.dumps({'error': 'job_id not found'}), 404, headers if not is_authorized(job): return json.dumps({'error': 'not authorized'}), 403, headers trans = conn.begin() try: conn.execute( db.JOBS_TABLE.delete().where(db.JOBS_TABLE.c.job_id == job_id)) trans.commit() return json.dumps({'success': True}), 200, headers except Exception, e: trans.rollback() return json.dumps({'error': str(e)}), 409, headers
def job_listener(event): '''Listens to completed job''' job_id = event.job.args[0] if event.code == events.EVENT_JOB_MISSED: db.mark_job_as_missed(job_id) elif event.exception: if isinstance(event.exception, util.JobError): error_object = event.exception.as_dict() else: error_object = "\n".join(traceback.format_tb(event.traceback) + [repr(event.exception)]) db.mark_job_as_errored(job_id, error_object) else: db.mark_job_as_completed(job_id, event.retval) api_key = db.get_job(job_id)["api_key"] result_ok = send_result(job_id, api_key) if not result_ok: db.mark_job_as_failed_to_post_result(job_id) # Optionally notify tests that job_listener() has finished. if "_TEST_CALLBACK_URL" in app.config: requests.get(app.config["_TEST_CALLBACK_URL"])
def work_and_save(job_id): db.set_job_status(job_id, 'running') job = db.get_job(job_id) result = work(job.input) db.set_job_status_and_output(job_id, 'done', result)
def xloader_status(context, data_dict): ''' Get the status of a ckanext-xloader job for a certain resource. :param resource_id: The resource id of the resource that you want the status for. :type resource_id: string ''' p.toolkit.check_access('xloader_status', context, data_dict) if 'id' in data_dict: data_dict['resource_id'] = data_dict['id'] res_id = _get_or_bust(data_dict, 'resource_id') task = p.toolkit.get_action('task_status_show')(context, { 'entity_id': res_id, 'task_type': 'xloader', 'key': 'xloader' }) datapusher_url = config.get('ckan.datapusher.url') if not datapusher_url: raise p.toolkit.ValidationError( {'configuration': ['ckan.datapusher.url not in config file']}) value = json.loads(task['value']) job_id = value.get('job_id') url = None job_detail = None if job_id: # get logs from the xloader db db.init(config) job_detail = db.get_job(job_id) # timestamp is a date, so not sure why this code was there # for log in job_detail['logs']: # if 'timestamp' in log: # date = time.strptime( # log['timestamp'], "%Y-%m-%dT%H:%M:%S.%f") # date = datetime.datetime.utcfromtimestamp( # time.mktime(date)) # log['timestamp'] = date try: error = json.loads(task['error']) except ValueError: # this happens occasionally, such as when the job times out error = task['error'] return { 'status': task['state'], 'job_id': job_id, 'job_url': url, 'last_updated': task['last_updated'], 'task_info': job_detail, 'error': error, }
def handle_phone_response(data): device_id = data['device_id'] job_id = data['job_id'] job = db.get_job(job_id=job_id) job.status = db.Job.ASSIGNED job.assigned_device = device_id job.num_attempts += 1 job.save() checker.remove_pending_acknowledgement(job.id) print(f"Device id={device_id} has acknowledged job id={job_id}")
def cancel_and_reschedule_job(self, job_id): job = db.get_job(job_id) job_json = job.to_json() # cancel the job job.cancel() print(f"[JOB {job.id}] cancelled and rescheduled.") # Reschedule the job for another device, if and only if the number of retries don't go past 3. if job.can_be_retried: if (schedule_job(job)): print(f"[JOB {job.id}] Rescheduled") else: print(f"[JOB {job.id}] Reschedule Limit Hit")
def handle_phone_cancel_job_response(data): success = data['success'] device_id = data['device_id'] job_id = data['job_id'] if success: # the phone was able to stop this job job = db.get_job(job_id=job_id) job.status = db.Job.FAILED job.assigned_device = None job.save() print(f"Device id={device_id} was able to cancel job id={job_id}") else: # the phone could not cancel this job # in this case, we don't do anything on this part of the SERVER side, since we should have a separate server cron process that handles failed jobs # (e.g., picks such failed jobs up and retry them if needed) # the device, however, would need to somehow end this task. print(f"Device id={device_id} was NOT able to cancel job id={job_id}")
def job_status(job_id, show_job_key=False, ignore_auth=False): '''Show a specific job. :param limit: Limit the number of logs :type limit: integer **Results:** :rtype: A dictionary with the following keys :param status: Status of job (complete, error) :type status: string :param sent_data: Input data for job :type sent_data: json encodable data :param job_id: An identifier for the job :type job_id: string :param result_url: Callback url :type result_url: url string :param data: Results from job. :type data: json encodable data :param error: Error raised during job execution :type error: string :param metadata: Metadata provided when submitting job. :type metadata: list of key - value pairs :param requested_timestamp: Time the job started :type requested_timestamp: timestamp :param finished_timestamp: Time the job finished :type finished_timestamp: timestamp :statuscode 200: no error :statuscode 403: not authorized to view the job's data :statuscode 404: job id not found :statuscode 409: an error occurred ''' limit = flask.request.args.get('limit') job_dict = db.get_job(job_id, limit=limit) if not job_dict: return json.dumps({'error': 'job_id not found'}), 404, headers if not ignore_auth and not is_authorized(job_dict): return json.dumps({'error': 'not authorized'}), 403, headers job_dict.pop('api_key', None) if not show_job_key: job_dict.pop('job_key', None) return flask.Response(json.dumps(job_dict, cls=DatetimeJsonEncoder), mimetype='application/json')
def send_result(job_id, api_key=None): ''' Send results to where requested. If api_key is provided, it is used, otherwiese the key from the job will be used. ''' job_dict = db.get_job(job_id) result_url = job_dict.get('result_url') if not result_url: # A job with an API key (for using when posting to the callback URL) # but no callback URL is weird, but it can happen. db.delete_api_key(job_id) return True api_key_from_job = job_dict.pop('api_key', None) if not api_key: api_key = api_key_from_job headers = {'Content-Type': 'application/json'} if api_key: if ':' in api_key: header, key = api_key.split(':') else: header, key = 'Authorization', api_key headers[header] = key try: result = requests.post(result_url, data=json.dumps(job_dict, cls=DatetimeJsonEncoder), headers=headers, verify=SSL_VERIFY) db.delete_api_key(job_id) except requests.ConnectionError: return False return result.status_code == requests.codes.ok
def job_status(job_id, show_job_key=False, ignore_auth=False): '''Show a specific job. **Results:** :rtype: A dictionary with the following keys :param status: Status of job (complete, error) :type status: string :param sent_data: Input data for job :type sent_data: json encodable data :param job_id: An identifier for the job :type job_id: string :param result_url: Callback url :type result_url: url string :param data: Results from job. :type data: json encodable data :param error: Error raised during job execution :type error: string :param metadata: Metadata provided when submitting job. :type metadata: list of key - value pairs :param requested_timestamp: Time the job started :type requested_timestamp: timestamp :param finished_timestamp: Time the job finished :type finished_timestamp: timestamp :statuscode 200: no error :statuscode 403: not authorized to view the job's data :statuscode 404: job id not found :statuscode 409: an error occurred ''' job_dict = db.get_job(job_id) if not job_dict: return json.dumps({'error': 'job_id not found'}), 404, headers if not ignore_auth and not is_authorized(job_dict): return json.dumps({'error': 'not authorized'}), 403, headers job_dict.pop('api_key', None) if not show_job_key: job_dict.pop('job_key', None) return flask.Response(json.dumps(job_dict, cls=DatetimeJsonEncoder), mimetype='application/json')
def send_result(job_id, api_key=None): ''' Send results to where requested. If api_key is provided, it is used, otherwiese the key from the job will be used. ''' job_dict = db.get_job(job_id) result_url = job_dict.get('result_url') if not result_url: # A job with an API key (for using when posting to the callback URL) # but no callback URL is weird, but it can happen. db.delete_api_key(job_id) return True api_key_from_job = job_dict.pop('api_key', None) if not api_key: api_key = api_key_from_job headers = {'Content-Type': 'application/json'} if api_key: if ':' in api_key: header, key = api_key.split(':') else: header, key = 'Authorization', api_key headers[header] = key try: result = requests.post( result_url, data=json.dumps(job_dict, cls=DatetimeJsonEncoder), headers=headers, verify=SSL_VERIFY) db.delete_api_key(job_id) except requests.ConnectionError: return False return result.status_code == requests.codes.ok
def job_update_status(job_id): body = request.get_json() assert "device_id" in body assert "status" in body device_id = body['device_id'] status = body['status'] result = body.get("result") device = db.get_device(device_id) if not device: return jsonify(success=False, error_code="INVALID_DEVICE_ID"), 400 job = db.get_job(job_id) if not job: return jsonify(success=False, error_code="INVALID_JOB_ID"), 400 if not job.assigned_device: return jsonify(success=False, error_code="CANNOT_UPDATE_UNASSIGNED_JOB") if job.assigned_device.id != device_id: return jsonify(success=False, error_code="JOB_ASSIGNED_TO_ANOTHER_DEVICE"), 400 job.status = status if status == db.Job.SUCCEEDED or status == db.Job.FAILED: # This job has finished, so it's no longer assigned to a device job.assigned_device = None if result: print(f"\n\n[JOB RESULT] Received output for job id {job_id}:") print(result, "\n\n") job.save() return jsonify(success=True)
def job_data(job_id): '''Get the raw data that the job returned. The mimetype will be the value provided in the metdata for the key ``mimetype``. **Results:** :rtype: string :statuscode 200: no error :statuscode 403: not authorized to view the job's data :statuscode 404: job id not found :statuscode 409: an error occurred ''' job_dict = db.get_job(job_id) if not job_dict: return json.dumps({'error': 'job_id not found'}), 404, headers if not is_authorized(job_dict): return json.dumps({'error': 'not authorized'}), 403, headers if job_dict['error']: return json.dumps({'error': job_dict['error']}), 409, headers content_type = job_dict['metadata'].get('mimetype') return flask.Response(job_dict['data'], mimetype=content_type)
def undermind_client(): for job_id, generation_id, strain1_id, model1_data, strain2_id, model2_data in db.get_job( ): try: print( f"[Undermind client] Generation {generation_id} job {job_id} started" ) (result1, result2) = openbw(model1_data, model2_data, strain1_id, strain2_id, generation_id) print( f"[Undermind client] Generation {generation_id} job {job_id} done" ) mr1 = db.create_model(strain1_id, result1) mr2 = db.create_model(strain2_id, result2) print(f"[Undermind client] Models created") db.create_result(job_id, generation_id, mr1) db.create_result(job_id, generation_id, mr2) db.finish_job(job_id) print(f"[Undermind client] Results created") except Exception as e: print(f"[Undermind client] Job {job_id} has no pants!") traceback.print_exc() db.reset_job(job_id)
result = job(job_id, input) if hasattr(result, "__call__"): db.mark_job_as_completed(job_id) return flask.Response(result(), mimetype='application/json') else: db.mark_job_as_completed(job_id, result) except util.JobError, e: db.mark_job_as_errored(job_id, e.as_dict()) except Exception, e: db.mark_job_as_errored( job_id, traceback.format_tb(sys.exc_traceback)[-1] + repr(e)) api_key = db.get_job(job_id)['api_key'] result_ok = send_result(job_id, api_key) if not result_ok: db.mark_job_as_failed_to_post_result(job_id) return job_status(job_id=job_id, show_job_key=True, ignore_auth=True) def run_asynchronous_job(job, job_id, job_key, input): if not scheduler.running: scheduler.start() try: db.add_pending_job(job_id, job_key, **input) except sa.exc.IntegrityError: error_string = 'job_id {} already exists'.format(job_id)
try: result = job(job_id, input) if hasattr(result, "__call__"): db.mark_job_as_completed(job_id) return flask.Response(result(), mimetype='application/json') else: db.mark_job_as_completed(job_id, result) except util.JobError, e: db.mark_job_as_errored(job_id, e.as_dict()) except Exception, e: db.mark_job_as_errored( job_id, traceback.format_tb(sys.exc_traceback)[-1] + repr(e)) api_key = db.get_job(job_id)['api_key'] result_ok = send_result(job_id, api_key) if not result_ok: db.mark_job_as_failed_to_post_result(job_id) return job_status(job_id=job_id, show_job_key=True, ignore_auth=True) def run_asynchronous_job(job, job_id, job_key, input): if not scheduler.running: scheduler.start() try: db.add_pending_job(job_id, job_key, **input) except sa.exc.IntegrityError: error_string = 'job_id {} already exists'.format(job_id)