def post_jobs():
"""
Create a job to create preferences/predictions for a custom sequence using the specified model(model_name).
request['sequence_id'] str: uuid of the custom sequence to process
request['job_type'] str: see config.DataType properties for values
request['model_name'] str: name of the model to use
:return: json response with id of the job
"""
required_prop_names = ["sequence_id", "job_type", "model_name"]
(sequence_id, job_type,
model_name) = get_required_json_props(request, required_prop_names)
try:
seq = SequenceList(sequence_id)
seq.load(get_db())
except KeyError as ex:
raise ClientException("Unable to find sequence. It may have purged.",
ErrorType.SEQUENCE_NOT_FOUND,
error_data=sequence_id)
job = CustomJob.find_existing_job(get_db(), job_type, sequence_id,
model_name)
status_code = None
if not job:
status_code = None
job = CustomJob.create_job(get_db(), job_type, sequence_id, model_name)
return make_ok_json_response({'id': job.uuid}, status_code)
def test_custom_job_no_data(self):
FASTA_DATA1 = """>someseq\nAAACCCGGGGTT"""
db = create_db_connection(TestWithPostgres.config.dbconfig)
# upload FASTA file
sequence_list = SequenceList.create_with_content_and_title(db, FASTA_DATA1, "somelist")
# create a job to determine predictions for a sequence_list
job_uuid = CustomJob.create_job(db, DataType.PREDICTION, sequence_list, model_name='E2f1').uuid
# mark job as running
CustomJob.set_job_running(db, job_uuid)
# upload file
BED_DATA = ''
result_uuid = CustomResultData.new_uuid()
result = CustomResultData(db, result_uuid, job_uuid, model_name='E2f1', bed_data=BED_DATA)
result.save()
predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=False,
limit=None, offset=None)
self.assertEqual(1, len(predictions))
first = predictions[0]
self.assertEqual('someseq', first['name'])
self.assertEqual('None', first['max'])
self.assertEqual([], first['values'])
self.assertEqual('AAACCCGGGGTT', first['sequence'])
# Make sure we can convert predictions to JSON
json_version = json.dumps({'data': predictions})
self.assertEqual('{"data', json_version[:6])
def test_custom_job_no_data(self):
FASTA_DATA1 = """>someseq\nAAACCCGGGGTT"""
db = create_db_connection(TestWithPostgres.config.dbconfig)
# upload FASTA file
sequence_list = SequenceList.create_with_content_and_title(db, FASTA_DATA1, "somelist")
# create a job to determine predictions for a sequence_list
job_uuid = CustomJob.create_job(db, DataType.PREDICTION, sequence_list, model_name='E2f1').uuid
# mark job as running
CustomJob.set_job_running(db, job_uuid)
# upload file
BED_DATA = ''
result_uuid = CustomResultData.new_uuid()
result = CustomResultData(db, result_uuid, job_uuid, model_name='E2f1', bed_data=BED_DATA)
result.save()
predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=False,
limit=None, offset=None)
self.assertEqual(1, len(predictions))
first = predictions[0]
self.assertEqual('someseq', first['name'])
self.assertEqual('None', first['max'])
self.assertEqual([], first['values'])
self.assertEqual('AAACCCGGGGTT', first['sequence'])
# Make sure we can convert predictions to JSON
json_version = json.dumps({'data': predictions})
self.assertEqual('{"data', json_version[:6])
def delete_old_items(cur):
print("Deleting old custom lists.")
cur.execute(DELETE_OLD_LISTS_SQL, [])
print("Deleting old jobs.")
CustomJob.delete_old_jobs(cur, DELETE_AFTER_HOURS)
print("Deleting old custom sequences.")
SequenceList.delete_old_and_unattached(cur, DELETE_AFTER_HOURS)
def get_job(job_uuid):
"""
Retrieve details about a specific job.
:param job_uuid: str: uuid of the job returned from the POST to /jobs.
:return: json response
"""
job = CustomJob(job_uuid)
job.load(get_db())
return make_json_response(job.get_dict())
def get_job(job_uuid):
"""
Retrieve details about a specific job.
:param job_uuid: str: uuid of the job returned from the POST to /jobs.
:return: json response
"""
job = CustomJob(job_uuid)
job.load(get_db())
return make_json_response(job.get_dict())
def test_custom_job_normal_workflow(self):
SHORT_SEQUENCE = 'AAACCCGGGGTT'
LONG_SEQUENCE = 'AAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTT' \
'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
FASTA_DATA1 = '>someseq\n' + SHORT_SEQUENCE + '\n' \
'>someseq2\n' + LONG_SEQUENCE
db = create_db_connection(TestWithPostgres.config.dbconfig)
# upload FASTA file
sequence_list = SequenceList.create_with_content_and_title(db, FASTA_DATA1, "sometitle")
# create a job to determine predictions for a sequence_list
job_uuid = CustomJob.create_job(db, DataType.PREDICTION, sequence_list, model_name="E2f1").uuid
# mark job as running
CustomJob.set_job_running(db, job_uuid)
# upload file
BED_DATA = """
someseq\t0\t10\t12.5\tAAACCCGGGG
someseq2\t20\t30\t4.5\tGGTTAAACCC
someseq2\t60\t75\t15.5\tAAAAAAAAAAAAAAA
""".strip()
result_uuid = CustomResultData.new_uuid()
result = CustomResultData(db, result_uuid, job_uuid, model_name='E2f1', bed_data=BED_DATA)
result.save()
self.assertEqual(BED_DATA, CustomResultData.bed_file_contents(db, result_uuid).strip())
predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=False,
limit=None, offset=None)
self.assertEqual(2, len(predictions))
first = predictions[0]
self.assertEqual('someseq', first['name'])
self.assertEqual(12.5, float(first['max']))
self.assertEqual([{u'start': 0, u'end': 10, u'value': 12.5}], first['values'])
self.assertEqual(SHORT_SEQUENCE, first['sequence'])
second = predictions[1]
self.assertEqual('someseq2', second['name'])
self.assertEqual(15.5, float(second['max']))
self.assertEqual(LONG_SEQUENCE, second['sequence'])
predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=True,
limit=None, offset=None)
self.assertEqual(2, len(predictions))
self.assertEqual(15.5, float(predictions[0]['max']))
self.assertEqual(12.5, float(predictions[1]['max']))
predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=True,
limit=1, offset=1)
self.assertEqual(1, len(predictions))
self.assertEqual(12.5, float(predictions[0]['max']))
# Make sure we can convert predictions to JSON
json_version = json.dumps({'data': predictions})
self.assertEqual('{"data', json_version[:6])
def test_custom_job_normal_workflow(self):
FASTA_DATA1 = """>someseq\nAAACCCGGGGTT\n>someseq2\nAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTT"""
db = create_db_connection(TestWithPostgres.config.dbconfig)
# upload FASTA file
sequence_list = SequenceList.create_with_content_and_title(db, FASTA_DATA1, "sometitle")
# create a job to determine predictions for a sequence_list
job_uuid = CustomJob.create_job(db, DataType.PREDICTION, sequence_list, model_name="E2f1").uuid
# mark job as running
CustomJob.set_job_running(db, job_uuid)
# upload file
BED_DATA = """
someseq\t0\t10\t12.5
someseq2\t20\t30\t4.5
someseq2\t60\t75\t15.5
""".strip()
result_uuid = CustomResultData.new_uuid()
result = CustomResultData(db, result_uuid, job_uuid, model_name='E2f1', bed_data=BED_DATA)
result.save()
self.assertEqual(BED_DATA, CustomResultData.bed_file_contents(db, result_uuid).strip())
predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=False,
limit=None, offset=None)
self.assertEqual(2, len(predictions))
first = predictions[0]
self.assertEqual('someseq', first['name'])
self.assertEqual(12.5, float(first['max']))
self.assertEqual([{u'start': 0, u'end': 10, u'value': 12.5}], first['values'])
self.assertEqual('AAACCCGGGGTT', first['sequence'])
second = predictions[1]
self.assertEqual('someseq2', second['name'])
self.assertEqual(15.5, float(second['max']))
self.assertEqual('AAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTT', second['sequence'])
predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=True,
limit=None, offset=None)
self.assertEqual(2, len(predictions))
self.assertEqual(15.5, float(predictions[0]['max']))
self.assertEqual(12.5, float(predictions[1]['max']))
predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=True,
limit=1, offset=1)
self.assertEqual(1, len(predictions))
self.assertEqual(12.5, float(predictions[0]['max']))
# Make sure we can convert predictions to JSON
json_version = json.dumps({'data': predictions})
self.assertEqual('{"data', json_version[:6])
def get_jobs():
"""
Return a list of jobs with optional job_status filter.
request['job_status'] str: see customjob.JobStatus properties for values
:return: json response with 'result' array of jobs.
"""
job_status = request.args.get("job_status")
result = []
for job in CustomJob.find_jobs(get_db(), job_status):
result.append(job.get_dict())
return make_json_response({'result': result})
def get_jobs():
"""
Return a list of jobs with optional job_status filter.
request['job_status'] str: see customjob.JobStatus properties for values
:return: json response with 'result' array of jobs.
"""
job_status = request.args.get("job_status")
result = []
for job in CustomJob.find_jobs(get_db(), job_status):
result.append(job.get_dict())
return make_json_response({'result': result})
def post_jobs():
"""
Create a job to create preferences/predictions for a custom sequence using the specified model(model_name).
request['sequence_id'] str: uuid of the custom sequence to process
request['job_type'] str: see config.DataType properties for values
request['model_name'] str: name of the model to use
:return: json response with id of the job
"""
required_prop_names = ["sequence_id", "job_type", "model_name"]
(sequence_id, job_type, model_name) = get_required_json_props(request, required_prop_names)
try:
seq = SequenceList(sequence_id)
seq.load(get_db())
except KeyError as ex:
raise ClientException("Unable to find sequence. It may have purged.",
ErrorType.SEQUENCE_NOT_FOUND,
error_data=sequence_id)
job = CustomJob.find_existing_job(get_db(), job_type, sequence_id, model_name)
status_code = None
if not job:
status_code = None
job = CustomJob.create_job(get_db(), job_type, sequence_id, model_name)
return make_ok_json_response({'id': job.uuid}, status_code)
def put_job(job_uuid):
"""
Update job status.
Secured via apache config: production/imads.conf.
request['job_status'] str: value from customjob.JobStatus properties
:param job_uuid: str: uuid of the job we want to update status
:return: json response
"""
db = get_db()
(job_status,) = get_required_json_props(request, ["job_status"])
error_message = request.get_json().get("error_message")
if job_status == JobStatus.RUNNING:
CustomJob.set_job_running(db, job_uuid)
elif job_status == JobStatus.COMPLETE:
CustomJob.set_job_complete(db, job_uuid)
elif job_status == JobStatus.ERROR:
CustomJob.set_job_as_error(db, job_uuid, error_message)
else:
raise ValueError("Invalid job status:{} for job:{}".format(job_status, job_uuid))
return json_ok_result()
def put_job(job_uuid):
"""
Update job status.
Secured via apache config: production/tfpredictions.conf.
request['job_status'] str: value from customjob.JobStatus properties
:param job_uuid: str: uuid of the job we want to update status
:return: json response
"""
db = get_db()
(job_status, ) = get_required_json_props(request, ["job_status"])
error_message = request.get_json().get("error_message")
if job_status == JobStatus.RUNNING:
CustomJob.set_job_running(db, job_uuid)
elif job_status == JobStatus.COMPLETE:
CustomJob.set_job_complete(db, job_uuid)
elif job_status == JobStatus.ERROR:
CustomJob.set_job_as_error(db, job_uuid, error_message)
else:
raise ValueError("Invalid job status:{} for job:{}".format(
job_status, job_uuid))
return json_ok_result()
def test_customjob(self):
FASTA_DATA1 = """>stuff\nAAACCCGGGGTT"""
db = create_db_connection(TestWithPostgres.config.dbconfig)
update_database(db, """
delete from custom_result_row;
delete from custom_result;
delete from job;
delete from sequence_list_item;
delete from sequence_list;
""", [])
# start out finding no jobs
jobs = CustomJob.find_jobs(db, None)
self.assertEqual(len(jobs), 0)
# create a new job that should be NEW status
sequence_list = SequenceList.create_with_content_and_title(db, FASTA_DATA1, "somelist")
job_uuid = CustomJob.create_job(db, DataType.PREDICTION, sequence_list, model_name="E2f1").uuid
job = CustomJob.read_job(db, job_uuid)
self.assertEqual(job_uuid, job.uuid)
self.assertEqual(JobStatus.NEW, job.status)
self.assertEqual(DataType.PREDICTION, job.type)
self.assertEqual(sequence_list, job.sequence_list)
self.assertIsNotNone(job.created)
self.assertIsNone(job.finished)
# find NEW job without filters
jobs = CustomJob.find_jobs(db, None)
self.assertEqual(len(jobs), 1)
self.assertEqual(jobs[0].uuid, job_uuid)
# find no for RUNNING jobs
jobs = CustomJob.find_jobs(db, JobStatus.RUNNING)
self.assertEqual(len(jobs), 0)
# find 1 for NEW jobs
jobs = CustomJob.find_jobs(db, JobStatus.NEW)
self.assertEqual(len(jobs), 1)
# Jobs can be set to running only once (when in NEW state)
CustomJob.set_job_running(db, job_uuid)
job = CustomJob.read_job(db, job_uuid)
self.assertEqual(JobStatus.RUNNING, job.status)
self.assertIsNone(job.finished)
# Disallow setting a job running twice (prevents two workers working on the same job)
with self.assertRaises(ValueError):
CustomJob.set_job_running(db, job_uuid)
# find 0 for NEW jobs
jobs = CustomJob.find_jobs(db, JobStatus.NEW)
self.assertEqual(len(jobs), 0)
# Jobs can be set to complete from RUNNING state
CustomJob.set_job_complete(db, job_uuid)
job = CustomJob.read_job(db, job_uuid)
self.assertEqual(JobStatus.COMPLETE, job.status)
self.assertIsNotNone(job.finished)
# find 0 for NEW jobs
jobs = CustomJob.find_jobs(db, JobStatus.NEW)
self.assertEqual(len(jobs), 0)
# Jobs can be set to complete from ERROR state
CustomJob.set_job_as_error(db, job_uuid, "Something failed.")
job = CustomJob.read_job(db, job_uuid)
self.assertEqual(JobStatus.ERROR, job.status)
self.assertEqual("Something failed.", job.error_msg)
self.assertIsNotNone(job.finished)
# find 0 for NEW jobs
jobs = CustomJob.find_jobs(db, JobStatus.NEW)
self.assertEqual(len(jobs), 0)
def test_customjob(self):
FASTA_DATA1 = """>stuff\nAAACCCGGGGTT"""
db = create_db_connection(TestWithPostgres.config.dbconfig)
update_database(db, """
delete from custom_result_row;
delete from custom_result;
delete from job;
delete from sequence_list_item;
delete from sequence_list;
""", [])
# start out finding no jobs
jobs = CustomJob.find_jobs(db, None)
self.assertEqual(len(jobs), 0)
# create a new job that should be NEW status
sequence_list = SequenceList.create_with_content_and_title(db, FASTA_DATA1, "somelist")
job_uuid = CustomJob.create_job(db, DataType.PREDICTION, sequence_list, model_name="E2f1").uuid
job = CustomJob.read_job(db, job_uuid)
self.assertEqual(job_uuid, job.uuid)
self.assertEqual(JobStatus.NEW, job.status)
self.assertEqual(DataType.PREDICTION, job.type)
self.assertEqual(sequence_list, job.sequence_list)
self.assertIsNotNone(job.created)
self.assertIsNone(job.finished)
# find NEW job without filters
jobs = CustomJob.find_jobs(db, None)
self.assertEqual(len(jobs), 1)
self.assertEqual(jobs[0].uuid, job_uuid)
# find no for RUNNING jobs
jobs = CustomJob.find_jobs(db, JobStatus.RUNNING)
self.assertEqual(len(jobs), 0)
# find 1 for NEW jobs
jobs = CustomJob.find_jobs(db, JobStatus.NEW)
self.assertEqual(len(jobs), 1)
# Jobs can be set to running only once (when in NEW state)
CustomJob.set_job_running(db, job_uuid)
job = CustomJob.read_job(db, job_uuid)
self.assertEqual(JobStatus.RUNNING, job.status)
self.assertIsNone(job.finished)
# Disallow setting a job running twice (prevents two workers working on the same job)
with self.assertRaises(ValueError):
CustomJob.set_job_running(db, job_uuid)
# find 0 for NEW jobs
jobs = CustomJob.find_jobs(db, JobStatus.NEW)
self.assertEqual(len(jobs), 0)
# Jobs can be set to complete from RUNNING state
CustomJob.set_job_complete(db, job_uuid)
job = CustomJob.read_job(db, job_uuid)
self.assertEqual(JobStatus.COMPLETE, job.status)
self.assertIsNotNone(job.finished)
# find 0 for NEW jobs
jobs = CustomJob.find_jobs(db, JobStatus.NEW)
self.assertEqual(len(jobs), 0)
# Jobs can be set to complete from ERROR state
CustomJob.set_job_as_error(db, job_uuid, "Something failed.")
job = CustomJob.read_job(db, job_uuid)
self.assertEqual(JobStatus.ERROR, job.status)
self.assertEqual("Something failed.", job.error_msg)
self.assertIsNotNone(job.finished)
# find 0 for NEW jobs
jobs = CustomJob.find_jobs(db, JobStatus.NEW)
self.assertEqual(len(jobs), 0)
