def test_custom_job_no_data(self): FASTA_DATA1 = """>someseq\nAAACCCGGGGTT""" db = create_db_connection(TestWithPostgres.config.dbconfig) # upload FASTA file sequence_list = SequenceList.create_with_content_and_title(db, FASTA_DATA1, "somelist") # create a job to determine predictions for a sequence_list job_uuid = CustomJob.create_job(db, DataType.PREDICTION, sequence_list, model_name='E2f1').uuid # mark job as running CustomJob.set_job_running(db, job_uuid) # upload file BED_DATA = '' result_uuid = CustomResultData.new_uuid() result = CustomResultData(db, result_uuid, job_uuid, model_name='E2f1', bed_data=BED_DATA) result.save() predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=False, limit=None, offset=None) self.assertEqual(1, len(predictions)) first = predictions[0] self.assertEqual('someseq', first['name']) self.assertEqual('None', first['max']) self.assertEqual([], first['values']) self.assertEqual('AAACCCGGGGTT', first['sequence']) # Make sure we can convert predictions to JSON json_version = json.dumps({'data': predictions}) self.assertEqual('{"data', json_version[:6])
def get_custom_result_raw_data(result_id): bed_file_contents = CustomResultData.bed_file_contents(get_db(), result_id) def gen(): yield bed_file_contents return download_file_response("data.bed", gen())
def search_custom_results(result_id): """ Search a result for predictions. request['maxPredictionSort'] - when true sort by max prediction request['all'] - include values in download request['page'] - which page of results to show request['perPage'] - items per page to show :param result_id: str: uuid of the custom_predictions/custom_preferences we want to search :return: json response with 'result' property containing an array of predictions """ args = request.args format = args.get('format') sort_by_max = args.get('maxPredictionSort') if sort_by_max == 'false': sort_by_max = None all_values = args.get('all') page = get_optional_int(args, 'page') per_page = get_optional_int(args, 'per_page') offset = None if page and per_page: offset = (page - 1) * per_page predictions = CustomResultData.get_predictions(get_db(), result_id, sort_by_max, per_page, offset) if format == 'tsv' or format == 'csv': filename = "custom_result.{}".format(format) separator = ',' if format == 'tsv': separator = '\t' return download_file_response(filename, make_download_custom_result(separator, all_values, predictions)) else: return make_ok_json_response({ 'result': predictions})
def search_custom_results(result_id): """ Search a result for predictions. request['maxPredictionSort'] - when true sort by max prediction request['all'] - include values in download request['page'] - which page of results to show request['perPage'] - items per page to show :param result_id: str: uuid of the custom_predictions/custom_preferences we want to search :return: json response with 'result' property containing an array of predictions """ args = request.args format = args.get('format') sort_by_max = args.get('maxPredictionSort') if sort_by_max == 'false': sort_by_max = None all_values = args.get('all') page = get_optional_int(args, 'page') per_page = get_optional_int(args, 'per_page') offset = None if page and per_page: offset = (page - 1) * per_page predictions = CustomResultData.get_predictions(get_db(), result_id, sort_by_max, per_page, offset) if format == 'tsv' or format == 'csv': filename = "custom_result.{}".format(format) separator = ',' if format == 'tsv': separator = '\t' return download_file_response( filename, make_download_custom_result(separator, all_values, predictions)) else: return make_ok_json_response({'result': predictions})
def test_custom_job_normal_workflow(self): SHORT_SEQUENCE = 'AAACCCGGGGTT' LONG_SEQUENCE = 'AAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTT' \ 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' FASTA_DATA1 = '>someseq\n' + SHORT_SEQUENCE + '\n' \ '>someseq2\n' + LONG_SEQUENCE db = create_db_connection(TestWithPostgres.config.dbconfig) # upload FASTA file sequence_list = SequenceList.create_with_content_and_title(db, FASTA_DATA1, "sometitle") # create a job to determine predictions for a sequence_list job_uuid = CustomJob.create_job(db, DataType.PREDICTION, sequence_list, model_name="E2f1").uuid # mark job as running CustomJob.set_job_running(db, job_uuid) # upload file BED_DATA = """ someseq\t0\t10\t12.5\tAAACCCGGGG someseq2\t20\t30\t4.5\tGGTTAAACCC someseq2\t60\t75\t15.5\tAAAAAAAAAAAAAAA """.strip() result_uuid = CustomResultData.new_uuid() result = CustomResultData(db, result_uuid, job_uuid, model_name='E2f1', bed_data=BED_DATA) result.save() self.assertEqual(BED_DATA, CustomResultData.bed_file_contents(db, result_uuid).strip()) predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=False, limit=None, offset=None) self.assertEqual(2, len(predictions)) first = predictions[0] self.assertEqual('someseq', first['name']) self.assertEqual(12.5, float(first['max'])) self.assertEqual([{u'start': 0, u'end': 10, u'value': 12.5}], first['values']) self.assertEqual(SHORT_SEQUENCE, first['sequence']) second = predictions[1] self.assertEqual('someseq2', second['name']) self.assertEqual(15.5, float(second['max'])) self.assertEqual(LONG_SEQUENCE, second['sequence']) predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=True, limit=None, offset=None) self.assertEqual(2, len(predictions)) self.assertEqual(15.5, float(predictions[0]['max'])) self.assertEqual(12.5, float(predictions[1]['max'])) predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=True, limit=1, offset=1) self.assertEqual(1, len(predictions)) self.assertEqual(12.5, float(predictions[0]['max'])) # Make sure we can convert predictions to JSON json_version = json.dumps({'data': predictions}) self.assertEqual('{"data', json_version[:6])
def post_custom_result(): """ Save custom prediction/preferences results. Secured via apache config: production/imads.conf. request['job_id'] - str: uuid of the job associated with these results request['bed_data'] - str: data that makes up the results request['model_name'] - str: name of the model used to build these results :return: json response with uuid of result stored in 'id' field """ required_prop_names = ["job_id", "model_name"] (job_id, model_name) = get_required_json_props(request, required_prop_names) bed_data = request.get_json().get('bed_data') decoded_bed_data = base64.b64decode(bed_data) result_uuid = CustomResultData.new_uuid() result_data = CustomResultData(get_db(), result_uuid, job_id, model_name, decoded_bed_data) result_data.save() return make_json_response({'result': 'ok', 'id': result_uuid})
def post_custom_result(): """ Save custom prediction/preferences results. Secured via apache config: production/tfpredictions.conf. request['job_id'] - str: uuid of the job associated with these results request['bed_data'] - str: data that makes up the results request['model_name'] - str: name of the model used to build these results :return: json response with uuid of result stored in 'id' field """ required_prop_names = ["job_id", "model_name"] (job_id, model_name) = get_required_json_props(request, required_prop_names) bed_data = request.get_json().get('bed_data') decoded_bed_data = base64.b64decode(bed_data) result_uuid = CustomResultData.new_uuid() result_data = CustomResultData(get_db(), result_uuid, job_id, model_name, decoded_bed_data) result_data.save() return make_json_response({'result': 'ok', 'id': result_uuid})
def find_custom_result(): """ Find a single prediction for a sequence_id and model_name. request['sequence_id'] str: uuid of the custom sequence to look for request['model_name'] str: name of the model we are looking for a :return: json response with id field that is either None or the uuid of the custom_predictions/custom_preferences. """ sequence_id = request.args['sequence_id'] model_name = request.args['model_name'] custom_result_id = CustomResultData.find_one(get_db(), sequence_id, model_name) return make_ok_json_response({'id': custom_result_id})
def find_custom_result(): """ Find a single prediction for a sequence_id and model_name. request['sequence_id'] str: uuid of the custom sequence to look for request['model_name'] str: name of the model we are looking for a :return: json response with id field that is either None or the uuid of the custom_predictions/custom_preferences. """ sequence_id = request.args['sequence_id'] model_name = request.args['model_name'] custom_result_id = CustomResultData.find_one(get_db(), sequence_id, model_name) return make_ok_json_response({'id': custom_result_id})
def test_custom_job_no_data(self): FASTA_DATA1 = """>someseq\nAAACCCGGGGTT""" db = create_db_connection(TestWithPostgres.config.dbconfig) # upload FASTA file sequence_list = SequenceList.create_with_content_and_title(db, FASTA_DATA1, "somelist") # create a job to determine predictions for a sequence_list job_uuid = CustomJob.create_job(db, DataType.PREDICTION, sequence_list, model_name='E2f1').uuid # mark job as running CustomJob.set_job_running(db, job_uuid) # upload file BED_DATA = '' result_uuid = CustomResultData.new_uuid() result = CustomResultData(db, result_uuid, job_uuid, model_name='E2f1', bed_data=BED_DATA) result.save() predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=False, limit=None, offset=None) self.assertEqual(1, len(predictions)) first = predictions[0] self.assertEqual('someseq', first['name']) self.assertEqual('None', first['max']) self.assertEqual([], first['values']) self.assertEqual('AAACCCGGGGTT', first['sequence']) # Make sure we can convert predictions to JSON json_version = json.dumps({'data': predictions}) self.assertEqual('{"data', json_version[:6])
def test_determine_last_page(self, mock_read_database): test_data = [ # (num_items, per_page, expected_last_page) (0, 10, 0), (1, 10, 1), (10, 10, 1), (11, 10, 2), (22, 3, 8), ] for num_items, per_page, expected_last_page in test_data: mock_read_database.return_value = [[num_items]] last_page = CustomResultData.determine_last_page(db=None, result_uuid='123-456-780', per_page=per_page) self.assertEqual(last_page, expected_last_page)
def test_custom_result_sequence_lookup(self, mock_read_database): mock_read_database.return_value = [ ('wild', 'attattattatt'), ('normal', 'catcatcatcat') ] sequence_lookup = CustomResultData.custom_result_sequence_lookup(db=None, result_id='456') self.assertEqual(sequence_lookup, {'normal': 'catcatcatcat', 'wild': 'attattattatt'}) args, kwargs = mock_read_database.call_args db, sql, params = args self.assertEqual(params, ['456']) self.assertIn('select sequence_list_item.name, sequence_list_item.sequence', sql)
def test_custom_job_normal_workflow(self): FASTA_DATA1 = """>someseq\nAAACCCGGGGTT\n>someseq2\nAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTT""" db = create_db_connection(TestWithPostgres.config.dbconfig) # upload FASTA file sequence_list = SequenceList.create_with_content_and_title(db, FASTA_DATA1, "sometitle") # create a job to determine predictions for a sequence_list job_uuid = CustomJob.create_job(db, DataType.PREDICTION, sequence_list, model_name="E2f1").uuid # mark job as running CustomJob.set_job_running(db, job_uuid) # upload file BED_DATA = """ someseq\t0\t10\t12.5 someseq2\t20\t30\t4.5 someseq2\t60\t75\t15.5 """.strip() result_uuid = CustomResultData.new_uuid() result = CustomResultData(db, result_uuid, job_uuid, model_name='E2f1', bed_data=BED_DATA) result.save() self.assertEqual(BED_DATA, CustomResultData.bed_file_contents(db, result_uuid).strip()) predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=False, limit=None, offset=None) self.assertEqual(2, len(predictions)) first = predictions[0] self.assertEqual('someseq', first['name']) self.assertEqual(12.5, float(first['max'])) self.assertEqual([{u'start': 0, u'end': 10, u'value': 12.5}], first['values']) self.assertEqual('AAACCCGGGGTT', first['sequence']) second = predictions[1] self.assertEqual('someseq2', second['name']) self.assertEqual(15.5, float(second['max'])) self.assertEqual('AAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTTAAACCCGGGGTT', second['sequence']) predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=True, limit=None, offset=None) self.assertEqual(2, len(predictions)) self.assertEqual(15.5, float(predictions[0]['max'])) self.assertEqual(12.5, float(predictions[1]['max'])) predictions = CustomResultData.get_predictions(db, result_uuid, sort_max_value=True, limit=1, offset=1) self.assertEqual(1, len(predictions)) self.assertEqual(12.5, float(predictions[0]['max'])) # Make sure we can convert predictions to JSON json_version = json.dumps({'data': predictions}) self.assertEqual('{"data', json_version[:6])
def find_custom_results_for_sequence_and_model_name(): """ Find a custom results for a sequence_id. request['sequence_id'] str: sequence id to use when searching custom results request['model_name'] str: optional model name to filter with :return: json response with results array of dict with keys resultId,modelName,sequenceId """ sequence_id = request.args.get('sequence_id') model_name = request.args.get('model_name') if not sequence_id: raise ValueError("Missing required sequence_id field.") custom_result_ids = CustomResultData.find(get_db(), sequence_id, model_name) return make_ok_json_response({'results': custom_result_ids})
def test_bed_file_contents(self, mock_read_database): mock_read_database.side_effect = [ [ # name dna sequence (custom_result_sequence_lookup query response data) ('wild', 'attattattatt'), ('normal', 'catcatcatcat') ], [ # name, start, stop, value (bed_file_contents query response data) ('wild', 4, 8, 0.9), ('normal', 7, 11, 0.4), ] ] bed_contents = CustomResultData.bed_file_contents(db=None, result_id='123') expected = """ wild\t4\t8\t0.9\tttat normal\t7\t11\t0.4\tatca """ self.assertEqual(bed_contents.strip(), expected.strip())
def test_find_with_model_filter(self): self.query_sql = '' self.query_params = '' def fake_read_database(db, sql, params): self.query_sql = sql self.query_params = params return [ ('123', 'ELK1') ] sv_read_database = pred.webserver.customresult.read_database try: pred.webserver.customresult.read_database = fake_read_database uid = '8B9836B5-8E3D-4346-AB12-69DD10313C77' results = CustomResultData.find(db=None, sequence_id=uid, model_name='ELK1') self.assertEqual(1, len(results)) item = results[0] self.assertEqual('123', item['resultId']) self.assertEqual('ELK1', item['modelName']) self.assertEqual('8B9836B5-8E3D-4346-AB12-69DD10313C77', item['sequenceId']) self.assertIn('custom_result.model_name =', self.query_sql) self.assertEqual(['8B9836B5-8E3D-4346-AB12-69DD10313C77', 'ELK1'], self.query_params) finally: pred.webserver.customresult.read_database = sv_read_database
def get_custom_result_raw_data(result_id): bed_file_contents = CustomResultData.bed_file_contents(get_db(), result_id) def gen(): yield bed_file_contents return download_file_response("data.bed", gen())
def test_last_page_query_and_params(self): query, params = CustomResultData.last_page_query_and_params(result_uuid='123-456-789') self.assertIn('select count(*)', query) self.assertIn('from custom_result', query) self.assertIn('custom_result.id = %s', query) self.assertEqual(params, ['123-456-789'])
def delete_old_jobs(cur, hours): for old_job in CustomJob.find_old_jobs(cur, hours): CustomResultData.delete_for_job(cur, old_job.uuid) old_job.delete(cur) return None
def delete_old_jobs(cur, hours): for old_job in CustomJob.find_old_jobs(cur, hours): CustomResultData.delete_for_job(cur, old_job.uuid) old_job.delete(cur) return None
def test_find_with_invalid_uuid(self): with self.assertRaises(ClientException): CustomResultData.find(db=None, sequence_id='', model_name=None)