def from_json(self, is_create, json_obj, sess_obj):
# do base attrs
self.base_from_json(is_create, json_obj)
self.set_attrs_from_json(json_obj, self.LIB_TYPE)
self.set_attrs_from_json(json_obj, self.LIB_INSERT_LEN)
self.set_attrs_from_json(json_obj, self.SAMPLE)
# stage_name == 'upload'
stage_name_list = json_obj[self.SEQUENCESET_ID_ARRAY]
#print "stage_name_list "+str(stage_name_list)
sequence_file_names = []
sequence_file_names = [x['file_name'] for x in stage_name_list if x['stage_name'] == 'upload']
sequence_file_ids = []
sequence_file_ids = [x['id'] for x in stage_name_list if x['stage_name'] == 'upload']
# sequence_file_names = [x['file_name'] for x in stage_name_list if x['stage_name'] == 'upload']
# for dict in stage_name_list:
# if dict['stage_name'] == 'upload':
# stage_name = dict['file_name']
# sequence_file_names.append(stage_name)
# stage_name = json_obj[self.SEQUENCESET_ID_ARRAY]['stage_name']
# self.sequence_set_ids = ",".join(json_obj[self.SEQUENCESET_ID_ARRAY]) # just keep this as a string everywhere until being used
self.sequence_set_ids = ",".join([str(i) for i in sequence_file_ids])
self.sequence_set_names = ",".join([str(i) for i in sequence_file_names])
mobedac_logger.info("library has sequence set ids: " + self.sequence_set_ids)
# print self.sequence_set_ids
# now put the objects into the real child collection
# have to do deletes on all existing child sample associations that are not in the new sample set
# BaseMoBEDAC.update_child_collection(SequenceSetORM, self.sequencesets, json_obj[LibraryORM.SEQUENCESET_IDS], sess_obj)
return self
def handle_POST(self, current_session, *vpath, **params):
""" Create an object
"""
try:
if self.orm_class is SubmissionORM:
submission_obj = self.orm_class({})
mobedac_logger.info("In handle_POST1:")
# AV here when get first send
json_obj = self.json_from_body()
submission_obj.from_json(True, json_obj, current_session)
submission_obj.initialize_for_processing(current_session)
return str(submission_obj.id)
else:
# create the object
new_obj = self.orm_class({})
mobedac_logger.info("In handle_POST2:")
json_obj = self.json_from_body()
new_obj.from_json(True, json_obj, current_session)
current_session.add(new_obj)
current_session.commit()
return new_obj.id
except Exception as e:
self.log_exception("There was an error attempting to service the POST request")
current_session.rollback()
cherrypy.response.status = 500
return "There was an error attempting to service the request"
def json_from_body(self):
cl = cherrypy.request.headers['Content-Length']
rawbody = cherrypy.request.body.read(int(cl))
decoded_rawbody = unidecode(rawbody)
mobedac_logger.info("restresource got JSON string: " + decoded_rawbody)
json_obj = json.loads(decoded_rawbody)
return json_obj
def getFromMOBEDAC(cls, id, source, sess_obj):
conn = None
complete_url = ""
try:
# dev mode?
my_remote_objects_are_local = get_parm("remote_objects_are_local")
if get_parm("remote_objects_are_local").lower() == 'true':
new_obj = cls.get_instance(id, sess_obj)
if new_obj == None:
raise ObjectRetrievalException(
"Unable to retrieve object: " + id + " from db")
else:
headers = {'content-type': 'application/json'}
conn = httplib.HTTPConnection(get_parm("mobedac_host"))
object_path = cls.get_REST_sub_path()
# complete_url = get_parm("mobedac_base_path") + object_path + "/" + id + "?auth=" + get_parm("mobedac_auth_key")
# complete_url = get_parm("mobedac_base_path") + object_path + "/" + id + "?auth=" + get_parm("mobedac_auth_ 38 key")
complete_url = get_parm(
"mobedac_base_path") + "r/" + object_path + "/" + id
if (conn.host == "localhost"
or conn.host == "api.metagenomics.anl.gov"):
complete_url = get_parm(
"mobedac_base_path"
) + object_path + "/" + id + "?auth=" + get_parm(
"mobedac_auth_key")
complete_url += "&template=1"
conn.request("GET", complete_url)
response = conn.getresponse()
data = response.read()
# if all went ok then build an object
if response.status != httplib.OK:
raise ObjectRetrievalException(
"Unable to retrieve object: " + id + " from host: " +
get_parm('mobedac_host') + " url: " + complete_url)
new_obj = cls({})
decoded_data = unidecode(data).strip()
mobedac_logger.info("complete_url: " + complete_url)
mobedac_logger.info("json string for object id_1: " + id +
" json: " + decoded_data)
json_obj = json.loads(decoded_data)
#mobedac_logger.info("json obj: " + str(json_obj))
new_obj.from_json(True, json_obj, sess_obj)
return new_obj
except ObjectRetrievalException as ore:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
raise ore
except Exception as e:
raise ObjectRetrievalException("Unable to retrieve object: " + id +
" from host: " +
get_parm('mobedac_host') +
" url: " + complete_url +
" error: " + str(e))
finally:
if conn != None:
conn.close()
def getFromMOBEDAC(cls, id, source, sess_obj):
conn = None
complete_url = ""
try:
# dev mode?
my_remote_objects_are_local = get_parm("remote_objects_are_local")
if get_parm("remote_objects_are_local").lower() == 'true':
new_obj = cls.get_instance(id, sess_obj)
if new_obj == None:
raise ObjectRetrievalException("Unable to retrieve object: " + id + " from db");
else:
headers = {'content-type': 'application/json'}
conn = httplib.HTTPConnection(get_parm("mobedac_host"))
object_path = cls.get_REST_sub_path()
# complete_url = get_parm("mobedac_base_path") + object_path + "/" + id + "?auth=" + get_parm("mobedac_auth_key")
# complete_url = get_parm("mobedac_base_path") + object_path + "/" + id + "?auth=" + get_parm("mobedac_auth_ 38 key")
complete_url = get_parm("mobedac_base_path") + "r/" + object_path + "/" + id
if (conn.host == "localhost" or conn.host == "api.metagenomics.anl.gov"):
complete_url = get_parm("mobedac_base_path") + object_path + "/" + id + "?auth=" + get_parm("mobedac_auth_key")
complete_url += "&template=1"
conn.request("GET", complete_url)
response = conn.getresponse()
data = response.read()
# if all went ok then build an object
if response.status != httplib.OK:
raise ObjectRetrievalException("Unable to retrieve object: " + id + " from host: " + get_parm('mobedac_host') + " url: " + complete_url)
new_obj = cls({})
decoded_data = unidecode(data).strip()
mobedac_logger.info("complete_url: "+complete_url)
mobedac_logger.info("json string for object id_1: " + id + " json: " + decoded_data)
json_obj = json.loads(decoded_data)
#mobedac_logger.info("json obj: " + str(json_obj))
new_obj.from_json(True, json_obj, sess_obj)
return new_obj
except ObjectRetrievalException as ore:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
raise ore
except Exception as e:
raise ObjectRetrievalException("Unable to retrieve object: " + id + " from host: " + get_parm('mobedac_host') + " url: " + complete_url + " error: " + str(e))
finally:
if conn != None:
conn.close()
def from_json(self, is_create, json_obj, sess_obj):
# do base attrs
self.base_from_json(is_create, json_obj)
self.set_attrs_from_json(json_obj, self.LIB_TYPE)
self.set_attrs_from_json(json_obj, self.LIB_INSERT_LEN)
self.set_attrs_from_json(json_obj, self.SAMPLE)
# stage_name == 'upload'
stage_name_list = json_obj[self.SEQUENCESET_ID_ARRAY]
#print "stage_name_list "+str(stage_name_list)
sequence_file_names = []
sequence_file_names = [
x['file_name'] for x in stage_name_list
if x['stage_name'] == 'upload'
]
sequence_file_ids = []
sequence_file_ids = [
x['id'] for x in stage_name_list if x['stage_name'] == 'upload'
]
# sequence_file_names = [x['file_name'] for x in stage_name_list if x['stage_name'] == 'upload']
# for dict in stage_name_list:
# if dict['stage_name'] == 'upload':
# stage_name = dict['file_name']
# sequence_file_names.append(stage_name)
# stage_name = json_obj[self.SEQUENCESET_ID_ARRAY]['stage_name']
# self.sequence_set_ids = ",".join(json_obj[self.SEQUENCESET_ID_ARRAY]) # just keep this as a string everywhere until being used
self.sequence_set_ids = ",".join([str(i) for i in sequence_file_ids])
self.sequence_set_names = ",".join(
[str(i) for i in sequence_file_names])
mobedac_logger.info("library has sequence set ids: " +
self.sequence_set_ids)
# print self.sequence_set_ids
# now put the objects into the real child collection
# have to do deletes on all existing child sample associations that are not in the new sample set
# BaseMoBEDAC.update_child_collection(SequenceSetORM, self.sequencesets, json_obj[LibraryORM.SEQUENCESET_IDS], sess_obj)
return self
def initialize_for_processing(self, sess_obj):
# need to call back to mobedac and figure out all the other objects etc
libraries = self.library_ids
# now loop and gather all info
project_hash = {}
sample_hash = {}
library_hash = {}
detail_objs = []
for lib_id in libraries:
try:
mobedac_logger.info("\nRetrieving remote library: " + str(lib_id))
library_hash[lib_id] = curr_library = LibraryORM.getFromMOBEDAC(lib_id, None, sess_obj)
except Exception as e:
# some kind of error
raise SubmissionException("There was an error retrieving library: " + str(lib_id) + " error: " + str(e))
mobedac_logger.info("sample: " + str(curr_library.sample))
sample_id = str(curr_library.sample)
mobedac_logger.info("file_name(s) " + str(curr_library.sequence_set_names))
#
# # THIS DOESN'T WORK 2013-3-27
# try:
# mobedac_logger.info("Retrieving remote sample: " + str(sample_id));
# # getFromMOBEDAC is in basmobedac.py
# sample_hash[sample_id] = curr_sample = SampleORM.getFromMOBEDAC(sample_id, None, sess_obj)
# except Exception as e:
# # some kind of error
# raise SubmissionException("There was an error retrieving sample: " + str(sample_id) + " error: " + str(e))
#
# project_id = curr_sample.project
# try:
# mobedac_logger.info("Retrieving remote project: " + str(project_id));
# project_hash[project_id] = curr_project = ProjectORM.getFromMOBEDAC(project_id, None, sess_obj)
# mobedac_logger.info("done Retrieving remote project: " + str(project_id));
# except Exception as e:
# # some kind of error
# raise SubmissionException("There was an error retrieving project: " + str(project_id) + " error: " + str(e))
# do some sanity check/validation on the library and project information
# mobedac_logger.info("library domain: " + curr_library.get_domain())
# mobedac_logger.info("library run_key: " + curr_library.get_run_key())
# mobedac_logger.info("library region: " + curr_library.get_region())
# mobedac_logger.info("project metadatastr: " + curr_project.mbd_metadata)
# if not(curr_library.get_run_key()):
# raise SubmissionException("The library: " + str(lib_id) + " is missing a run key")
# if not(curr_library.get_domain()):
# raise SubmissionException("The library: " + str(lib_id) + " is missing a domain")
# if not(curr_library.get_region()):
# raise SubmissionException("The library: " + str(lib_id) + " is missing a region")
# #if not(curr_project.get_metadata_json()['vamps_id']):
# raise SubmissionException("The project: " + str(lib_id) + " is missing a vamps_id")
# check the primers
# primers = curr_library.get_primers()
# if only 1 primer and it isn't a BOTH direction then complain
# forward_found = False
# reverse_found = False
mobedac_logger.info("done Retrieving remote objects")
# for primer in primers:
# dir = primer['direction'].lower()
# if dir == 'f':
# forward_found = True
# elif dir == 'r':
# reverse_found = True
# if not(forward_found) or not(reverse_found):
# raise SubmissionException("You must supply at least 1 forward primer and 1 reverse primer.")
# mobedac_logger.info("done with primers");
# now what will the official project name be in vamps for this library?
# curr_library_domain = curr_library.get_domain()
# curr_library_region = curr_library.get_region()
# domain_region_suffix = '_' + curr_library_domain[0].upper() + curr_library_region.lower()
# vamps_project_name = curr_project.get_metadata_json()['vamps_id']['value'] + domain_region_suffix
# self.set_attrs_from_json(json_obj, self.SAMPLE)
## stage_name == 'upload'
# stage_name_list = json_obj[self.SEQUENCESET_ID_ARRAY]
# sequence_file_names = []
# sequence_file_names = [x['file_name'] for x in stage_name_list if x['stage_name'] == 'upload']
#
mobedac_logger.info("preparing the submission detail object")
# now create the submission details objects
new_details = SubmissionDetailsORM(None)
# new_details.project_id = project_id
new_details.project_id = "XX_ID"
vamps_project_name = "MOBEDAC_TEST_" + new_details.project_id
new_details.vamps_project_name = vamps_project_name
new_details.library_id = lib_id
new_details.sample_id = sample_id
# new_details.region = curr_library_region.lower()
new_details.region = ""
new_details.sequence_file_url = curr_library.sequence_set_names
detail_objs.append(new_details)
# mobedac uses periods...we don't want them
dataset_name = str(curr_library.id).replace(".", "_")
new_details.vamps_dataset_name = dataset_name
new_details.sequenceset_id = curr_library.get_sequence_set_id_array()[0] # just take the first one for now
mobedac_logger.info("new_details has sequence set id: " + new_details.sequenceset_id)
# this is the starting value for next_action....call mobedac and download the sequence data
new_details.next_action = SubmissionDetailsORM.ACTION_DOWNLOAD
mobedac_logger.info("DONE preparing the submission detail object")
try:
# make sure to write this submission out before anything
sess_obj.add(self)
mobedac_logger.info("committing all submission detail objects")
sess_obj.commit()
# now set the submission id into each of these submission_detail objects since everything worked out ok
# because only now do we have a submission id (self)
for new_detail in detail_objs:
new_detail.submission_id = self.id
sess_obj.add(new_detail)
sess_obj.commit()
mobedac_logger.info("committing submission object got id: " + str(self.id))
except Exception as e:
raise SubmissionException("There was an error during submission: %s" % e)
def __init__(self, value):
super(SubmissionException, self).__init__(value)
self.value = value
mobedac_logger.info(value)
mobedac_logger.exception(value)
def initialize_for_processing(self, sess_obj):
# need to call back to mobedac and figure out all the other objects etc
libraries = self.library_ids
# now loop and gather all info
project_hash = {}
sample_hash = {}
library_hash = {}
detail_objs = []
for lib_id in libraries:
try:
mobedac_logger.info("\nRetrieving remote library: " + str(lib_id));
library_hash[lib_id] = curr_library = LibraryORM.getFromMOBEDAC(lib_id, None, sess_obj)
except Exception as e:
# some kind of error
raise SubmissionException("There was an error retrieving library: " + str(lib_id) + " error: " + str(e))
mobedac_logger.info("sample: " + str(curr_library.sample) );
sample_id = str(curr_library.sample)
mobedac_logger.info('file_name(s) '+str(curr_library.sequence_set_names))
#
# # THIS DOESN'T WORK 2013-3-27
# try:
# mobedac_logger.info("Retrieving remote sample: " + str(sample_id));
# # getFromMOBEDAC is in basmobedac.py
# sample_hash[sample_id] = curr_sample = SampleORM.getFromMOBEDAC(sample_id, None, sess_obj)
# except Exception as e:
# # some kind of error
# raise SubmissionException("There was an error retrieving sample: " + str(sample_id) + " error: " + str(e))
#
# project_id = curr_sample.project
# try:
# mobedac_logger.info("Retrieving remote project: " + str(project_id));
# project_hash[project_id] = curr_project = ProjectORM.getFromMOBEDAC(project_id, None, sess_obj)
# mobedac_logger.info("done Retrieving remote project: " + str(project_id));
# except Exception as e:
# # some kind of error
# raise SubmissionException("There was an error retrieving project: " + str(project_id) + " error: " + str(e))
# do some sanity check/validation on the library and project information
#mobedac_logger.info("library domain: " + curr_library.get_domain())
#mobedac_logger.info("library run_key: " + curr_library.get_run_key())
#mobedac_logger.info("library region: " + curr_library.get_region())
#mobedac_logger.info("project metadatastr: " + curr_project.mbd_metadata)
# if not(curr_library.get_run_key()):
# raise SubmissionException("The library: " + str(lib_id) + " is missing a run key")
# if not(curr_library.get_domain()):
# raise SubmissionException("The library: " + str(lib_id) + " is missing a domain")
# if not(curr_library.get_region()):
# raise SubmissionException("The library: " + str(lib_id) + " is missing a region")
# #if not(curr_project.get_metadata_json()['vamps_id']):
# raise SubmissionException("The project: " + str(lib_id) + " is missing a vamps_id")
#check the primers
# primers = curr_library.get_primers()
# if only 1 primer and it isn't a BOTH direction then complain
# forward_found = False
# reverse_found = False
mobedac_logger.info("done Retrieving remote objects");
# for primer in primers:
# dir = primer['direction'].lower()
# if dir == 'f':
# forward_found = True
# elif dir == 'r':
# reverse_found = True
# if not(forward_found) or not(reverse_found):
# raise SubmissionException("You must supply at least 1 forward primer and 1 reverse primer.")
# mobedac_logger.info("done with primers");
# now what will the official project name be in vamps for this library?
# curr_library_domain = curr_library.get_domain()
# curr_library_region = curr_library.get_region()
# domain_region_suffix = '_' + curr_library_domain[0].upper() + curr_library_region.lower()
# vamps_project_name = curr_project.get_metadata_json()['vamps_id']['value'] + domain_region_suffix
# self.set_attrs_from_json(json_obj, self.SAMPLE)
## stage_name == 'upload'
# stage_name_list = json_obj[self.SEQUENCESET_ID_ARRAY]
# sequence_file_names = []
# sequence_file_names = [x['file_name'] for x in stage_name_list if x['stage_name'] == 'upload']
#
mobedac_logger.info("preparing the submission detail object");
# now create the submission details objects
new_details = SubmissionDetailsORM(None)
#new_details.project_id = project_id
new_details.project_id = "XX_ID"
vamps_project_name = "MOBEDAC_TEST_"+new_details.project_id
new_details.vamps_project_name = vamps_project_name
new_details.library_id = lib_id
new_details.sample_id = sample_id
#new_details.region = curr_library_region.lower()
new_details.region = ''
new_details.sequence_file_url = curr_library.sequence_set_names
detail_objs.append(new_details)
# mobedac uses periods...we don't want them
dataset_name = str(curr_library.id).replace('.', '_')
new_details.vamps_dataset_name = dataset_name
new_details.sequenceset_id = curr_library.get_sequence_set_id_array()[0] # just take the first one for now
mobedac_logger.info("new_details has sequence set id: " + new_details.sequenceset_id)
# this is the starting value for next_action....call mobedac and download the sequence data
new_details.next_action = SubmissionDetailsORM.ACTION_DOWNLOAD
mobedac_logger.info("DONE preparing the submission detail object");
try:
# make sure to write this submission out before anything
sess_obj.add(self)
mobedac_logger.info("committing all submission detail objects");
sess_obj.commit()
# now set the submission id into each of these submission_detail objects since everything worked out ok
# because only now do we have a submission id (self)
for new_detail in detail_objs:
new_detail.submission_id = self.id
sess_obj.add(new_detail)
sess_obj.commit()
mobedac_logger.info("committing submission object got id: " + str(self.id));
except Exception as e:
raise SubmissionException("There was an error during submission: %s" % e)