def from_json(self, is_create, json_obj, sess_obj): # do base attrs self.base_from_json(is_create, json_obj) self.set_attrs_from_json(json_obj, self.LIB_TYPE) self.set_attrs_from_json(json_obj, self.LIB_INSERT_LEN) self.set_attrs_from_json(json_obj, self.SAMPLE) # stage_name == 'upload' stage_name_list = json_obj[self.SEQUENCESET_ID_ARRAY] #print "stage_name_list "+str(stage_name_list) sequence_file_names = [] sequence_file_names = [x['file_name'] for x in stage_name_list if x['stage_name'] == 'upload'] sequence_file_ids = [] sequence_file_ids = [x['id'] for x in stage_name_list if x['stage_name'] == 'upload'] # sequence_file_names = [x['file_name'] for x in stage_name_list if x['stage_name'] == 'upload'] # for dict in stage_name_list: # if dict['stage_name'] == 'upload': # stage_name = dict['file_name'] # sequence_file_names.append(stage_name) # stage_name = json_obj[self.SEQUENCESET_ID_ARRAY]['stage_name'] # self.sequence_set_ids = ",".join(json_obj[self.SEQUENCESET_ID_ARRAY]) # just keep this as a string everywhere until being used self.sequence_set_ids = ",".join([str(i) for i in sequence_file_ids]) self.sequence_set_names = ",".join([str(i) for i in sequence_file_names]) mobedac_logger.info("library has sequence set ids: " + self.sequence_set_ids) # print self.sequence_set_ids # now put the objects into the real child collection # have to do deletes on all existing child sample associations that are not in the new sample set # BaseMoBEDAC.update_child_collection(SequenceSetORM, self.sequencesets, json_obj[LibraryORM.SEQUENCESET_IDS], sess_obj) return self
def handle_POST(self, current_session, *vpath, **params): """ Create an object """ try: if self.orm_class is SubmissionORM: submission_obj = self.orm_class({}) mobedac_logger.info("In handle_POST1:") # AV here when get first send json_obj = self.json_from_body() submission_obj.from_json(True, json_obj, current_session) submission_obj.initialize_for_processing(current_session) return str(submission_obj.id) else: # create the object new_obj = self.orm_class({}) mobedac_logger.info("In handle_POST2:") json_obj = self.json_from_body() new_obj.from_json(True, json_obj, current_session) current_session.add(new_obj) current_session.commit() return new_obj.id except Exception as e: self.log_exception("There was an error attempting to service the POST request") current_session.rollback() cherrypy.response.status = 500 return "There was an error attempting to service the request"
def json_from_body(self): cl = cherrypy.request.headers['Content-Length'] rawbody = cherrypy.request.body.read(int(cl)) decoded_rawbody = unidecode(rawbody) mobedac_logger.info("restresource got JSON string: " + decoded_rawbody) json_obj = json.loads(decoded_rawbody) return json_obj
def getFromMOBEDAC(cls, id, source, sess_obj): conn = None complete_url = "" try: # dev mode? my_remote_objects_are_local = get_parm("remote_objects_are_local") if get_parm("remote_objects_are_local").lower() == 'true': new_obj = cls.get_instance(id, sess_obj) if new_obj == None: raise ObjectRetrievalException( "Unable to retrieve object: " + id + " from db") else: headers = {'content-type': 'application/json'} conn = httplib.HTTPConnection(get_parm("mobedac_host")) object_path = cls.get_REST_sub_path() # complete_url = get_parm("mobedac_base_path") + object_path + "/" + id + "?auth=" + get_parm("mobedac_auth_key") # complete_url = get_parm("mobedac_base_path") + object_path + "/" + id + "?auth=" + get_parm("mobedac_auth_ 38 key") complete_url = get_parm( "mobedac_base_path") + "r/" + object_path + "/" + id if (conn.host == "localhost" or conn.host == "api.metagenomics.anl.gov"): complete_url = get_parm( "mobedac_base_path" ) + object_path + "/" + id + "?auth=" + get_parm( "mobedac_auth_key") complete_url += "&template=1" conn.request("GET", complete_url) response = conn.getresponse() data = response.read() # if all went ok then build an object if response.status != httplib.OK: raise ObjectRetrievalException( "Unable to retrieve object: " + id + " from host: " + get_parm('mobedac_host') + " url: " + complete_url) new_obj = cls({}) decoded_data = unidecode(data).strip() mobedac_logger.info("complete_url: " + complete_url) mobedac_logger.info("json string for object id_1: " + id + " json: " + decoded_data) json_obj = json.loads(decoded_data) #mobedac_logger.info("json obj: " + str(json_obj)) new_obj.from_json(True, json_obj, sess_obj) return new_obj except ObjectRetrievalException as ore: exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback) raise ore except Exception as e: raise ObjectRetrievalException("Unable to retrieve object: " + id + " from host: " + get_parm('mobedac_host') + " url: " + complete_url + " error: " + str(e)) finally: if conn != None: conn.close()
def getFromMOBEDAC(cls, id, source, sess_obj): conn = None complete_url = "" try: # dev mode? my_remote_objects_are_local = get_parm("remote_objects_are_local") if get_parm("remote_objects_are_local").lower() == 'true': new_obj = cls.get_instance(id, sess_obj) if new_obj == None: raise ObjectRetrievalException("Unable to retrieve object: " + id + " from db"); else: headers = {'content-type': 'application/json'} conn = httplib.HTTPConnection(get_parm("mobedac_host")) object_path = cls.get_REST_sub_path() # complete_url = get_parm("mobedac_base_path") + object_path + "/" + id + "?auth=" + get_parm("mobedac_auth_key") # complete_url = get_parm("mobedac_base_path") + object_path + "/" + id + "?auth=" + get_parm("mobedac_auth_ 38 key") complete_url = get_parm("mobedac_base_path") + "r/" + object_path + "/" + id if (conn.host == "localhost" or conn.host == "api.metagenomics.anl.gov"): complete_url = get_parm("mobedac_base_path") + object_path + "/" + id + "?auth=" + get_parm("mobedac_auth_key") complete_url += "&template=1" conn.request("GET", complete_url) response = conn.getresponse() data = response.read() # if all went ok then build an object if response.status != httplib.OK: raise ObjectRetrievalException("Unable to retrieve object: " + id + " from host: " + get_parm('mobedac_host') + " url: " + complete_url) new_obj = cls({}) decoded_data = unidecode(data).strip() mobedac_logger.info("complete_url: "+complete_url) mobedac_logger.info("json string for object id_1: " + id + " json: " + decoded_data) json_obj = json.loads(decoded_data) #mobedac_logger.info("json obj: " + str(json_obj)) new_obj.from_json(True, json_obj, sess_obj) return new_obj except ObjectRetrievalException as ore: exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback) raise ore except Exception as e: raise ObjectRetrievalException("Unable to retrieve object: " + id + " from host: " + get_parm('mobedac_host') + " url: " + complete_url + " error: " + str(e)) finally: if conn != None: conn.close()
def from_json(self, is_create, json_obj, sess_obj): # do base attrs self.base_from_json(is_create, json_obj) self.set_attrs_from_json(json_obj, self.LIB_TYPE) self.set_attrs_from_json(json_obj, self.LIB_INSERT_LEN) self.set_attrs_from_json(json_obj, self.SAMPLE) # stage_name == 'upload' stage_name_list = json_obj[self.SEQUENCESET_ID_ARRAY] #print "stage_name_list "+str(stage_name_list) sequence_file_names = [] sequence_file_names = [ x['file_name'] for x in stage_name_list if x['stage_name'] == 'upload' ] sequence_file_ids = [] sequence_file_ids = [ x['id'] for x in stage_name_list if x['stage_name'] == 'upload' ] # sequence_file_names = [x['file_name'] for x in stage_name_list if x['stage_name'] == 'upload'] # for dict in stage_name_list: # if dict['stage_name'] == 'upload': # stage_name = dict['file_name'] # sequence_file_names.append(stage_name) # stage_name = json_obj[self.SEQUENCESET_ID_ARRAY]['stage_name'] # self.sequence_set_ids = ",".join(json_obj[self.SEQUENCESET_ID_ARRAY]) # just keep this as a string everywhere until being used self.sequence_set_ids = ",".join([str(i) for i in sequence_file_ids]) self.sequence_set_names = ",".join( [str(i) for i in sequence_file_names]) mobedac_logger.info("library has sequence set ids: " + self.sequence_set_ids) # print self.sequence_set_ids # now put the objects into the real child collection # have to do deletes on all existing child sample associations that are not in the new sample set # BaseMoBEDAC.update_child_collection(SequenceSetORM, self.sequencesets, json_obj[LibraryORM.SEQUENCESET_IDS], sess_obj) return self
def initialize_for_processing(self, sess_obj): # need to call back to mobedac and figure out all the other objects etc libraries = self.library_ids # now loop and gather all info project_hash = {} sample_hash = {} library_hash = {} detail_objs = [] for lib_id in libraries: try: mobedac_logger.info("\nRetrieving remote library: " + str(lib_id)) library_hash[lib_id] = curr_library = LibraryORM.getFromMOBEDAC(lib_id, None, sess_obj) except Exception as e: # some kind of error raise SubmissionException("There was an error retrieving library: " + str(lib_id) + " error: " + str(e)) mobedac_logger.info("sample: " + str(curr_library.sample)) sample_id = str(curr_library.sample) mobedac_logger.info("file_name(s) " + str(curr_library.sequence_set_names)) # # # THIS DOESN'T WORK 2013-3-27 # try: # mobedac_logger.info("Retrieving remote sample: " + str(sample_id)); # # getFromMOBEDAC is in basmobedac.py # sample_hash[sample_id] = curr_sample = SampleORM.getFromMOBEDAC(sample_id, None, sess_obj) # except Exception as e: # # some kind of error # raise SubmissionException("There was an error retrieving sample: " + str(sample_id) + " error: " + str(e)) # # project_id = curr_sample.project # try: # mobedac_logger.info("Retrieving remote project: " + str(project_id)); # project_hash[project_id] = curr_project = ProjectORM.getFromMOBEDAC(project_id, None, sess_obj) # mobedac_logger.info("done Retrieving remote project: " + str(project_id)); # except Exception as e: # # some kind of error # raise SubmissionException("There was an error retrieving project: " + str(project_id) + " error: " + str(e)) # do some sanity check/validation on the library and project information # mobedac_logger.info("library domain: " + curr_library.get_domain()) # mobedac_logger.info("library run_key: " + curr_library.get_run_key()) # mobedac_logger.info("library region: " + curr_library.get_region()) # mobedac_logger.info("project metadatastr: " + curr_project.mbd_metadata) # if not(curr_library.get_run_key()): # raise SubmissionException("The library: " + str(lib_id) + " is missing a run key") # if not(curr_library.get_domain()): # raise SubmissionException("The library: " + str(lib_id) + " is missing a domain") # if not(curr_library.get_region()): # raise SubmissionException("The library: " + str(lib_id) + " is missing a region") # #if not(curr_project.get_metadata_json()['vamps_id']): # raise SubmissionException("The project: " + str(lib_id) + " is missing a vamps_id") # check the primers # primers = curr_library.get_primers() # if only 1 primer and it isn't a BOTH direction then complain # forward_found = False # reverse_found = False mobedac_logger.info("done Retrieving remote objects") # for primer in primers: # dir = primer['direction'].lower() # if dir == 'f': # forward_found = True # elif dir == 'r': # reverse_found = True # if not(forward_found) or not(reverse_found): # raise SubmissionException("You must supply at least 1 forward primer and 1 reverse primer.") # mobedac_logger.info("done with primers"); # now what will the official project name be in vamps for this library? # curr_library_domain = curr_library.get_domain() # curr_library_region = curr_library.get_region() # domain_region_suffix = '_' + curr_library_domain[0].upper() + curr_library_region.lower() # vamps_project_name = curr_project.get_metadata_json()['vamps_id']['value'] + domain_region_suffix # self.set_attrs_from_json(json_obj, self.SAMPLE) ## stage_name == 'upload' # stage_name_list = json_obj[self.SEQUENCESET_ID_ARRAY] # sequence_file_names = [] # sequence_file_names = [x['file_name'] for x in stage_name_list if x['stage_name'] == 'upload'] # mobedac_logger.info("preparing the submission detail object") # now create the submission details objects new_details = SubmissionDetailsORM(None) # new_details.project_id = project_id new_details.project_id = "XX_ID" vamps_project_name = "MOBEDAC_TEST_" + new_details.project_id new_details.vamps_project_name = vamps_project_name new_details.library_id = lib_id new_details.sample_id = sample_id # new_details.region = curr_library_region.lower() new_details.region = "" new_details.sequence_file_url = curr_library.sequence_set_names detail_objs.append(new_details) # mobedac uses periods...we don't want them dataset_name = str(curr_library.id).replace(".", "_") new_details.vamps_dataset_name = dataset_name new_details.sequenceset_id = curr_library.get_sequence_set_id_array()[0] # just take the first one for now mobedac_logger.info("new_details has sequence set id: " + new_details.sequenceset_id) # this is the starting value for next_action....call mobedac and download the sequence data new_details.next_action = SubmissionDetailsORM.ACTION_DOWNLOAD mobedac_logger.info("DONE preparing the submission detail object") try: # make sure to write this submission out before anything sess_obj.add(self) mobedac_logger.info("committing all submission detail objects") sess_obj.commit() # now set the submission id into each of these submission_detail objects since everything worked out ok # because only now do we have a submission id (self) for new_detail in detail_objs: new_detail.submission_id = self.id sess_obj.add(new_detail) sess_obj.commit() mobedac_logger.info("committing submission object got id: " + str(self.id)) except Exception as e: raise SubmissionException("There was an error during submission: %s" % e)
def __init__(self, value): super(SubmissionException, self).__init__(value) self.value = value mobedac_logger.info(value) mobedac_logger.exception(value)
def initialize_for_processing(self, sess_obj): # need to call back to mobedac and figure out all the other objects etc libraries = self.library_ids # now loop and gather all info project_hash = {} sample_hash = {} library_hash = {} detail_objs = [] for lib_id in libraries: try: mobedac_logger.info("\nRetrieving remote library: " + str(lib_id)); library_hash[lib_id] = curr_library = LibraryORM.getFromMOBEDAC(lib_id, None, sess_obj) except Exception as e: # some kind of error raise SubmissionException("There was an error retrieving library: " + str(lib_id) + " error: " + str(e)) mobedac_logger.info("sample: " + str(curr_library.sample) ); sample_id = str(curr_library.sample) mobedac_logger.info('file_name(s) '+str(curr_library.sequence_set_names)) # # # THIS DOESN'T WORK 2013-3-27 # try: # mobedac_logger.info("Retrieving remote sample: " + str(sample_id)); # # getFromMOBEDAC is in basmobedac.py # sample_hash[sample_id] = curr_sample = SampleORM.getFromMOBEDAC(sample_id, None, sess_obj) # except Exception as e: # # some kind of error # raise SubmissionException("There was an error retrieving sample: " + str(sample_id) + " error: " + str(e)) # # project_id = curr_sample.project # try: # mobedac_logger.info("Retrieving remote project: " + str(project_id)); # project_hash[project_id] = curr_project = ProjectORM.getFromMOBEDAC(project_id, None, sess_obj) # mobedac_logger.info("done Retrieving remote project: " + str(project_id)); # except Exception as e: # # some kind of error # raise SubmissionException("There was an error retrieving project: " + str(project_id) + " error: " + str(e)) # do some sanity check/validation on the library and project information #mobedac_logger.info("library domain: " + curr_library.get_domain()) #mobedac_logger.info("library run_key: " + curr_library.get_run_key()) #mobedac_logger.info("library region: " + curr_library.get_region()) #mobedac_logger.info("project metadatastr: " + curr_project.mbd_metadata) # if not(curr_library.get_run_key()): # raise SubmissionException("The library: " + str(lib_id) + " is missing a run key") # if not(curr_library.get_domain()): # raise SubmissionException("The library: " + str(lib_id) + " is missing a domain") # if not(curr_library.get_region()): # raise SubmissionException("The library: " + str(lib_id) + " is missing a region") # #if not(curr_project.get_metadata_json()['vamps_id']): # raise SubmissionException("The project: " + str(lib_id) + " is missing a vamps_id") #check the primers # primers = curr_library.get_primers() # if only 1 primer and it isn't a BOTH direction then complain # forward_found = False # reverse_found = False mobedac_logger.info("done Retrieving remote objects"); # for primer in primers: # dir = primer['direction'].lower() # if dir == 'f': # forward_found = True # elif dir == 'r': # reverse_found = True # if not(forward_found) or not(reverse_found): # raise SubmissionException("You must supply at least 1 forward primer and 1 reverse primer.") # mobedac_logger.info("done with primers"); # now what will the official project name be in vamps for this library? # curr_library_domain = curr_library.get_domain() # curr_library_region = curr_library.get_region() # domain_region_suffix = '_' + curr_library_domain[0].upper() + curr_library_region.lower() # vamps_project_name = curr_project.get_metadata_json()['vamps_id']['value'] + domain_region_suffix # self.set_attrs_from_json(json_obj, self.SAMPLE) ## stage_name == 'upload' # stage_name_list = json_obj[self.SEQUENCESET_ID_ARRAY] # sequence_file_names = [] # sequence_file_names = [x['file_name'] for x in stage_name_list if x['stage_name'] == 'upload'] # mobedac_logger.info("preparing the submission detail object"); # now create the submission details objects new_details = SubmissionDetailsORM(None) #new_details.project_id = project_id new_details.project_id = "XX_ID" vamps_project_name = "MOBEDAC_TEST_"+new_details.project_id new_details.vamps_project_name = vamps_project_name new_details.library_id = lib_id new_details.sample_id = sample_id #new_details.region = curr_library_region.lower() new_details.region = '' new_details.sequence_file_url = curr_library.sequence_set_names detail_objs.append(new_details) # mobedac uses periods...we don't want them dataset_name = str(curr_library.id).replace('.', '_') new_details.vamps_dataset_name = dataset_name new_details.sequenceset_id = curr_library.get_sequence_set_id_array()[0] # just take the first one for now mobedac_logger.info("new_details has sequence set id: " + new_details.sequenceset_id) # this is the starting value for next_action....call mobedac and download the sequence data new_details.next_action = SubmissionDetailsORM.ACTION_DOWNLOAD mobedac_logger.info("DONE preparing the submission detail object"); try: # make sure to write this submission out before anything sess_obj.add(self) mobedac_logger.info("committing all submission detail objects"); sess_obj.commit() # now set the submission id into each of these submission_detail objects since everything worked out ok # because only now do we have a submission id (self) for new_detail in detail_objs: new_detail.submission_id = self.id sess_obj.add(new_detail) sess_obj.commit() mobedac_logger.info("committing submission object got id: " + str(self.id)); except Exception as e: raise SubmissionException("There was an error during submission: %s" % e)