def adminServletGetMessage(request, response): try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** message_type = request.form.get("message_type") if (message_type != "starting_message"): logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') UserSessionManager().isValidUser(userID, sessionToken) #**************************************************************** # Step 1.GET THE MESSAGES FROM THE DATABASE #**************************************************************** daoInstance = MessageDAO() matchedMessages = daoInstance.findAll( otherParams={"message_type": message_type}) daoInstance.closeConnection() response.setContent({"success": True, "messageList": matchedMessages}) except Exception as ex: handleException(response, ex, __file__, "adminServletGetMessage") finally: return response
def adminServletDeleteMessage(request, response, message_type=None): try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') userName = request.cookies.get('userName') UserSessionManager().isValidAdminUser(userID, userName, sessionToken) #**************************************************************** # Step 1.GET THE MESSAGES FROM THE DATABASE #**************************************************************** if message_type == None: message_type = request.form.get("message_type") daoInstance = MessageDAO() daoInstance.removeAll(otherParams={"message_type": message_type}) daoInstance.closeConnection() response.setContent({"success": True}) except Exception as ex: handleException(response, ex, __file__, "adminServletDeleteMessage") finally: return response
def userManagementNewNoLoginSession(request, response): try: #**************************************************************** # Step 1. INITIALIZE EMPTY USER INSTANCE #**************************************************************** logging.info("STEP1 - START 'NO LOGIN' session...") initializeUserDirectories(None) # sessionToken = UserSessionManager().registerNewUser("" + str(userID)) response.setContent({ "success": True, "userID": None, "userName": None, "sessionToken": None, "p": None }) except Exception as ex: handleException(response, ex, __file__, "userManagementNewNoLoginSession") finally: return response
def adminCleanDatabases(request, response): """ This function... @param {Request} request, the request object @param {Response} response, the response object """ try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') userName = request.cookies.get('userName') UserSessionManager().isValidAdminUser(userID, userName, sessionToken) #**************************************************************** # Step 1. RUN THE SCRIPT #**************************************************************** from src.AdminTools.scripts.clean_databases import cleanDatabases as clean_databases_routine clean_databases_routine(force=True) response.setContent({"success": True}) except Exception as ex: handleException(response, ex, __file__, "cleanDatabases") finally: return response
def userManagementSignOut(request, response): userInstance = None daoInstance = None try: #**************************************************************** # Step 1.READ PARAMS #**************************************************************** logging.info("STEP1 - READ PARAMS...") formFields = request.form userID = formFields.get("userID") sessionToken = formFields.get("sessionToken") #**************************************************************** # Step 2. CLOSE SESSION #**************************************************************** logging.info("STEP2 - REMOVING USER..") UserSessionManager().removeUser(userID, sessionToken) response.setContent({"success": True}) except Exception as ex: handleException(response, ex, __file__, "userManagementSignOut") finally: if (daoInstance != None): daoInstance.closeConnection() return response
def pathwayAcquisitionSaveImage(request, response): jobID = "" try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** # logging.info("STEP0 - CHECK IF VALID USER....") # userID = request.cookies.get('userID') # sessionToken = request.cookies.get('sessionToken') # UserSessionManager().isValidUser(userID, sessionToken) jobID = request.form.get("jobID") jobInstance = JobInformationManager().loadJobInstance(jobID) svgData = request.form.get("svgCode") fileName = "paintomics_" + request.form.get("fileName").replace( " ", "_") + "_" + jobID fileFormat = request.form.get("format") userID = jobInstance.getUserID() userDirID = userID if userID is not None else "nologin" path = CLIENT_TMP_DIR + userDirID + jobInstance.getOutputDir().replace( CLIENT_TMP_DIR + userDirID, "") if (fileFormat == "png"): def createImage(svgData): svg = rsvg.Handle(data=svgData) width = svg.props.width height = svg.props.height surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, width, height) context = cairo.Context(surface) open(path + fileName + "." + fileFormat, 'a').close() svg.render_cairo(context) surface.write_to_png(path + fileName + "." + fileFormat) try: logging.info("TRYING...") createImage(svgData=svgData) except Exception as ex: logging.info("TRYING again...") createImage(svgData=svgData) elif (fileFormat == "svg"): file_ = open(path + fileName + "." + fileFormat, 'w') file_.write(svgData) file_.close() path = "/get_cluster_image/" + jobID + "/output/" response.setContent({ "success": True, "filepath": path + fileName + "." + fileFormat }) except Exception as ex: handleException(response, ex, __file__, "pathwayAcquisitionSaveImage") finally: return response
def pathwayAcquisitionMetagenes_PART1(REQUEST, RESPONSE, QUEUE_INSTANCE, JOB_ID, ROOT_DIRECTORY): # **************************************************************** # Step 0. VARIABLE DECLARATION # The following variables are defined: # - jobInstance: instance of the PathwayAcquisitionJob class. # Contains all the information for the current job. # - userID: the ID for the user # **************************************************************** jobInstance = None userID = None try: # **************************************************************** # Step 1. CHECK IF VALID USER SESSION # **************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = REQUEST.cookies.get('userID') sessionToken = REQUEST.cookies.get('sessionToken') UserSessionManager().isValidUser(userID, sessionToken) # **************************************************************** # Step 2. LOAD THE JOB INSTANCE AND RETRIEVE FORM INFO # **************************************************************** savedJobID = REQUEST.form.get("jobID") savedJobInstance = JobInformationManager().loadJobInstance(savedJobID) if savedJobInstance.getReadOnly() and str( savedJobInstance.getUserID()) != str(userID): raise Exception("Invalid user for the job generating metagenes.") omicName = REQUEST.form.get("omic") clusterNumber = int(REQUEST.form.get("number")) # Make sure the number of clusters is inside [1, 20] clusterNumber = 1 if clusterNumber < 1 else 20 if clusterNumber > 20 else clusterNumber # ************************************************************************ # Step 4. Queue job # ************************************************************************ QUEUE_INSTANCE.enqueue(fn=pathwayAcquisitionMetagenes_PART2, args=(ROOT_DIRECTORY, userID, savedJobInstance, omicName, clusterNumber, RESPONSE), timeout=600, job_id=JOB_ID) # ************************************************************************ # Step 5. Return the Job ID # ************************************************************************ RESPONSE.setContent({"success": True, "jobID": JOB_ID}) except Exception as ex: handleException(RESPONSE, ex, __file__, "pathwayAcquisitionMetagenes_PART1", userID=userID) finally: return RESPONSE
def adminServletDeleteUser(request, response, toDeleteUserID): """ This function... @param {Request} request, the request object @param {Response} response, the response object """ try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') userName = request.cookies.get('userName') UserSessionManager().isValidAdminUser(userID, userName, sessionToken) if toDeleteUserID == "0": response.setContent({"success": False}) else: jobDAOInstance = JobDAO() filesDAOInstance = FileDAO() userDAOInstance = UserDAO() logging.info("STEP1 - CLEANING DATA FOR " + toDeleteUserID + "...") #**************************************************************** # Step 1. DELETE ALL JOBS FOR THE USER #**************************************************************** allJobs = jobDAOInstance.findAll( otherParams={"userID": toDeleteUserID}) jobID = "" for jobInstance in allJobs: jobID = jobInstance.getJobID() logging.info("STEP2 - REMOVING " + jobID + " FROM DATABASE...") jobDAOInstance.remove(jobInstance.getJobID(), otherParams={"userID": toDeleteUserID}) #**************************************************************** # Step 3. DELETE ALL FILES FOR THE USER #**************************************************************** logging.info("STEP3 - REMOVING ALL FILES FROM DATABASE...") filesDAOInstance.removeAll(otherParams={"userID": toDeleteUserID}) logging.info("STEP3 - REMOVING ALL FILES FROM USER DIRECTORY...") if os_path.isdir(CLIENT_TMP_DIR + toDeleteUserID): shutil_rmtree(CLIENT_TMP_DIR + toDeleteUserID) #**************************************************************** # Step 4. DELETE THE USER INSTANCE FROM DATABASE #**************************************************************** logging.info("STEP6 - REMOVING ALL FILES FROM DATABASE...") userDAOInstance.remove(int(toDeleteUserID)) response.setContent({"success": True}) except Exception as ex: handleException(response, ex, __file__, "adminServletDeleteUser") finally: return response
def dataManagementGetMyFiles(request, response, DESTINATION_DIR, MAX_CLIENT_SPACE, isReference=False): #VARIABLE DECLARATION fileInstance = None fileInstances = [] daoInstance = None try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') UserSessionManager().isValidUser(userID, sessionToken) if not isReference: DESTINATION_DIR += userID else: userID = "-1" DESTINATION_DIR += "GTF/" #**************************************************************** # Step 1.GET THE LIST OF FILES #**************************************************************** logging.info("STEP1 - GET MY FILE LIST REQUEST RECEIVED") daoInstance = FileDAO() matchedFiles = daoInstance.findAll(otherParams={"userID": userID}) logging.info("STEP1 - GET MY FILE LIST REQUEST RECEIVED...DONE") #**************************************************************** # Step 2.CALCULATE USED SPACE #**************************************************************** logging.info("STEP2 - GET THE CURRENT USED SPACE...") dataSummary = { "usedSpace": dir_total_size(DESTINATION_DIR), "availableSpace": MAX_CLIENT_SPACE } logging.info("STEP2 - GET THE CURRENT USED SPACE...DONE") response.setContent({ "success": True, "fileList": matchedFiles, "dataSummary": dataSummary }) except Exception as ex: handleException(response, ex, __file__, "dataManagementGetMyFiles") finally: if (daoInstance != None): daoInstance.closeConnection() return response
def pathwayAcquisitionTouchJob(request, response): try: jobID = request.form.get("jobID") JobInformationManager().touchAccessDate(jobID) response.setContent({"success": True}) except Exception as ex: handleException(response, ex, __file__, "pathwayAcquisitionTouchJob", jobID=jobID) finally: return response
def pathwayAcquisitionSaveVisualOptions(request, response): #VARIABLE DECLARATION visualOptionsInstance = None jobID = "" userID = "" try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') UserSessionManager().isValidUser(userID, sessionToken) #**************************************************************** # Step 1.GET THE INSTANCE OF visual Options #**************************************************************** visualOptions = request.get_json() jobID = visualOptions.get("jobID") jobInstance = JobInformationManager().loadJobInstance(jobID) if jobInstance.getReadOnly() and str( jobInstance.getUserID()) != str(userID): raise Exception("Invalid user for the job saving visual options") newTimestamp = int(time()) visualOptions["timestamp"] = newTimestamp #************************************************************************ # Step 3. Save the visual Options in the MongoDB #************************************************************************ logging.info("STEP 3 - SAVING VISUAL OPTIONS FOR JOB " + jobID + "...") JobInformationManager().storeVisualOptions(jobID, visualOptions) logging.info("STEP 3 - SAVING VISUAL OPTIONS FOR JOB " + jobID + "...DONE") response.setContent({"success": True, "timestamp": newTimestamp}) except Exception as ex: handleException(response, ex, __file__, "pathwayAcquisitionSaveVisualOptions", userID=userID) finally: return response
def adminServletSystemInformation(request, response): """ This function... @param {Request} request, the request object @param {Response} response, the response object """ try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') userName = request.cookies.get('userName') UserSessionManager().isValidAdminUser(userID, userName, sessionToken) disk_use = [] try: df = subprocess.Popen(["df", "-h"], stdout=subprocess.PIPE) output = df.communicate()[0] output = output.split("\n") output.pop(0) for line in output: disk_use.append(line.split()) except Exception as e: pass return response.setContent({ 'cpu_count': psutil.cpu_count(), "cpu_use": psutil.cpu_percent(), "mem_total": psutil.virtual_memory().total / (1024.0**3), "mem_use": psutil.virtual_memory().percent, "swap_total": psutil.swap_memory().total / (1024.0**3), "swap_use": psutil.swap_memory().percent, "disk_use": disk_use }).getResponse() except Exception as ex: handleException(response, ex, __file__, "monitorCPU") finally: return response
def pathwayAcquisitionSaveSharingOptions(request, response): #VARIABLE DECLARATION jobID = "" userID = "" try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') UserSessionManager().isValidUser(userID, sessionToken) #**************************************************************** # Step 1.GET THE INSTANCE OF sharing options #**************************************************************** jobID = request.form.get("jobID") jobInstance = JobInformationManager().loadJobInstance(jobID) if str(jobInstance.getUserID()) != str(userID): raise Exception("Invalid user for this jobID") #************************************************************************ # Step 3. Save the visual Options in the MongoDB #************************************************************************ jobInstance.setAllowSharing( request.form.get("allowSharing", 'false') == 'true') jobInstance.setReadOnly( request.form.get("readOnly", 'false') == 'true') logging.info("STEP 3 - SAVING SHARING OPTIONS FOR JOB " + jobID + "...") JobInformationManager().storeSharingOptions(jobInstance) logging.info("STEP 3 - SAVING SHARING OPTIONS FOR JOB " + jobID + "...DONE") response.setContent({"success": True}) except Exception as ex: handleException(response, ex, __file__, "pathwayAcquisitionSaveSharingOptions", userID=userID) finally: return response
def userManagementChangePassword(request, response): # VARIABLE DECLARATION userInstance = None daoInstance = None try: #**************************************************************** # Step 1. CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') UserSessionManager().isValidUser(userID, sessionToken) # **************************************************************** # Step 2.READ THE NEW PASS # **************************************************************** logging.info("STEP1 - READ PARAMS AND CHECK IF USER ALREADY EXISTS...") password = request.form.get("password") from hashlib import sha1 password = sha1(password.encode('ascii')).hexdigest() daoInstance = UserDAO() userInstance = daoInstance.findByID(userID) if userInstance == None: raise CredentialException( "The email or password you entered is incorrect.") # **************************************************************** # Step 3. UPDATE THE MODEL # **************************************************************** userInstance.setPassword(password) daoInstance.update(userInstance, {}) response.setContent({"success": True}) except CredentialException as ex: handleException(response, ex, __file__, "userManagementChangePassword", 200) except Exception as ex: handleException(response, ex, __file__, "userManagementChangePassword") finally: if (daoInstance != None): daoInstance.closeConnection() return response
def adminServletRestoreData(request, response): """ This function... @param {Request} request, the request object @param {Response} response, the response object """ try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') userName = request.cookies.get('userName') UserSessionManager().isValidAdminUser(userID, userName, sessionToken) #**************************************************************** # Step 1.GET THE SPECIE CODE AND THE UPDATE OPTION #**************************************************************** formFields = request.form from subprocess import check_output, CalledProcessError, STDOUT logging.info("STARTING DBManager Restore PROCESS.") scriptArgs = [ ROOT_DIRECTORY + "AdminTools/DBManager.py", "restore", "--remove=1", "--force=1" ] try: check_output(scriptArgs, stderr=STDOUT) except CalledProcessError as exc: raise Exception( "Error while calling DBManager Restore: Exit status " + str(exc.returncode) + ". Error message: " + exc.output) logging.info("FINISHED DBManager Restore PROCESS.") response.setContent({"success": True}) except Exception as ex: handleException(response, ex, __file__, "adminServletRestoreData") finally: return response
def adminServletGetAllUsers(request, response): """ This function obtains a list of all the users registered in the system including different details such as the used space, the registration date, etc. @param {Request} request, the request object @param {Response} response, the response object """ try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') userName = request.cookies.get('userName') UserSessionManager().isValidAdminUser(userID, userName, sessionToken) #**************************************************************** # Step 1. GET THE LIST OF ALL USERS #**************************************************************** logging.info("STEP1 - GET THE LIST OF ALL USERS...") userList = UserDAO().findAll() for userInstance in userList: userInstance.usedSpace = 0 if os_path.isdir(CLIENT_TMP_DIR + str(userInstance.getUserId())): userInstance.usedSpace = dir_total_size( CLIENT_TMP_DIR + str(userInstance.getUserId())) response.setContent({ "success": True, "userList": userList, "availableSpace": MAX_CLIENT_SPACE, "max_jobs_days": MAX_JOB_DAYS, "max_guest_days": MAX_GUEST_DAYS }) except Exception as ex: handleException(response, ex, __file__, "adminServletGetAllUsers") finally: return response
def dataManagementGetMyJobs(request, response): #VARIABLE DECLARATION jobInstance = None jobInstances = [] daoInstance = None try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') UserSessionManager().isValidUser(userID, sessionToken) if (userID is None): response.setContent({ "success": False, "errorMessage": "Log in required</br>Sorry but the feature you are requesting is only available to registered accounts." }) else: #**************************************************************** # Step 2.GET THE LIST OF JOBS FOR GIVEN USER #**************************************************************** logging.info("STEP1 - GET MY JOB LIST REQUEST RECEIVED") daoInstance = JobDAO() matchedFiles = daoInstance.findAll(otherParams={"userID": userID}) logging.info("STEP1 - GET MY JOB LIST REQUEST RECEIVED...DONE") response.setContent({"success": True, "jobList": matchedFiles}) except Exception as ex: handleException(response, ex, __file__, "dataManagementGetMyJobs") finally: if (daoInstance != None): daoInstance.closeConnection() return response
def pathwayAcquisitionMetagenes_PART2(ROOT_DIRECTORY, userID, savedJobInstance, omicName, clusterNumber, RESPONSE): #VARIABLE DECLARATION jobID = "" userID = "" try: #************************************************************************ # Step 3. Save the visual Options in the MongoDB #************************************************************************ logging.info("UPDATE METAGENES - STEP 2 FOR JOB " + jobID + "...") savedJobInstance.generateMetagenesList(ROOT_DIRECTORY, {omicName: clusterNumber}, [omicName]) logging.info("UPDATE METAGENES - STEP 2 FOR JOB " + jobID + "...DONE") JobInformationManager().storePathways(savedJobInstance) matchedPathwaysJSONList = [] for matchedPathway in savedJobInstance.getMatchedPathways().itervalues( ): matchedPathwaysJSONList.append(matchedPathway.toBSON()) RESPONSE.setContent({ "success": True, "jobID": jobID, # "timestamp": newTimestamp, "pathwaysInfo": matchedPathwaysJSONList }) except Exception as ex: handleException(RESPONSE, ex, __file__, "pathwayAcquisitionMetagenes_PART2", userID=userID) finally: return RESPONSE
def pathwayAcquisitionAdjustPvalues(request, response): try: #**************************************************************** # Step 1.GET THE INFO #**************************************************************** formFields = request.get_json() #request.form # List of pathway => {pvalues} pvalues = formFields.get("pValues") # Check what kind of p-value we want to update if "stoufferWeights" in formFields: newStoufferWeights = formFields.get("stoufferWeights") visiblePathways = formFields.get("visiblePathways") newStoufferPvalues = defaultdict(dict) newAdjustedStoufferPvalues = defaultdict(dict) # Iterate over each database (adjusting it independently) for db_name, db_pvalues in pvalues.iteritems(): # Each pathway has a different set of matching omics and thus, Stouffer weights. # The new Stouffer p-value will be computed for each pathway, even those that are currently hidden. for pathway_id, pathway_pvalues in db_pvalues.iteritems(): # Select those with a proper p-value number and present in Stouffer weights valid_pvalues = { omic: pvalue for omic, pvalue in pathway_pvalues.iteritems() if pvalue != "-" and omic in newStoufferWeights.keys() } # Make sure to pass the Stouffer weights in the same order as the p-values newStoufferValue = calculateStoufferCombinedPvalue( valid_pvalues.values(), [ newStoufferWeights[omicName] for omicName in valid_pvalues.keys() ]) newStoufferPvalues[db_name][pathway_id] = newStoufferValue # Adjust the new Stouffer p-values passing only those pathways that are currently visible newAdjustedStoufferPvalues[db_name] = adjustPvalues({ pathway: pvalue for pathway, pvalue in newStoufferPvalues[db_name].iteritems() if pathway in visiblePathways }) response.setContent({ "success": True, "stoufferPvalues": newStoufferPvalues, "adjustedStoufferPvalues": newAdjustedStoufferPvalues }) else: # No new stouffer weights, just recalculate the provided p-values # Iterate over each database (adjusting it independently) adjustedPvaluesByOmic = defaultdict() for db_name, db_pvalues in pvalues.iteritems(): pvaluesByOmic = defaultdict(dict) for pathway, pathwayPvalues in db_pvalues.iteritems(): for omic, omicPvalue in pathwayPvalues.iteritems(): # Skip those in which there is no pValue (no matching in the pathway for that omic) if omicPvalue != '-': pvaluesByOmic[omic][pathway] = omicPvalue adjustedPvaluesByOmic[db_name] = { omic: adjustPvalues(omic_pvalues) for omic, omic_pvalues in pvaluesByOmic.iteritems() } response.setContent({ "success": True, "adjustedPvalues": adjustedPvaluesByOmic }) except Exception as ex: handleException(response, ex, __file__, "pathwayAcquisitionAdjustPvalues") finally: return response
def adminServletGetAvailableOrganisms(request, response): """ This function... @param {Request} request, the request object @param {Response} response, the response object """ try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') userName = request.cookies.get('userName') UserSessionManager().isValidAdminUser(userID, userName, sessionToken) #**************************************************************** # Step 1.GET THE LIST OF INSTALLED SPECIES (DATABASES and SPECIES.JSON) #**************************************************************** import csv databaseList = [] with open(KEGG_DATA_DIR + 'current/common/organisms_all.list') as availableSpeciesFile: reader = csv.reader(availableSpeciesFile, delimiter='\t') for row in reader: organism_code = row[1] # Step 2.4 Check if the organism has non installed data available if os_path.isfile(KEGG_DATA_DIR + 'download/' + organism_code + '/VERSION'): downloaded = True elif os_path.isfile(KEGG_DATA_DIR + 'download/' + organism_code + '/DOWNLOADING'): downloaded = "downloading" else: downloaded = False #Erroneous download not removed --> remove if os_path.isdir(KEGG_DATA_DIR + 'download/' + organism_code): shutil_rmtree(KEGG_DATA_DIR + 'download/' + organism_code) databaseList.append({ "organism_name": row[2], "organism_code": organism_code, "categories": row[3].split(";"), "organism_id": row[0], "downloaded": downloaded }) availableSpeciesFile.close() response.setContent({ "databaseList": databaseList, "download_log": KEGG_DATA_DIR + "download/download.log", "install_log": KEGG_DATA_DIR + "current/install.log" }) except Exception as ex: handleException(response, ex, __file__, "adminServletGetInstalledOrganisms") finally: return response
def fromBEDtoGenes_STEP1(REQUEST, RESPONSE, QUEUE_INSTANCE, JOB_ID, EXAMPLE_FILES_DIR, exampleMode=False): """ This function corresponds to FIRST PART of the FIRST step in the Bed2Genes process. First, it takes a Request object which contains the fields of the form that started the process. This is a summary for the steps in the process: Step 0. VARIABLE DECLARATION Step 1. CHECK IF VALID USER SESSION Step 2. CREATE THE NEW INSTANCE OF JOB Step 3. SAVE THE UPLOADED FILES Step 4. READ PARAMS Step 5. QUEUE THE JOB INSTANCE Step 6. RETURN THE NEW JOB ID @param {Request} REQUEST @param {Response} RESPONSE @param {RQ QUEUE} QUEUE_INSTANCE @param {String} JOB_ID @param {Boolean} exampleMode @returns Response """ #TODO: ALLOWED_EXTENSIONS http://flask.pocoo.org/docs/0.10/patterns/fileuploads/ #TODO: secure_filename #**************************************************************** #Step 0. VARIABLE DECLARATION #The following variables are defined: # - jobInstance: instance of the Bed2GeneJob class. Contains all the information for the current job. # - userID: the ID for the user #**************************************************************** jobInstance = None userID = None try: #**************************************************************** # Step 1. CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = REQUEST.cookies.get('userID') sessionToken = REQUEST.cookies.get('sessionToken') UserSessionManager().isValidUser(userID, sessionToken) #**************************************************************** # Step 2. CREATE THE NEW INSTANCE OF JOB #**************************************************************** jobInstance = Bed2GeneJob(JOB_ID, userID, CLIENT_TMP_DIR) jobInstance.initializeDirectories() logging.info("STEP1 - NEW JOB SUBMITTED " + jobInstance.getJobID()) #**************************************************************** # Step 3. SAVE THE UPLOADED FILES #**************************************************************** formFields = REQUEST.form if (exampleMode == False): logging.info("STEP1 - FILE UPLOADING REQUEST RECEIVED") uploadedFiles = REQUEST.files logging.info("STEP1 - READING FILES....") JobInformationManager().saveFiles(uploadedFiles, formFields, userID, jobInstance, CLIENT_TMP_DIR, EXAMPLE_FILES_DIR) logging.info("STEP1 - READING FILES....DONE") elif (exampleMode == "example"): #**************************************************************** # Step 2.SAVE THE UPLOADED FILES #**************************************************************** logging.info("STEP1 - EXAMPLE MODE SELECTED") logging.info("STEP1 - COPYING FILES....") exampleOmics = ["DNase unmapped"] for omicName in exampleOmics: dataFileName = omicName.replace(" ", "_").lower() + "_values.tab" logging.info( "STEP1 - USING ALREADY SUBMITTED FILE (data file) " + EXAMPLE_FILES_DIR + dataFileName + " FOR " + omicName) relevantFileName = omicName.replace( " ", "_").lower() + "_relevant.tab" logging.info( "STEP1 - USING ALREADY SUBMITTED FILE (relevant features file) " + EXAMPLE_FILES_DIR + relevantFileName + " FOR " + omicName) jobInstance.addGeneBasedInputOmic({ "omicName": omicName, "inputDataFile": EXAMPLE_FILES_DIR + dataFileName, "relevantFeaturesFile": EXAMPLE_FILES_DIR + relevantFileName, "isExample": True }) jobInstance.addReferenceInput({ "omicName": omicName, "fileType": "Reference file", "inputDataFile": EXAMPLE_FILES_DIR + "GTF/sorted_mmu.gtf" }) specie = "mmu" jobInstance.setOrganism(specie) else: raise NotImplementedError #**************************************************************** # Step 4. READ PARAMS #**************************************************************** namePrefix = formFields.get("name_prefix") logging.info("STEP2 - INPUT VALUES ARE:") jobInstance.omicName = formFields.get(namePrefix + "_omic_name", "DNase-seq") logging.info(" - omicName :" + jobInstance.omicName) jobInstance.presortedGTF = formFields.get(namePrefix + "_presortedGTF", False) logging.info(" - presortedGTF :" + str(jobInstance.presortedGTF)) jobInstance.report = formFields.get(namePrefix + "_report", "gene") logging.info(" - report :" + jobInstance.report) jobInstance.distance = formFields.get(namePrefix + "_distance", 10) logging.info(" - distance :" + str(jobInstance.distance)) jobInstance.tss = formFields.get(namePrefix + "_tss", 200) logging.info(" - tss :" + str(jobInstance.tss)) jobInstance.promoter = formFields.get(namePrefix + "_promoter", 1300) logging.info(" - promoter :" + str(jobInstance.promoter)) jobInstance.geneAreaPercentage = formFields.get( namePrefix + "_geneAreaPercentage", 90) logging.info(" - geneAreaPercentage :" + str(jobInstance.geneAreaPercentage)) jobInstance.regionAreaPercentage = formFields.get( namePrefix + "_regionAreaPercentage", 50) logging.info(" - regionAreaPercentage :" + str(jobInstance.regionAreaPercentage)) jobInstance.ignoreMissing = True if namePrefix + "_ignoremissing" in formFields.keys( ) else False logging.info(" - ignore missing :" + str(jobInstance.ignoreMissing)) jobInstance.featureEnrichment = formFields.get( namePrefix + "_feature_enrichment_pre", False) logging.info(" - Feature Enrichment :" + str(jobInstance.featureEnrichment)) #rules jobInstance.geneIDtag = formFields.get(namePrefix + "_geneIDtag", "gene_id") logging.info(" - geneIDtag :" + str(jobInstance.geneIDtag)) jobInstance.summarizationMethod = formFields.get( namePrefix + "_summarization_method", "mean") logging.info(" - summarization_method :" + jobInstance.summarizationMethod) jobInstance.reportRegions = formFields.getlist(namePrefix + "_reportRegions") if len(jobInstance.reportRegions) == 0: jobInstance.reportRegions = ["all"] logging.info(" - reportRegions :" + str(jobInstance.reportRegions)) #************************************************************************ # Step 4. Queue job #************************************************************************ QUEUE_INSTANCE.enqueue(fn=fromBEDtoGenes_STEP2, args=(jobInstance, userID, exampleMode, RESPONSE), timeout=600, job_id=JOB_ID) #************************************************************************ # Step 5. Return the Job ID #************************************************************************ RESPONSE.setContent({"success": True, "jobID": JOB_ID}) except Exception as ex: handleException(RESPONSE, ex, __file__, "fromBEDtoGenes_STEP1") finally: return RESPONSE
def adminServletGetInstalledOrganisms(request, response): """ This function... @param {Request} request, the request object @param {Response} response, the response object """ try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') userName = request.cookies.get('userName') UserSessionManager().isValidAdminUser(userID, userName, sessionToken) #**************************************************************** # Step 1.GET THE LIST OF INSTALLED SPECIES (DATABASES and SPECIES.JSON) #**************************************************************** organisms_names = {} with open(KEGG_DATA_DIR + 'current/common/organisms_all.list') as organisms_all: reader = csv.reader(organisms_all, delimiter='\t') for row in reader: organisms_names[row[1]] = row[2] organisms_all.close() installedSpecies = [] from pymongo import MongoClient client = MongoClient(MONGODB_HOST, MONGODB_PORT) databases = client.database_names() #**************************************************************** # Step 2.FOR EACH INSTALLED DATABASE GET THE INFORMATION #**************************************************************** databaseList = [] common_info_date = "" for database in databases: if not "-paintomics" in database: continue elif "global-paintomics" == database: db = client[database] common_info_date = db.versions.find({"name": "COMMON"})[0].get("date") # Step 2.4 Check if the organism has non installed data available if os_path.isfile(KEGG_DATA_DIR + 'download/common/VERSION'): downloaded = True elif os_path.isfile(KEGG_DATA_DIR + 'download/common/DOWNLOADING'): downloaded = "downloading" else: downloaded = False #Erroneous download not removed --> remove if os_path.isdir(KEGG_DATA_DIR + 'download/common/'): shutil_rmtree(KEGG_DATA_DIR + 'download/common/') databaseList.append({ "organism_name": "Common KEGG data", "organism_code": "common", "kegg_date": common_info_date, "downloaded": downloaded }) else: # Step 2.1 GET THE SPECIE CODE organism_code = database.replace("-paintomics", "") # Step 2.2 GET THE SPECIE NAME organism_name = organisms_names.get(organism_code, "Unknown specie") # Step 2.3 GET THE SPECIE VERSIONS db = client[database] kegg_date = db.versions.find({"name": "KEGG"})[0].get("date") mapping_date = db.versions.find({"name": "MAPPING"})[0].get("date") acceptedIDs = db.versions.find({"name": "ACCEPTED_IDS"}) if acceptedIDs.count() > 0: acceptedIDs = acceptedIDs[0].get("ids") else: acceptedIDs = "" # Step 2.4 Check if the organism has non installed data available if os_path.isfile(KEGG_DATA_DIR + 'download/' + organism_code + '/VERSION'): downloaded = True elif os_path.isfile(KEGG_DATA_DIR + 'download/' + organism_code + '/DOWNLOADING'): downloaded = "downloading" else: downloaded = False #Erroneous download not removed --> remove if os_path.isdir(KEGG_DATA_DIR + 'download/' + organism_code): shutil_rmtree(KEGG_DATA_DIR + 'download/' + organism_code) databaseList.append({ "organism_name": organism_name, "organism_code": organism_code, "kegg_date": kegg_date, "mapping_date": mapping_date, "acceptedIDs": acceptedIDs, "downloaded": downloaded }) client.close() response.setContent({ "common_info_date": common_info_date, "databaseList": databaseList }) except Exception as ex: handleException(response, ex, __file__, "adminServletGetInstalledOrganisms") finally: return response
def adminServletInstallOrganism(request, response, organism_code, ROOT_DIRECTORY): """ This function manages an 'Install/Update Organism' request by calling to the DBManager tool. @param {Request} request, the request object @param {Response} response, the response object @param {String} organism_code, @param {String} ROOT_DIRECTORY, """ try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') userName = request.cookies.get('userName') UserSessionManager().isValidAdminUser(userID, userName, sessionToken) #**************************************************************** # Step 1.GET THE SPECIE CODE AND THE UPDATE OPTION #**************************************************************** download = json.loads(request.data).get("download") update_kegg = 1 update_mapping = 1 common = 0 if organism_code == "common": common = 1 organism_code = "#common" from subprocess import check_output, CalledProcessError, STDOUT #**************************************************************** # Step 2a. IF THE SELECTED OPTION IS DOWNLOAD #**************************************************************** if download: logging.info("STARTING DBManager download PROCESS.") scriptArgs = [ ROOT_DIRECTORY + "AdminTools/DBManager.py", "download", "--specie=" + organism_code, "--kegg=" + str(update_kegg), "--mapping=" + str(update_mapping), "--common=" + str(common) ] try: check_output(scriptArgs, stderr=STDOUT) except CalledProcessError as exc: raise Exception( "Error while calling DBManager download: Exit status " + str(exc.returncode) + ". Error message: " + exc.output) logging.info("FINISHED DBManager Download PROCESS.") # **************************************************************** # Step 2B. IF THE SELECTED OPTION IS INSTALL # **************************************************************** else: logging.info("STARTING DBManager Install PROCESS.") scriptArgs = [ ROOT_DIRECTORY + "AdminTools/DBManager.py", "install", "--specie=" + organism_code, "--common=" + str(common) ] try: check_output(scriptArgs, stderr=STDOUT) except CalledProcessError as exc: raise Exception( "Error while calling DBManager Install: Exit status " + str(exc.returncode) + ". Error message: " + exc.output) logging.info("FINISHED DBManager Install PROCESS.") response.setContent({"success": True}) except Exception as ex: handleException(response, ex, __file__, "adminServletUpdateOrganism") finally: return response
def pathwayAcquisitionRecoverJob(request, response, QUEUE_INSTANCE): #VARIABLE DECLARATION jobInstance = None jobID = "" userID = "" #TODO: COMPROBAR OWNERS try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') UserSessionManager().isValidUser(userID, sessionToken) #**************************************************************** # Step 1.LOAD THE INSTANCE OF JOB #**************************************************************** formFields = request.form jobID = formFields.get("jobID") logging.info("RECOVER_JOB - LOADING JOB " + jobID + "...") jobInstance = JobInformationManager().loadJobInstance(jobID) queueJob = QUEUE_INSTANCE.fetch_job(jobID) if (queueJob is not None and not queueJob.is_finished()): logging.info("RECOVER_JOB - JOB " + jobID + " HAS NOT FINISHED ") response.setContent({ "success": False, "message": "Your job " + jobID + " is still running in the queue. Please, try again later to check if it has finished." }) return response if (jobInstance == None): #TODO DIAS BORRADO? logging.info("RECOVER_JOB - JOB " + jobID + " NOT FOUND AT DATABASE.") response.setContent({ "success": False, "errorMessage": "Job " + jobID + " not found at database.<br>Please, note that jobs are automatically removed after 7 days for guests and 14 days for registered users." }) return response # Allow "no user" jobs to be viewed by anyone, logged or not if (str(jobInstance.getUserID()) != 'None' and jobInstance.getUserID() != userID and not jobInstance.getAllowSharing()): logging.info("RECOVER_JOB - JOB " + jobID + " DOES NOT BELONG TO USER " + str(userID) + " JOB HAS USER " + str(jobInstance.getUserID())) response.setContent({ "success": False, "errorMessage": "Invalid Job ID (" + jobID + ") for current user.<br>Please, check the Job ID and try again." }) return response logging.info("RECOVER_JOB - JOB " + jobInstance.getJobID() + " LOADED SUCCESSFULLY.") matchedCompoundsJSONList = map( lambda foundFeature: foundFeature.toBSON(), jobInstance.getFoundCompounds()) matchedPathwaysJSONList = [] for matchedPathway in jobInstance.getMatchedPathways().itervalues(): matchedPathwaysJSONList.append(matchedPathway.toBSON()) logging.info("RECOVER_JOB - GENERATING PATHWAYS INFORMATION...DONE") if (len(matchedCompoundsJSONList) == 0 and jobInstance.getLastStep() == 2 and len(jobInstance.getCompoundBasedInputOmics()) > 0): logging.info( "RECOVER_JOB - JOB " + jobID + " DOES NOT CONTAINS FOUND COMPOUNDS (STEP 2: OLD FORMAT?).") response.setContent({ "success": False, "errorMessage": "Job " + jobID + " does not contains saved information about the found compounds, please run it again." }) elif (len(matchedPathwaysJSONList) == 0 and jobInstance.getLastStep() > 2): logging.info("RECOVER_JOB - JOB " + jobID + " DOES NOT CONTAINS PATHWAYS.") response.setContent({ "success": False, "errorMessage": "Job " + jobID + " does not contains information about pathways. Please, run it again." }) else: response.setContent({ "success": True, "jobID": jobInstance.getJobID(), "userID": jobInstance.getUserID(), "pathwaysInfo": matchedPathwaysJSONList, "geneBasedInputOmics": jobInstance.getGeneBasedInputOmics(), "compoundBasedInputOmics": jobInstance.getCompoundBasedInputOmics(), "organism": jobInstance.getOrganism(), "summary": jobInstance.summary, "visualOptions": JobInformationManager().getVisualOptions(jobID), "databases": jobInstance.getDatabases(), "matchedMetabolites": matchedCompoundsJSONList, "stepNumber": jobInstance.getLastStep(), "name": jobInstance.getName(), "timestamp": int(time()), "allowSharing": jobInstance.getAllowSharing(), "readOnly": jobInstance.getReadOnly(), "omicsValuesID": jobInstance.getValueIdTable() }) except Exception as ex: handleException(response, ex, __file__, "pathwayAcquisitionRecoverJob", userID=userID) finally: return response
def adminServletSendReport(request, response, ROOT_DIRECTORY): """ This function... @param {Request} request, the request object @param {Response} response, the response object """ try: #logging.info("STEP0 - CHECK IF VALID USER....") #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** userID = request.cookies.get('userID') #sessionToken = request.cookies.get('sessionToken') #UserSessionManager().isValidUser(userID, sessionToken) #**************************************************************** # Step 1.GET THE SPECIE CODE AND THE UPDATE OPTION #**************************************************************** formFields = request.form if userID is not None: userEmail = UserDAO().findByID(userID) userName = userEmail.getUserName() userEmail = userEmail.getEmail() else: userEmail = formFields.get("fromEmail", smpt_sender) userName = formFields.get("fromName", "No name provided") type = formFields.get("type") _message = formFields.get("message") title = "Other request" color = "#333" if type == "error": type = "Error notification" title = "<h1>New error notification</h1>" color = "#f95959" elif type == "specie_request": type = "New specie requesting" title = "<h1>New organism requested</h1>" color = "#0090ff" else: type = "Other request" message = '<html><body>' message += "<a href='" + "http://bioinfo.cipf.es/paintomics/" + "' target='_blank'>" message += " <img src='cid:image1' border='0' width='auto' height='50' alt='Paintomics 3 logo'>" message += "</a>" message += "<div style='width:100%; height:10px; border-top: 1px dotted #333; margin-top:20px; margin-bottom:30px;'></div>" message += "<h1>" + title + "</h1>" message += "<p>Thanks for the report, " + userName + "!</p>" message += "<p><b>Username:</b> " + userEmail + "</p></br>" message += "<div style='width:100%; border: 1px solid " + color + "; padding:10px;font-family: monospace;color:" + color + ";'>" + _message + "</div>" message += "<p>We will contact you as soon as possible.</p>" message += "<p>Best regards,</p>" message += "<p>The Paintomics developers team.</p>" message += "<div style='width:100%; height:10px; border-top: 1px dotted #333; margin-top:20px; margin-bottom:30px;'></div>" message += "<p>Problems? E-mail <a href='mailto:" + "*****@*****.**" + "'>" + "*****@*****.**" + "</a></p>" message += '</body></html>' sendEmail(ROOT_DIRECTORY, smpt_sender, smpt_sender_name, type, message, fromEmail=userEmail, fromName=userName, isHTML=True) response.setContent({"success": True}) except Exception as ex: handleException(response, ex, __file__, "adminServletSendReport") finally: return response
def pathwayAcquisitionStep3(request, response): #VARIABLE DECLARATION jobInstance = None jobID = "" userID = "" try: #**************************************************************** # Step 0.CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = request.cookies.get('userID') sessionToken = request.cookies.get('sessionToken') UserSessionManager().isValidUser(userID, sessionToken) #**************************************************************** # Step 1.LOAD THE INSTANCE OF JOB #**************************************************************** formFields = request.form jobID = formFields.get("jobID") #TODO: IN PREVIOUS STEPS THE USER COULD SPECIFY THE DEFAULT OMICS TO SHOW visibleOmics = [] logging.info("STEP3 - LOADING JOB " + jobID + "...") jobInstance = JobInformationManager().loadJobInstance(jobID) jobInstance.setUserID(userID) if (jobInstance == None): raise UserWarning("Job " + jobID + " was not found at database.") logging.info("STEP3 - JOB " + jobInstance.getJobID() + " LOADED SUCCESSFULLY.") #**************************************************************** # Step 2.READ THE SELECTED PATHWAYS #**************************************************************** logging.info("STEP3 - GENERATING PATHWAYS INFORMATION...") selectedPathways = formFields.getlist("selectedPathways") #TODO: SOLO GENERAR INFO PARA LAS QUE NO LA TENGAN YA GUARDADA EN LA BBDD [ selectedPathwayInstances, graphicalOptionsInstancesBSON, omicsValuesSubset ] = jobInstance.generateSelectedPathwaysInformation( selectedPathways, visibleOmics, True) logging.info("STEP3 - GENERATING PATHWAYS INFORMATION...DONE") #************************************************************************ # Step 3. Save the jobInstance in the MongoDB #************************************************************************ logging.info("STEP 3 - SAVING NEW JOB DATA...") JobInformationManager().storeJobInstance(jobInstance, 3) logging.info("STEP 3 - SAVING NEW JOB DATA...DONE") response.setContent({ "success": True, "jobID": jobInstance.getJobID(), "graphicalOptionsInstances": graphicalOptionsInstancesBSON, "omicsValues": omicsValuesSubset, "organism": jobInstance.getOrganism() }) except Exception as ex: handleException(response, ex, __file__, "pathwayAcquisitionStep3", userID=userID) finally: return response
def pathwayAcquisitionStep1_PART1(REQUEST, RESPONSE, QUEUE_INSTANCE, JOB_ID, EXAMPLE_FILES_DIR, exampleMode): """ This function corresponds to FIRST PART of the FIRST step in the Pathways acquisition process. First, it takes a Request object which contains the fields of the form that started the process. This is a summarization for the steps in the process: Step 0. VARIABLE DECLARATION Step 1. CHECK IF VALID USER SESSION Step 2. CREATE THE NEW INSTANCE OF JOB Step 3. SAVE THE UPLOADED FILES Step 4. QUEUE THE JOB INSTANCE Step 5. RETURN THE NEW JOB ID @param {Request} REQUEST @param {Response} RESPONSE @param {RQ QUEUE} QUEUE_INSTANCE @param {String} JOB_ID @param {Boolean} exampleMode @returns Response """ #TODO:ALLOWED_EXTENSIONS http://flask.pocoo.org/docs/0.10/patterns/fileuploads/ #TODO: secure_filename #**************************************************************** #Step 0. VARIABLE DECLARATION #The following variables are defined: # - jobInstance: instance of the PathwayAcquisitionJob class. # Contains all the information for the current job. # - userID: the ID for the user #**************************************************************** jobInstance = None userID = None try: #**************************************************************** # Step 1. CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = REQUEST.cookies.get('userID') sessionToken = REQUEST.cookies.get('sessionToken') UserSessionManager().isValidUser(userID, sessionToken) #**************************************************************** # Step 2. CREATE THE NEW INSTANCE OF JOB #**************************************************************** jobInstance = PathwayAcquisitionJob(JOB_ID, userID, CLIENT_TMP_DIR) jobInstance.initializeDirectories() logging.info("STEP1 - NEW JOB SUBMITTED " + jobInstance.getJobID()) #**************************************************************** # Step 3. SAVE THE UPLOADED FILES #**************************************************************** if (exampleMode == False): logging.info("STEP1 - FILE UPLOADING REQUEST RECEIVED") uploadedFiles = REQUEST.files formFields = REQUEST.form jobInstance.description = "" jobInstance.setName(formFields.get("jobDescription", "")[:100]) specie = formFields.get("specie") #GET THE SPECIE NAME databases = REQUEST.form.getlist('databases[]') jobInstance.setOrganism(specie) # Check the available databases for species organismDB = set(dicDatabases.get(specie, [{}])[0].keys()) # TODO: disabled multiple databases for the moment # jobInstance.setDatabases(list(set([u'KEGG']) | set(databases).intersection(organismDB))) jobInstance.setDatabases([u'KEGG']) logging.info("STEP1 - SELECTED SPECIE IS " + specie) logging.info("STEP1 - READING FILES....") JobInformationManager().saveFiles(uploadedFiles, formFields, userID, jobInstance, CLIENT_TMP_DIR) logging.info("STEP1 - READING FILES....DONE") elif (exampleMode == "example"): #**************************************************************** # Step 2.SAVE THE UPLOADED FILES #**************************************************************** logging.info("STEP1 - EXAMPLE MODE SELECTED") logging.info("STEP1 - COPYING FILES....") exampleOmics = { "Gene expression": False, "Metabolomics": True, "Proteomics": True, "miRNA-seq": False, "DNase-seq": False } for omicName, featureEnrichment in exampleOmics.iteritems(): dataFileName = omicName.replace(" ", "_").replace( "-seq", "").lower() + "_values.tab" logging.info( "STEP1 - USING ALREADY SUBMITTED FILE (data file) " + EXAMPLE_FILES_DIR + dataFileName + " FOR " + omicName) relevantFileName = omicName.replace(" ", "_").replace( "-seq", "").lower() + "_relevant.tab" logging.info( "STEP1 - USING ALREADY SUBMITTED FILE (relevant features file) " + EXAMPLE_FILES_DIR + relevantFileName + " FOR " + omicName) if (["Metabolomics"].count(omicName)): jobInstance.addCompoundBasedInputOmic({ "omicName": omicName, "inputDataFile": EXAMPLE_FILES_DIR + dataFileName, "relevantFeaturesFile": EXAMPLE_FILES_DIR + relevantFileName, "isExample": True, "featureEnrichment": featureEnrichment }) else: jobInstance.addGeneBasedInputOmic({ "omicName": omicName, "inputDataFile": EXAMPLE_FILES_DIR + dataFileName, "relevantFeaturesFile": EXAMPLE_FILES_DIR + relevantFileName, "isExample": True, "featureEnrichment": featureEnrichment }) specie = "mmu" jobInstance.setOrganism(specie) jobInstance.setDatabases(['KEGG']) else: raise NotImplementedError #************************************************************************ # Step 4. Queue job #************************************************************************ QUEUE_INSTANCE.enqueue(fn=pathwayAcquisitionStep1_PART2, args=(jobInstance, userID, exampleMode, RESPONSE), timeout=600, job_id=JOB_ID) #************************************************************************ # Step 5. Return the Job ID #************************************************************************ RESPONSE.setContent({"success": True, "jobID": JOB_ID}) except Exception as ex: if (jobInstance != None): jobInstance.cleanDirectories(remove_output=True) handleException(RESPONSE, ex, __file__, "pathwayAcquisitionStep1_PART1", userID=userID) finally: return RESPONSE
def pathwayAcquisitionStep2_PART2(jobID, userID, selectedCompounds, clusterNumber, RESPONSE, ROOT_DIRECTORY): """ This function corresponds to SECOND PART of the SECOND step in the Pathways acquisition process. Given a JOB INSTANCE, first processes the uploaded files (identifiers matching and compound list generation) and finally generates the response. This code is executed at the Redis Queue. This is a summarization for the steps in the process: Step 1. READ AND UPDATE THE SELECTED METABOLITES Step 2. GENERATE PATHWAYS INFORMATION Step 3. GENERATE THE METAGENES INFORMATION Step 4. UPDATE JOB INFORMATION AT DATABASE Step 5. GENERATE RESPONSE AND FINISH @param {PathwayAcquisitionJob} jobInstance @param {String[]} selectedCompounds @param {Response} RESPONSE @param {String} userID @param {Boolean} exampleMode @returns Response """ jobInstance = None try: logging.info("STEP2 - LOADING JOB " + jobID + "...") jobInstance = JobInformationManager().loadJobInstance(jobID) jobInstance.setUserID(userID) if (jobInstance == None): raise UserWarning("Job " + jobID + " was not found at database.") jobInstance.setDirectories(CLIENT_TMP_DIR) jobInstance.initializeDirectories() logging.info("STEP2 - JOB " + jobInstance.getJobID() + " LOADED SUCCESSFULLY.") #**************************************************************** # Step 1.READ THE SELECTED METABOLITES #**************************************************************** logging.info("STEP2 - UPDATING SELECTED COMPOUNDS LIST...") #TODO: CHANGE THIS TO ALLOW BACK jobInstance.updateSubmitedCompoundsList(selectedCompounds) logging.info("STEP2 - UPDATING SELECTED COMPOUNDS LIST...DONE") #**************************************************************** # Step 2. GENERATING PATHWAYS INFORMATION #**************************************************************** logging.info("STEP2 - GENERATING PATHWAYS INFORMATION...") summary = jobInstance.generatePathwaysList() logging.info("STEP2 - GENERATING PATHWAYS INFORMATION...DONE") #**************************************************************** # Step 3. GENERATING METAGENES INFORMATION #**************************************************************** logging.info("STEP2 - GENERATING METAGENES INFORMATION...") jobInstance.generateMetagenesList(ROOT_DIRECTORY, clusterNumber) logging.info("STEP2 - GENERATING METAGENES INFORMATION...DONE") jobInstance.setLastStep(3) #************************************************************************ # Step 4. Save the all the Matched Compounds and pathways in MongoDB #************************************************************************ logging.info("STEP2 - SAVING NEW JOB DATA...") JobInformationManager().storeJobInstance(jobInstance, 2) logging.info("STEP2 - SAVING NEW JOB DATA...DONE") #************************************************************************ # Step 5. Update the response content #************************************************************************ matchedPathwaysJSONList = [] for matchedPathway in jobInstance.getMatchedPathways().itervalues(): matchedPathwaysJSONList.append(matchedPathway.toBSON()) RESPONSE.setContent({ "success": True, "organism": jobInstance.getOrganism(), "jobID": jobInstance.getJobID(), "summary": summary, "pathwaysInfo": matchedPathwaysJSONList, "geneBasedInputOmics": jobInstance.getGeneBasedInputOmics(), "compoundBasedInputOmics": jobInstance.getCompoundBasedInputOmics(), "databases": jobInstance.getDatabases(), "omicsValuesID": jobInstance.getValueIdTable(), "timestamp": int(time()) }) except Exception as ex: handleException(RESPONSE, ex, __file__, "pathwayAcquisitionStep2_PART2", userID=userID) finally: jobInstance.cleanDirectories() return RESPONSE
def pathwayAcquisitionStep2_PART1(REQUEST, RESPONSE, QUEUE_INSTANCE, ROOT_DIRECTORY): """ This function corresponds to FIRST PART of the SECOND step in the Pathways acquisition process. First, it takes a Request object which contains the fields of the form that started the process. This is a summary for the steps in the process: Step 0. VARIABLE DECLARATION Step 1. CHECK IF VALID USER SESSION Step 2. LOAD THE INSTANCE OF JOB Step 3. QUEUE THE JOB INSTANCE Step 4. RETURN THE JOB ID @param {Request} REQUEST @param {Response} RESPONSE @param {RQ QUEUE} QUEUE_INSTANCE @returns Response """ #**************************************************************** #Step 0. VARIABLE DECLARATION #The following variables are defined: # - jobID: the ID for the job instance # - userID: the ID for the user #**************************************************************** jobID = "" userID = "" try: #**************************************************************** # Step 1. CHECK IF VALID USER SESSION #**************************************************************** logging.info("STEP0 - CHECK IF VALID USER....") userID = REQUEST.cookies.get('userID') sessionToken = REQUEST.cookies.get('sessionToken') UserSessionManager().isValidUser(userID, sessionToken) #**************************************************************** # Step 2.LOAD THE INSTANCE OF JOB #**************************************************************** formFields = REQUEST.form jobID = formFields.get("jobID") selectedCompounds = REQUEST.form.getlist("selectedCompounds[]") # Retrieve the number of cluster on a per omic basis # Note: this will contain the omic name transformed to remove spaces and special chars clusterNumber = { key.replace("clusterNumber:", ""): value for key, value in formFields.iteritems() if key.startswith("clusterNumber:") } #************************************************************************ # Step 3. Queue job #************************************************************************ QUEUE_INSTANCE.enqueue(fn=pathwayAcquisitionStep2_PART2, args=( jobID, userID, selectedCompounds, clusterNumber, RESPONSE, ROOT_DIRECTORY, ), timeout=600, job_id=jobID) #************************************************************************ # Step 5. Return the Job ID #************************************************************************ RESPONSE.setContent({"success": True, "jobID": jobID}) except Exception as ex: handleException(RESPONSE, ex, __file__, "pathwayAcquisitionStep2_PART1", userID=userID) finally: return RESPONSE
def pathwayAcquisitionStep1_PART2(jobInstance, userID, exampleMode, RESPONSE): """ This function corresponds to SECOND PART of the FIRST step in the Pathways acquisition process. Given a JOB INSTANCE, first processes the uploaded files (identifiers matching and compound list generation) and finally generates the response. This code is executed at the PyQlite Queue. This is a summarization for the steps in the process: Step 0. CHECK FILES CONTENT Step 1. PROCESS THE FILES DATA Step 2. SAVE THE JOB INSTANCE AT THE DATABASE Step 3. GENERATE RESPONSE AND FINISH @param {PathwayAcquisitionJob} jobInstance @param {Response} RESPONSE @param {String} userID @param {Boolean} exampleMode @returns Response """ try: #**************************************************************** # Step 0.VALIDATE THE FILES DATA #**************************************************************** logging.info("STEP0 - VALIDATING INPUT...") jobInstance.validateInput() logging.info("STEP1 - VALIDATING INPUT...DONE") #**************************************************************** # Step 1.PROCESS THE FILES DATA #**************************************************************** logging.info("STEP1 - PROCESSING FILES...") matchedMetabolites = jobInstance.processFilesContent( ) #This function processes all the files and returns a checkboxes list to show to the user logging.info("STEP1 - PROCESSING FILES...DONE") #************************************************************************ # Step 2. Save the jobInstance in the MongoDB #************************************************************************ logging.info("STEP1 - SAVING JOB DATA...") jobInstance.setLastStep(2) jobInstance.getJobDescription(True, exampleMode == "example") JobInformationManager().storeJobInstance(jobInstance, 1) logging.info("STEP1 - SAVING JOB DATA...DONE") jobInstance.cleanDirectories() #************************************************************************ # Step 3. Update the response content #************************************************************************ RESPONSE.setContent({ "success": True, "organism": jobInstance.getOrganism(), "jobID": jobInstance.getJobID(), "userID": jobInstance.getUserID(), "matchedMetabolites": map(lambda foundFeature: foundFeature.toBSON(), matchedMetabolites), "geneBasedInputOmics": jobInstance.getGeneBasedInputOmics(), "compoundBasedInputOmics": jobInstance.getCompoundBasedInputOmics(), "databases": jobInstance.getDatabases(), "name": jobInstance.getName(), "timestamp": int(time()) }) except Exception as ex: jobInstance.cleanDirectories(remove_output=True) # TODO: at this point we should notify the queue system about the error, or else # will keep returning success to the job. handleException(RESPONSE, ex, __file__, "pathwayAcquisitionStep1_PART2", userID=userID) finally: return RESPONSE