def adminServletGetMessage(request, response):
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
message_type = request.form.get("message_type")
if (message_type != "starting_message"):
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
UserSessionManager().isValidUser(userID, sessionToken)
#****************************************************************
# Step 1.GET THE MESSAGES FROM THE DATABASE
#****************************************************************
daoInstance = MessageDAO()
matchedMessages = daoInstance.findAll(
otherParams={"message_type": message_type})
daoInstance.closeConnection()
response.setContent({"success": True, "messageList": matchedMessages})
except Exception as ex:
handleException(response, ex, __file__, "adminServletGetMessage")
finally:
return response
def adminServletDeleteMessage(request, response, message_type=None):
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
userName = request.cookies.get('userName')
UserSessionManager().isValidAdminUser(userID, userName, sessionToken)
#****************************************************************
# Step 1.GET THE MESSAGES FROM THE DATABASE
#****************************************************************
if message_type == None:
message_type = request.form.get("message_type")
daoInstance = MessageDAO()
daoInstance.removeAll(otherParams={"message_type": message_type})
daoInstance.closeConnection()
response.setContent({"success": True})
except Exception as ex:
handleException(response, ex, __file__, "adminServletDeleteMessage")
finally:
return response
def userManagementNewNoLoginSession(request, response):
try:
#****************************************************************
# Step 1. INITIALIZE EMPTY USER INSTANCE
#****************************************************************
logging.info("STEP1 - START 'NO LOGIN' session...")
initializeUserDirectories(None)
# sessionToken = UserSessionManager().registerNewUser("" + str(userID))
response.setContent({
"success": True,
"userID": None,
"userName": None,
"sessionToken": None,
"p": None
})
except Exception as ex:
handleException(response, ex, __file__,
"userManagementNewNoLoginSession")
finally:
return response
def adminCleanDatabases(request, response):
"""
This function...
@param {Request} request, the request object
@param {Response} response, the response object
"""
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
userName = request.cookies.get('userName')
UserSessionManager().isValidAdminUser(userID, userName, sessionToken)
#****************************************************************
# Step 1. RUN THE SCRIPT
#****************************************************************
from src.AdminTools.scripts.clean_databases import cleanDatabases as clean_databases_routine
clean_databases_routine(force=True)
response.setContent({"success": True})
except Exception as ex:
handleException(response, ex, __file__, "cleanDatabases")
finally:
return response
def userManagementSignOut(request, response):
userInstance = None
daoInstance = None
try:
#****************************************************************
# Step 1.READ PARAMS
#****************************************************************
logging.info("STEP1 - READ PARAMS...")
formFields = request.form
userID = formFields.get("userID")
sessionToken = formFields.get("sessionToken")
#****************************************************************
# Step 2. CLOSE SESSION
#****************************************************************
logging.info("STEP2 - REMOVING USER..")
UserSessionManager().removeUser(userID, sessionToken)
response.setContent({"success": True})
except Exception as ex:
handleException(response, ex, __file__, "userManagementSignOut")
finally:
if (daoInstance != None):
daoInstance.closeConnection()
return response
def pathwayAcquisitionSaveImage(request, response):
jobID = ""
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
# logging.info("STEP0 - CHECK IF VALID USER....")
# userID = request.cookies.get('userID')
# sessionToken = request.cookies.get('sessionToken')
# UserSessionManager().isValidUser(userID, sessionToken)
jobID = request.form.get("jobID")
jobInstance = JobInformationManager().loadJobInstance(jobID)
svgData = request.form.get("svgCode")
fileName = "paintomics_" + request.form.get("fileName").replace(
" ", "_") + "_" + jobID
fileFormat = request.form.get("format")
userID = jobInstance.getUserID()
userDirID = userID if userID is not None else "nologin"
path = CLIENT_TMP_DIR + userDirID + jobInstance.getOutputDir().replace(
CLIENT_TMP_DIR + userDirID, "")
if (fileFormat == "png"):
def createImage(svgData):
svg = rsvg.Handle(data=svgData)
width = svg.props.width
height = svg.props.height
surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, width,
height)
context = cairo.Context(surface)
open(path + fileName + "." + fileFormat, 'a').close()
svg.render_cairo(context)
surface.write_to_png(path + fileName + "." + fileFormat)
try:
logging.info("TRYING...")
createImage(svgData=svgData)
except Exception as ex:
logging.info("TRYING again...")
createImage(svgData=svgData)
elif (fileFormat == "svg"):
file_ = open(path + fileName + "." + fileFormat, 'w')
file_.write(svgData)
file_.close()
path = "/get_cluster_image/" + jobID + "/output/"
response.setContent({
"success": True,
"filepath": path + fileName + "." + fileFormat
})
except Exception as ex:
handleException(response, ex, __file__, "pathwayAcquisitionSaveImage")
finally:
return response
def pathwayAcquisitionMetagenes_PART1(REQUEST, RESPONSE, QUEUE_INSTANCE,
JOB_ID, ROOT_DIRECTORY):
# ****************************************************************
# Step 0. VARIABLE DECLARATION
# The following variables are defined:
# - jobInstance: instance of the PathwayAcquisitionJob class.
# Contains all the information for the current job.
# - userID: the ID for the user
# ****************************************************************
jobInstance = None
userID = None
try:
# ****************************************************************
# Step 1. CHECK IF VALID USER SESSION
# ****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = REQUEST.cookies.get('userID')
sessionToken = REQUEST.cookies.get('sessionToken')
UserSessionManager().isValidUser(userID, sessionToken)
# ****************************************************************
# Step 2. LOAD THE JOB INSTANCE AND RETRIEVE FORM INFO
# ****************************************************************
savedJobID = REQUEST.form.get("jobID")
savedJobInstance = JobInformationManager().loadJobInstance(savedJobID)
if savedJobInstance.getReadOnly() and str(
savedJobInstance.getUserID()) != str(userID):
raise Exception("Invalid user for the job generating metagenes.")
omicName = REQUEST.form.get("omic")
clusterNumber = int(REQUEST.form.get("number"))
# Make sure the number of clusters is inside [1, 20]
clusterNumber = 1 if clusterNumber < 1 else 20 if clusterNumber > 20 else clusterNumber
# ************************************************************************
# Step 4. Queue job
# ************************************************************************
QUEUE_INSTANCE.enqueue(fn=pathwayAcquisitionMetagenes_PART2,
args=(ROOT_DIRECTORY, userID, savedJobInstance,
omicName, clusterNumber, RESPONSE),
timeout=600,
job_id=JOB_ID)
# ************************************************************************
# Step 5. Return the Job ID
# ************************************************************************
RESPONSE.setContent({"success": True, "jobID": JOB_ID})
except Exception as ex:
handleException(RESPONSE,
ex,
__file__,
"pathwayAcquisitionMetagenes_PART1",
userID=userID)
finally:
return RESPONSE
def adminServletDeleteUser(request, response, toDeleteUserID):
"""
This function...
@param {Request} request, the request object
@param {Response} response, the response object
"""
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
userName = request.cookies.get('userName')
UserSessionManager().isValidAdminUser(userID, userName, sessionToken)
if toDeleteUserID == "0":
response.setContent({"success": False})
else:
jobDAOInstance = JobDAO()
filesDAOInstance = FileDAO()
userDAOInstance = UserDAO()
logging.info("STEP1 - CLEANING DATA FOR " + toDeleteUserID + "...")
#****************************************************************
# Step 1. DELETE ALL JOBS FOR THE USER
#****************************************************************
allJobs = jobDAOInstance.findAll(
otherParams={"userID": toDeleteUserID})
jobID = ""
for jobInstance in allJobs:
jobID = jobInstance.getJobID()
logging.info("STEP2 - REMOVING " + jobID + " FROM DATABASE...")
jobDAOInstance.remove(jobInstance.getJobID(),
otherParams={"userID": toDeleteUserID})
#****************************************************************
# Step 3. DELETE ALL FILES FOR THE USER
#****************************************************************
logging.info("STEP3 - REMOVING ALL FILES FROM DATABASE...")
filesDAOInstance.removeAll(otherParams={"userID": toDeleteUserID})
logging.info("STEP3 - REMOVING ALL FILES FROM USER DIRECTORY...")
if os_path.isdir(CLIENT_TMP_DIR + toDeleteUserID):
shutil_rmtree(CLIENT_TMP_DIR + toDeleteUserID)
#****************************************************************
# Step 4. DELETE THE USER INSTANCE FROM DATABASE
#****************************************************************
logging.info("STEP6 - REMOVING ALL FILES FROM DATABASE...")
userDAOInstance.remove(int(toDeleteUserID))
response.setContent({"success": True})
except Exception as ex:
handleException(response, ex, __file__, "adminServletDeleteUser")
finally:
return response
def dataManagementGetMyFiles(request,
response,
DESTINATION_DIR,
MAX_CLIENT_SPACE,
isReference=False):
#VARIABLE DECLARATION
fileInstance = None
fileInstances = []
daoInstance = None
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
UserSessionManager().isValidUser(userID, sessionToken)
if not isReference:
DESTINATION_DIR += userID
else:
userID = "-1"
DESTINATION_DIR += "GTF/"
#****************************************************************
# Step 1.GET THE LIST OF FILES
#****************************************************************
logging.info("STEP1 - GET MY FILE LIST REQUEST RECEIVED")
daoInstance = FileDAO()
matchedFiles = daoInstance.findAll(otherParams={"userID": userID})
logging.info("STEP1 - GET MY FILE LIST REQUEST RECEIVED...DONE")
#****************************************************************
# Step 2.CALCULATE USED SPACE
#****************************************************************
logging.info("STEP2 - GET THE CURRENT USED SPACE...")
dataSummary = {
"usedSpace": dir_total_size(DESTINATION_DIR),
"availableSpace": MAX_CLIENT_SPACE
}
logging.info("STEP2 - GET THE CURRENT USED SPACE...DONE")
response.setContent({
"success": True,
"fileList": matchedFiles,
"dataSummary": dataSummary
})
except Exception as ex:
handleException(response, ex, __file__, "dataManagementGetMyFiles")
finally:
if (daoInstance != None):
daoInstance.closeConnection()
return response
def pathwayAcquisitionTouchJob(request, response):
try:
jobID = request.form.get("jobID")
JobInformationManager().touchAccessDate(jobID)
response.setContent({"success": True})
except Exception as ex:
handleException(response,
ex,
__file__,
"pathwayAcquisitionTouchJob",
jobID=jobID)
finally:
return response
def pathwayAcquisitionSaveVisualOptions(request, response):
#VARIABLE DECLARATION
visualOptionsInstance = None
jobID = ""
userID = ""
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
UserSessionManager().isValidUser(userID, sessionToken)
#****************************************************************
# Step 1.GET THE INSTANCE OF visual Options
#****************************************************************
visualOptions = request.get_json()
jobID = visualOptions.get("jobID")
jobInstance = JobInformationManager().loadJobInstance(jobID)
if jobInstance.getReadOnly() and str(
jobInstance.getUserID()) != str(userID):
raise Exception("Invalid user for the job saving visual options")
newTimestamp = int(time())
visualOptions["timestamp"] = newTimestamp
#************************************************************************
# Step 3. Save the visual Options in the MongoDB
#************************************************************************
logging.info("STEP 3 - SAVING VISUAL OPTIONS FOR JOB " + jobID + "...")
JobInformationManager().storeVisualOptions(jobID, visualOptions)
logging.info("STEP 3 - SAVING VISUAL OPTIONS FOR JOB " + jobID +
"...DONE")
response.setContent({"success": True, "timestamp": newTimestamp})
except Exception as ex:
handleException(response,
ex,
__file__,
"pathwayAcquisitionSaveVisualOptions",
userID=userID)
finally:
return response
def adminServletSystemInformation(request, response):
"""
This function...
@param {Request} request, the request object
@param {Response} response, the response object
"""
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
userName = request.cookies.get('userName')
UserSessionManager().isValidAdminUser(userID, userName, sessionToken)
disk_use = []
try:
df = subprocess.Popen(["df", "-h"], stdout=subprocess.PIPE)
output = df.communicate()[0]
output = output.split("\n")
output.pop(0)
for line in output:
disk_use.append(line.split())
except Exception as e:
pass
return response.setContent({
'cpu_count':
psutil.cpu_count(),
"cpu_use":
psutil.cpu_percent(),
"mem_total":
psutil.virtual_memory().total / (1024.0**3),
"mem_use":
psutil.virtual_memory().percent,
"swap_total":
psutil.swap_memory().total / (1024.0**3),
"swap_use":
psutil.swap_memory().percent,
"disk_use":
disk_use
}).getResponse()
except Exception as ex:
handleException(response, ex, __file__, "monitorCPU")
finally:
return response
def pathwayAcquisitionSaveSharingOptions(request, response):
#VARIABLE DECLARATION
jobID = ""
userID = ""
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
UserSessionManager().isValidUser(userID, sessionToken)
#****************************************************************
# Step 1.GET THE INSTANCE OF sharing options
#****************************************************************
jobID = request.form.get("jobID")
jobInstance = JobInformationManager().loadJobInstance(jobID)
if str(jobInstance.getUserID()) != str(userID):
raise Exception("Invalid user for this jobID")
#************************************************************************
# Step 3. Save the visual Options in the MongoDB
#************************************************************************
jobInstance.setAllowSharing(
request.form.get("allowSharing", 'false') == 'true')
jobInstance.setReadOnly(
request.form.get("readOnly", 'false') == 'true')
logging.info("STEP 3 - SAVING SHARING OPTIONS FOR JOB " + jobID +
"...")
JobInformationManager().storeSharingOptions(jobInstance)
logging.info("STEP 3 - SAVING SHARING OPTIONS FOR JOB " + jobID +
"...DONE")
response.setContent({"success": True})
except Exception as ex:
handleException(response,
ex,
__file__,
"pathwayAcquisitionSaveSharingOptions",
userID=userID)
finally:
return response
def userManagementChangePassword(request, response):
# VARIABLE DECLARATION
userInstance = None
daoInstance = None
try:
#****************************************************************
# Step 1. CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
UserSessionManager().isValidUser(userID, sessionToken)
# ****************************************************************
# Step 2.READ THE NEW PASS
# ****************************************************************
logging.info("STEP1 - READ PARAMS AND CHECK IF USER ALREADY EXISTS...")
password = request.form.get("password")
from hashlib import sha1
password = sha1(password.encode('ascii')).hexdigest()
daoInstance = UserDAO()
userInstance = daoInstance.findByID(userID)
if userInstance == None:
raise CredentialException(
"The email or password you entered is incorrect.")
# ****************************************************************
# Step 3. UPDATE THE MODEL
# ****************************************************************
userInstance.setPassword(password)
daoInstance.update(userInstance, {})
response.setContent({"success": True})
except CredentialException as ex:
handleException(response, ex, __file__, "userManagementChangePassword",
200)
except Exception as ex:
handleException(response, ex, __file__, "userManagementChangePassword")
finally:
if (daoInstance != None):
daoInstance.closeConnection()
return response
def adminServletRestoreData(request, response):
"""
This function...
@param {Request} request, the request object
@param {Response} response, the response object
"""
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
userName = request.cookies.get('userName')
UserSessionManager().isValidAdminUser(userID, userName, sessionToken)
#****************************************************************
# Step 1.GET THE SPECIE CODE AND THE UPDATE OPTION
#****************************************************************
formFields = request.form
from subprocess import check_output, CalledProcessError, STDOUT
logging.info("STARTING DBManager Restore PROCESS.")
scriptArgs = [
ROOT_DIRECTORY + "AdminTools/DBManager.py", "restore",
"--remove=1", "--force=1"
]
try:
check_output(scriptArgs, stderr=STDOUT)
except CalledProcessError as exc:
raise Exception(
"Error while calling DBManager Restore: Exit status " +
str(exc.returncode) + ". Error message: " + exc.output)
logging.info("FINISHED DBManager Restore PROCESS.")
response.setContent({"success": True})
except Exception as ex:
handleException(response, ex, __file__, "adminServletRestoreData")
finally:
return response
def adminServletGetAllUsers(request, response):
"""
This function obtains a list of all the users registered in the system including different details
such as the used space, the registration date, etc.
@param {Request} request, the request object
@param {Response} response, the response object
"""
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
userName = request.cookies.get('userName')
UserSessionManager().isValidAdminUser(userID, userName, sessionToken)
#****************************************************************
# Step 1. GET THE LIST OF ALL USERS
#****************************************************************
logging.info("STEP1 - GET THE LIST OF ALL USERS...")
userList = UserDAO().findAll()
for userInstance in userList:
userInstance.usedSpace = 0
if os_path.isdir(CLIENT_TMP_DIR + str(userInstance.getUserId())):
userInstance.usedSpace = dir_total_size(
CLIENT_TMP_DIR + str(userInstance.getUserId()))
response.setContent({
"success": True,
"userList": userList,
"availableSpace": MAX_CLIENT_SPACE,
"max_jobs_days": MAX_JOB_DAYS,
"max_guest_days": MAX_GUEST_DAYS
})
except Exception as ex:
handleException(response, ex, __file__, "adminServletGetAllUsers")
finally:
return response
def dataManagementGetMyJobs(request, response):
#VARIABLE DECLARATION
jobInstance = None
jobInstances = []
daoInstance = None
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
UserSessionManager().isValidUser(userID, sessionToken)
if (userID is None):
response.setContent({
"success":
False,
"errorMessage":
"Log in required</br>Sorry but the feature you are requesting is only available to registered accounts."
})
else:
#****************************************************************
# Step 2.GET THE LIST OF JOBS FOR GIVEN USER
#****************************************************************
logging.info("STEP1 - GET MY JOB LIST REQUEST RECEIVED")
daoInstance = JobDAO()
matchedFiles = daoInstance.findAll(otherParams={"userID": userID})
logging.info("STEP1 - GET MY JOB LIST REQUEST RECEIVED...DONE")
response.setContent({"success": True, "jobList": matchedFiles})
except Exception as ex:
handleException(response, ex, __file__, "dataManagementGetMyJobs")
finally:
if (daoInstance != None):
daoInstance.closeConnection()
return response
def pathwayAcquisitionMetagenes_PART2(ROOT_DIRECTORY, userID, savedJobInstance,
omicName, clusterNumber, RESPONSE):
#VARIABLE DECLARATION
jobID = ""
userID = ""
try:
#************************************************************************
# Step 3. Save the visual Options in the MongoDB
#************************************************************************
logging.info("UPDATE METAGENES - STEP 2 FOR JOB " + jobID + "...")
savedJobInstance.generateMetagenesList(ROOT_DIRECTORY,
{omicName: clusterNumber},
[omicName])
logging.info("UPDATE METAGENES - STEP 2 FOR JOB " + jobID + "...DONE")
JobInformationManager().storePathways(savedJobInstance)
matchedPathwaysJSONList = []
for matchedPathway in savedJobInstance.getMatchedPathways().itervalues(
):
matchedPathwaysJSONList.append(matchedPathway.toBSON())
RESPONSE.setContent({
"success": True,
"jobID": jobID,
# "timestamp": newTimestamp,
"pathwaysInfo": matchedPathwaysJSONList
})
except Exception as ex:
handleException(RESPONSE,
ex,
__file__,
"pathwayAcquisitionMetagenes_PART2",
userID=userID)
finally:
return RESPONSE
def pathwayAcquisitionAdjustPvalues(request, response):
try:
#****************************************************************
# Step 1.GET THE INFO
#****************************************************************
formFields = request.get_json() #request.form
# List of pathway => {pvalues}
pvalues = formFields.get("pValues")
# Check what kind of p-value we want to update
if "stoufferWeights" in formFields:
newStoufferWeights = formFields.get("stoufferWeights")
visiblePathways = formFields.get("visiblePathways")
newStoufferPvalues = defaultdict(dict)
newAdjustedStoufferPvalues = defaultdict(dict)
# Iterate over each database (adjusting it independently)
for db_name, db_pvalues in pvalues.iteritems():
# Each pathway has a different set of matching omics and thus, Stouffer weights.
# The new Stouffer p-value will be computed for each pathway, even those that are currently hidden.
for pathway_id, pathway_pvalues in db_pvalues.iteritems():
# Select those with a proper p-value number and present in Stouffer weights
valid_pvalues = {
omic: pvalue
for omic, pvalue in pathway_pvalues.iteritems()
if pvalue != "-" and omic in newStoufferWeights.keys()
}
# Make sure to pass the Stouffer weights in the same order as the p-values
newStoufferValue = calculateStoufferCombinedPvalue(
valid_pvalues.values(), [
newStoufferWeights[omicName]
for omicName in valid_pvalues.keys()
])
newStoufferPvalues[db_name][pathway_id] = newStoufferValue
# Adjust the new Stouffer p-values passing only those pathways that are currently visible
newAdjustedStoufferPvalues[db_name] = adjustPvalues({
pathway: pvalue
for pathway, pvalue in
newStoufferPvalues[db_name].iteritems()
if pathway in visiblePathways
})
response.setContent({
"success":
True,
"stoufferPvalues":
newStoufferPvalues,
"adjustedStoufferPvalues":
newAdjustedStoufferPvalues
})
else:
# No new stouffer weights, just recalculate the provided p-values
# Iterate over each database (adjusting it independently)
adjustedPvaluesByOmic = defaultdict()
for db_name, db_pvalues in pvalues.iteritems():
pvaluesByOmic = defaultdict(dict)
for pathway, pathwayPvalues in db_pvalues.iteritems():
for omic, omicPvalue in pathwayPvalues.iteritems():
# Skip those in which there is no pValue (no matching in the pathway for that omic)
if omicPvalue != '-':
pvaluesByOmic[omic][pathway] = omicPvalue
adjustedPvaluesByOmic[db_name] = {
omic: adjustPvalues(omic_pvalues)
for omic, omic_pvalues in pvaluesByOmic.iteritems()
}
response.setContent({
"success": True,
"adjustedPvalues": adjustedPvaluesByOmic
})
except Exception as ex:
handleException(response, ex, __file__,
"pathwayAcquisitionAdjustPvalues")
finally:
return response
def adminServletGetAvailableOrganisms(request, response):
"""
This function...
@param {Request} request, the request object
@param {Response} response, the response object
"""
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
userName = request.cookies.get('userName')
UserSessionManager().isValidAdminUser(userID, userName, sessionToken)
#****************************************************************
# Step 1.GET THE LIST OF INSTALLED SPECIES (DATABASES and SPECIES.JSON)
#****************************************************************
import csv
databaseList = []
with open(KEGG_DATA_DIR +
'current/common/organisms_all.list') as availableSpeciesFile:
reader = csv.reader(availableSpeciesFile, delimiter='\t')
for row in reader:
organism_code = row[1]
# Step 2.4 Check if the organism has non installed data available
if os_path.isfile(KEGG_DATA_DIR + 'download/' + organism_code +
'/VERSION'):
downloaded = True
elif os_path.isfile(KEGG_DATA_DIR + 'download/' +
organism_code + '/DOWNLOADING'):
downloaded = "downloading"
else:
downloaded = False
#Erroneous download not removed --> remove
if os_path.isdir(KEGG_DATA_DIR + 'download/' +
organism_code):
shutil_rmtree(KEGG_DATA_DIR + 'download/' +
organism_code)
databaseList.append({
"organism_name": row[2],
"organism_code": organism_code,
"categories": row[3].split(";"),
"organism_id": row[0],
"downloaded": downloaded
})
availableSpeciesFile.close()
response.setContent({
"databaseList":
databaseList,
"download_log":
KEGG_DATA_DIR + "download/download.log",
"install_log":
KEGG_DATA_DIR + "current/install.log"
})
except Exception as ex:
handleException(response, ex, __file__,
"adminServletGetInstalledOrganisms")
finally:
return response
def fromBEDtoGenes_STEP1(REQUEST,
RESPONSE,
QUEUE_INSTANCE,
JOB_ID,
EXAMPLE_FILES_DIR,
exampleMode=False):
"""
This function corresponds to FIRST PART of the FIRST step in the Bed2Genes process.
First, it takes a Request object which contains the fields of the form that started the process.
This is a summary for the steps in the process:
Step 0. VARIABLE DECLARATION
Step 1. CHECK IF VALID USER SESSION
Step 2. CREATE THE NEW INSTANCE OF JOB
Step 3. SAVE THE UPLOADED FILES
Step 4. READ PARAMS
Step 5. QUEUE THE JOB INSTANCE
Step 6. RETURN THE NEW JOB ID
@param {Request} REQUEST
@param {Response} RESPONSE
@param {RQ QUEUE} QUEUE_INSTANCE
@param {String} JOB_ID
@param {Boolean} exampleMode
@returns Response
"""
#TODO: ALLOWED_EXTENSIONS http://flask.pocoo.org/docs/0.10/patterns/fileuploads/
#TODO: secure_filename
#****************************************************************
#Step 0. VARIABLE DECLARATION
#The following variables are defined:
# - jobInstance: instance of the Bed2GeneJob class. Contains all the information for the current job.
# - userID: the ID for the user
#****************************************************************
jobInstance = None
userID = None
try:
#****************************************************************
# Step 1. CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = REQUEST.cookies.get('userID')
sessionToken = REQUEST.cookies.get('sessionToken')
UserSessionManager().isValidUser(userID, sessionToken)
#****************************************************************
# Step 2. CREATE THE NEW INSTANCE OF JOB
#****************************************************************
jobInstance = Bed2GeneJob(JOB_ID, userID, CLIENT_TMP_DIR)
jobInstance.initializeDirectories()
logging.info("STEP1 - NEW JOB SUBMITTED " + jobInstance.getJobID())
#****************************************************************
# Step 3. SAVE THE UPLOADED FILES
#****************************************************************
formFields = REQUEST.form
if (exampleMode == False):
logging.info("STEP1 - FILE UPLOADING REQUEST RECEIVED")
uploadedFiles = REQUEST.files
logging.info("STEP1 - READING FILES....")
JobInformationManager().saveFiles(uploadedFiles, formFields,
userID, jobInstance,
CLIENT_TMP_DIR,
EXAMPLE_FILES_DIR)
logging.info("STEP1 - READING FILES....DONE")
elif (exampleMode == "example"):
#****************************************************************
# Step 2.SAVE THE UPLOADED FILES
#****************************************************************
logging.info("STEP1 - EXAMPLE MODE SELECTED")
logging.info("STEP1 - COPYING FILES....")
exampleOmics = ["DNase unmapped"]
for omicName in exampleOmics:
dataFileName = omicName.replace(" ",
"_").lower() + "_values.tab"
logging.info(
"STEP1 - USING ALREADY SUBMITTED FILE (data file) " +
EXAMPLE_FILES_DIR + dataFileName + " FOR " + omicName)
relevantFileName = omicName.replace(
" ", "_").lower() + "_relevant.tab"
logging.info(
"STEP1 - USING ALREADY SUBMITTED FILE (relevant features file) "
+ EXAMPLE_FILES_DIR + relevantFileName + " FOR " +
omicName)
jobInstance.addGeneBasedInputOmic({
"omicName":
omicName,
"inputDataFile":
EXAMPLE_FILES_DIR + dataFileName,
"relevantFeaturesFile":
EXAMPLE_FILES_DIR + relevantFileName,
"isExample":
True
})
jobInstance.addReferenceInput({
"omicName":
omicName,
"fileType":
"Reference file",
"inputDataFile":
EXAMPLE_FILES_DIR + "GTF/sorted_mmu.gtf"
})
specie = "mmu"
jobInstance.setOrganism(specie)
else:
raise NotImplementedError
#****************************************************************
# Step 4. READ PARAMS
#****************************************************************
namePrefix = formFields.get("name_prefix")
logging.info("STEP2 - INPUT VALUES ARE:")
jobInstance.omicName = formFields.get(namePrefix + "_omic_name",
"DNase-seq")
logging.info(" - omicName :" + jobInstance.omicName)
jobInstance.presortedGTF = formFields.get(namePrefix + "_presortedGTF",
False)
logging.info(" - presortedGTF :" +
str(jobInstance.presortedGTF))
jobInstance.report = formFields.get(namePrefix + "_report", "gene")
logging.info(" - report :" + jobInstance.report)
jobInstance.distance = formFields.get(namePrefix + "_distance", 10)
logging.info(" - distance :" + str(jobInstance.distance))
jobInstance.tss = formFields.get(namePrefix + "_tss", 200)
logging.info(" - tss :" + str(jobInstance.tss))
jobInstance.promoter = formFields.get(namePrefix + "_promoter", 1300)
logging.info(" - promoter :" + str(jobInstance.promoter))
jobInstance.geneAreaPercentage = formFields.get(
namePrefix + "_geneAreaPercentage", 90)
logging.info(" - geneAreaPercentage :" +
str(jobInstance.geneAreaPercentage))
jobInstance.regionAreaPercentage = formFields.get(
namePrefix + "_regionAreaPercentage", 50)
logging.info(" - regionAreaPercentage :" +
str(jobInstance.regionAreaPercentage))
jobInstance.ignoreMissing = True if namePrefix + "_ignoremissing" in formFields.keys(
) else False
logging.info(" - ignore missing :" +
str(jobInstance.ignoreMissing))
jobInstance.featureEnrichment = formFields.get(
namePrefix + "_feature_enrichment_pre", False)
logging.info(" - Feature Enrichment :" +
str(jobInstance.featureEnrichment))
#rules
jobInstance.geneIDtag = formFields.get(namePrefix + "_geneIDtag",
"gene_id")
logging.info(" - geneIDtag :" + str(jobInstance.geneIDtag))
jobInstance.summarizationMethod = formFields.get(
namePrefix + "_summarization_method", "mean")
logging.info(" - summarization_method :" +
jobInstance.summarizationMethod)
jobInstance.reportRegions = formFields.getlist(namePrefix +
"_reportRegions")
if len(jobInstance.reportRegions) == 0:
jobInstance.reportRegions = ["all"]
logging.info(" - reportRegions :" + str(jobInstance.reportRegions))
#************************************************************************
# Step 4. Queue job
#************************************************************************
QUEUE_INSTANCE.enqueue(fn=fromBEDtoGenes_STEP2,
args=(jobInstance, userID, exampleMode,
RESPONSE),
timeout=600,
job_id=JOB_ID)
#************************************************************************
# Step 5. Return the Job ID
#************************************************************************
RESPONSE.setContent({"success": True, "jobID": JOB_ID})
except Exception as ex:
handleException(RESPONSE, ex, __file__, "fromBEDtoGenes_STEP1")
finally:
return RESPONSE
def adminServletGetInstalledOrganisms(request, response):
"""
This function...
@param {Request} request, the request object
@param {Response} response, the response object
"""
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
userName = request.cookies.get('userName')
UserSessionManager().isValidAdminUser(userID, userName, sessionToken)
#****************************************************************
# Step 1.GET THE LIST OF INSTALLED SPECIES (DATABASES and SPECIES.JSON)
#****************************************************************
organisms_names = {}
with open(KEGG_DATA_DIR +
'current/common/organisms_all.list') as organisms_all:
reader = csv.reader(organisms_all, delimiter='\t')
for row in reader:
organisms_names[row[1]] = row[2]
organisms_all.close()
installedSpecies = []
from pymongo import MongoClient
client = MongoClient(MONGODB_HOST, MONGODB_PORT)
databases = client.database_names()
#****************************************************************
# Step 2.FOR EACH INSTALLED DATABASE GET THE INFORMATION
#****************************************************************
databaseList = []
common_info_date = ""
for database in databases:
if not "-paintomics" in database:
continue
elif "global-paintomics" == database:
db = client[database]
common_info_date = db.versions.find({"name":
"COMMON"})[0].get("date")
# Step 2.4 Check if the organism has non installed data available
if os_path.isfile(KEGG_DATA_DIR + 'download/common/VERSION'):
downloaded = True
elif os_path.isfile(KEGG_DATA_DIR +
'download/common/DOWNLOADING'):
downloaded = "downloading"
else:
downloaded = False
#Erroneous download not removed --> remove
if os_path.isdir(KEGG_DATA_DIR + 'download/common/'):
shutil_rmtree(KEGG_DATA_DIR + 'download/common/')
databaseList.append({
"organism_name": "Common KEGG data",
"organism_code": "common",
"kegg_date": common_info_date,
"downloaded": downloaded
})
else:
# Step 2.1 GET THE SPECIE CODE
organism_code = database.replace("-paintomics", "")
# Step 2.2 GET THE SPECIE NAME
organism_name = organisms_names.get(organism_code,
"Unknown specie")
# Step 2.3 GET THE SPECIE VERSIONS
db = client[database]
kegg_date = db.versions.find({"name": "KEGG"})[0].get("date")
mapping_date = db.versions.find({"name":
"MAPPING"})[0].get("date")
acceptedIDs = db.versions.find({"name": "ACCEPTED_IDS"})
if acceptedIDs.count() > 0:
acceptedIDs = acceptedIDs[0].get("ids")
else:
acceptedIDs = ""
# Step 2.4 Check if the organism has non installed data available
if os_path.isfile(KEGG_DATA_DIR + 'download/' + organism_code +
'/VERSION'):
downloaded = True
elif os_path.isfile(KEGG_DATA_DIR + 'download/' +
organism_code + '/DOWNLOADING'):
downloaded = "downloading"
else:
downloaded = False
#Erroneous download not removed --> remove
if os_path.isdir(KEGG_DATA_DIR + 'download/' +
organism_code):
shutil_rmtree(KEGG_DATA_DIR + 'download/' +
organism_code)
databaseList.append({
"organism_name": organism_name,
"organism_code": organism_code,
"kegg_date": kegg_date,
"mapping_date": mapping_date,
"acceptedIDs": acceptedIDs,
"downloaded": downloaded
})
client.close()
response.setContent({
"common_info_date": common_info_date,
"databaseList": databaseList
})
except Exception as ex:
handleException(response, ex, __file__,
"adminServletGetInstalledOrganisms")
finally:
return response
def adminServletInstallOrganism(request, response, organism_code,
ROOT_DIRECTORY):
"""
This function manages an 'Install/Update Organism' request by calling to the
DBManager tool.
@param {Request} request, the request object
@param {Response} response, the response object
@param {String} organism_code,
@param {String} ROOT_DIRECTORY,
"""
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
userName = request.cookies.get('userName')
UserSessionManager().isValidAdminUser(userID, userName, sessionToken)
#****************************************************************
# Step 1.GET THE SPECIE CODE AND THE UPDATE OPTION
#****************************************************************
download = json.loads(request.data).get("download")
update_kegg = 1
update_mapping = 1
common = 0
if organism_code == "common":
common = 1
organism_code = "#common"
from subprocess import check_output, CalledProcessError, STDOUT
#****************************************************************
# Step 2a. IF THE SELECTED OPTION IS DOWNLOAD
#****************************************************************
if download:
logging.info("STARTING DBManager download PROCESS.")
scriptArgs = [
ROOT_DIRECTORY + "AdminTools/DBManager.py", "download",
"--specie=" + organism_code, "--kegg=" + str(update_kegg),
"--mapping=" + str(update_mapping), "--common=" + str(common)
]
try:
check_output(scriptArgs, stderr=STDOUT)
except CalledProcessError as exc:
raise Exception(
"Error while calling DBManager download: Exit status " +
str(exc.returncode) + ". Error message: " + exc.output)
logging.info("FINISHED DBManager Download PROCESS.")
# ****************************************************************
# Step 2B. IF THE SELECTED OPTION IS INSTALL
# ****************************************************************
else:
logging.info("STARTING DBManager Install PROCESS.")
scriptArgs = [
ROOT_DIRECTORY + "AdminTools/DBManager.py", "install",
"--specie=" + organism_code, "--common=" + str(common)
]
try:
check_output(scriptArgs, stderr=STDOUT)
except CalledProcessError as exc:
raise Exception(
"Error while calling DBManager Install: Exit status " +
str(exc.returncode) + ". Error message: " + exc.output)
logging.info("FINISHED DBManager Install PROCESS.")
response.setContent({"success": True})
except Exception as ex:
handleException(response, ex, __file__, "adminServletUpdateOrganism")
finally:
return response
def pathwayAcquisitionRecoverJob(request, response, QUEUE_INSTANCE):
#VARIABLE DECLARATION
jobInstance = None
jobID = ""
userID = ""
#TODO: COMPROBAR OWNERS
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
UserSessionManager().isValidUser(userID, sessionToken)
#****************************************************************
# Step 1.LOAD THE INSTANCE OF JOB
#****************************************************************
formFields = request.form
jobID = formFields.get("jobID")
logging.info("RECOVER_JOB - LOADING JOB " + jobID + "...")
jobInstance = JobInformationManager().loadJobInstance(jobID)
queueJob = QUEUE_INSTANCE.fetch_job(jobID)
if (queueJob is not None and not queueJob.is_finished()):
logging.info("RECOVER_JOB - JOB " + jobID + " HAS NOT FINISHED ")
response.setContent({
"success":
False,
"message":
"Your job " + jobID +
" is still running in the queue. Please, try again later to check if it has finished."
})
return response
if (jobInstance == None):
#TODO DIAS BORRADO?
logging.info("RECOVER_JOB - JOB " + jobID +
" NOT FOUND AT DATABASE.")
response.setContent({
"success":
False,
"errorMessage":
"Job " + jobID +
" not found at database.<br>Please, note that jobs are automatically removed after 7 days for guests and 14 days for registered users."
})
return response
# Allow "no user" jobs to be viewed by anyone, logged or not
if (str(jobInstance.getUserID()) != 'None'
and jobInstance.getUserID() != userID
and not jobInstance.getAllowSharing()):
logging.info("RECOVER_JOB - JOB " + jobID +
" DOES NOT BELONG TO USER " + str(userID) +
" JOB HAS USER " + str(jobInstance.getUserID()))
response.setContent({
"success":
False,
"errorMessage":
"Invalid Job ID (" + jobID +
") for current user.<br>Please, check the Job ID and try again."
})
return response
logging.info("RECOVER_JOB - JOB " + jobInstance.getJobID() +
" LOADED SUCCESSFULLY.")
matchedCompoundsJSONList = map(
lambda foundFeature: foundFeature.toBSON(),
jobInstance.getFoundCompounds())
matchedPathwaysJSONList = []
for matchedPathway in jobInstance.getMatchedPathways().itervalues():
matchedPathwaysJSONList.append(matchedPathway.toBSON())
logging.info("RECOVER_JOB - GENERATING PATHWAYS INFORMATION...DONE")
if (len(matchedCompoundsJSONList) == 0
and jobInstance.getLastStep() == 2
and len(jobInstance.getCompoundBasedInputOmics()) > 0):
logging.info(
"RECOVER_JOB - JOB " + jobID +
" DOES NOT CONTAINS FOUND COMPOUNDS (STEP 2: OLD FORMAT?).")
response.setContent({
"success":
False,
"errorMessage":
"Job " + jobID +
" does not contains saved information about the found compounds, please run it again."
})
elif (len(matchedPathwaysJSONList) == 0
and jobInstance.getLastStep() > 2):
logging.info("RECOVER_JOB - JOB " + jobID +
" DOES NOT CONTAINS PATHWAYS.")
response.setContent({
"success":
False,
"errorMessage":
"Job " + jobID +
" does not contains information about pathways. Please, run it again."
})
else:
response.setContent({
"success":
True,
"jobID":
jobInstance.getJobID(),
"userID":
jobInstance.getUserID(),
"pathwaysInfo":
matchedPathwaysJSONList,
"geneBasedInputOmics":
jobInstance.getGeneBasedInputOmics(),
"compoundBasedInputOmics":
jobInstance.getCompoundBasedInputOmics(),
"organism":
jobInstance.getOrganism(),
"summary":
jobInstance.summary,
"visualOptions":
JobInformationManager().getVisualOptions(jobID),
"databases":
jobInstance.getDatabases(),
"matchedMetabolites":
matchedCompoundsJSONList,
"stepNumber":
jobInstance.getLastStep(),
"name":
jobInstance.getName(),
"timestamp":
int(time()),
"allowSharing":
jobInstance.getAllowSharing(),
"readOnly":
jobInstance.getReadOnly(),
"omicsValuesID":
jobInstance.getValueIdTable()
})
except Exception as ex:
handleException(response,
ex,
__file__,
"pathwayAcquisitionRecoverJob",
userID=userID)
finally:
return response
def adminServletSendReport(request, response, ROOT_DIRECTORY):
"""
This function...
@param {Request} request, the request object
@param {Response} response, the response object
"""
try:
#logging.info("STEP0 - CHECK IF VALID USER....")
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
userID = request.cookies.get('userID')
#sessionToken = request.cookies.get('sessionToken')
#UserSessionManager().isValidUser(userID, sessionToken)
#****************************************************************
# Step 1.GET THE SPECIE CODE AND THE UPDATE OPTION
#****************************************************************
formFields = request.form
if userID is not None:
userEmail = UserDAO().findByID(userID)
userName = userEmail.getUserName()
userEmail = userEmail.getEmail()
else:
userEmail = formFields.get("fromEmail", smpt_sender)
userName = formFields.get("fromName", "No name provided")
type = formFields.get("type")
_message = formFields.get("message")
title = "Other request"
color = "#333"
if type == "error":
type = "Error notification"
title = "<h1>New error notification</h1>"
color = "#f95959"
elif type == "specie_request":
type = "New specie requesting"
title = "<h1>New organism requested</h1>"
color = "#0090ff"
else:
type = "Other request"
message = '<html><body>'
message += "<a href='" + "http://bioinfo.cipf.es/paintomics/" + "' target='_blank'>"
message += " <img src='cid:image1' border='0' width='auto' height='50' alt='Paintomics 3 logo'>"
message += "</a>"
message += "<div style='width:100%; height:10px; border-top: 1px dotted #333; margin-top:20px; margin-bottom:30px;'></div>"
message += "<h1>" + title + "</h1>"
message += "<p>Thanks for the report, " + userName + "!</p>"
message += "<p><b>Username:</b> " + userEmail + "</p></br>"
message += "<div style='width:100%; border: 1px solid " + color + "; padding:10px;font-family: monospace;color:" + color + ";'>" + _message + "</div>"
message += "<p>We will contact you as soon as possible.</p>"
message += "<p>Best regards,</p>"
message += "<p>The Paintomics developers team.</p>"
message += "<div style='width:100%; height:10px; border-top: 1px dotted #333; margin-top:20px; margin-bottom:30px;'></div>"
message += "<p>Problems? E-mail <a href='mailto:" + "*****@*****.**" + "'>" + "*****@*****.**" + "</a></p>"
message += '</body></html>'
sendEmail(ROOT_DIRECTORY,
smpt_sender,
smpt_sender_name,
type,
message,
fromEmail=userEmail,
fromName=userName,
isHTML=True)
response.setContent({"success": True})
except Exception as ex:
handleException(response, ex, __file__, "adminServletSendReport")
finally:
return response
def pathwayAcquisitionStep3(request, response):
#VARIABLE DECLARATION
jobInstance = None
jobID = ""
userID = ""
try:
#****************************************************************
# Step 0.CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = request.cookies.get('userID')
sessionToken = request.cookies.get('sessionToken')
UserSessionManager().isValidUser(userID, sessionToken)
#****************************************************************
# Step 1.LOAD THE INSTANCE OF JOB
#****************************************************************
formFields = request.form
jobID = formFields.get("jobID")
#TODO: IN PREVIOUS STEPS THE USER COULD SPECIFY THE DEFAULT OMICS TO SHOW
visibleOmics = []
logging.info("STEP3 - LOADING JOB " + jobID + "...")
jobInstance = JobInformationManager().loadJobInstance(jobID)
jobInstance.setUserID(userID)
if (jobInstance == None):
raise UserWarning("Job " + jobID + " was not found at database.")
logging.info("STEP3 - JOB " + jobInstance.getJobID() +
" LOADED SUCCESSFULLY.")
#****************************************************************
# Step 2.READ THE SELECTED PATHWAYS
#****************************************************************
logging.info("STEP3 - GENERATING PATHWAYS INFORMATION...")
selectedPathways = formFields.getlist("selectedPathways")
#TODO: SOLO GENERAR INFO PARA LAS QUE NO LA TENGAN YA GUARDADA EN LA BBDD
[
selectedPathwayInstances, graphicalOptionsInstancesBSON,
omicsValuesSubset
] = jobInstance.generateSelectedPathwaysInformation(
selectedPathways, visibleOmics, True)
logging.info("STEP3 - GENERATING PATHWAYS INFORMATION...DONE")
#************************************************************************
# Step 3. Save the jobInstance in the MongoDB
#************************************************************************
logging.info("STEP 3 - SAVING NEW JOB DATA...")
JobInformationManager().storeJobInstance(jobInstance, 3)
logging.info("STEP 3 - SAVING NEW JOB DATA...DONE")
response.setContent({
"success": True,
"jobID": jobInstance.getJobID(),
"graphicalOptionsInstances": graphicalOptionsInstancesBSON,
"omicsValues": omicsValuesSubset,
"organism": jobInstance.getOrganism()
})
except Exception as ex:
handleException(response,
ex,
__file__,
"pathwayAcquisitionStep3",
userID=userID)
finally:
return response
def pathwayAcquisitionStep1_PART1(REQUEST, RESPONSE, QUEUE_INSTANCE, JOB_ID,
EXAMPLE_FILES_DIR, exampleMode):
"""
This function corresponds to FIRST PART of the FIRST step in the Pathways acquisition process.
First, it takes a Request object which contains the fields of the form that started the process.
This is a summarization for the steps in the process:
Step 0. VARIABLE DECLARATION
Step 1. CHECK IF VALID USER SESSION
Step 2. CREATE THE NEW INSTANCE OF JOB
Step 3. SAVE THE UPLOADED FILES
Step 4. QUEUE THE JOB INSTANCE
Step 5. RETURN THE NEW JOB ID
@param {Request} REQUEST
@param {Response} RESPONSE
@param {RQ QUEUE} QUEUE_INSTANCE
@param {String} JOB_ID
@param {Boolean} exampleMode
@returns Response
"""
#TODO:ALLOWED_EXTENSIONS http://flask.pocoo.org/docs/0.10/patterns/fileuploads/
#TODO: secure_filename
#****************************************************************
#Step 0. VARIABLE DECLARATION
#The following variables are defined:
# - jobInstance: instance of the PathwayAcquisitionJob class.
# Contains all the information for the current job.
# - userID: the ID for the user
#****************************************************************
jobInstance = None
userID = None
try:
#****************************************************************
# Step 1. CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = REQUEST.cookies.get('userID')
sessionToken = REQUEST.cookies.get('sessionToken')
UserSessionManager().isValidUser(userID, sessionToken)
#****************************************************************
# Step 2. CREATE THE NEW INSTANCE OF JOB
#****************************************************************
jobInstance = PathwayAcquisitionJob(JOB_ID, userID, CLIENT_TMP_DIR)
jobInstance.initializeDirectories()
logging.info("STEP1 - NEW JOB SUBMITTED " + jobInstance.getJobID())
#****************************************************************
# Step 3. SAVE THE UPLOADED FILES
#****************************************************************
if (exampleMode == False):
logging.info("STEP1 - FILE UPLOADING REQUEST RECEIVED")
uploadedFiles = REQUEST.files
formFields = REQUEST.form
jobInstance.description = ""
jobInstance.setName(formFields.get("jobDescription", "")[:100])
specie = formFields.get("specie") #GET THE SPECIE NAME
databases = REQUEST.form.getlist('databases[]')
jobInstance.setOrganism(specie)
# Check the available databases for species
organismDB = set(dicDatabases.get(specie, [{}])[0].keys())
# TODO: disabled multiple databases for the moment
# jobInstance.setDatabases(list(set([u'KEGG']) | set(databases).intersection(organismDB)))
jobInstance.setDatabases([u'KEGG'])
logging.info("STEP1 - SELECTED SPECIE IS " + specie)
logging.info("STEP1 - READING FILES....")
JobInformationManager().saveFiles(uploadedFiles, formFields,
userID, jobInstance,
CLIENT_TMP_DIR)
logging.info("STEP1 - READING FILES....DONE")
elif (exampleMode == "example"):
#****************************************************************
# Step 2.SAVE THE UPLOADED FILES
#****************************************************************
logging.info("STEP1 - EXAMPLE MODE SELECTED")
logging.info("STEP1 - COPYING FILES....")
exampleOmics = {
"Gene expression": False,
"Metabolomics": True,
"Proteomics": True,
"miRNA-seq": False,
"DNase-seq": False
}
for omicName, featureEnrichment in exampleOmics.iteritems():
dataFileName = omicName.replace(" ", "_").replace(
"-seq", "").lower() + "_values.tab"
logging.info(
"STEP1 - USING ALREADY SUBMITTED FILE (data file) " +
EXAMPLE_FILES_DIR + dataFileName + " FOR " + omicName)
relevantFileName = omicName.replace(" ", "_").replace(
"-seq", "").lower() + "_relevant.tab"
logging.info(
"STEP1 - USING ALREADY SUBMITTED FILE (relevant features file) "
+ EXAMPLE_FILES_DIR + relevantFileName + " FOR " +
omicName)
if (["Metabolomics"].count(omicName)):
jobInstance.addCompoundBasedInputOmic({
"omicName":
omicName,
"inputDataFile":
EXAMPLE_FILES_DIR + dataFileName,
"relevantFeaturesFile":
EXAMPLE_FILES_DIR + relevantFileName,
"isExample":
True,
"featureEnrichment":
featureEnrichment
})
else:
jobInstance.addGeneBasedInputOmic({
"omicName":
omicName,
"inputDataFile":
EXAMPLE_FILES_DIR + dataFileName,
"relevantFeaturesFile":
EXAMPLE_FILES_DIR + relevantFileName,
"isExample":
True,
"featureEnrichment":
featureEnrichment
})
specie = "mmu"
jobInstance.setOrganism(specie)
jobInstance.setDatabases(['KEGG'])
else:
raise NotImplementedError
#************************************************************************
# Step 4. Queue job
#************************************************************************
QUEUE_INSTANCE.enqueue(fn=pathwayAcquisitionStep1_PART2,
args=(jobInstance, userID, exampleMode,
RESPONSE),
timeout=600,
job_id=JOB_ID)
#************************************************************************
# Step 5. Return the Job ID
#************************************************************************
RESPONSE.setContent({"success": True, "jobID": JOB_ID})
except Exception as ex:
if (jobInstance != None):
jobInstance.cleanDirectories(remove_output=True)
handleException(RESPONSE,
ex,
__file__,
"pathwayAcquisitionStep1_PART1",
userID=userID)
finally:
return RESPONSE
def pathwayAcquisitionStep2_PART2(jobID, userID, selectedCompounds,
clusterNumber, RESPONSE, ROOT_DIRECTORY):
"""
This function corresponds to SECOND PART of the SECOND step in the Pathways acquisition process.
Given a JOB INSTANCE, first processes the uploaded files (identifiers matching and compound list generation)
and finally generates the response.
This code is executed at the Redis Queue.
This is a summarization for the steps in the process:
Step 1. READ AND UPDATE THE SELECTED METABOLITES
Step 2. GENERATE PATHWAYS INFORMATION
Step 3. GENERATE THE METAGENES INFORMATION
Step 4. UPDATE JOB INFORMATION AT DATABASE
Step 5. GENERATE RESPONSE AND FINISH
@param {PathwayAcquisitionJob} jobInstance
@param {String[]} selectedCompounds
@param {Response} RESPONSE
@param {String} userID
@param {Boolean} exampleMode
@returns Response
"""
jobInstance = None
try:
logging.info("STEP2 - LOADING JOB " + jobID + "...")
jobInstance = JobInformationManager().loadJobInstance(jobID)
jobInstance.setUserID(userID)
if (jobInstance == None):
raise UserWarning("Job " + jobID + " was not found at database.")
jobInstance.setDirectories(CLIENT_TMP_DIR)
jobInstance.initializeDirectories()
logging.info("STEP2 - JOB " + jobInstance.getJobID() +
" LOADED SUCCESSFULLY.")
#****************************************************************
# Step 1.READ THE SELECTED METABOLITES
#****************************************************************
logging.info("STEP2 - UPDATING SELECTED COMPOUNDS LIST...")
#TODO: CHANGE THIS TO ALLOW BACK
jobInstance.updateSubmitedCompoundsList(selectedCompounds)
logging.info("STEP2 - UPDATING SELECTED COMPOUNDS LIST...DONE")
#****************************************************************
# Step 2. GENERATING PATHWAYS INFORMATION
#****************************************************************
logging.info("STEP2 - GENERATING PATHWAYS INFORMATION...")
summary = jobInstance.generatePathwaysList()
logging.info("STEP2 - GENERATING PATHWAYS INFORMATION...DONE")
#****************************************************************
# Step 3. GENERATING METAGENES INFORMATION
#****************************************************************
logging.info("STEP2 - GENERATING METAGENES INFORMATION...")
jobInstance.generateMetagenesList(ROOT_DIRECTORY, clusterNumber)
logging.info("STEP2 - GENERATING METAGENES INFORMATION...DONE")
jobInstance.setLastStep(3)
#************************************************************************
# Step 4. Save the all the Matched Compounds and pathways in MongoDB
#************************************************************************
logging.info("STEP2 - SAVING NEW JOB DATA...")
JobInformationManager().storeJobInstance(jobInstance, 2)
logging.info("STEP2 - SAVING NEW JOB DATA...DONE")
#************************************************************************
# Step 5. Update the response content
#************************************************************************
matchedPathwaysJSONList = []
for matchedPathway in jobInstance.getMatchedPathways().itervalues():
matchedPathwaysJSONList.append(matchedPathway.toBSON())
RESPONSE.setContent({
"success":
True,
"organism":
jobInstance.getOrganism(),
"jobID":
jobInstance.getJobID(),
"summary":
summary,
"pathwaysInfo":
matchedPathwaysJSONList,
"geneBasedInputOmics":
jobInstance.getGeneBasedInputOmics(),
"compoundBasedInputOmics":
jobInstance.getCompoundBasedInputOmics(),
"databases":
jobInstance.getDatabases(),
"omicsValuesID":
jobInstance.getValueIdTable(),
"timestamp":
int(time())
})
except Exception as ex:
handleException(RESPONSE,
ex,
__file__,
"pathwayAcquisitionStep2_PART2",
userID=userID)
finally:
jobInstance.cleanDirectories()
return RESPONSE
def pathwayAcquisitionStep2_PART1(REQUEST, RESPONSE, QUEUE_INSTANCE,
ROOT_DIRECTORY):
"""
This function corresponds to FIRST PART of the SECOND step in the Pathways acquisition process.
First, it takes a Request object which contains the fields of the form that started the process.
This is a summary for the steps in the process:
Step 0. VARIABLE DECLARATION
Step 1. CHECK IF VALID USER SESSION
Step 2. LOAD THE INSTANCE OF JOB
Step 3. QUEUE THE JOB INSTANCE
Step 4. RETURN THE JOB ID
@param {Request} REQUEST
@param {Response} RESPONSE
@param {RQ QUEUE} QUEUE_INSTANCE
@returns Response
"""
#****************************************************************
#Step 0. VARIABLE DECLARATION
#The following variables are defined:
# - jobID: the ID for the job instance
# - userID: the ID for the user
#****************************************************************
jobID = ""
userID = ""
try:
#****************************************************************
# Step 1. CHECK IF VALID USER SESSION
#****************************************************************
logging.info("STEP0 - CHECK IF VALID USER....")
userID = REQUEST.cookies.get('userID')
sessionToken = REQUEST.cookies.get('sessionToken')
UserSessionManager().isValidUser(userID, sessionToken)
#****************************************************************
# Step 2.LOAD THE INSTANCE OF JOB
#****************************************************************
formFields = REQUEST.form
jobID = formFields.get("jobID")
selectedCompounds = REQUEST.form.getlist("selectedCompounds[]")
# Retrieve the number of cluster on a per omic basis
# Note: this will contain the omic name transformed to remove spaces and special chars
clusterNumber = {
key.replace("clusterNumber:", ""): value
for key, value in formFields.iteritems()
if key.startswith("clusterNumber:")
}
#************************************************************************
# Step 3. Queue job
#************************************************************************
QUEUE_INSTANCE.enqueue(fn=pathwayAcquisitionStep2_PART2,
args=(
jobID,
userID,
selectedCompounds,
clusterNumber,
RESPONSE,
ROOT_DIRECTORY,
),
timeout=600,
job_id=jobID)
#************************************************************************
# Step 5. Return the Job ID
#************************************************************************
RESPONSE.setContent({"success": True, "jobID": jobID})
except Exception as ex:
handleException(RESPONSE,
ex,
__file__,
"pathwayAcquisitionStep2_PART1",
userID=userID)
finally:
return RESPONSE
def pathwayAcquisitionStep1_PART2(jobInstance, userID, exampleMode, RESPONSE):
"""
This function corresponds to SECOND PART of the FIRST step in the Pathways acquisition process.
Given a JOB INSTANCE, first processes the uploaded files (identifiers matching and compound list generation)
and finally generates the response.
This code is executed at the PyQlite Queue.
This is a summarization for the steps in the process:
Step 0. CHECK FILES CONTENT
Step 1. PROCESS THE FILES DATA
Step 2. SAVE THE JOB INSTANCE AT THE DATABASE
Step 3. GENERATE RESPONSE AND FINISH
@param {PathwayAcquisitionJob} jobInstance
@param {Response} RESPONSE
@param {String} userID
@param {Boolean} exampleMode
@returns Response
"""
try:
#****************************************************************
# Step 0.VALIDATE THE FILES DATA
#****************************************************************
logging.info("STEP0 - VALIDATING INPUT...")
jobInstance.validateInput()
logging.info("STEP1 - VALIDATING INPUT...DONE")
#****************************************************************
# Step 1.PROCESS THE FILES DATA
#****************************************************************
logging.info("STEP1 - PROCESSING FILES...")
matchedMetabolites = jobInstance.processFilesContent(
) #This function processes all the files and returns a checkboxes list to show to the user
logging.info("STEP1 - PROCESSING FILES...DONE")
#************************************************************************
# Step 2. Save the jobInstance in the MongoDB
#************************************************************************
logging.info("STEP1 - SAVING JOB DATA...")
jobInstance.setLastStep(2)
jobInstance.getJobDescription(True, exampleMode == "example")
JobInformationManager().storeJobInstance(jobInstance, 1)
logging.info("STEP1 - SAVING JOB DATA...DONE")
jobInstance.cleanDirectories()
#************************************************************************
# Step 3. Update the response content
#************************************************************************
RESPONSE.setContent({
"success":
True,
"organism":
jobInstance.getOrganism(),
"jobID":
jobInstance.getJobID(),
"userID":
jobInstance.getUserID(),
"matchedMetabolites":
map(lambda foundFeature: foundFeature.toBSON(),
matchedMetabolites),
"geneBasedInputOmics":
jobInstance.getGeneBasedInputOmics(),
"compoundBasedInputOmics":
jobInstance.getCompoundBasedInputOmics(),
"databases":
jobInstance.getDatabases(),
"name":
jobInstance.getName(),
"timestamp":
int(time())
})
except Exception as ex:
jobInstance.cleanDirectories(remove_output=True)
# TODO: at this point we should notify the queue system about the error, or else
# will keep returning success to the job.
handleException(RESPONSE,
ex,
__file__,
"pathwayAcquisitionStep1_PART2",
userID=userID)
finally:
return RESPONSE
