def process(self):
try:
logging.debug('in process')
originalFilename = self.getOriginalFileName(self.queueModel.getFileName(), self.queueModel.getPath())
self.logNumberOfLines(originalFilename, self.queueModel.getFileName(), self.queueModel.getPath())
self.generateMetadataFile()
outputFile = self.queueModel.getOutputDir() + '/' + PropertyUtil.getAttribute('output.file.pre') + self.queueModel.getFileName()
outputFile2 = self.queueModel.getOutputDir() + '/' + self.outputFile
outputFile3 = self.queueModel.getOutputDir() + '/' + self.outputFile + '.png'
fullCmd = PropertyUtil.getAttribute('fullCmd') + ' ' + self.queueModel.getPath() + '/' + self.queueModel.getFileName() + \
' ' + outputFile + ' ' + outputFile3
logging.debug(fullCmd)
os.system(fullCmd)
os.system('mv '+ outputFile + ' ' + outputFile2)
logging.debug('emailing ...')
mailContent = "The file (" \
+ self.queueModel.getFileName() \
+ ") you uploaded on " \
+ self.queueModel.getTimeStamp() \
+ " has been processed.\n" \
+ "\nYou can view the result page at: " \
+ self.queueModel.getAccessLink() \
+ ". This link will expire two weeks from today." \
+ "\r\n\r\n - SOCcer Team \r\n (Note: Please do not reply to this email. If you need assistance, please contact [email protected])"
msg = MIMEText(mailContent)
msg['Subject'] = 'Your request has been processed'
msg['From'] = 'SOCcer <*****@*****.**>'
msg['To'] = self.queueModel.getEmail()
MailUtil.composeMail(msg)
except:
logging.error('Unable to process the request...')
logging.error(traceback.format_exc())
def __init__(self):
rs3dConfig = PropertyUtil(r"config.ini")
app.config[rs3d.UPLOAD_FOLDER] = rs3dConfig.getAsString(
rs3d.UPLOAD_FOLDER)
app.config[rs3d.QUEUE_CONFIG] = StompConfig(
rs3dConfig.getAsString(rs3d.QUEUE_URL))
app.config[rs3d.QUEUE_NAME] = rs3dConfig.getAsString(rs3d.QUEUE_NAME)
def __init__(self):
super(RequestProcessor,self).__init__(self.consume)
config = PropertyUtil(r"config.ini")
config[RequestProcessor.CONFIG] = StompConfig(uri="failover:("+config.getAsString(RequestProcessor.URL)+")?startupMaxReconnectAttempts=-1,initialReconnectDelay=300000")
self.CONFIG = config
robjects.r('''source('ARTPWrapper.R')''')
self.r_runARTP = robjects.r['runARTPWithHandlers']
def logNumberOfLines(self, originalFileName, fileName, path):
logfile = PropertyUtil.getAttribute('deploy.target.dir') + '/' + PropertyUtil.getAttribute('log.fileName')
if not os.path.exists(logfile):
with open(logfile,"w") as logF:
logF.write("Timestamp, Input file Name, Number of Lines")
logLineContent = '\n' + str(datetime.now()) + '\t' + originalFileName + '\t' + str(self.getNumberOfLines(path+'/'+fileName))
with open(logfile,"a") as logF:
logF.write(logLineContent)
def __init__(self):
config = PropertyUtil(r"config.ini")
config[RequestProcessor.CONFIG] = StompConfig(
uri="failover:(" + config.getAsString(RequestProcessor.URL) +
")?startupMaxReconnectAttempts=-1,initialReconnectDelay=300000")
self.CONFIG = config
robjects.r('''source('ARTPWrapper.R')''')
self.r_runARTP = robjects.r['runARTPWithHandlers']
def __init__(self, config=None):
# Initialize Connections to ActiveMQ
self.QUEUE=PropertyUtil.getAttribute('queue.remote.name')
self.ERROR_QUEUE=PropertyUtil.getAttribute('queue.remote.error.name')
if config is None:
config = StompConfig(PropertyUtil.getAttribute('queye.remote.url'))
self.config = config
def __init__(self, config=None):
# Initialize Connections to ActiveMQ
self.QUEUE = PropertyUtil.getAttribute('queue.remote.name')
self.ERROR_QUEUE = PropertyUtil.getAttribute('queue.remote.error.name')
if config is None:
config = StompConfig(PropertyUtil.getAttribute('queye.remote.url'))
self.config = config
def logNumberOfLines(self, originalFileName, fileName, path):
logfile = PropertyUtil.getAttribute(
'deploy.target.dir') + '/' + PropertyUtil.getAttribute(
'log.fileName')
if not os.path.exists(logfile):
with open(logfile, "w") as logF:
logF.write("Timestamp, Input file Name, Number of Lines")
logLineContent = '\n' + str(
datetime.now()) + '\t' + originalFileName + '\t' + str(
self.getNumberOfLines(path + '/' + fileName))
with open(logfile, "a") as logF:
logF.write(logLineContent)
def composeMail(msg):
try:
s = smtplib.SMTP(PropertyUtil.getAttribute('mail.smtp.host'))
logging.debug('email result to ' + msg['To'])
s.sendmail(msg['From'], msg['To'].split(), msg.as_string())
except:
logging.error('Unable to send mail to ' + msg['To'])
logging.error(traceback.format_exc())
def composeMail(msg):
try:
s = smtplib.SMTP(PropertyUtil.getAttribute('mail.smtp.host'))
logging.debug('email result to ' + msg['To'])
s.sendmail(msg['From'],msg['To'].split(),msg.as_string())
except:
logging.error('Unable to send mail to ' + msg['To'])
logging.error(traceback.format_exc())
def __init__(self):
pathwayConfig = PropertyUtil(r"config.ini")
pathwayConfig[Pathway.QUEUE_CONFIG] = StompConfig(pathwayConfig.getAsString(Pathway.QUEUE_URL))
app.config[Pathway.CONFIG] = pathwayConfig
app.config['PATHWAY_FOLDER'] = os.path.join(pathwayConfig.getAsString(Pathway.FOLDERROOT),pathwayConfig.getAsString(Pathway.PATHWAY_FOLDER))
app.config['POPULATION_FOLDER'] = os.path.join(pathwayConfig.getAsString(Pathway.FOLDERROOT), pathwayConfig.getAsString(Pathway.POPULATION_FOLDER))
app.config['UPLOAD_FOLDER'] = os.path.join(pathwayConfig.getAsString(Pathway.FOLDERROOT), pathwayConfig.getAsString(Pathway.UPLOAD_FOLDER))
app.config['OUT_FOLDER'] = os.path.join(pathwayConfig.getAsString(Pathway.FOLDERROOT),pathwayConfig.getAsString(Pathway.OUT_FOLDER))
app.config['PLINK_PATTERN'] = os.path.join(pathwayConfig.getAsString(Pathway.FOLDERROOT),pathwayConfig.getAsString(Pathway.PLINK_PATTERN))
if not os.path.exists(app.config['OUT_FOLDER']):
os.makedirs(app.config['OUT_FOLDER'])
if not os.path.exists(app.config['UPLOAD_FOLDER']):
os.makedirs(app.config['UPLOAD_FOLDER'])
app.run(host='0.0.0.0', port=pathwayConfig.getAsInt(Pathway.PORT), debug=pathwayConfig.getAsBoolean(Pathway.DEBUG))
def composeMail(recipient, message, file):
config = PropertyUtil(r"config.ini")
recipient = recipient
packet = MIMEMultipart()
packet['Subject'] = "eConsent Agreement Form"
packet['From'] = "eConsent <*****@*****.**>"
packet['To'] = recipient
packet.attach(MIMEText(message, 'html'))
with open(file, "rb") as openfile:
packet.attach(
MIMEApplication(openfile.read(),
Content_Disposition='attachment; filename="%s"' %
os.path.basename(file),
Name=os.path.basename(file)))
MAIL_HOST = config.getAsString('mail.host')
print MAIL_HOST
smtp = smtplib.SMTP(MAIL_HOST)
smtp.sendmail("*****@*****.**", recipient, packet.as_string())
print "sent email"
def process(self):
try:
logging.debug('in process')
originalFilename = self.getOriginalFileName(
self.queueModel.getFileName(), self.queueModel.getPath())
self.logNumberOfLines(originalFilename,
self.queueModel.getFileName(),
self.queueModel.getPath())
self.generateMetadataFile()
outputFile = self.queueModel.getOutputDir(
) + '/' + PropertyUtil.getAttribute(
'output.file.pre') + self.queueModel.getFileName()
outputFile2 = self.queueModel.getOutputDir(
) + '/' + self.outputFile
outputFile3 = self.queueModel.getOutputDir(
) + '/' + self.outputFile + '.png'
fullCmd = PropertyUtil.getAttribute('fullCmd') + ' ' + self.queueModel.getPath() + '/' + self.queueModel.getFileName() + \
' ' + outputFile + ' ' + outputFile3
logging.debug(fullCmd)
os.system(fullCmd)
os.system('mv ' + outputFile + ' ' + outputFile2)
logging.debug('emailing ...')
mailContent = "The file (" \
+ self.queueModel.getFileName() \
+ ") you uploaded on " \
+ self.queueModel.getTimeStamp() \
+ " has been processed.\n" \
+ "\nYou can view the result page at: " \
+ self.queueModel.getAccessLink() \
+ ". This link will expire two weeks from today." \
+ "\r\n\r\n - SOCcer Team \r\n (Note: Please do not reply to this email. If you need assistance, please contact [email protected])"
msg = MIMEText(mailContent)
msg['Subject'] = 'Your request has been processed'
msg['From'] = 'SOCcer <*****@*****.**>'
msg['To'] = self.queueModel.getEmail()
MailUtil.composeMail(msg)
except:
logging.error('Unable to process the request...')
logging.error(traceback.format_exc())
def __init__(self):
config = PropertyUtil(r"config.ini")
# Initialize Connections to ActiveMQ
self.QUEUE = config.getAsString(RequestProcessor.NAME)
self.ERROR_QUEUE = config.getAsString('queue.error.name')
config = StompConfig(config.getAsString(RequestProcessor.URL))
self.config = config
def composeMail(self,recipients,message,files=[]):
config = PropertyUtil(r"config.ini")
logging.info("sending message")
if not isinstance(recipients,list):
recipients = [recipients]
packet = MIMEMultipart()
packet['Subject'] = "Subject: Pathway Analysis Results"
packet['From'] = "Pathway Analysis Tool <*****@*****.**>"
packet['To'] = ", ".join(recipients)
logging.info(recipients)
# print message
packet.attach(MIMEText(message))
for file in files:
with open(file,"rb") as openfile:
packet.attach(MIMEApplication(
openfile.read(),
Content_Disposition='attachment; filename="%s"' % os.path.basename(file),
Name=os.path.basename(file)
))
MAIL_HOST=config.getAsString('mail.host')
logging.info(MAIL_HOST)
smtp = smtplib.SMTP(MAIL_HOST)
smtp.sendmail("*****@*****.**",recipients,packet.as_string())
def composeMail(self, recipients, message, files=[]):
config = PropertyUtil(r"config.ini")
print "sending message"
if not isinstance(recipients, list):
recipients = [recipients]
packet = MIMEMultipart()
packet['Subject'] = "Your request has been processed"
packet['From'] = "SOCcer<*****@*****.**>"
packet['To'] = ", ".join(recipients)
print recipients
print message
packet.attach(MIMEText(message, 'html'))
for file in files:
with open(file, "rb") as openfile:
packet.attach(
MIMEApplication(
openfile.read(),
Content_Disposition='attachment; filename="%s"' %
os.path.basename(file),
Name=os.path.basename(file)))
MAIL_HOST = config.getAsString('mail.host')
print MAIL_HOST
smtp = smtplib.SMTP(MAIL_HOST)
smtp.sendmail("*****@*****.**", recipients, packet.as_string())
def __init__(self):
# super(pathwayProcessor,self).__init__(self.consume)
config = PropertyUtil(r"config.ini")
# config[pathwayProcessor.CONFIG] = StompConfig(uri="failover:("+config.getAsString(pathwayProcessor.URL)+")?startupMaxReconnectAttempts=-1,initialReconnectDelay=300000")
self.QUEUE=config.getAsString(pathwayProcessor.NAME)
self.ERROR_QUEUE=config.getAsString('queue.error.name')
# config[pathwayProcessor.CONFIG] = StompConfig(config.getAsString(pathwayProcessor.URL))
config = StompConfig(config.getAsString(pathwayProcessor.URL))
self.config = config
robjects.r('''source('ARTPWrapper.R')''')
self.r_runARTP = robjects.r['runARTPWithHandlers']
def main():
pathwayConfig = PropertyUtil(r"config.ini")
pathwayConfig[QUEUE_CONFIG] = StompConfig(
pathwayConfig.getAsString(QUEUE_URL))
app.config[CONFIG] = pathwayConfig
app.config['PATHWAY_FOLDER'] = os.path.join(
pathwayConfig.getAsString(FOLDERROOT),
pathwayConfig.getAsString(PATHWAY_FOLDER))
app.config['POPULATION_FOLDER'] = os.path.join(
pathwayConfig.getAsString(FOLDERROOT),
pathwayConfig.getAsString(POPULATION_FOLDER))
app.config['UPLOAD_FOLDER'] = os.path.join(
pathwayConfig.getAsString(FOLDERROOT),
pathwayConfig.getAsString(UPLOAD_FOLDER))
app.config['OUT_FOLDER'] = os.path.join(
pathwayConfig.getAsString(FOLDERROOT),
pathwayConfig.getAsString(OUT_FOLDER))
app.config['PLINK_PATTERN'] = os.path.join(
pathwayConfig.getAsString(FOLDERROOT),
pathwayConfig.getAsString(PLINK_PATTERN))
if not os.path.exists(app.config['OUT_FOLDER']):
os.makedirs(app.config['OUT_FOLDER'])
if not os.path.exists(app.config['UPLOAD_FOLDER']):
os.makedirs(app.config['UPLOAD_FOLDER'])
OptionGenerator()
def consume(self, client, frame):
logging.info("In consume")
config = PropertyUtil(r"config.ini")
starttime = str(time.time())
parameters = json.loads(frame.body)
timestamp = frame.headers['timestamp']
outfileString = os.path.join(parameters['outdir'], str(parameters['idstr']) + '.json')
jsonout = {
'submittedTime': parameters['idstr'],
'queuedTime':timestamp,
'payload':parameters,
'processStartTime': starttime
}
with open(outfileString,'w') as outfile:
json.dump(jsonout,outfile)
try:
# Run R-Script
artpResult = json.loads(self.r_runARTP(json.dumps(parameters))[0])
except Exception as e:
artpResult = {}
artpResult["error"] = str(e)
jsonout["processStopTime"] = str(time.time())
message = ""
if "warnings" in artpResult:
jsonout["warnings"] = artpResult["warnings"]
with open(outfileString,'w') as outfile:
json.dump(jsonout,outfile)
message += "\nWarnings:\n"
if (isinstance(artpResult["warnings"],list)):
for warning in artpResult["warnings"]:
message += warning.strip() + "\n\n"
else:
message += artpResult["warnings"].strip() + "\n\n"
if "error" in artpResult:
jsonout["error"] = artpResult["error"]
jsonout["status"] = "error"
with open(outfileString,'w') as outfile:
json.dump(jsonout,outfile)
message = "Error: " + artpResult["error"].strip() + "\n" + message + "\n\n" +frame.body
print(message)
self.composeMail(config.getAsString(pathwayProcessor.MAIL_ADMIN).split(","), message)
self.composeMail(parameters["email"], "Unfortunately there was an error processing your request. The site administrators have been alerted to the problem. Please contact " + config.getAsString(pathwayProcessor.MAIL_ADMIN) + " if any question.\n\n" + message)
return
# email results
files = [ os.path.join(parameters['outdir'],parameters['idstr']+'.Rdata') ]
saveValue = artpResult["saveValue"]
jsonout["saveValue"] = saveValue
jsonout["status"] = "success"
with open(outfileString,'w') as outfile:
json.dump(jsonout,outfile)
message = ("Dear User,\n\n" +
"We have analyzed your data using the ARTP2 package (version: " + ".".join([str(x) for x in saveValue['options']['version'][0]]) + "). " +
"The sARTP test returned a pathway p-value " + str(round(saveValue['pathway.pvalue'],1-int(math.floor(math.log10(saveValue['pathway.pvalue']))))) + ". The p-value was estimated by " + str(saveValue['options']['nperm']) + " resampling steps.\n\n" +
"Several gene/SNP filters were applied to the data based on specified options. " +
"There are " + str(self.rLength(saveValue['deleted.genes'].get('Gene',None))) + " genes and " + str(self.rLength(set(saveValue['deleted.snps'].get('SNP',None)))) + " SNPs that were excluded from the analysis. " +
"After that, " + str(self.rLength(saveValue['gene.pvalue']['Chr'])) + " unique genes and " + str(self.rLength(set(saveValue['pathway']['SNP']))) + " unique SNPs were used in testing. " +
"Some warning messages (if any) can be found at the end of this email.\n\n" +
"More detailed result of this pathway analysis is saved as an R list, saveValue, in the attached file. You can read it in R through function load(). For example, the pathway p-value is\n\n" +
"saveValue$pathway.pvalue\n\n" +
"and the gene-level p-values are saved in a data frame\n\n" +
"saveValue$gene.pvalue\n\n" +
"To understand the reason for each SNP/gene excluded from the analysis, check\n\n" +
"saveValue$deleted.snps\n\n" +
"and\n\n" +
"saveValue$deleted.genes\n\n" +
"A full list of genes and SNPs that were used in testing is saved in a data frame\n\n" +
"saveValue$pathway\n\n" +
"All the options are saved in \n\n" +
"saveValue$options\n\n" +
"For more information, please refer to the help document of function sARTP in R package ARTP2.\n\n" +
message)
print(message)
self.composeMail(parameters['email'], message, files)
# remove the already used files
for study in parameters['studies']:
os.remove(study['filename'])
if (parameters['pathway_type'] == 'file_pathway'):
os.remove(parameters['file_pathway'])
for file in files:
os.remove(file)
def __init__(self):
config = PropertyUtil(r"config.ini")
config[RequestProcessor.CONFIG] = StompConfig(uri="failover:("+config.getAsString(RequestProcessor.URL)+")?startupMaxReconnectAttempts=-1,initialReconnectDelay=300000")
self.CONFIG = config
from flask import Flask, request, redirect, current_app
from PropertyUtil import PropertyUtil
from rpy2.robjects import r
from stompest.config import StompConfig
from stompest.sync import Stomp
from werkzeug import secure_filename
import os.path
from shutil import copytree, ignore_patterns
app = Flask(__name__, static_folder='', static_url_path='/')
if not os.path.exists('tmp'):
os.makedirs('tmp')
QUEUE_NAME = 'queue.name'
QUEUE_URL = 'queue.url'
jpsurvConfig = PropertyUtil(r"config.ini")
UPLOAD_DIR = os.path.join(os.getcwd(), 'tmp')
print 'JPSurv is starting...'
#COLORS TO Make logging Mover visible
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
ENDC = '\033[0m'
def main():
pathwayConfig = PropertyUtil(r"config.ini")
pathwayConfig[QUEUE_CONFIG] = StompConfig(pathwayConfig.getAsString(QUEUE_URL))
app.config[CONFIG] = pathwayConfig
app.config['PATHWAY_FOLDER'] = os.path.join(pathwayConfig.getAsString(FOLDERROOT),pathwayConfig.getAsString(PATHWAY_FOLDER))
app.config['POPULATION_FOLDER'] = os.path.join(pathwayConfig.getAsString(FOLDERROOT), pathwayConfig.getAsString(POPULATION_FOLDER))
app.config['UPLOAD_FOLDER'] = os.path.join(pathwayConfig.getAsString(FOLDERROOT), pathwayConfig.getAsString(UPLOAD_FOLDER))
app.config['OUT_FOLDER'] = os.path.join(pathwayConfig.getAsString(FOLDERROOT),pathwayConfig.getAsString(OUT_FOLDER))
app.config['PLINK_PATTERN'] = os.path.join(pathwayConfig.getAsString(FOLDERROOT),pathwayConfig.getAsString(PLINK_PATTERN))
if not os.path.exists(app.config['OUT_FOLDER']):
os.makedirs(app.config['OUT_FOLDER'])
if not os.path.exists(app.config['UPLOAD_FOLDER']):
os.makedirs(app.config['UPLOAD_FOLDER'])
OptionGenerator()
def __init__(self):
self.config = PropertyUtil(r"config.ini")
with open("pathway_options.json","w") as file:
file.write(json.dumps(self.pathway_options()))
with open("population_options.json","w") as file:
file.write(json.dumps(self.population_options()))
import json
import os
import subprocess
import sys
import time
from flask import Flask, jsonify, request, Response, send_from_directory
import uuid
from stompest.config import StompConfig
from stompest.sync import Stomp
from PropertyUtil import PropertyUtil
from werkzeug.utils import secure_filename
app = Flask(__name__, static_folder='', static_url_path='/')
QUEUE_NAME = 'queue.name'
QUEUE_URL = 'queue.url'
soccerConfig = PropertyUtil(r"config.ini")
RESULTS_PATH = soccerConfig.getAsString("soccer.folder.out")
@app.route('/upload', methods=["POST"])
def upload():
try:
filename = None
if len(request.files) > 0 and 'fileSelect' in request.files:
userFile = request.files['fileSelect']
socSystem = request.form["socSystem"]
originalFileName = userFile.filename
# check for correct file extension
fileName = str(uuid.uuid4())