def process(self): try: logging.debug('in process') originalFilename = self.getOriginalFileName(self.queueModel.getFileName(), self.queueModel.getPath()) self.logNumberOfLines(originalFilename, self.queueModel.getFileName(), self.queueModel.getPath()) self.generateMetadataFile() outputFile = self.queueModel.getOutputDir() + '/' + PropertyUtil.getAttribute('output.file.pre') + self.queueModel.getFileName() outputFile2 = self.queueModel.getOutputDir() + '/' + self.outputFile outputFile3 = self.queueModel.getOutputDir() + '/' + self.outputFile + '.png' fullCmd = PropertyUtil.getAttribute('fullCmd') + ' ' + self.queueModel.getPath() + '/' + self.queueModel.getFileName() + \ ' ' + outputFile + ' ' + outputFile3 logging.debug(fullCmd) os.system(fullCmd) os.system('mv '+ outputFile + ' ' + outputFile2) logging.debug('emailing ...') mailContent = "The file (" \ + self.queueModel.getFileName() \ + ") you uploaded on " \ + self.queueModel.getTimeStamp() \ + " has been processed.\n" \ + "\nYou can view the result page at: " \ + self.queueModel.getAccessLink() \ + ". This link will expire two weeks from today." \ + "\r\n\r\n - SOCcer Team \r\n (Note: Please do not reply to this email. If you need assistance, please contact [email protected])" msg = MIMEText(mailContent) msg['Subject'] = 'Your request has been processed' msg['From'] = 'SOCcer <*****@*****.**>' msg['To'] = self.queueModel.getEmail() MailUtil.composeMail(msg) except: logging.error('Unable to process the request...') logging.error(traceback.format_exc())
def __init__(self): rs3dConfig = PropertyUtil(r"config.ini") app.config[rs3d.UPLOAD_FOLDER] = rs3dConfig.getAsString( rs3d.UPLOAD_FOLDER) app.config[rs3d.QUEUE_CONFIG] = StompConfig( rs3dConfig.getAsString(rs3d.QUEUE_URL)) app.config[rs3d.QUEUE_NAME] = rs3dConfig.getAsString(rs3d.QUEUE_NAME)
def __init__(self): super(RequestProcessor,self).__init__(self.consume) config = PropertyUtil(r"config.ini") config[RequestProcessor.CONFIG] = StompConfig(uri="failover:("+config.getAsString(RequestProcessor.URL)+")?startupMaxReconnectAttempts=-1,initialReconnectDelay=300000") self.CONFIG = config robjects.r('''source('ARTPWrapper.R')''') self.r_runARTP = robjects.r['runARTPWithHandlers']
def logNumberOfLines(self, originalFileName, fileName, path): logfile = PropertyUtil.getAttribute('deploy.target.dir') + '/' + PropertyUtil.getAttribute('log.fileName') if not os.path.exists(logfile): with open(logfile,"w") as logF: logF.write("Timestamp, Input file Name, Number of Lines") logLineContent = '\n' + str(datetime.now()) + '\t' + originalFileName + '\t' + str(self.getNumberOfLines(path+'/'+fileName)) with open(logfile,"a") as logF: logF.write(logLineContent)
def __init__(self): config = PropertyUtil(r"config.ini") config[RequestProcessor.CONFIG] = StompConfig( uri="failover:(" + config.getAsString(RequestProcessor.URL) + ")?startupMaxReconnectAttempts=-1,initialReconnectDelay=300000") self.CONFIG = config robjects.r('''source('ARTPWrapper.R')''') self.r_runARTP = robjects.r['runARTPWithHandlers']
def __init__(self, config=None): # Initialize Connections to ActiveMQ self.QUEUE=PropertyUtil.getAttribute('queue.remote.name') self.ERROR_QUEUE=PropertyUtil.getAttribute('queue.remote.error.name') if config is None: config = StompConfig(PropertyUtil.getAttribute('queye.remote.url')) self.config = config
def __init__(self, config=None): # Initialize Connections to ActiveMQ self.QUEUE = PropertyUtil.getAttribute('queue.remote.name') self.ERROR_QUEUE = PropertyUtil.getAttribute('queue.remote.error.name') if config is None: config = StompConfig(PropertyUtil.getAttribute('queye.remote.url')) self.config = config
def logNumberOfLines(self, originalFileName, fileName, path): logfile = PropertyUtil.getAttribute( 'deploy.target.dir') + '/' + PropertyUtil.getAttribute( 'log.fileName') if not os.path.exists(logfile): with open(logfile, "w") as logF: logF.write("Timestamp, Input file Name, Number of Lines") logLineContent = '\n' + str( datetime.now()) + '\t' + originalFileName + '\t' + str( self.getNumberOfLines(path + '/' + fileName)) with open(logfile, "a") as logF: logF.write(logLineContent)
def composeMail(msg): try: s = smtplib.SMTP(PropertyUtil.getAttribute('mail.smtp.host')) logging.debug('email result to ' + msg['To']) s.sendmail(msg['From'], msg['To'].split(), msg.as_string()) except: logging.error('Unable to send mail to ' + msg['To']) logging.error(traceback.format_exc())
def composeMail(msg): try: s = smtplib.SMTP(PropertyUtil.getAttribute('mail.smtp.host')) logging.debug('email result to ' + msg['To']) s.sendmail(msg['From'],msg['To'].split(),msg.as_string()) except: logging.error('Unable to send mail to ' + msg['To']) logging.error(traceback.format_exc())
def __init__(self): pathwayConfig = PropertyUtil(r"config.ini") pathwayConfig[Pathway.QUEUE_CONFIG] = StompConfig(pathwayConfig.getAsString(Pathway.QUEUE_URL)) app.config[Pathway.CONFIG] = pathwayConfig app.config['PATHWAY_FOLDER'] = os.path.join(pathwayConfig.getAsString(Pathway.FOLDERROOT),pathwayConfig.getAsString(Pathway.PATHWAY_FOLDER)) app.config['POPULATION_FOLDER'] = os.path.join(pathwayConfig.getAsString(Pathway.FOLDERROOT), pathwayConfig.getAsString(Pathway.POPULATION_FOLDER)) app.config['UPLOAD_FOLDER'] = os.path.join(pathwayConfig.getAsString(Pathway.FOLDERROOT), pathwayConfig.getAsString(Pathway.UPLOAD_FOLDER)) app.config['OUT_FOLDER'] = os.path.join(pathwayConfig.getAsString(Pathway.FOLDERROOT),pathwayConfig.getAsString(Pathway.OUT_FOLDER)) app.config['PLINK_PATTERN'] = os.path.join(pathwayConfig.getAsString(Pathway.FOLDERROOT),pathwayConfig.getAsString(Pathway.PLINK_PATTERN)) if not os.path.exists(app.config['OUT_FOLDER']): os.makedirs(app.config['OUT_FOLDER']) if not os.path.exists(app.config['UPLOAD_FOLDER']): os.makedirs(app.config['UPLOAD_FOLDER']) app.run(host='0.0.0.0', port=pathwayConfig.getAsInt(Pathway.PORT), debug=pathwayConfig.getAsBoolean(Pathway.DEBUG))
def composeMail(recipient, message, file): config = PropertyUtil(r"config.ini") recipient = recipient packet = MIMEMultipart() packet['Subject'] = "eConsent Agreement Form" packet['From'] = "eConsent <*****@*****.**>" packet['To'] = recipient packet.attach(MIMEText(message, 'html')) with open(file, "rb") as openfile: packet.attach( MIMEApplication(openfile.read(), Content_Disposition='attachment; filename="%s"' % os.path.basename(file), Name=os.path.basename(file))) MAIL_HOST = config.getAsString('mail.host') print MAIL_HOST smtp = smtplib.SMTP(MAIL_HOST) smtp.sendmail("*****@*****.**", recipient, packet.as_string()) print "sent email"
def process(self): try: logging.debug('in process') originalFilename = self.getOriginalFileName( self.queueModel.getFileName(), self.queueModel.getPath()) self.logNumberOfLines(originalFilename, self.queueModel.getFileName(), self.queueModel.getPath()) self.generateMetadataFile() outputFile = self.queueModel.getOutputDir( ) + '/' + PropertyUtil.getAttribute( 'output.file.pre') + self.queueModel.getFileName() outputFile2 = self.queueModel.getOutputDir( ) + '/' + self.outputFile outputFile3 = self.queueModel.getOutputDir( ) + '/' + self.outputFile + '.png' fullCmd = PropertyUtil.getAttribute('fullCmd') + ' ' + self.queueModel.getPath() + '/' + self.queueModel.getFileName() + \ ' ' + outputFile + ' ' + outputFile3 logging.debug(fullCmd) os.system(fullCmd) os.system('mv ' + outputFile + ' ' + outputFile2) logging.debug('emailing ...') mailContent = "The file (" \ + self.queueModel.getFileName() \ + ") you uploaded on " \ + self.queueModel.getTimeStamp() \ + " has been processed.\n" \ + "\nYou can view the result page at: " \ + self.queueModel.getAccessLink() \ + ". This link will expire two weeks from today." \ + "\r\n\r\n - SOCcer Team \r\n (Note: Please do not reply to this email. If you need assistance, please contact [email protected])" msg = MIMEText(mailContent) msg['Subject'] = 'Your request has been processed' msg['From'] = 'SOCcer <*****@*****.**>' msg['To'] = self.queueModel.getEmail() MailUtil.composeMail(msg) except: logging.error('Unable to process the request...') logging.error(traceback.format_exc())
def __init__(self): config = PropertyUtil(r"config.ini") # Initialize Connections to ActiveMQ self.QUEUE = config.getAsString(RequestProcessor.NAME) self.ERROR_QUEUE = config.getAsString('queue.error.name') config = StompConfig(config.getAsString(RequestProcessor.URL)) self.config = config
def composeMail(self,recipients,message,files=[]): config = PropertyUtil(r"config.ini") logging.info("sending message") if not isinstance(recipients,list): recipients = [recipients] packet = MIMEMultipart() packet['Subject'] = "Subject: Pathway Analysis Results" packet['From'] = "Pathway Analysis Tool <*****@*****.**>" packet['To'] = ", ".join(recipients) logging.info(recipients) # print message packet.attach(MIMEText(message)) for file in files: with open(file,"rb") as openfile: packet.attach(MIMEApplication( openfile.read(), Content_Disposition='attachment; filename="%s"' % os.path.basename(file), Name=os.path.basename(file) )) MAIL_HOST=config.getAsString('mail.host') logging.info(MAIL_HOST) smtp = smtplib.SMTP(MAIL_HOST) smtp.sendmail("*****@*****.**",recipients,packet.as_string())
def composeMail(self, recipients, message, files=[]): config = PropertyUtil(r"config.ini") print "sending message" if not isinstance(recipients, list): recipients = [recipients] packet = MIMEMultipart() packet['Subject'] = "Your request has been processed" packet['From'] = "SOCcer<*****@*****.**>" packet['To'] = ", ".join(recipients) print recipients print message packet.attach(MIMEText(message, 'html')) for file in files: with open(file, "rb") as openfile: packet.attach( MIMEApplication( openfile.read(), Content_Disposition='attachment; filename="%s"' % os.path.basename(file), Name=os.path.basename(file))) MAIL_HOST = config.getAsString('mail.host') print MAIL_HOST smtp = smtplib.SMTP(MAIL_HOST) smtp.sendmail("*****@*****.**", recipients, packet.as_string())
def __init__(self): # super(pathwayProcessor,self).__init__(self.consume) config = PropertyUtil(r"config.ini") # config[pathwayProcessor.CONFIG] = StompConfig(uri="failover:("+config.getAsString(pathwayProcessor.URL)+")?startupMaxReconnectAttempts=-1,initialReconnectDelay=300000") self.QUEUE=config.getAsString(pathwayProcessor.NAME) self.ERROR_QUEUE=config.getAsString('queue.error.name') # config[pathwayProcessor.CONFIG] = StompConfig(config.getAsString(pathwayProcessor.URL)) config = StompConfig(config.getAsString(pathwayProcessor.URL)) self.config = config robjects.r('''source('ARTPWrapper.R')''') self.r_runARTP = robjects.r['runARTPWithHandlers']
def main(): pathwayConfig = PropertyUtil(r"config.ini") pathwayConfig[QUEUE_CONFIG] = StompConfig( pathwayConfig.getAsString(QUEUE_URL)) app.config[CONFIG] = pathwayConfig app.config['PATHWAY_FOLDER'] = os.path.join( pathwayConfig.getAsString(FOLDERROOT), pathwayConfig.getAsString(PATHWAY_FOLDER)) app.config['POPULATION_FOLDER'] = os.path.join( pathwayConfig.getAsString(FOLDERROOT), pathwayConfig.getAsString(POPULATION_FOLDER)) app.config['UPLOAD_FOLDER'] = os.path.join( pathwayConfig.getAsString(FOLDERROOT), pathwayConfig.getAsString(UPLOAD_FOLDER)) app.config['OUT_FOLDER'] = os.path.join( pathwayConfig.getAsString(FOLDERROOT), pathwayConfig.getAsString(OUT_FOLDER)) app.config['PLINK_PATTERN'] = os.path.join( pathwayConfig.getAsString(FOLDERROOT), pathwayConfig.getAsString(PLINK_PATTERN)) if not os.path.exists(app.config['OUT_FOLDER']): os.makedirs(app.config['OUT_FOLDER']) if not os.path.exists(app.config['UPLOAD_FOLDER']): os.makedirs(app.config['UPLOAD_FOLDER']) OptionGenerator()
def consume(self, client, frame): logging.info("In consume") config = PropertyUtil(r"config.ini") starttime = str(time.time()) parameters = json.loads(frame.body) timestamp = frame.headers['timestamp'] outfileString = os.path.join(parameters['outdir'], str(parameters['idstr']) + '.json') jsonout = { 'submittedTime': parameters['idstr'], 'queuedTime':timestamp, 'payload':parameters, 'processStartTime': starttime } with open(outfileString,'w') as outfile: json.dump(jsonout,outfile) try: # Run R-Script artpResult = json.loads(self.r_runARTP(json.dumps(parameters))[0]) except Exception as e: artpResult = {} artpResult["error"] = str(e) jsonout["processStopTime"] = str(time.time()) message = "" if "warnings" in artpResult: jsonout["warnings"] = artpResult["warnings"] with open(outfileString,'w') as outfile: json.dump(jsonout,outfile) message += "\nWarnings:\n" if (isinstance(artpResult["warnings"],list)): for warning in artpResult["warnings"]: message += warning.strip() + "\n\n" else: message += artpResult["warnings"].strip() + "\n\n" if "error" in artpResult: jsonout["error"] = artpResult["error"] jsonout["status"] = "error" with open(outfileString,'w') as outfile: json.dump(jsonout,outfile) message = "Error: " + artpResult["error"].strip() + "\n" + message + "\n\n" +frame.body print(message) self.composeMail(config.getAsString(pathwayProcessor.MAIL_ADMIN).split(","), message) self.composeMail(parameters["email"], "Unfortunately there was an error processing your request. The site administrators have been alerted to the problem. Please contact " + config.getAsString(pathwayProcessor.MAIL_ADMIN) + " if any question.\n\n" + message) return # email results files = [ os.path.join(parameters['outdir'],parameters['idstr']+'.Rdata') ] saveValue = artpResult["saveValue"] jsonout["saveValue"] = saveValue jsonout["status"] = "success" with open(outfileString,'w') as outfile: json.dump(jsonout,outfile) message = ("Dear User,\n\n" + "We have analyzed your data using the ARTP2 package (version: " + ".".join([str(x) for x in saveValue['options']['version'][0]]) + "). " + "The sARTP test returned a pathway p-value " + str(round(saveValue['pathway.pvalue'],1-int(math.floor(math.log10(saveValue['pathway.pvalue']))))) + ". The p-value was estimated by " + str(saveValue['options']['nperm']) + " resampling steps.\n\n" + "Several gene/SNP filters were applied to the data based on specified options. " + "There are " + str(self.rLength(saveValue['deleted.genes'].get('Gene',None))) + " genes and " + str(self.rLength(set(saveValue['deleted.snps'].get('SNP',None)))) + " SNPs that were excluded from the analysis. " + "After that, " + str(self.rLength(saveValue['gene.pvalue']['Chr'])) + " unique genes and " + str(self.rLength(set(saveValue['pathway']['SNP']))) + " unique SNPs were used in testing. " + "Some warning messages (if any) can be found at the end of this email.\n\n" + "More detailed result of this pathway analysis is saved as an R list, saveValue, in the attached file. You can read it in R through function load(). For example, the pathway p-value is\n\n" + "saveValue$pathway.pvalue\n\n" + "and the gene-level p-values are saved in a data frame\n\n" + "saveValue$gene.pvalue\n\n" + "To understand the reason for each SNP/gene excluded from the analysis, check\n\n" + "saveValue$deleted.snps\n\n" + "and\n\n" + "saveValue$deleted.genes\n\n" + "A full list of genes and SNPs that were used in testing is saved in a data frame\n\n" + "saveValue$pathway\n\n" + "All the options are saved in \n\n" + "saveValue$options\n\n" + "For more information, please refer to the help document of function sARTP in R package ARTP2.\n\n" + message) print(message) self.composeMail(parameters['email'], message, files) # remove the already used files for study in parameters['studies']: os.remove(study['filename']) if (parameters['pathway_type'] == 'file_pathway'): os.remove(parameters['file_pathway']) for file in files: os.remove(file)
def __init__(self): config = PropertyUtil(r"config.ini") config[RequestProcessor.CONFIG] = StompConfig(uri="failover:("+config.getAsString(RequestProcessor.URL)+")?startupMaxReconnectAttempts=-1,initialReconnectDelay=300000") self.CONFIG = config
from flask import Flask, request, redirect, current_app from PropertyUtil import PropertyUtil from rpy2.robjects import r from stompest.config import StompConfig from stompest.sync import Stomp from werkzeug import secure_filename import os.path from shutil import copytree, ignore_patterns app = Flask(__name__, static_folder='', static_url_path='/') if not os.path.exists('tmp'): os.makedirs('tmp') QUEUE_NAME = 'queue.name' QUEUE_URL = 'queue.url' jpsurvConfig = PropertyUtil(r"config.ini") UPLOAD_DIR = os.path.join(os.getcwd(), 'tmp') print 'JPSurv is starting...' #COLORS TO Make logging Mover visible HEADER = '\033[95m' OKBLUE = '\033[94m' OKGREEN = '\033[92m' WARNING = '\033[93m' FAIL = '\033[91m' BOLD = '\033[1m' UNDERLINE = '\033[4m' ENDC = '\033[0m'
def main(): pathwayConfig = PropertyUtil(r"config.ini") pathwayConfig[QUEUE_CONFIG] = StompConfig(pathwayConfig.getAsString(QUEUE_URL)) app.config[CONFIG] = pathwayConfig app.config['PATHWAY_FOLDER'] = os.path.join(pathwayConfig.getAsString(FOLDERROOT),pathwayConfig.getAsString(PATHWAY_FOLDER)) app.config['POPULATION_FOLDER'] = os.path.join(pathwayConfig.getAsString(FOLDERROOT), pathwayConfig.getAsString(POPULATION_FOLDER)) app.config['UPLOAD_FOLDER'] = os.path.join(pathwayConfig.getAsString(FOLDERROOT), pathwayConfig.getAsString(UPLOAD_FOLDER)) app.config['OUT_FOLDER'] = os.path.join(pathwayConfig.getAsString(FOLDERROOT),pathwayConfig.getAsString(OUT_FOLDER)) app.config['PLINK_PATTERN'] = os.path.join(pathwayConfig.getAsString(FOLDERROOT),pathwayConfig.getAsString(PLINK_PATTERN)) if not os.path.exists(app.config['OUT_FOLDER']): os.makedirs(app.config['OUT_FOLDER']) if not os.path.exists(app.config['UPLOAD_FOLDER']): os.makedirs(app.config['UPLOAD_FOLDER']) OptionGenerator()
def __init__(self): self.config = PropertyUtil(r"config.ini") with open("pathway_options.json","w") as file: file.write(json.dumps(self.pathway_options())) with open("population_options.json","w") as file: file.write(json.dumps(self.population_options()))
import json import os import subprocess import sys import time from flask import Flask, jsonify, request, Response, send_from_directory import uuid from stompest.config import StompConfig from stompest.sync import Stomp from PropertyUtil import PropertyUtil from werkzeug.utils import secure_filename app = Flask(__name__, static_folder='', static_url_path='/') QUEUE_NAME = 'queue.name' QUEUE_URL = 'queue.url' soccerConfig = PropertyUtil(r"config.ini") RESULTS_PATH = soccerConfig.getAsString("soccer.folder.out") @app.route('/upload', methods=["POST"]) def upload(): try: filename = None if len(request.files) > 0 and 'fileSelect' in request.files: userFile = request.files['fileSelect'] socSystem = request.form["socSystem"] originalFileName = userFile.filename # check for correct file extension fileName = str(uuid.uuid4())