def run(self, retry, smtpServer, emailFrom, emailTo, archive_file): self._buildInputfile() IOService.run(self, retry) if not os.path.exists(self.output_file): logging.warning("The Daily Archiving failed, missing output file") subject = 'Daily Archiving Failed' body = 'The Daily Archiving failed with error: missing output file. Imagine logfile attached if it exists.' send_email(smtpServer, emailFrom, emailTo, subject, body, {os.path.dirname(self.log_file):os.path.basename(self.log_file)}) return paths = os.path.split(archive_file) archive_file = os.path.join(paths[0], "%s_%s" % (self.asofdate, paths[1])) outputFile= file(archive_file, 'wb') for tempFile in [self.output_file]: logging.info('Reading part file %s' % tempFile) tfile = file(tempFile, 'rb') line = tfile.readline() while line: outputFile.write(line) line = tfile.readline() outputFile.close() logging.info("Done") return archive_file
def run(self, retry, smtpServer, emailFrom, emailTo): """Runs the service to update security level credit spread overrides in Imagine.""" # Get indices of arguments we'll need to change ios = self.args.index("-ios") + 1 f = self.args.index("-f") + 1 o = self.args.index("-o") + 1 try: # Run print portfolio i/o service self._buildPPInputFile() result = IOService.run(self, retry) if result != 0: raise Exception, "IO Service failure" # Run update security environment i/o service self.args[ios] = "updateSecurityEnvironment" self.args[f] = self.use_input_file self.args[o] = self.use_output_file self._parsePPOutputFile() self._buildUSEInputFile(credit_spread=0.01) result = IOService.run(self, retry) if result != 0: raise Exception, "IO Service failure" # Run print portfolio i/o service again self.args[ios] = "printPortfolio" self.args[f] = self.pp_input_file self.args[o] = self.pp_output_file result = IOService.run(self, retry) if result != 0: raise Exception, "IO Service failure" # Run update security environment i/o service with Implied Credit Spreads inserted self.args[ios] = "updateSecurityEnvironment" self.args[f] = self.use_input_file self.args[o] = self.use_output_file self._parsePPOutputFile() self._buildUSEInputFile() result = IOService.run(self, retry) if result != 0: raise Exception, "IO Service failure" except NoCreditSpreadHoldings: logging.info("All credit spread holdings in imagine have zero quantity, bypassing further downloads.") except Exception, e: logging.exception("The Credit Spread extract failed with error %s" % e) subject = "Credit Spread extract failed" body = "The Credit Spread service failed with error message %s. Imagine logfile attached." % e send_email( smtpServer, emailFrom, emailTo, subject, body, {os.path.dirname(self.log_file): os.path.basename(self.log_file)}, )
def run(self, retry, smtpServer, emailFrom, emailTo, archive_file): self._buildInputfile() IOService.run(self, retry) if not os.path.exists(self.output_file): logging.warning("The Trade Extract failed, missing output file") subject = "Trade Extract Failed" body = "The Trade Extract failed, missing output file. Imagine logfile attached." send_email( smtpServer, emailFrom, emailTo, subject, body, {os.path.dirname(self.log_file): os.path.basename(self.log_file)}, ) return paths = os.path.split(archive_file) archive_file = os.path.join(paths[0], "%s_%s" % (self.asofdate, paths[1])) self._formatOutputFile(archive_file) logging.info("Done") return archive_file
def run(self): logging.info("downloadIMSCash extract as at %s" % (self.asofdate)) inputFilename = os.path.join(self.working_path, "_%s_ims_cash_input.txt" % self.asofdate) outputFilename = os.path.join(self.working_path, "%s_ims_cash_output.csv" % self.asofdate) if os.path.exists(outputFilename) and os.path.getsize(outputFilename) > 0: if not self.replace_file: logging.info("output file exists, skipping download of %s" % outputFilename) return self._buildInputfile(inputFilename, self.asofdate.strftime("%Y%m%d")) ios = IOService(service = "downloadPayments", props = self.props, input_file = inputFilename, output_file = outputFilename, log_file = self.log_file, outDelim = "|", lib_path=self.lib_path, java_path=self.java_path) ios.run() if not os.path.exists(outputFilename): logging.warning("The Imagine Holdings download failed, missing output file") msg = MIMEText("The Imagine Holdings download failed, missing output file") msg['Subject'] = "Extract Issue - downloadIMSCash" smtp = smtplib.SMTP(self.smtpServer) smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string()) smtp.quit() return paths = os.path.split(self.output_file) archiveFilename = os.path.join(paths[0], "%s_%s" % (self.asofdate, paths[1])) outputFile= file(archiveFilename, 'wb') for tempFile in [outputFilename]: logging.info('Reading part file %s' % tempFile) tfile = file(tempFile, 'rb') line = tfile.readline() while line: outputFile.write(line) line = tfile.readline() outputFile.close() logging.info("Done") return archiveFilename
def run(self, retry, smtpServer, emailFrom, emailTo): """Runs the daily movers service.""" self._buildInputfile() IOService.run(self, retry) if not os.path.exists(self.output_file): logging.warning("The Daily Movers extract failed, missing output file") subject = "Daily Movers extract failed" body = "The Daily Movers failed, missing output file. Imagine logfile attached." send_email( smtpServer, emailFrom, emailTo, subject, body, {os.path.dirname(self.log_file): os.path.basename(self.log_file)}, ) return body = self._parseOutputFile() subject = "Daily Movers %s" % self.asofdate.strftime("%d-%b-%Y") send_email(smtpServer, emailFrom, emailTo, subject, body, {})
def run(self, retry, smtpServer, emailFrom, emailTo, reports_db): """Runs the service to generate the flash report from Imagine.""" try: #Run print portfolio i/o service self._buildInputFile() result = IOService.run(self, retry) if result != 0: raise Exception, "IO Service failure" logging.info("Establishing session with database %s" % reports_db) engine = create_engine(reports_db) Session = sessionmaker(bind=engine) session = Session() self._loadDatabase(session) self._generateReport(session) self._emailReport(smtpServer, emailFrom, emailTo) except Exception, e: logging.exception("The Daily Flash extract failed with error %s" % e) subject = 'Daily Flash extract failed' body = 'The Daily Flash service failed with error message "%s". Imagine logfile attached.' % e send_email(smtpServer, emailFrom, emailTo, subject, body, {os.path.dirname(self.log_file):os.path.basename(self.log_file)}) return
def run(self): """ Creates sub ioservice extracts as size of payments data can exceed java memory heap. initially split by Month, the split is now bymonthly. """ # Loop through months; fire download once per month testDate = self.startdate failedCounter = 0 counterA = 0 while testDate <= self.asofdate: counterA += 1 # Get first and last of month firstOM = testDate if firstOM.day == 1: lastOM = datetime.date(firstOM.year, firstOM.month, 15) #Day before 16 which would be start of new period else: lastOM = datetime.date(firstOM.year, firstOM.month, calendar.monthrange(firstOM.year, firstOM.month)[1]) if self.asofdate <= lastOM: lastOM = self.asofdate firstOMstr = firstOM.strftime("%Y%m%d") lastOMstr = lastOM.strftime("%Y%m%d") logging.info("downloadPayments extract for %s to %s" % (firstOMstr, lastOMstr)) if firstOM.month == 12 and firstOM.day == 16: testDate = datetime.date(firstOM.year+1, 1, 1) else: if firstOM.day == 1: testDate = datetime.date(firstOM.year, firstOM.month, 16) else: testDate = datetime.date(firstOM.year, firstOM.month+1, 1) inputFilename = os.path.join(self.working_path, "_%sinput.txt" % lastOMstr) outputFilename = os.path.join(self.working_path, "%sdownload.csv" % lastOMstr) if os.path.exists(outputFilename) and os.path.getsize(outputFilename) > 0: if not self.replace_file: logging.info("output file exists, skipping download of %s" % outputFilename) continue self._buildInputfile(inputFilename, firstOMstr, lastOMstr) ios = IOService(service = "downloadPayments", props = self.props, input_file = inputFilename, output_file = outputFilename, log_file = self.log_file, outDelim = "|", columms = self.columns, lib_path=self.lib_path, java_path=self.java_path) #if 0: # def run(): # return -1 # ios.run = run result = ios.run() if result != 0: failedCounter += 1 """ if result != 0 and self.retry > 0: logging.warning("First IOService first attempt failed, will make another %s attempts" % self.retry) while result != 0 and self.retry > 0: logging.warning("IO Service failure [%s], retrying" % result) result = ios.run() self.retry += -1 if result != 0: logging.warning("All IOService attempts failed for %s" % lastOMstr) failedCounter += 1 """ if failedCounter: logging.warning("%s of the Imagine extracts failed. Exiting Process." % failedCounter) msg = MIMEText("%s of the Imagine extracts failed to download from Imagine. The process has exited, please rerun download." % failedCounter) msg['Subject'] = "Download Failures - downloadPayments" smtp = smtplib.SMTP(self.smtpServer) smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string()) smtp.quit() return logging.info("Stiching together imagine extracts") tt = os.path.join(self.working_path, "20*.csv") outputFiles = glob.glob(tt) if len(outputFiles) != counterA: logging.warning("Error with the Imagine extracts. Exiting Process. %s, %s" % (len(outputFiles), counterA)) msg = MIMEText("Error with the Imagine download payments extracts. The process has exited, please rerun download.") msg['Subject'] = "Stitching Issue - downloadPayments" if send_email: smtp = smtplib.SMTP(self.smtpServer) smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string()) smtp.quit() return #outputFilename = os.path.join(self.working_path, "%sdownload.csv" % lastOMstr) outputFilename = os.path.join(self.working_path, self.output_file) logging.info("writing to %s" % outputFilename) outputFile= file(outputFilename, 'wb') counterB = 0 for tempFile in outputFiles: logging.info('Reading part file %s' % tempFile) tfile = file(tempFile, 'rb') if counterB != 0: #Skip headers of all but first file logging.debug("Skipping headers in %s" % tempFile) line = tfile.readline() linecount = 1 while line and linecount < 6: line = tfile.readline() linecount += 1 else: line = "seed" counterB += 1 while line: line = tfile.readline() if line and (line.strip() != "**End Table**"): outputFile.write(line) # Check Size logging.info("Checking output file size...") size = os.path.getsize(outputFilename) / float(1024 * 1024) if size > _maxOutputMB: logging.warning("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB)) msg = MIMEText("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB)) if send_email: msg['Subject'] = "File Size Issue - downloadPayments" smtp = smtplib.SMTP(self.smtpServer) smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string()) smtp.quit() logging.info("Done")
def run(self): counterA = 1 logging.info("PrintPortfolio extract as at %s" % (self.asofdate)) inputFilename = os.path.join(self.working_path, "_%s_%s_%s_input.txt" % (self.asofdate,self.filtertype,self.filter)) outputFilename = os.path.join(self.working_path, "%s_%s_%s_download.csv" % (self.asofdate,self.filtertype,self.filter)) if os.path.exists(outputFilename) and os.path.getsize(outputFilename) > 0: if not self.replace_file: logging.info("output file exists, skipping download of %s" % outputFilename) return self._buildInputfile(inputFilename, self.asofdate.strftime("%Y%m%d"), self.filter, self.filtertype) ios = IOService(service = "printPortfolio", props = self.props, input_file = inputFilename, output_file = outputFilename, log_file = self.log_file, trans='ioservice.ReportTranslator', outDelim = ",", columms = self.columns, lib_path=self.lib_path, java_path=self.java_path) attempts = 0 while attempts < self.retry: result = ios.run() file_generated = os.path.exists(outputFilename) and os.path.getsize(outputFilename) > 0 if result == 0 and file_generated: #All OK break else: attempts += 1 if result != 0 or not file_generated: logging.warning("Imagine extract failed. Exiting Process.") msg = MIMEText("The Imagine PrintPortfolio process failed. The process has exited, please rerun download.") msg['Subject'] = "Extract Issue - PrintPortfolio" smtp = smtplib.SMTP(self.smtpServer) smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string()) smtp.quit() raise Exception("Imagine Print portfolio error") orig_name = outputFilename outputFilename = os.path.join(self.working_path, self.output_file) outputFile= file(outputFilename, 'wb') for tempFile in [orig_name]: logging.info('Reading part file %s' % tempFile) tfile = file(tempFile, 'rb') line = tfile.readline() while line: outputFile.write(line) line = tfile.readline() # Check Size logging.info("Checking output file size...") size = os.path.getsize(outputFilename) / float(1024 * 1024) if size > _maxOutputMB: logging.warning("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB)) msg = MIMEText("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB)) msg['Subject'] = "File Size Issue - PrintPortfolio" smtp = smtplib.SMTP(self.smtpServer) smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string()) smtp.quit() logging.info("Done")
def run(self): counterA = 1 logging.info("DownloadVAR extract as at %s" % (self.asofdate)) inputFilename = os.path.join(self.working_path, "_%sinput.txt" % self.asofdate) outputFilename = os.path.join(self.working_path, "%sdownload.csv" % self.asofdate) extracted = False if os.path.exists(outputFilename) and os.path.getsize(outputFilename) > 0: if not self.replace_file: logging.info("output file exists, skipping download of %s" % outputFilename) extracted = True if not extracted: self._buildInputfile(inputFilename, self.asofdate.strftime("%Y%m%d"), portfolio=self.portfolio, vartype=self.vartype) ios = IOService(service = "runBatchCmd", props = self.props, input_file = inputFilename, output_file = outputFilename, log_file = self.log_file, outDelim = "|", columms = self.columns, lib_path=self.lib_path, java_path=self.java_path) ios.run() tt = os.path.join(self.working_path, "20*.*") outputFiles = glob.glob(tt) if len(outputFiles) != 1: logging.warning("The Imagine VAR extract failed. Exiting Process.") msg = MIMEText("The Imagine VAR extract failed. The process has exited, please rerun.") msg['Subject'] = "Extract Issue - DownloadVAR" smtp = smtplib.SMTP(self.smtpServer) smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string()) smtp.quit() return #outputFilename = os.path.join(self.working_path, "%sdownload.csv" % lastOMstr) outputFilename = os.path.join(self.working_path, self.output_file) outputFile= file(outputFilename, 'wb') counterB = 0 for tempFile in outputFiles: logging.info('Reading part file %s' % tempFile) tfile = file(tempFile, 'rb') line = tfile.readline() while line: outputFile.write(line) line = tfile.readline() # Check Size logging.info("Checking output file size...") size = os.path.getsize(outputFilename) / float(1024 * 1024) if size > _maxOutputMB: logging.warning("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB)) msg = MIMEText("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB)) msg['Subject'] = "File Size Issue - DownloadVAR" smtp = smtplib.SMTP(self.smtpServer) smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string()) smtp.quit() logging.info("Done")
def run(self): # Loop through months; fire download once per month testDate = self.startdate counterA = 1 logging.info("downloadTrades extract until %s" % (self.enddate)) inputFilename = os.path.join(self.working_path, "_%sinput.txt" % self.enddate) outputFilename = os.path.join(self.working_path, "%sdownload.csv" % self.enddate) extract = True if os.path.exists(outputFilename) and os.path.getsize(outputFilename) > 0: if not self.replace_file: logging.info("output file exists, skipping download of %s" % outputFilename) extract = False if extract: self._buildInputfile(inputFilename, self.startdate.strftime("%Y%m%d"), self.enddate.strftime("%Y%m%d")) ios = IOService(service = "downloadTrades", props = self.props, input_file = inputFilename, output_file = outputFilename, log_file = self.log_file, outDelim = "|", columms = self.columns, lib_path=self.lib_path, java_path=self.java_path) #if 0: # def run(): # return -1 # ios.run = run result = ios.run() if result != 0 and self.retry > 0: logging.warning("First IOService first attempt failed, will make another %s attempts" % self.retry) while result != 0 and self.retry > 0: logging.warning("IO Service failure [%s], retrying" % result) result = ios.run() self.retry += -1 tt = os.path.join(self.working_path, "20*.csv") outputFiles = glob.glob(tt) if len(outputFiles) != 1: logging.warning("Some of the Imagine extracts failed. Exiting Process.") msg = MIMEText("One or more files for the Download Trades process failed to download from Imagine. The process has exited, please rerun download.") msg['Subject'] = "Extract Issue - downloadTrades" smtp = smtplib.SMTP(self.smtpServer) smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string()) smtp.quit() return #outputFilename = os.path.join(self.working_path, "%sdownload.csv" % lastOMstr) outputFilename = os.path.join(self.working_path, self.output_file) outputFile= file(outputFilename, 'wb') counterB = 0 for tempFile in outputFiles: logging.info('Reading part file %s' % tempFile) tfile = file(tempFile, 'rb') line = tfile.readline() while line: outputFile.write(line) line = tfile.readline() # Check Size logging.info("Checking output file size...") size = os.path.getsize(outputFilename) / float(1024 * 1024) if size > _maxOutputMB: logging.warning("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB)) msg = MIMEText("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB)) msg['Subject'] = "File Size Issue - downloadTrades" smtp = smtplib.SMTP(self.smtpServer) smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string()) smtp.quit() logging.info("Done")