def __init__(self, props, working_path, output_dir, asofdate, lib_path, java_path, reports_url):
datetoday = datetime.date.today()
self.asofdate = asofdate
self.reports_url = reports_url
self.working_path = os.path.join(working_path, datetoday.strftime("%Y%m%d"), "flashpnl")
self.output_dir = output_dir
try:
os.makedirs(self.working_path)
except:
pass
self.log_file = os.path.join(self.working_path, "%s_daily_flash_v2.log" % self.asofdate)
self.input_file = os.path.join(self.working_path, "%s_daily_flash_v2_input.txt" % self.asofdate)
self.output_file = os.path.join(self.working_path, "%s_daily_flash_v2_output.csv" % self.asofdate)
IOService.__init__(self,
service="printPortfolio",
props=props,
input_file=self.input_file,
output_file=self.output_file,
log_file=self.log_file,
outDelim=",",
columms=None,
lib_path=lib_path,
java_path=java_path,
trans='ioservice.ReportTranslator',
aggreg=False)
def run(self, retry, smtpServer, emailFrom, emailTo, archive_file):
self._buildInputfile()
IOService.run(self, retry)
if not os.path.exists(self.output_file):
logging.warning("The Daily Archiving failed, missing output file")
subject = 'Daily Archiving Failed'
body = 'The Daily Archiving failed with error: missing output file. Imagine logfile attached if it exists.'
send_email(smtpServer, emailFrom, emailTo, subject, body, {os.path.dirname(self.log_file):os.path.basename(self.log_file)})
return
paths = os.path.split(archive_file)
archive_file = os.path.join(paths[0], "%s_%s" % (self.asofdate, paths[1]))
outputFile= file(archive_file, 'wb')
for tempFile in [self.output_file]:
logging.info('Reading part file %s' % tempFile)
tfile = file(tempFile, 'rb')
line = tfile.readline()
while line:
outputFile.write(line)
line = tfile.readline()
outputFile.close()
logging.info("Done")
return archive_file
def run(self, retry, smtpServer, emailFrom, emailTo):
"""Runs the service to update security level credit spread overrides in Imagine."""
# Get indices of arguments we'll need to change
ios = self.args.index("-ios") + 1
f = self.args.index("-f") + 1
o = self.args.index("-o") + 1
try:
# Run print portfolio i/o service
self._buildPPInputFile()
result = IOService.run(self, retry)
if result != 0:
raise Exception, "IO Service failure"
# Run update security environment i/o service
self.args[ios] = "updateSecurityEnvironment"
self.args[f] = self.use_input_file
self.args[o] = self.use_output_file
self._parsePPOutputFile()
self._buildUSEInputFile(credit_spread=0.01)
result = IOService.run(self, retry)
if result != 0:
raise Exception, "IO Service failure"
# Run print portfolio i/o service again
self.args[ios] = "printPortfolio"
self.args[f] = self.pp_input_file
self.args[o] = self.pp_output_file
result = IOService.run(self, retry)
if result != 0:
raise Exception, "IO Service failure"
# Run update security environment i/o service with Implied Credit Spreads inserted
self.args[ios] = "updateSecurityEnvironment"
self.args[f] = self.use_input_file
self.args[o] = self.use_output_file
self._parsePPOutputFile()
self._buildUSEInputFile()
result = IOService.run(self, retry)
if result != 0:
raise Exception, "IO Service failure"
except NoCreditSpreadHoldings:
logging.info("All credit spread holdings in imagine have zero quantity, bypassing further downloads.")
except Exception, e:
logging.exception("The Credit Spread extract failed with error %s" % e)
subject = "Credit Spread extract failed"
body = "The Credit Spread service failed with error message %s. Imagine logfile attached." % e
send_email(
smtpServer,
emailFrom,
emailTo,
subject,
body,
{os.path.dirname(self.log_file): os.path.basename(self.log_file)},
)
def __init__(self, props, working_path, asofdate, lib_path, java_path):
datetoday = datetime.date.today()
self.asofdate = asofdate
working_path = os.path.join(working_path, datetoday.strftime("%Y%m%d"), "dailyarchive")
try:
os.makedirs(working_path)
except:
pass
self.log_file = os.path.join(working_path, "%s_daily_archive.log" % self.asofdate)
self.input_file = os.path.join(working_path, "%s_daily_archive_input.txt" % self.asofdate)
self.output_file = os.path.join(working_path, "%s_daily_archive_output.csv" % self.asofdate)
IOService.__init__(self,
service="runBatchCmd",
props=props,
input_file=self.input_file,
output_file=self.output_file,
log_file=self.log_file,
outDelim="|",
columms=None,
lib_path=lib_path,
java_path=java_path,
trans=None,
aggreg=False)
def __init__(self, props, working_path, asofdate, lib_path, java_path):
datetoday = datetime.date.today()
self.asofdate = asofdate
working_path = os.path.join(working_path, datetoday.strftime("%Y%m%d"), "tradeextract")
try:
os.makedirs(working_path)
except:
pass
self.log_file = os.path.join(working_path, "%s_trade_extract.log" % self.asofdate)
self.input_file = os.path.join(working_path, "%s_trade_extract_input.txt" % self.asofdate)
self.output_file = os.path.join(working_path, "%s_trade_extract_output.txt" % self.asofdate)
columns = os.path.join(os.path.abspath(os.path.dirname(__file__)), "columns", "tradeExtractColumns.txt")
IOService.__init__(
self,
service="downloadTrades",
props=props,
input_file=self.input_file,
output_file=self.output_file,
log_file=self.log_file,
outDelim="|",
columms=columns,
lib_path=lib_path,
java_path=java_path,
trans="ioservice.GenericTradeTranslator",
aggreg=True,
)
def __init__(self, props, working_path, asofdate, lib_path, java_path):
datetoday = datetime.date.today()
self.asofdate = asofdate
working_path = os.path.join(working_path, datetoday.strftime("%Y%m%d"), "creditspread")
try:
os.makedirs(working_path)
except:
pass
self.log_file = os.path.join(working_path, "%s_credit_spread.log" % self.asofdate)
self.pp_input_file = os.path.join(working_path, "%s_credit_spread_pp_input.txt" % self.asofdate)
self.pp_output_file = os.path.join(working_path, "%s_credit_spread_pp_output.csv" % self.asofdate)
self.use_input_file = os.path.join(working_path, "%s_credit_spread_use_input.txt" % self.asofdate)
self.use_output_file = os.path.join(working_path, "%s_credit_spread_use_output.txt" % self.asofdate)
self.props = props
IOService.__init__(
self,
service="printPortfolio",
props=self.props,
input_file=self.pp_input_file,
output_file=self.pp_output_file,
log_file=self.log_file,
outDelim=",",
columms=None,
lib_path=lib_path,
java_path=java_path,
trans="ioservice.ReportTranslator",
aggreg=False,
)
def run(self, retry, smtpServer, emailFrom, emailTo, archive_file):
self._buildInputfile()
IOService.run(self, retry)
if not os.path.exists(self.output_file):
logging.warning("The Trade Extract failed, missing output file")
subject = "Trade Extract Failed"
body = "The Trade Extract failed, missing output file. Imagine logfile attached."
send_email(
smtpServer,
emailFrom,
emailTo,
subject,
body,
{os.path.dirname(self.log_file): os.path.basename(self.log_file)},
)
return
paths = os.path.split(archive_file)
archive_file = os.path.join(paths[0], "%s_%s" % (self.asofdate, paths[1]))
self._formatOutputFile(archive_file)
logging.info("Done")
return archive_file
def run(self):
logging.info("downloadIMSCash extract as at %s" % (self.asofdate))
inputFilename = os.path.join(self.working_path, "_%s_ims_cash_input.txt" % self.asofdate)
outputFilename = os.path.join(self.working_path, "%s_ims_cash_output.csv" % self.asofdate)
if os.path.exists(outputFilename) and os.path.getsize(outputFilename) > 0:
if not self.replace_file:
logging.info("output file exists, skipping download of %s" % outputFilename)
return
self._buildInputfile(inputFilename, self.asofdate.strftime("%Y%m%d"))
ios = IOService(service = "downloadPayments",
props = self.props,
input_file = inputFilename,
output_file = outputFilename,
log_file = self.log_file,
outDelim = "|",
lib_path=self.lib_path,
java_path=self.java_path)
ios.run()
if not os.path.exists(outputFilename):
logging.warning("The Imagine Holdings download failed, missing output file")
msg = MIMEText("The Imagine Holdings download failed, missing output file")
msg['Subject'] = "Extract Issue - downloadIMSCash"
smtp = smtplib.SMTP(self.smtpServer)
smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string())
smtp.quit()
return
paths = os.path.split(self.output_file)
archiveFilename = os.path.join(paths[0], "%s_%s" % (self.asofdate, paths[1]))
outputFile= file(archiveFilename, 'wb')
for tempFile in [outputFilename]:
logging.info('Reading part file %s' % tempFile)
tfile = file(tempFile, 'rb')
line = tfile.readline()
while line:
outputFile.write(line)
line = tfile.readline()
outputFile.close()
logging.info("Done")
return archiveFilename
def run(self, retry, smtpServer, emailFrom, emailTo):
"""Runs the daily movers service."""
self._buildInputfile()
IOService.run(self, retry)
if not os.path.exists(self.output_file):
logging.warning("The Daily Movers extract failed, missing output file")
subject = "Daily Movers extract failed"
body = "The Daily Movers failed, missing output file. Imagine logfile attached."
send_email(
smtpServer,
emailFrom,
emailTo,
subject,
body,
{os.path.dirname(self.log_file): os.path.basename(self.log_file)},
)
return
body = self._parseOutputFile()
subject = "Daily Movers %s" % self.asofdate.strftime("%d-%b-%Y")
send_email(smtpServer, emailFrom, emailTo, subject, body, {})
def run(self, retry, smtpServer, emailFrom, emailTo, reports_db):
"""Runs the service to generate the flash report from Imagine."""
try:
#Run print portfolio i/o service
self._buildInputFile()
result = IOService.run(self, retry)
if result != 0:
raise Exception, "IO Service failure"
logging.info("Establishing session with database %s" % reports_db)
engine = create_engine(reports_db)
Session = sessionmaker(bind=engine)
session = Session()
self._loadDatabase(session)
self._generateReport(session)
self._emailReport(smtpServer, emailFrom, emailTo)
except Exception, e:
logging.exception("The Daily Flash extract failed with error %s" % e)
subject = 'Daily Flash extract failed'
body = 'The Daily Flash service failed with error message "%s". Imagine logfile attached.' % e
send_email(smtpServer, emailFrom, emailTo, subject, body, {os.path.dirname(self.log_file):os.path.basename(self.log_file)})
return
def run(self):
"""
Creates sub ioservice extracts as size of payments data can exceed java memory heap.
initially split by Month, the split is now bymonthly.
"""
# Loop through months; fire download once per month
testDate = self.startdate
failedCounter = 0
counterA = 0
while testDate <= self.asofdate:
counterA += 1
# Get first and last of month
firstOM = testDate
if firstOM.day == 1:
lastOM = datetime.date(firstOM.year, firstOM.month, 15) #Day before 16 which would be start of new period
else:
lastOM = datetime.date(firstOM.year, firstOM.month, calendar.monthrange(firstOM.year, firstOM.month)[1])
if self.asofdate <= lastOM:
lastOM = self.asofdate
firstOMstr = firstOM.strftime("%Y%m%d")
lastOMstr = lastOM.strftime("%Y%m%d")
logging.info("downloadPayments extract for %s to %s" % (firstOMstr, lastOMstr))
if firstOM.month == 12 and firstOM.day == 16:
testDate = datetime.date(firstOM.year+1, 1, 1)
else:
if firstOM.day == 1:
testDate = datetime.date(firstOM.year, firstOM.month, 16)
else:
testDate = datetime.date(firstOM.year, firstOM.month+1, 1)
inputFilename = os.path.join(self.working_path, "_%sinput.txt" % lastOMstr)
outputFilename = os.path.join(self.working_path, "%sdownload.csv" % lastOMstr)
if os.path.exists(outputFilename) and os.path.getsize(outputFilename) > 0:
if not self.replace_file:
logging.info("output file exists, skipping download of %s" % outputFilename)
continue
self._buildInputfile(inputFilename, firstOMstr, lastOMstr)
ios = IOService(service = "downloadPayments",
props = self.props,
input_file = inputFilename,
output_file = outputFilename,
log_file = self.log_file,
outDelim = "|",
columms = self.columns,
lib_path=self.lib_path,
java_path=self.java_path)
#if 0:
# def run():
# return -1
# ios.run = run
result = ios.run()
if result != 0:
failedCounter += 1
"""
if result != 0 and self.retry > 0:
logging.warning("First IOService first attempt failed, will make another %s attempts" % self.retry)
while result != 0 and self.retry > 0:
logging.warning("IO Service failure [%s], retrying" % result)
result = ios.run()
self.retry += -1
if result != 0:
logging.warning("All IOService attempts failed for %s" % lastOMstr)
failedCounter += 1
"""
if failedCounter:
logging.warning("%s of the Imagine extracts failed. Exiting Process." % failedCounter)
msg = MIMEText("%s of the Imagine extracts failed to download from Imagine. The process has exited, please rerun download." % failedCounter)
msg['Subject'] = "Download Failures - downloadPayments"
smtp = smtplib.SMTP(self.smtpServer)
smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string())
smtp.quit()
return
logging.info("Stiching together imagine extracts")
tt = os.path.join(self.working_path, "20*.csv")
outputFiles = glob.glob(tt)
if len(outputFiles) != counterA:
logging.warning("Error with the Imagine extracts. Exiting Process. %s, %s" % (len(outputFiles), counterA))
msg = MIMEText("Error with the Imagine download payments extracts. The process has exited, please rerun download.")
msg['Subject'] = "Stitching Issue - downloadPayments"
if send_email:
smtp = smtplib.SMTP(self.smtpServer)
smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string())
smtp.quit()
return
#outputFilename = os.path.join(self.working_path, "%sdownload.csv" % lastOMstr)
outputFilename = os.path.join(self.working_path, self.output_file)
logging.info("writing to %s" % outputFilename)
outputFile= file(outputFilename, 'wb')
counterB = 0
for tempFile in outputFiles:
logging.info('Reading part file %s' % tempFile)
tfile = file(tempFile, 'rb')
if counterB != 0: #Skip headers of all but first file
logging.debug("Skipping headers in %s" % tempFile)
line = tfile.readline()
linecount = 1
while line and linecount < 6:
line = tfile.readline()
linecount += 1
else:
line = "seed"
counterB += 1
while line:
line = tfile.readline()
if line and (line.strip() != "**End Table**"):
outputFile.write(line)
# Check Size
logging.info("Checking output file size...")
size = os.path.getsize(outputFilename) / float(1024 * 1024)
if size > _maxOutputMB:
logging.warning("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB))
msg = MIMEText("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB))
if send_email:
msg['Subject'] = "File Size Issue - downloadPayments"
smtp = smtplib.SMTP(self.smtpServer)
smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string())
smtp.quit()
logging.info("Done")
def run(self):
counterA = 1
logging.info("PrintPortfolio extract as at %s" % (self.asofdate))
inputFilename = os.path.join(self.working_path, "_%s_%s_%s_input.txt" % (self.asofdate,self.filtertype,self.filter))
outputFilename = os.path.join(self.working_path, "%s_%s_%s_download.csv" % (self.asofdate,self.filtertype,self.filter))
if os.path.exists(outputFilename) and os.path.getsize(outputFilename) > 0:
if not self.replace_file:
logging.info("output file exists, skipping download of %s" % outputFilename)
return
self._buildInputfile(inputFilename, self.asofdate.strftime("%Y%m%d"), self.filter, self.filtertype)
ios = IOService(service = "printPortfolio",
props = self.props,
input_file = inputFilename,
output_file = outputFilename,
log_file = self.log_file,
trans='ioservice.ReportTranslator',
outDelim = ",",
columms = self.columns,
lib_path=self.lib_path,
java_path=self.java_path)
attempts = 0
while attempts < self.retry:
result = ios.run()
file_generated = os.path.exists(outputFilename) and os.path.getsize(outputFilename) > 0
if result == 0 and file_generated:
#All OK
break
else:
attempts += 1
if result != 0 or not file_generated:
logging.warning("Imagine extract failed. Exiting Process.")
msg = MIMEText("The Imagine PrintPortfolio process failed. The process has exited, please rerun download.")
msg['Subject'] = "Extract Issue - PrintPortfolio"
smtp = smtplib.SMTP(self.smtpServer)
smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string())
smtp.quit()
raise Exception("Imagine Print portfolio error")
orig_name = outputFilename
outputFilename = os.path.join(self.working_path, self.output_file)
outputFile= file(outputFilename, 'wb')
for tempFile in [orig_name]:
logging.info('Reading part file %s' % tempFile)
tfile = file(tempFile, 'rb')
line = tfile.readline()
while line:
outputFile.write(line)
line = tfile.readline()
# Check Size
logging.info("Checking output file size...")
size = os.path.getsize(outputFilename) / float(1024 * 1024)
if size > _maxOutputMB:
logging.warning("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB))
msg = MIMEText("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB))
msg['Subject'] = "File Size Issue - PrintPortfolio"
smtp = smtplib.SMTP(self.smtpServer)
smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string())
smtp.quit()
logging.info("Done")
def run(self):
counterA = 1
logging.info("DownloadVAR extract as at %s" % (self.asofdate))
inputFilename = os.path.join(self.working_path, "_%sinput.txt" % self.asofdate)
outputFilename = os.path.join(self.working_path, "%sdownload.csv" % self.asofdate)
extracted = False
if os.path.exists(outputFilename) and os.path.getsize(outputFilename) > 0:
if not self.replace_file:
logging.info("output file exists, skipping download of %s" % outputFilename)
extracted = True
if not extracted:
self._buildInputfile(inputFilename, self.asofdate.strftime("%Y%m%d"), portfolio=self.portfolio, vartype=self.vartype)
ios = IOService(service = "runBatchCmd",
props = self.props,
input_file = inputFilename,
output_file = outputFilename,
log_file = self.log_file,
outDelim = "|",
columms = self.columns,
lib_path=self.lib_path,
java_path=self.java_path)
ios.run()
tt = os.path.join(self.working_path, "20*.*")
outputFiles = glob.glob(tt)
if len(outputFiles) != 1:
logging.warning("The Imagine VAR extract failed. Exiting Process.")
msg = MIMEText("The Imagine VAR extract failed. The process has exited, please rerun.")
msg['Subject'] = "Extract Issue - DownloadVAR"
smtp = smtplib.SMTP(self.smtpServer)
smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string())
smtp.quit()
return
#outputFilename = os.path.join(self.working_path, "%sdownload.csv" % lastOMstr)
outputFilename = os.path.join(self.working_path, self.output_file)
outputFile= file(outputFilename, 'wb')
counterB = 0
for tempFile in outputFiles:
logging.info('Reading part file %s' % tempFile)
tfile = file(tempFile, 'rb')
line = tfile.readline()
while line:
outputFile.write(line)
line = tfile.readline()
# Check Size
logging.info("Checking output file size...")
size = os.path.getsize(outputFilename) / float(1024 * 1024)
if size > _maxOutputMB:
logging.warning("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB))
msg = MIMEText("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB))
msg['Subject'] = "File Size Issue - DownloadVAR"
smtp = smtplib.SMTP(self.smtpServer)
smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string())
smtp.quit()
logging.info("Done")
def run(self):
# Loop through months; fire download once per month
testDate = self.startdate
counterA = 1
logging.info("downloadTrades extract until %s" % (self.enddate))
inputFilename = os.path.join(self.working_path, "_%sinput.txt" % self.enddate)
outputFilename = os.path.join(self.working_path, "%sdownload.csv" % self.enddate)
extract = True
if os.path.exists(outputFilename) and os.path.getsize(outputFilename) > 0:
if not self.replace_file:
logging.info("output file exists, skipping download of %s" % outputFilename)
extract = False
if extract:
self._buildInputfile(inputFilename, self.startdate.strftime("%Y%m%d"), self.enddate.strftime("%Y%m%d"))
ios = IOService(service = "downloadTrades",
props = self.props,
input_file = inputFilename,
output_file = outputFilename,
log_file = self.log_file,
outDelim = "|",
columms = self.columns,
lib_path=self.lib_path,
java_path=self.java_path)
#if 0:
# def run():
# return -1
# ios.run = run
result = ios.run()
if result != 0 and self.retry > 0:
logging.warning("First IOService first attempt failed, will make another %s attempts" % self.retry)
while result != 0 and self.retry > 0:
logging.warning("IO Service failure [%s], retrying" % result)
result = ios.run()
self.retry += -1
tt = os.path.join(self.working_path, "20*.csv")
outputFiles = glob.glob(tt)
if len(outputFiles) != 1:
logging.warning("Some of the Imagine extracts failed. Exiting Process.")
msg = MIMEText("One or more files for the Download Trades process failed to download from Imagine. The process has exited, please rerun download.")
msg['Subject'] = "Extract Issue - downloadTrades"
smtp = smtplib.SMTP(self.smtpServer)
smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string())
smtp.quit()
return
#outputFilename = os.path.join(self.working_path, "%sdownload.csv" % lastOMstr)
outputFilename = os.path.join(self.working_path, self.output_file)
outputFile= file(outputFilename, 'wb')
counterB = 0
for tempFile in outputFiles:
logging.info('Reading part file %s' % tempFile)
tfile = file(tempFile, 'rb')
line = tfile.readline()
while line:
outputFile.write(line)
line = tfile.readline()
# Check Size
logging.info("Checking output file size...")
size = os.path.getsize(outputFilename) / float(1024 * 1024)
if size > _maxOutputMB:
logging.warning("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB))
msg = MIMEText("File size (%s MB) exceeded %s MB threshold. Sending File Size E-Mail..." % (size, _maxOutputMB))
msg['Subject'] = "File Size Issue - downloadTrades"
smtp = smtplib.SMTP(self.smtpServer)
smtp.sendmail(self.emailFrom, self.emailTo, msg.as_string())
smtp.quit()
logging.info("Done")
