def setUpClass(cls):
'''
Open up a bunch of statments and add them to a test database for testing stuff
TODO: Develop the randomtradgenerator write trades to the db for more generic testing
'''
settings = QSettings('zero_substance', 'structjour')
for i, name in enumerate(cls.infiles):
name = os.path.join(cls.datadir, name)
x, cls.inputType = getStatementType(name)
print(cls.inputType)
if cls.inputType == 'DAS':
ds = DasStatement(name, settings, cls.thedates[i])
ds.getTrades(testFileLoc=cls.datadir, testdb=cls.testdb)
elif cls.inputType == "IB_CSV":
ibs = IbStatement(db=cls.testdb)
ibs.openIBStatement(name)
else:
continue
# self.assertTrue(4 == 5, "Unsupported file type in test_TheTradeObject")
statement = StatementDB(db=cls.testdb)
df = statement.getStatement(cls.thedates[i])
# self.assertFalse(df.empty, f"Found no trades in db on {daDate}")
dtrade = DefineTrades(cls.inputType)
dframe, ldf = dtrade.processDBTrades(df)
tto = TheTradeObject(ldf[0], False, SumReqFields())
cls.ttos.append(tto)
def setUpClass(cls):
bu = Backup()
bu.backup()
if ModelBase.session:
ModelBase.session.rollback()
ddiirr = os.path.dirname(__file__)
os.chdir(os.path.realpath(ddiirr + '/../'))
outdir = 'test/out'
cls.outdir = os.path.realpath(outdir)
cls.db = 'data/testdb.sqlite'
cls.db = os.path.realpath(cls.db)
if os.path.exists(cls.db):
clearTables(cls.db)
cls.rtg = RTG(db=cls.db)
# cls.dates = ['20200203 09:30', '20200204 07:30', '20200205 09:35', '20200206 11:40', '20200207 10:39']
cls.dates = ['20200207 10:39']
cls.infiles = cls.rtg.saveSomeTestFiles(cls.dates,
cls.outdir,
strict=True,
overwrite=True)
settings = QSettings('zero_substance', 'structjour')
for i, name in enumerate(cls.infiles):
name = os.path.join(cls.outdir, name)
x, cls.inputType = getStatementType(name)
# print(cls.inputType)
if cls.inputType == 'DAS':
ds = DasStatement(name, settings, cls.dates[i])
ds.getTrades(testFileLoc=cls.outdir, testdb=cls.db)
elif cls.inputType == "IB_CSV":
ibs = IbStatement(db=cls.db)
ibs.openIBStatement(name)
else:
continue
# self.assertTrue(4 == 5, "Unsupported file type in test_TheTradeObject")
statement = StatementDB(db=cls.db)
df = statement.getStatement(cls.dates[i])
# self.assertFalse(df.empty, f"Found no trades in db on {daDate}")
dtrade = DefineTrades(cls.inputType)
dframe, ldf = dtrade.processDBTrades(df)
# tto = TheTradeObject(ldf[0], False, SumReqFields())
jf = JournalFiles(indir=cls.outdir,
outdir=outdir,
theDate=cls.dates[i],
infile=name)
cls.sc = SumControl()
lf = LayoutForms(cls.sc, jf, dframe)
lf.runTtoSummaries(ldf)
cls.jfs.append(jf)
cls.dframes.append(dframe)
# cls.ttos.append(tto)
cls.ldfs.append(ldf)
cls.lfs.append(lf)
def test_figureBAPL(self):
'''
figureBAPL is called by openIBStatement and is give the trade tables and
position tables from an IB statement. figureBAPL fills in Balance, Average
and pnl. Every Balance entry should be filled. There could be blanks for
Average. Here we can test all balance entries made it to the db
'''
ibs = IbStatement()
ibdb = StatementDB()
ibdb.reinitializeTradeTables()
# These two files are a multiday flex and an activity.
# Find a way to find recent files to test
# trades1, meta1 = ibs.openIBStatement(self.testfile1)
trades2, meta2 = ibs.openIBStatement(self.testfile2)
len2 = len(trades2['TRNT']) if 'TRNT' in trades2.keys() else len(
trades2['Trades'])
ModelBase.connect(new_session=True)
session = ModelBase.session
q = session.query(Trade).all()
q2 = session.query(Trade).filter(Trade.balance is None).all()
self.assertEqual(len2, len(q))
self.assertEqual(len(q2), 0)
bu = Backup()
bu.restore()
def setUpClass(cls):
bu = Backup()
bu.backup()
ddiirr = os.path.dirname(__file__)
os.chdir(os.path.realpath(ddiirr))
os.chdir(os.path.realpath('../'))
cls.outdir = os.path.realpath(cls.outdir)
cls.db = os.path.realpath(cls.db)
if os.path.exists(cls.db):
clearTables(cls.db)
cls.rtg = RTG(db=cls.db, overnight=100)
cls.theDate = '20200207 10:39'
cls.infile = cls.rtg.saveSomeTestFiles([cls.theDate],
cls.outdir,
strict=True,
overwrite=False)[0]
settings = QSettings('zero_substance', 'structjour')
# for i, name in enumerate(cls.infiles):
name = os.path.join(cls.outdir, cls.infile)
x, cls.inputType = getStatementType(name)
if cls.inputType == 'DAS':
ds = DasStatement(name, settings, cls.theDate)
ds.getTrades(testFileLoc=cls.outdir, testdb=cls.db)
elif cls.inputType == "IB_CSV":
ibs = IbStatement(db=cls.db)
ibs.openIBStatement(name)
else:
raise ValueError(f'Unsupported File type: {cls.inputType}')
statement = StatementDB(db=cls.db)
cls.df = statement.getStatement(cls.theDate)
cls.dtrades = DefineTrades(cls.inputType)
cls.rc = FinReqCol(cls.inputType)
cls.trades = cls.dtrades.addFinReqCol(cls.df)
rccolumns = cls.rc.columns.copy()
rccolumns = cls.dtrades.appendCols(rccolumns)
cls.trades = cls.trades[rccolumns]
cls.trades.copy()
cls.trades = cls.trades.sort_values(
[cls.rc.ticker, cls.rc.acct, cls.rc.date])
def test_openIbStatement(self):
ibs = IbStatement(db=self.db)
ibdb = StatementDB(db=self.db)
ibdb.reinitializeTradeTables()
x = ibs.openIBStatement(self.testfile1)
self.assertIsNotNone(x)
self.assertIsInstance(x[0], dict)
self.assertTrue('TRNT' in x[0].keys() or 'Trades' in x[0].keys())
st = ibdb.getStatement(self.theDate)
self.assertIsInstance(st, pd.DataFrame)
self.assertGreater(len(st), 0)
def openStuff(self, allofit=None):
'''Site specific testing stuff-- open a multi day statement'''
# Site specific stuff here Open a monthly or yearly statemnet into the test db.
# Currently set to search the journal root dir, so it may load an annual or two.
bdir = ff.getBaseDir()
fs = ff.findFilesInDir(bdir, 'U242.csv', True)
ibs = IbStatement()
for f in fs:
x = ibs.openIBStatement(f)
delt = ibs.endDate - ibs.beginDate
assert delt.days > 20
if not allofit:
break
return ibs, x
def test_openIbStatement_notcsv(self):
ibs = IbStatement(db=self.db)
ibdb = StatementDB(db=self.db)
ibdb.reinitializeTradeTables()
x = ibs.openIBStatement('data\alittleOrgTODO.txt')
self.assertEqual(len(x[0]), 0)
def runnit(self, loaditrun=False):
'''
Load an initial input file and process it.
'''
self.sc.dateInSync = True
if self.sc.ui.useDatabase.isChecked() and loaditrun is False:
if not self.gotTrades():
return
return self.loadit()
self.sc.doWeSave()
self.initialize()
if not self.indir:
logging.info('No file to load?')
return
jf = JournalFiles(indir=self.indir,
outdir=self.outdir,
theDate=self.theDate,
infile=self.infile,
inputType=self.inputtype,
infile2=self.positions,
mydevel=True)
if self.inputtype == 'DB':
self.runDBInput(self.theDate, jf)
windowTitle = f'{self.sc.baseWindowTitle}: {self.sc.ui.infileEdit.text()}: no user data'
self.sc.setWindowTitle(windowTitle)
if self.gotTrades():
self.sc.ui.useDatabase.setChecked(True)
self.sc.dbDefault(True)
return
local = os.path.normpath(self.ui.infileEdit.text())
if os.path.normpath(jf.inpathfile) != local:
if os.path.exists(local):
d, jf.infile = os.path.split(local)
jf.inpathfile = local
x, inputType = getStatementType(jf.inpathfile)
if not inputType:
msg = f'<h3>No trades found. File does not appear to be a statement</h3><ul> '
msg += f'<div><strong>{jf.inpathfile}</strong></div>'
msgbx = QMessageBox()
msgbx.setIconPixmap(QPixmap("structjour/images/ZSLogo.png"))
msgbx.setText(msg)
msgbx.exec()
return
self.inputtype = inputType
windowTitle = self.sc.baseWindowTitle + ': ' + self.sc.ui.infileEdit.text(
) + ': no user data'
self.sc.setWindowTitle(windowTitle)
if self.inputtype == 'IB_HTML' or self.inputtype == 'IB_CSV':
jf.inputType = self.inputtype
ibs = IbStatement()
x = ibs.openIBStatement(jf.inpathfile)
msg = ''
if x[0]:
tkey = 'Trades' if 'Trades' in x[0].keys(
) else 'TRNT' if 'TRNT' in x[0].keys() else None
if not tkey:
raise ValueError(
f'Error in processing statemnt {jf.inpathfile}')
numtickets = len(x[0][tkey])
gotToday = self.runDBInput(self.theDate, jf)
if gotToday:
if self.gotTrades():
self.sc.ui.useDatabase.setChecked(True)
self.sc.dbDefault(True)
else:
msg = f'<h3>No trades found on date {self.theDate.date()}</h3><ul> '
msg += f'<div><strong>{jf.inpathfile}</strong></div>'
msg += f'<div>Found {numtickets} tickets. They are now in DB</div>'
msg += f'<div>{list(x[1].keys())}</div>'
else:
msg = f'<h3>No trades recorded from the file:</h3><ul> '
msg = msg + f'<div><strong>{jf.inpathfile}</strong></div>'
msg = msg + f'<div>{x[1]}</div>'
msgbx = QMessageBox()
msgbx.setIconPixmap(QPixmap("structjour/images/ZSLogo.png"))
msgbx.setText(msg)
msgbx.exec()
return
elif self.inputtype == 'DAS':
x = checkDateDir(jf.inpathfile)
if not x:
msg = "<h3>The date for this DAS statement is not clear</h3>"
msg += "<div>Please enter the date for this statement</div>"
msg += f'<div><strong>{jf.inpathfile}</strong></div>'
msg += '<div>(YYYYMMDD) ex: 20190113</div>'
theDate = getDate(msg)
if theDate:
self.settings.setValue('theDate', theDate)
self.sc.ui.dateEdit.setDate(pd2qtime(theDate, qdate=True))
else:
return
ds = DasStatement(jf.infile, self.settings, self.theDate)
ds.getTrades()
self.runDBInput(self.theDate, jf)
if self.gotTrades():
self.sc.ui.useDatabase.setChecked(True)
self.sc.dbDefault(True)
return
else:
msg = '<h3>Unrecognized input:</h3><ul> '
msgbx = QMessageBox()
msgbx.setIconPixmap(QPixmap("structjour/images/ZSLogo.png"))
msgbx.setText(msg)
msgbx.exec()
return
def setUpClass(cls):
ddiirr = os.path.dirname(__file__)
os.chdir(os.path.realpath(ddiirr + '/../'))
outdir = 'test/out'
cls.outdir = os.path.realpath(outdir)
cls.db = 'data/testdb.sqlite'
cls.db = os.path.realpath(cls.db)
if os.path.exists(cls.db):
clearTables(cls.db)
cls.rtg = RTG(db=cls.db)
# cls.dates = ['20200203 09:30', '20200204 07:30', '20200205 09:35', '20200206 11:40', '20200207 10:39']
cls.theDate = '20200207 10:39'
cls.infile = cls.rtg.saveSomeTestFiles([cls.theDate], cls.outdir)[0]
settings = QSettings('zero_substance', 'structjour')
# for i, name in enumerate(cls.infiles):
name = os.path.join(cls.outdir, cls.infile)
x, inputType = getStatementType(name)
if inputType == 'DAS':
ds = DasStatement(name, settings, cls.theDate)
ds.getTrades(testFileLoc=cls.outdir, testdb=cls.db)
elif inputType == "IB_CSV":
ibs = IbStatement(db=cls.db)
ibs.openIBStatement(name)
else:
raise ValueError(f'Unsupported File type: {inputType}')
statement = StatementDB(db=cls.db)
df = statement.getStatement(cls.theDate)
dtrade = DefineTrades(inputType)
dframe, cls.ldf = dtrade.processDBTrades(df)
cls.jf = JournalFiles(indir=cls.outdir,
outdir=outdir,
theDate=cls.theDate,
infile=name)
cls.sc = SumControl()
lf = LayoutForms(cls.sc, cls.jf, dframe)
lf.runTtoSummaries(cls.ldf)
# Setup mistake note fields to test
cls.note = 'Ground control to Major Tom'
for i, key in enumerate(lf.ts):
tto = lf.ts[key]
notex = cls.note + str(i + 1)
tto['MstkNote'] = notex
# Setup a couple images to add
imdir = 'images/'
img1 = os.path.join(imdir, 'fractal-art-fractals.jpg')
img2 = os.path.join(imdir, 'psych.jpg')
assert os.path.exists(img1)
assert os.path.exists(img2)
for key in lf.ts:
tto = lf.ts[key]['chart1'] = img1
tto = lf.ts[key]['chart2'] = img2
t = ExportToExcel(lf.ts, cls.jf, df)
imageNames = t.getImageNamesFromTS()
cls.imageLocation, dframe = t.layoutExcelData(t.df, cls.ldf,
imageNames)
assert len(cls.ldf) == len(cls.imageLocation)
# Create an openpyxl wb from the dataframe
ls = LayoutSheet(t.topMargin, len(t.df))
cls.wb, ws, nt = ls.createWorkbook(dframe)
# Place both forms 2 cells to the right of the main table
mstkAnchor = (len(dframe.columns) + 2, 1)
cls.mistake = MistakeSummary(numTrades=len(cls.ldf), anchor=mstkAnchor)
cls.t = t
cls.a = mstkAnchor
def test_openIbStatement_notcsv(self):
ibs = IbStatement()
ibdb = StatementDB()
x = ibs.openIBStatement('data\alittleOrgTODO.txt')
self.assertEqual(len(x[0]), 0)