def setUp(self):
ddiirr = os.path.dirname(__file__)
os.chdir(os.path.realpath(ddiirr))
os.chdir(os.path.realpath('../'))
self.inputtype = 'IB_CSV'
indir = 'data/'
# f2 = 'ActivityStatement.20190313_PL.html'
jf = JournalFiles(indir=indir, infile=self.f1, theDate=self.theDate, inputType='IB_CSV', mydevel=False)
jf.inputType = 'IB_CSV'
# statement = Statement_IBActivity(jf)
# df = statement.getTrades_IBActivity(jf.inpathfile)
ibs = IbStatement(db=self.testdb)
ibdb = StatementDB(self.testdb)
ibdb.reinitializeTradeTables()
ibs.openIBStatementCSV(jf.inpathfile)
df2 = ibdb.getStatement(self.theDate)
if df2.empty:
sdate = self.theDate.strftime('%Y-%m-%d')
msg = f'In test_dailycontrol.setup: Error: found no trades in db for {sdate}'
self.assertTrue(not df2.empty, msg)
tu = DefineTrades(jf.inputType)
self.df, ldf = tu.processDBTrades(df2)
sc = SumControl()
lf = LayoutForms(sc, jf, self.df)
lf.runTtoSummaries(ldf)
self.ts = lf.ts
ibdb.addTradeSummariesSA(self.ts, ldf)
def setUpClass(cls):
'''
Open up a bunch of statments and add them to a test database for testing stuff
TODO: Develop the randomtradgenerator write trades to the db for more generic testing
'''
settings = QSettings('zero_substance', 'structjour')
for i, name in enumerate(cls.infiles):
name = os.path.join(cls.datadir, name)
x, cls.inputType = getStatementType(name)
print(cls.inputType)
if cls.inputType == 'DAS':
ds = DasStatement(name, settings, cls.thedates[i])
ds.getTrades(testFileLoc=cls.datadir, testdb=cls.testdb)
elif cls.inputType == "IB_CSV":
ibs = IbStatement(db=cls.testdb)
ibs.openIBStatement(name)
else:
continue
# self.assertTrue(4 == 5, "Unsupported file type in test_TheTradeObject")
statement = StatementDB(db=cls.testdb)
df = statement.getStatement(cls.thedates[i])
# self.assertFalse(df.empty, f"Found no trades in db on {daDate}")
dtrade = DefineTrades(cls.inputType)
dframe, ldf = dtrade.processDBTrades(df)
tto = TheTradeObject(ldf[0], False, SumReqFields())
cls.ttos.append(tto)
def test_findTradeSummary(self):
'''
Test findTradeSummary, a helper method for addTradeSummaries and updateTradeSummaries.
Note that one of those needs to have run and succeeded inorder to test this method.
'''
infile = "data/flex.369463.ActivityFlexMonth.20191008.20191106.csv"
theDate = pd.Timestamp('2019-10-16')
# Create these three objects
ibs = IbStatement(db=self.db)
ibdb = StatementDB(self.db)
ibdb.reinitializeTradeTables()
trades = DefineTrades("DB")
# This call loads the statement into the db
ibs.openIBStatementCSV(infile)
# Here is an example of processing a single day of trades (3 calls)
# This gets a collection of trades from a single day that can become a trade_sum entry
df = ibdb.getStatement(theDate)
# The following method and function process the statement transactions into a collection
# of trades where each trade is a single row representing multiple transactions
dframe, ldf = trades.processDBTrades(df)
tradeSummaries, ts, entries, initialImageNames = runSummaries(ldf)
ibdb.addTradeSummaries(ts, ldf)
# The test database trades_sum should now only the trades from theDate, one
# entry per trade
for i, trade in enumerate(tradeSummaries):
x = ibdb.findTradeSummary(theDate, trade['Start'].unique()[0])
self.assertEqual(trade['Name'].unique()[0], x[1])
def test_figureBAPL(self):
'''
figureBAPL is called by openIBStatement and is give the trade tables and
position tables from an IB statement. figureBAPL fills in Balance, Average
and pnl. Every Balance entry should be filled. There could be blanks for
Average. Here we can test all balance entries made it to the db
'''
ibs = IbStatement()
ibdb = StatementDB()
ibdb.reinitializeTradeTables()
# These two files are a multiday flex and an activity.
# Find a way to find recent files to test
# trades1, meta1 = ibs.openIBStatement(self.testfile1)
trades2, meta2 = ibs.openIBStatement(self.testfile2)
len2 = len(trades2['TRNT']) if 'TRNT' in trades2.keys() else len(
trades2['Trades'])
ModelBase.connect(new_session=True)
session = ModelBase.session
q = session.query(Trade).all()
q2 = session.query(Trade).filter(Trade.balance is None).all()
self.assertEqual(len2, len(q))
self.assertEqual(len(q2), 0)
bu = Backup()
bu.restore()
def setUpClass(cls):
bu = Backup()
bu.backup()
if ModelBase.session:
ModelBase.session.rollback()
ddiirr = os.path.dirname(__file__)
os.chdir(os.path.realpath(ddiirr + '/../'))
outdir = 'test/out'
cls.outdir = os.path.realpath(outdir)
cls.db = 'data/testdb.sqlite'
cls.db = os.path.realpath(cls.db)
if os.path.exists(cls.db):
clearTables(cls.db)
cls.rtg = RTG(db=cls.db)
# cls.dates = ['20200203 09:30', '20200204 07:30', '20200205 09:35', '20200206 11:40', '20200207 10:39']
cls.dates = ['20200207 10:39']
cls.infiles = cls.rtg.saveSomeTestFiles(cls.dates,
cls.outdir,
strict=True,
overwrite=True)
settings = QSettings('zero_substance', 'structjour')
for i, name in enumerate(cls.infiles):
name = os.path.join(cls.outdir, name)
x, cls.inputType = getStatementType(name)
# print(cls.inputType)
if cls.inputType == 'DAS':
ds = DasStatement(name, settings, cls.dates[i])
ds.getTrades(testFileLoc=cls.outdir, testdb=cls.db)
elif cls.inputType == "IB_CSV":
ibs = IbStatement(db=cls.db)
ibs.openIBStatement(name)
else:
continue
# self.assertTrue(4 == 5, "Unsupported file type in test_TheTradeObject")
statement = StatementDB(db=cls.db)
df = statement.getStatement(cls.dates[i])
# self.assertFalse(df.empty, f"Found no trades in db on {daDate}")
dtrade = DefineTrades(cls.inputType)
dframe, ldf = dtrade.processDBTrades(df)
# tto = TheTradeObject(ldf[0], False, SumReqFields())
jf = JournalFiles(indir=cls.outdir,
outdir=outdir,
theDate=cls.dates[i],
infile=name)
cls.sc = SumControl()
lf = LayoutForms(cls.sc, jf, dframe)
lf.runTtoSummaries(ldf)
cls.jfs.append(jf)
cls.dframes.append(dframe)
# cls.ttos.append(tto)
cls.ldfs.append(ldf)
cls.lfs.append(lf)
def test_unifyDateFormat(self):
ts = ('2020-03-04, 15:30:45')
df = pd.DataFrame(data=[[
ts,
]], columns=[
'DateTime',
])
ibs = IbStatement()
df = ibs.unifyDateFormat(df)
self.assertTrue(df['DateTime'].unique()[0] == '20200304;153045')
def test_openIbStatement(self):
ibs = IbStatement(db=self.db)
ibdb = StatementDB(db=self.db)
ibdb.reinitializeTradeTables()
x = ibs.openIBStatement(self.testfile1)
self.assertIsNotNone(x)
self.assertIsInstance(x[0], dict)
self.assertTrue('TRNT' in x[0].keys() or 'Trades' in x[0].keys())
st = ibdb.getStatement(self.theDate)
self.assertIsInstance(st, pd.DataFrame)
self.assertGreater(len(st), 0)
def test_fixNumericTypes(self):
df = pd.DataFrame(data=[['2,344.78', '126.99', '3.50']],
columns=['Quantity', 'Price', 'Commission'])
df2 = pd.DataFrame(data=[[
'2,344.78',
]], columns=[
'Qty',
])
ibs = IbStatement()
t = ibs.fixNumericTypes(df)
t2 = ibs.fixNumericTypes(df2)
self.assertIsInstance(t['Quantity'].unique()[0], float)
self.assertIsInstance(t['Price'].unique()[0], float)
self.assertIsInstance(t['Commission'].unique()[0], float)
self.assertIsInstance(t2['Qty'].unique()[0], float)
def openStuff(self, allofit=None):
'''Site specific testing stuff-- open a multi day statement'''
# Site specific stuff here Open a monthly or yearly statemnet into the test db.
# Currently set to search the journal root dir, so it may load an annual or two.
bdir = ff.getBaseDir()
fs = ff.findFilesInDir(bdir, 'U242.csv', True)
ibs = IbStatement()
for f in fs:
x = ibs.openIBStatement(f)
delt = ibs.endDate - ibs.beginDate
assert delt.days > 20
if not allofit:
break
return ibs, x
def setUpClass(cls):
bu = Backup()
bu.backup()
ddiirr = os.path.dirname(__file__)
os.chdir(os.path.realpath(ddiirr))
os.chdir(os.path.realpath('../'))
cls.outdir = os.path.realpath(cls.outdir)
cls.db = os.path.realpath(cls.db)
if os.path.exists(cls.db):
clearTables(cls.db)
cls.rtg = RTG(db=cls.db, overnight=100)
cls.theDate = '20200207 10:39'
cls.infile = cls.rtg.saveSomeTestFiles([cls.theDate],
cls.outdir,
strict=True,
overwrite=False)[0]
settings = QSettings('zero_substance', 'structjour')
# for i, name in enumerate(cls.infiles):
name = os.path.join(cls.outdir, cls.infile)
x, cls.inputType = getStatementType(name)
if cls.inputType == 'DAS':
ds = DasStatement(name, settings, cls.theDate)
ds.getTrades(testFileLoc=cls.outdir, testdb=cls.db)
elif cls.inputType == "IB_CSV":
ibs = IbStatement(db=cls.db)
ibs.openIBStatement(name)
else:
raise ValueError(f'Unsupported File type: {cls.inputType}')
statement = StatementDB(db=cls.db)
cls.df = statement.getStatement(cls.theDate)
cls.dtrades = DefineTrades(cls.inputType)
cls.rc = FinReqCol(cls.inputType)
cls.trades = cls.dtrades.addFinReqCol(cls.df)
rccolumns = cls.rc.columns.copy()
rccolumns = cls.dtrades.appendCols(rccolumns)
cls.trades = cls.trades[rccolumns]
cls.trades.copy()
cls.trades = cls.trades.sort_values(
[cls.rc.ticker, cls.rc.acct, cls.rc.date])
def test_ibstatement(self):
'''Test basic usage loading an ib statement and getting a statement'''
infile = "data/flex.369463.ActivityFlexMonth.20191008.20191106.csv"
theDate = pd.Timestamp('2019-10-16')
# Create these two objects
ibs = IbStatement()
ibdb = StatementDB()
ibdb.reinitializeTradeTables()
# This call loads the statement into the db
ibs.openIBStatementCSV(infile)
# This call will then retrieve one day of trades as a dataframe. theDate is string or timestamp
df2 = ibdb.getStatement(theDate)
self.assertIsInstance(df2, pd.DataFrame)
self.assertFalse(df2.empty)
bu = Backup()
bu.restore()
def test_openIbStatement_notcsv(self):
ibs = IbStatement(db=self.db)
ibdb = StatementDB(db=self.db)
ibdb.reinitializeTradeTables()
x = ibs.openIBStatement('data\alittleOrgTODO.txt')
self.assertEqual(len(x[0]), 0)
def runnit(self, loaditrun=False):
'''
Load an initial input file and process it.
'''
self.sc.dateInSync = True
if self.sc.ui.useDatabase.isChecked() and loaditrun is False:
if not self.gotTrades():
return
return self.loadit()
self.sc.doWeSave()
self.initialize()
if not self.indir:
logging.info('No file to load?')
return
jf = JournalFiles(indir=self.indir,
outdir=self.outdir,
theDate=self.theDate,
infile=self.infile,
inputType=self.inputtype,
infile2=self.positions,
mydevel=True)
if self.inputtype == 'DB':
self.runDBInput(self.theDate, jf)
windowTitle = f'{self.sc.baseWindowTitle}: {self.sc.ui.infileEdit.text()}: no user data'
self.sc.setWindowTitle(windowTitle)
if self.gotTrades():
self.sc.ui.useDatabase.setChecked(True)
self.sc.dbDefault(True)
return
local = os.path.normpath(self.ui.infileEdit.text())
if os.path.normpath(jf.inpathfile) != local:
if os.path.exists(local):
d, jf.infile = os.path.split(local)
jf.inpathfile = local
x, inputType = getStatementType(jf.inpathfile)
if not inputType:
msg = f'<h3>No trades found. File does not appear to be a statement</h3><ul> '
msg += f'<div><strong>{jf.inpathfile}</strong></div>'
msgbx = QMessageBox()
msgbx.setIconPixmap(QPixmap("structjour/images/ZSLogo.png"))
msgbx.setText(msg)
msgbx.exec()
return
self.inputtype = inputType
windowTitle = self.sc.baseWindowTitle + ': ' + self.sc.ui.infileEdit.text(
) + ': no user data'
self.sc.setWindowTitle(windowTitle)
if self.inputtype == 'IB_HTML' or self.inputtype == 'IB_CSV':
jf.inputType = self.inputtype
ibs = IbStatement()
x = ibs.openIBStatement(jf.inpathfile)
msg = ''
if x[0]:
tkey = 'Trades' if 'Trades' in x[0].keys(
) else 'TRNT' if 'TRNT' in x[0].keys() else None
if not tkey:
raise ValueError(
f'Error in processing statemnt {jf.inpathfile}')
numtickets = len(x[0][tkey])
gotToday = self.runDBInput(self.theDate, jf)
if gotToday:
if self.gotTrades():
self.sc.ui.useDatabase.setChecked(True)
self.sc.dbDefault(True)
else:
msg = f'<h3>No trades found on date {self.theDate.date()}</h3><ul> '
msg += f'<div><strong>{jf.inpathfile}</strong></div>'
msg += f'<div>Found {numtickets} tickets. They are now in DB</div>'
msg += f'<div>{list(x[1].keys())}</div>'
else:
msg = f'<h3>No trades recorded from the file:</h3><ul> '
msg = msg + f'<div><strong>{jf.inpathfile}</strong></div>'
msg = msg + f'<div>{x[1]}</div>'
msgbx = QMessageBox()
msgbx.setIconPixmap(QPixmap("structjour/images/ZSLogo.png"))
msgbx.setText(msg)
msgbx.exec()
return
elif self.inputtype == 'DAS':
x = checkDateDir(jf.inpathfile)
if not x:
msg = "<h3>The date for this DAS statement is not clear</h3>"
msg += "<div>Please enter the date for this statement</div>"
msg += f'<div><strong>{jf.inpathfile}</strong></div>'
msg += '<div>(YYYYMMDD) ex: 20190113</div>'
theDate = getDate(msg)
if theDate:
self.settings.setValue('theDate', theDate)
self.sc.ui.dateEdit.setDate(pd2qtime(theDate, qdate=True))
else:
return
ds = DasStatement(jf.infile, self.settings, self.theDate)
ds.getTrades()
self.runDBInput(self.theDate, jf)
if self.gotTrades():
self.sc.ui.useDatabase.setChecked(True)
self.sc.dbDefault(True)
return
else:
msg = '<h3>Unrecognized input:</h3><ul> '
msgbx = QMessageBox()
msgbx.setIconPixmap(QPixmap("structjour/images/ZSLogo.png"))
msgbx.setText(msg)
msgbx.exec()
return
def setUpClass(cls):
ddiirr = os.path.dirname(__file__)
os.chdir(os.path.realpath(ddiirr + '/../'))
outdir = 'test/out'
cls.outdir = os.path.realpath(outdir)
cls.db = 'data/testdb.sqlite'
cls.db = os.path.realpath(cls.db)
if os.path.exists(cls.db):
clearTables(cls.db)
cls.rtg = RTG(db=cls.db)
# cls.dates = ['20200203 09:30', '20200204 07:30', '20200205 09:35', '20200206 11:40', '20200207 10:39']
cls.theDate = '20200207 10:39'
cls.infile = cls.rtg.saveSomeTestFiles([cls.theDate], cls.outdir)[0]
settings = QSettings('zero_substance', 'structjour')
# for i, name in enumerate(cls.infiles):
name = os.path.join(cls.outdir, cls.infile)
x, inputType = getStatementType(name)
if inputType == 'DAS':
ds = DasStatement(name, settings, cls.theDate)
ds.getTrades(testFileLoc=cls.outdir, testdb=cls.db)
elif inputType == "IB_CSV":
ibs = IbStatement(db=cls.db)
ibs.openIBStatement(name)
else:
raise ValueError(f'Unsupported File type: {inputType}')
statement = StatementDB(db=cls.db)
df = statement.getStatement(cls.theDate)
dtrade = DefineTrades(inputType)
dframe, cls.ldf = dtrade.processDBTrades(df)
cls.jf = JournalFiles(indir=cls.outdir,
outdir=outdir,
theDate=cls.theDate,
infile=name)
cls.sc = SumControl()
lf = LayoutForms(cls.sc, cls.jf, dframe)
lf.runTtoSummaries(cls.ldf)
# Setup mistake note fields to test
cls.note = 'Ground control to Major Tom'
for i, key in enumerate(lf.ts):
tto = lf.ts[key]
notex = cls.note + str(i + 1)
tto['MstkNote'] = notex
# Setup a couple images to add
imdir = 'images/'
img1 = os.path.join(imdir, 'fractal-art-fractals.jpg')
img2 = os.path.join(imdir, 'psych.jpg')
assert os.path.exists(img1)
assert os.path.exists(img2)
for key in lf.ts:
tto = lf.ts[key]['chart1'] = img1
tto = lf.ts[key]['chart2'] = img2
t = ExportToExcel(lf.ts, cls.jf, df)
imageNames = t.getImageNamesFromTS()
cls.imageLocation, dframe = t.layoutExcelData(t.df, cls.ldf,
imageNames)
assert len(cls.ldf) == len(cls.imageLocation)
# Create an openpyxl wb from the dataframe
ls = LayoutSheet(t.topMargin, len(t.df))
cls.wb, ws, nt = ls.createWorkbook(dframe)
# Place both forms 2 cells to the right of the main table
mstkAnchor = (len(dframe.columns) + 2, 1)
cls.mistake = MistakeSummary(numTrades=len(cls.ldf), anchor=mstkAnchor)
cls.t = t
cls.a = mstkAnchor
def test_openIbStatement_notcsv(self):
ibs = IbStatement()
ibdb = StatementDB()
x = ibs.openIBStatement('data\alittleOrgTODO.txt')
self.assertEqual(len(x[0]), 0)