def test_addTradeSummaries(self):
'''
Tests addTradeSummaries. The method requires trades are already in the database.
We achieve that with openStuff.
For this test, I load everything (openStuff) and run
addTradeSummaries on all covered days. Its slow. Could be partially sqlite but
all the APL BAPL stuff is probably the main crawler. In practice, this will add
daily or monthly statements. And in running the program there is no way to run
the trade summaries in mass. Its desinged to load up a single day. ITs day-trader
centric. Let it stay slow for now.
'''
ibdb = StatementDB(self.db)
self.clearTables()
ibs, x = self.openStuff()
# ibdb.getUncoveredDays
covered = ibdb.getCoveredDays()
for count, day in enumerate(covered):
df = ibdb.getStatement(day)
if not df.empty:
tu = DefineTrades("DB")
dframe, ldf = tu.processDBTrades(df)
tradeSummaries, ts, entries, initialImageNames = runSummaries(ldf)
ibdb.addTradeSummaries(ts, ldf)
summaries = ibdb.getTradeSumByDate(day)
for summary in summaries:
summary = ibdb.makeTradeSumDict(summary)
entryTrades = ibdb.getEntryTrades(summary['id'])
self.assertGreater(len(entryTrades), 0)
break # Use this to just test addTradeSummaries once
def test_findTradeSummary(self):
'''
Test findTradeSummary, a helper method for addTradeSummaries and updateTradeSummaries.
Note that one of those needs to have run and succeeded inorder to test this method.
'''
infile = "data/flex.369463.ActivityFlexMonth.20191008.20191106.csv"
theDate = pd.Timestamp('2019-10-16')
# Create these three objects
ibs = IbStatement(db=self.db)
ibdb = StatementDB(self.db)
ibdb.reinitializeTradeTables()
trades = DefineTrades("DB")
# This call loads the statement into the db
ibs.openIBStatementCSV(infile)
# Here is an example of processing a single day of trades (3 calls)
# This gets a collection of trades from a single day that can become a trade_sum entry
df = ibdb.getStatement(theDate)
# The following method and function process the statement transactions into a collection
# of trades where each trade is a single row representing multiple transactions
dframe, ldf = trades.processDBTrades(df)
tradeSummaries, ts, entries, initialImageNames = runSummaries(ldf)
ibdb.addTradeSummaries(ts, ldf)
# The test database trades_sum should now only the trades from theDate, one
# entry per trade
for i, trade in enumerate(tradeSummaries):
x = ibdb.findTradeSummary(theDate, trade['Start'].unique()[0])
self.assertEqual(trade['Name'].unique()[0], x[1])
def setUp(self):
ddiirr = os.path.dirname(__file__)
os.chdir(os.path.realpath(ddiirr))
os.chdir(os.path.realpath('../'))
self.conn = sqlite3.connect(self.testdb)
self.inputtype = 'IB_CSV'
indir = 'data/'
# f2 = 'ActivityStatement.20190313_PL.html'
jf = JournalFiles(indir=indir,
infile=self.f1,
theDate=self.theDate,
inputType='IB_CSV',
mydevel=False)
jf.inputType = 'IB_CSV'
# statement = Statement_IBActivity(jf)
# df = statement.getTrades_IBActivity(jf.inpathfile)
ibs = IbStatement(db=self.testdb)
ibdb = StatementDB(self.testdb)
ibdb.reinitializeTradeTables()
ibs.openIBStatementCSV(jf.inpathfile)
df2 = ibdb.getStatement(self.theDate)
if df2.empty:
sdate = self.theDate.strftime('%Y-%m-%d')
msg = f'In test_dailycontrol.setup: Error: found no trades in db for {sdate}'
self.assertTrue(not df2.empty, msg)
tu = DefineTrades(jf.inputType)
self.df, ldf = tu.processDBTrades(df2)
sc = SumControl()
lf = LayoutForms(sc, jf, self.df)
lf.runTtoSummaries(ldf)
self.ts = lf.ts
ibdb.addTradeSummaries(self.ts, ldf)
class runController:
'''
Programming notes-- minimize the use of the ui (self.ui). Instead create high level
interface in sc as needed.
:Settings-keys: ['theDate', 'setToday', scheme', 'journal', 'dasInfile', 'dasInfile2',
'ibInfile', outdir, 'interval', inputType]
'''
def __init__(self, sc):
self.sc = sc
self.ui = self.sc.ui
self.initialize()
self.inputtype = None
self.ui.goBtn.pressed.connect(self.runnit)
self.sc.ui.dateEdit.dateChanged.connect(self.theDateChanged)
# self.ui.loadBtn.pressed.connect(self.loadit)
self.loadedDate = None
self.statement = None
def theDateChanged(self, val):
self.sc.dateInSync = False
self.sc.theDateChanged(val)
if self.sc.ui.useDatabase.isChecked() and self.gotTrades():
self.runnit()
def initialize(self):
'''
Initialize the inputs and outs
'''
# Might blitz thes lines if JournalFiles gets an overhaul. For ease of transaiton
# We keep JournalFiles till its allworks into the Qt run
self.settings = self.sc.settings
self.inputtype = self.settings.value('inputType')
self.indir = self.sc.getDirectory()
inkey = ''
if self.inputtype == 'DAS':
inkey = 'dasInfile'
elif self.inputtype == 'IB_HTML':
inkey = 'ibInfileName'
if self.settings.value('outdirPolicy') == 'default':
self.outdir = None
else:
self.outdir = self.settings.value('outdir')
theDate = self.settings.value('theDate', pd.Timestamp.today())
if theDate and isinstance(theDate, (QDate, QDateTime)):
theDate = qtime2pd(theDate)
self.theDate = theDate
self.positions = self.settings.value('dasInfile2')
# end blitz
self.infile = self.settings.value(inkey)
self.inpathfile = self.ui.infileEdit.text()
if os.path.splitext(self.inpathfile)[1].lower() == ".csv":
self.infile = os.path.split(self.inpathfile)[1]
self.sc.setWindowTitle(self.sc.baseWindowTitle)
def loadit(self):
'''
Load saved objects
'''
self.sc.doWeSave()
daDate = self.ui.dateEdit.date()
daDate = qtime2pd(daDate)
self.loadedDate = daDate
self.settings.setValue('theDate', daDate)
self.initialize()
if not self.indir:
logging.info('No file to load?')
return
jf = JournalFiles(indir=self.indir,
outdir=self.outdir,
theDate=self.theDate,
infile=self.infile,
inputType=self.inputtype,
infile2=self.positions,
mydevel=True)
lf = LayoutForms(self.sc, jf, None)
if not lf.loadTradesFromDB(daDate):
msg = f'No user data has been saved for {daDate.strftime("%A %B %d")}. Loading trade data.'
logging.info(msg)
# msgbx = QMessageBox()
# msgbx.setIconPixmap(QPixmap("structjour/images/ZSLogo.png"))
# msgbx.setText(msg)
# msgbx.exec()
self.runnit(True)
def runDBInput(self, daDate, jf):
'''
Get the trades from daDate in the DB and process the trades
'''
self.statement = StatementDB()
daDate = qtime2pd(daDate)
df = self.statement.getStatement(daDate)
if df.empty:
return False
tu = DefineTrades(self.inputtype)
dframe, ldf = tu.processDBTrades(df)
lf = LayoutForms(self.sc, jf, dframe)
lf.pickleitnow()
lf.runTtoSummaries(ldf)
self.statement.addTradeSummaries(lf.ts, ldf)
return True
def gotTrades(self):
'''
From the text in the infileEdit box, determine if we have
trades in the db. We can tell because the first token is an
int showing how many trades are held.
'''
if self.sc.ui.useDatabase.isChecked():
text = self.sc.ui.infileEdit.text()
if len(text):
try:
num = int(text.split(' ')[0])
if num > 0:
return True
except Exception:
pass
if self.statement is not None:
count, countt = self.statement.getNumTicketsForDay(
qtime2pd(self.sc.ui.dateEdit.date()))
if count > 0 or countt > 0:
return True
return False
def runnit(self, loaditrun=False):
'''
Load an initial input file and process it.
'''
self.sc.dateInSync = True
if self.sc.ui.useDatabase.isChecked() and loaditrun is False:
if not self.gotTrades():
return
return self.loadit()
self.sc.doWeSave()
self.initialize()
if not self.indir:
logging.info('No file to load?')
return
jf = JournalFiles(indir=self.indir,
outdir=self.outdir,
theDate=self.theDate,
infile=self.infile,
inputType=self.inputtype,
infile2=self.positions,
mydevel=True)
if self.inputtype == 'DB':
self.runDBInput(self.theDate, jf)
windowTitle = f'{self.sc.baseWindowTitle}: {self.sc.ui.infileEdit.text()}: no user data'
self.sc.setWindowTitle(windowTitle)
if self.gotTrades():
self.sc.ui.useDatabase.setChecked(True)
self.sc.dbDefault(True)
return
local = os.path.normpath(self.ui.infileEdit.text())
if os.path.normpath(jf.inpathfile) != local:
if os.path.exists(local):
d, jf.infile = os.path.split(local)
jf.inpathfile = local
x, inputType = getStatementType(jf.inpathfile)
if not inputType:
msg = f'<h3>No trades found. File does not appear to be a statement</h3><ul> '
msg += f'<div><strong>{jf.inpathfile}</strong></div>'
msgbx = QMessageBox()
msgbx.setIconPixmap(QPixmap("structjour/images/ZSLogo.png"))
msgbx.setText(msg)
msgbx.exec()
return
self.inputtype = inputType
windowTitle = self.sc.baseWindowTitle + ': ' + self.sc.ui.infileEdit.text(
) + ': no user data'
self.sc.setWindowTitle(windowTitle)
if self.inputtype == 'IB_HTML' or self.inputtype == 'IB_CSV':
jf.inputType = self.inputtype
ibs = IbStatement()
x = ibs.openIBStatement(jf.inpathfile)
msg = ''
if x[0]:
tkey = 'Trades' if 'Trades' in x[0].keys(
) else 'TRNT' if 'TRNT' in x[0].keys() else None
if not tkey:
raise ValueError(
f'Error in processing statemnt {jf.inpathfile}')
numtickets = len(x[0][tkey])
gotToday = self.runDBInput(self.theDate, jf)
if gotToday:
if self.gotTrades():
self.sc.ui.useDatabase.setChecked(True)
self.sc.dbDefault(True)
else:
msg = f'<h3>No trades found on date {self.theDate.date()}</h3><ul> '
msg += f'<div><strong>{jf.inpathfile}</strong></div>'
msg += f'<div>Found {numtickets} tickets. They are now in DB</div>'
msg += f'<div>{list(x[1].keys())}</div>'
else:
msg = f'<h3>No trades recorded from the file:</h3><ul> '
msg = msg + f'<div><strong>{jf.inpathfile}</strong></div>'
msg = msg + f'<div>{x[1]}</div>'
msgbx = QMessageBox()
msgbx.setIconPixmap(QPixmap("structjour/images/ZSLogo.png"))
msgbx.setText(msg)
msgbx.exec()
return
elif self.inputtype == 'DAS':
x = checkDateDir(jf.inpathfile)
if not x:
msg = "<h3>The date for this DAS statement is not clear</h3>"
msg += "<div>Please enter the date for this statement</div>"
msg += f'<div><strong>{jf.inpathfile}</strong></div>'
msg += '<div>(YYYYMMDD) ex: 20190113</div>'
theDate = getDate(msg)
if theDate:
self.settings.setValue('theDate', theDate)
self.sc.ui.dateEdit.setDate(pd2qtime(theDate, qdate=True))
else:
return
ds = DasStatement(jf.infile, self.settings, self.theDate)
ds.getTrades()
self.runDBInput(self.theDate, jf)
if self.gotTrades():
self.sc.ui.useDatabase.setChecked(True)
self.sc.dbDefault(True)
return
else:
msg = '<h3>Unrecognized input:</h3><ul> '
msgbx = QMessageBox()
msgbx.setIconPixmap(QPixmap("structjour/images/ZSLogo.png"))
msgbx.setText(msg)
msgbx.exec()
return