def testCheckReqColumnsWithReqColSuccess(self):
'''Test return values of DataFrameUtil.checkRequiredInputFields'''
reqCol = ReqCol()
finReqCol = FinReqCol()
frc = pd.DataFrame(columns=finReqCol.columns)
t1 = False
t2 = False
try:
t1 = DataFrameUtil.checkRequiredInputFields(frc, finReqCol.columns)
t2 = DataFrameUtil.checkRequiredInputFields(frc, reqCol.columns)
except ValueError as ex:
print(ex)
self.assertTrue(t1)
self.assertTrue(t2)
def testCheckRequiredColumnsThrow(self):
'''Test DataFrameUtil.checkRequredInputFields for raising exceptions'''
vals = [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']]
apd = pd.DataFrame(vals)
apd.columns = [['Its', 'the', 'end', 'of',
'the', 'world', 'as', 'we', 'know', 'it']]
columns = ['Its', 'the', 'end', 'of', 'the',
'world', 'as', 'we', 'know', 'it', 'sofuckit']
# DataFrameUtil.checkRequiredInputFields(apd, columns)
try:
DataFrameUtil.checkRequiredInputFields(apd, columns)
except ValueError:
pass
except Exception as ex:
msg = "{0}{1}".format("Unexpected exception. ", ex)
self.fail(msg)
else:
self.fail("Failed to throw expected exception")
vals = [[1, 2, 3, 4, 5, 6, 7, 8, 9], [
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i']]
apd = pd.DataFrame(
vals, columns=['Its', 'the', 'end', 'of', 'world', 'as', 'we', 'know', 'it'])
gotve = False
try:
DataFrameUtil.checkRequiredInputFields(apd, columns)
except ValueError:
gotve = True
except Exception as ex:
msg = "Wrong exception was thrown" + ex
self.fail(msg)
finally:
self.assertTrue(gotve, "Failed to throw a Value Error Exception")
def testCheckrequiredColumnsWithReqColFail(self):
'''Test method DataFrameUtil.checkRequiredInputFields'''
reqCol = ReqCol()
finReqCol = FinReqCol()
fail = pd.DataFrame(
columns=['Time', 'Symb', 'Side', 'Price', 'Qty', 'Account'])
rc = pd.DataFrame(columns=reqCol.columns)
gotve = False
try:
DataFrameUtil.checkRequiredInputFields(fail, reqCol.columns)
except ValueError:
gotve = True
finally:
self.assertTrue(gotve, "Failed to throw value error")
gotve = False
try:
DataFrameUtil.checkRequiredInputFields(rc, finReqCol.columns)
except ValueError:
gotve = True
finally:
self.assertTrue(gotve, "Failed to throw a ValueError")
def getStatementType(infile):
'''
Determine if infile is a statement. If it is return a tuple (data, type) TODO: Not doing what I said...
If it is a DAS statement, determine if it matches the current date. As DAS statements do not
include dates, the date in structjour and the directory structure date must match.
If they don't match, the program, at a higher level, will pop a query to get the date of the
statement.
'''
file, ext = os.path.splitext(infile)
if not os.path.exists(infile) or (ext.lower() != '.csv'
and not ext.lower().startswith('.htm')):
return None, None
if ext.lower() == '.csv':
df = pd.read_csv(infile, names=[x for x in range(0, 100)])
if df.iloc[0][0] == 'BOF' or df.iloc[0][0] == 'HEADER' or (
df.iloc[0][0] == 'ClientAccountID') or (df.iloc[0][0]
== 'Statement'):
return df, "IB_CSV"
df = pd.read_csv(infile)
if not df.empty:
requiredFields = list(ReqCol().columns)
requiredFields.remove('Date')
# A small hack to allow tradesByTickets to pass as a DAS export
if 'PnL' not in df.columns:
requiredFields.remove('PnL')
requiredFields.append('P / L')
try:
if DataFrameUtil.checkRequiredInputFields(df, requiredFields):
if not checkDateDir(infile):
return None, 'DAS'
return df, 'DAS'
except ValueError:
pass
elif ext.lower().startswith('.htm'):
soup = BeautifulSoup(readit(infile), 'html.parser')
tbldivs = soup.find_all("div", id=lambda x: x and x.startswith('tbl'))
if tbldivs:
return tbldivs, 'IB_HTML'
return None, None