class TestOpenCSV(TestSetUp):
def setUp(self):
TestSetUp.setUp(self)
self.pos_file = test_pos_path + r'/2014-08-01-PositionStatement.csv'
self.acc_file = test_acc_path + r'/2014-07-23-AccountStatement.csv'
self.pos_data = open(self.pos_file, mode='r').read()
self.acc_data = open(self.acc_file, mode='r').read()
self.open_csv = OpenCSV(data=self.pos_data)
def test_property(self):
"""
Test property inside class
"""
print 'data: %s' % self.open_csv.lines
self.assertEqual(type(self.open_csv.lines), list)
self.assertGreater(len(self.open_csv.lines), 0)
def test_read_lines_from_file(self):
"""
Test read lines from file with test file
"""
self.open_csv.read_lines_from_file(fname=self.pos_file)
lines = self.open_csv.lines
print 'lines type: %s' % type(lines)
print 'lines length: %d\n' % len(lines)
print 'first 5 rows in lines:'
for line in lines[:5]:
print '\t"' + line + '"'
self.assertTrue(len(lines))
self.assertEqual(type(lines), list)
def test_replace_dash_inside_quote(self):
"""
Test replace dash inside quote
example "$1,254.00" become $1254.00
"""
line = 'OVERNIGHT FUTURES BP,"$1,653.36"'
result = self.open_csv.replace_dash_inside_quote(line)
print 'line: %s' % line
print 'result: %s' % result
self.assertNotIn('"', result)
def test_split_lines_with_dash(self):
"""
Test split line using dash into list of items
"""
line = 'TSLA,,,,,,5.18,-.12,1.29,-1.06,+4.46%,$8.00,$8.00,($250.00)'
result = self.open_csv.split_lines_with_dash(line)
print 'line: %s' % line
print 'result: %s' % result
print 'type: %s' % type(result)
print 'length: %d' % len(result)
self.assertEqual(len(result), 14)
self.assertEqual(type(result), list)
def test_remove_bracket_then_add_negative(self):
"""
Test remove brackets '()' on str item then add negative
"""
item = '($5609.52)'
result = self.open_csv.remove_bracket_then_add_negative(item)
print 'item: %s' % item
print 'result: %s' % result
self.assertNotIn('(', result)
self.assertNotIn(')', result)
self.assertIn('-', result)
def test_remove_dollar_symbols(self):
"""
Test remove dollar symbol on str item
"""
item = '3773.49'
result = self.open_csv.remove_dollar_symbols(item)
print 'item: %s' % item
print 'result: %s' % result
self.assertNotIn('$', result)
def test_remove_percent_symbols(self):
"""
Test remove dollar symbol on str item
"""
item = '+1.77%'
result = self.open_csv.remove_percent_symbols(item)
print 'item: %s' % item
print 'result: %s' % result
self.assertNotIn('%', result)
def test_split_str_with_space(self):
"""
Test split str with space
"""
items = [
'100 AUG 14 67.5 CALL', '100 (Weeklys) AUG1 14 50 PUT',
'100 AUG 14 26.5 CALL'
]
for item in items:
result = self.open_csv.split_str_with_space(item)
print 'item: %s' % item
print 'result: %s\n' % result
self.assertEqual(type(result), list)
self.assertIn(len(result), (5, 6))
def test_remove_brackets_only(self):
"""
Test remove brackets on options name
"""
items = [
'100 AUG 14 47 PUT',
'100 (Weeklys) AUG2 14 1990 CALL',
'100 (Mini) AUG1 14 20 PUT',
]
for item in items:
result = self.open_csv.remove_brackets_only(item)
print 'item: %s' % item
print 'result: %s' % result
self.assertNotIn('(', result)
self.assertNotIn(')', result)
def test_last_five_lines(self):
"""
Test using a list of lines get last 5 items
"""
lines = range(10)
result = self.open_csv.last_five_lines(lines)
print 'lines type: %s' % type(result)
print 'lines length: %s' % len(result)
self.assertEqual(type(result), list)
self.assertEqual(len(result), 5)
pprint(result)
pprint(self.open_csv.last_five_lines(self.open_csv.lines))
def test_convert_float_or_str(self):
"""
Test convert item into float or string
"""
items = ['12.24', 'VXX', '02:49:35', '956.98', 'JUL4 14']
for item in items:
result = self.open_csv.convert_float_or_str(item)
print 'item: %s, result: %s, type: %s' % (item, result,
type(result))
self.assertIn(type(result), [str, float])
def test_format_item(self):
"""
Test format item that remove all unnecessary symbol
"""
items = ['($56.50)', '-17.86%', '$0.00', '+1', 'AUG 14']
for item in items:
result = self.open_csv.format_item(item)
print 'item: %s, result: %s, type: %s' % (item, result,
type(result))
self.assertIn(type(result), [str, float])
result = str(result)
self.assertNotIn('%', result)
self.assertNotIn('$', result)
self.assertNotIn('(', result)
self.assertNotIn(')', result)
def test_get_lines(self):
"""
Test get lines section from lines
"""
self.open_csv = OpenCSV(self.acc_data)
phrases = [('Profits and Losses', 'OVERALL TOTALS'),
('Options', ',,,,,,'), ('Equities', 'OVERALL TOTALS'),
('Account Trade History', None),
('Account Order History', None), ('Forex Statements', None),
('Futures Statements', None), ('Cash Balance', 'TOTAL')]
for start, end in phrases:
lines = self.open_csv.get_lines(start, end)
print start, end
for line in lines:
print line
print '-' * 100
def test_make_dict(self):
"""
Test make dict using keys and values
"""
keys = [
'symbol', 'description', 'pl_open', 'pl_pct', 'pl_day', 'pl_ytd',
'margin_req'
]
values = ['AA', 'ALCOA INC COM', 0.0, 0.0, 0.0, -271.98, 0.0]
print 'keys: %s' % keys
print 'values: %s' % values
result = self.open_csv.make_dict(keys, values)
print 'result: %s' % result
for key, value in result.items():
self.assertIn(key, keys)
self.assertIn(value, values)
index = keys.index(key)
self.assertEqual(key, keys[index])
self.assertEqual(value, values[index])
def test_del_empty_keys(self):
"""
Test delete empty key in dict
"""
items = {'': '', 'ggg': 'hhh', 'kkk': 'lll'}
result = self.open_csv.del_empty_keys(items)
print 'items: %s' % items
print 'result: %s' % result
self.assertGreater(len(items), len(result))
self.assertNotIn('', result.keys())
def test_fillna_dict(self):
"""
Test fill empty or none value in dict
"""
self.open_csv = OpenCSV(self.acc_data)
self.open_csv.trade_history = list()
trade_history_keys = [
'', 'execute_time', 'spread', 'side', 'quantity', 'pos_effect',
'symbol', 'expire_date', 'strike', 'contract', 'price',
'net_price', 'order_type'
]
self.open_csv.set_values(start_phrase='Account Trade History',
end_phrase=None,
start_with=2,
end_until=-1,
prop_keys=trade_history_keys,
prop_name='trade_history')
before_fillna = self.open_csv.trade_history
after_fillna = self.open_csv.fillna_dict(self.open_csv.trade_history)
after_fillna = map(self.open_csv.del_empty_keys, after_fillna)
for tk1, tk2 in zip(before_fillna, after_fillna):
print tk1
print tk2
print ''
self.assertIn('', tk1.values())
self.assertNotIn('', tk2.values())
for value1, value2 in zip(tk1.values(), tk2.values()):
if value1 and (value1 != 'DEBIT' and value1 != 'CREDIT'):
self.assertIn(value1, tk2.values())
for key in tk1.keys():
if key:
self.assertIn(key, tk2.keys())
def test_fillna_dict_with_exists(self):
"""
Test fill empty or none value in dict
"""
self.acc_file = test_path + r'/2014-10-31/2014-10-31-AccountStatement.csv'
self.acc_data = open(self.acc_file, mode='r').read()
self.open_csv = OpenCSV(self.acc_data)
self.open_csv.trade_history = list()
trade_history_keys = [
'', 'execute_time', 'spread', 'side', 'quantity', 'pos_effect',
'symbol', 'expire_date', 'strike', 'contract', 'price',
'net_price', 'order_type'
]
self.open_csv.set_values(start_phrase='Account Trade History',
end_phrase=None,
start_with=2,
end_until=-1,
prop_keys=trade_history_keys,
prop_name='trade_history')
before_fillna = deepcopy(self.open_csv.trade_history)
for t in before_fillna:
print t['execute_time']
self.open_csv.fillna_dict_with_exists(
self.open_csv.trade_history, 'execute_time',
('execute_time', 'spread', 'order_type'))
after_fillna = self.open_csv.trade_history
after_fillna = map(self.open_csv.del_empty_keys, after_fillna)
for key, (tk1, tk2) in enumerate(zip(before_fillna, after_fillna)):
print tk1
print tk2
print ''
if not tk1['execute_time']:
self.assertEqual(tk1['execute_time'], '')
self.assertEqual(tk1['spread'], '')
self.assertEqual(tk1['order_type'], '')
self.assertEqual(tk2['execute_time'],
after_fillna[key - 1]['execute_time'])
self.assertEqual(tk2['spread'],
after_fillna[key - 1]['spread'])
self.assertEqual(tk2['order_type'],
after_fillna[key - 1]['order_type'])
def test_convert_specific_type(self):
"""
Test convert specific type for dict in a list
"""
items = [
{
'a': 0,
'ref_no': 793403165.0,
'c': 0
},
{
'a': 0,
'ref_no': 801854049.0,
'c': 0
},
{
'a': 0,
'ref_no': 802092476.0,
'c': 0
},
{
'a': 0,
'ref_no': 17.0,
'c': 0
},
{
'a': 0,
'ref_no': 123.0,
'c': 0
},
]
self.open_csv.cash_balance = [dict(i) for i in items]
self.open_csv.convert_specific_type(self.open_csv.cash_balance,
'ref_no', int, 0)
for item, result in zip(items, self.open_csv.cash_balance):
print 'dict: %s, result: %s' % (item, result)
self.assertEqual(item['ref_no'], result['ref_no'])
self.assertNotEqual(type(item['ref_no']), type(result['ref_no']))
def test_set_values(self):
"""
Test open files, get lines section, format data,
make dict and finally set values in property
"""
self.open_csv = OpenCSV(self.acc_data)
self.open_csv.trade_history = list()
trade_history_keys = [
'', 'execute_time', 'spread', 'side', 'quantity', 'pos_effect',
'symbol', 'expire_date', 'strike', 'contract', 'price',
'net_price', 'order_type'
]
start_phrase = 'Account Trade History'
end_phrase = None
start_add = 2
end_reduce = -1
prop_name = 'trade_history'
print 'start phrase: %s, end phrase: %s' % (start_phrase, end_phrase)
print 'start add: %d, end reduce: %d' % (start_add, end_reduce)
print 'property name: %s' % prop_name
self.open_csv.set_values(start_phrase=start_phrase,
end_phrase=end_phrase,
start_with=start_add,
end_until=end_reduce,
prop_keys=trade_history_keys,
prop_name=prop_name)
trade_history = self.open_csv.trade_history
lines = self.open_csv.get_lines('Account Trade History')
print ''
for tk in trade_history:
print tk
self.assertEqual(type(tk), dict)
self.assertEqual(len(tk), 13)
self.assertEqual(len(lines), len(trade_history) + 2 + 1)
self.assertEqual(type(trade_history), list)
class TestOpenCSV(TestSetUp):
def setUp(self):
TestSetUp.setUp(self)
self.pos_file = test_pos_path + r'/2014-08-01-PositionStatement.csv'
self.acc_file = test_acc_path + r'/2014-07-23-AccountStatement.csv'
self.pos_data = open(self.pos_file, mode='r').read()
self.acc_data = open(self.acc_file, mode='r').read()
self.open_csv = OpenCSV(data=self.pos_data)
def test_property(self):
"""
Test property inside class
"""
print 'data: %s' % self.open_csv.lines
self.assertEqual(type(self.open_csv.lines), list)
self.assertGreater(len(self.open_csv.lines), 0)
def test_read_lines_from_file(self):
"""
Test read lines from file with test file
"""
self.open_csv.read_lines_from_file(fname=self.pos_file)
lines = self.open_csv.lines
print 'lines type: %s' % type(lines)
print 'lines length: %d\n' % len(lines)
print 'first 5 rows in lines:'
for line in lines[:5]:
print '\t"' + line + '"'
self.assertTrue(len(lines))
self.assertEqual(type(lines), list)
def test_replace_dash_inside_quote(self):
"""
Test replace dash inside quote
example "$1,254.00" become $1254.00
"""
line = 'OVERNIGHT FUTURES BP,"$1,653.36"'
result = self.open_csv.replace_dash_inside_quote(line)
print 'line: %s' % line
print 'result: %s' % result
self.assertNotIn('"', result)
def test_split_lines_with_dash(self):
"""
Test split line using dash into list of items
"""
line = 'TSLA,,,,,,5.18,-.12,1.29,-1.06,+4.46%,$8.00,$8.00,($250.00)'
result = self.open_csv.split_lines_with_dash(line)
print 'line: %s' % line
print 'result: %s' % result
print 'type: %s' % type(result)
print 'length: %d' % len(result)
self.assertEqual(len(result), 14)
self.assertEqual(type(result), list)
def test_remove_bracket_then_add_negative(self):
"""
Test remove brackets '()' on str item then add negative
"""
item = '($5609.52)'
result = self.open_csv.remove_bracket_then_add_negative(item)
print 'item: %s' % item
print 'result: %s' % result
self.assertNotIn('(', result)
self.assertNotIn(')', result)
self.assertIn('-', result)
def test_remove_dollar_symbols(self):
"""
Test remove dollar symbol on str item
"""
item = '3773.49'
result = self.open_csv.remove_dollar_symbols(item)
print 'item: %s' % item
print 'result: %s' % result
self.assertNotIn('$', result)
def test_remove_percent_symbols(self):
"""
Test remove dollar symbol on str item
"""
item = '+1.77%'
result = self.open_csv.remove_percent_symbols(item)
print 'item: %s' % item
print 'result: %s' % result
self.assertNotIn('%', result)
def test_split_str_with_space(self):
"""
Test split str with space
"""
items = [
'100 AUG 14 67.5 CALL',
'100 (Weeklys) AUG1 14 50 PUT',
'100 AUG 14 26.5 CALL'
]
for item in items:
result = self.open_csv.split_str_with_space(item)
print 'item: %s' % item
print 'result: %s\n' % result
self.assertEqual(type(result), list)
self.assertIn(len(result), (5, 6))
def test_remove_brackets_only(self):
"""
Test remove brackets on options name
"""
items = [
'100 AUG 14 47 PUT',
'100 (Weeklys) AUG2 14 1990 CALL',
'100 (Mini) AUG1 14 20 PUT',
]
for item in items:
result = self.open_csv.remove_brackets_only(item)
print 'item: %s' % item
print 'result: %s' % result
self.assertNotIn('(', result)
self.assertNotIn(')', result)
def test_last_five_lines(self):
"""
Test using a list of lines get last 5 items
"""
lines = range(10)
result = self.open_csv.last_five_lines(lines)
print 'lines type: %s' % type(result)
print 'lines length: %s' % len(result)
self.assertEqual(type(result), list)
self.assertEqual(len(result), 5)
pprint(result)
pprint(self.open_csv.last_five_lines(self.open_csv.lines))
def test_convert_float_or_str(self):
"""
Test convert item into float or string
"""
items = ['12.24', 'VXX', '02:49:35', '956.98', 'JUL4 14']
for item in items:
result = self.open_csv.convert_float_or_str(item)
print 'item: %s, result: %s, type: %s' % (
item, result, type(result)
)
self.assertIn(type(result), [str, float])
def test_format_item(self):
"""
Test format item that remove all unnecessary symbol
"""
items = ['($56.50)', '-17.86%', '$0.00', '+1', 'AUG 14']
for item in items:
result = self.open_csv.format_item(item)
print 'item: %s, result: %s, type: %s' % (
item, result, type(result)
)
self.assertIn(type(result), [str, float])
result = str(result)
self.assertNotIn('%', result)
self.assertNotIn('$', result)
self.assertNotIn('(', result)
self.assertNotIn(')', result)
def test_get_lines(self):
"""
Test get lines section from lines
"""
self.open_csv = OpenCSV(self.acc_data)
phrases = [
('Profits and Losses', 'OVERALL TOTALS'),
('Options', ',,,,,,'),
('Equities', 'OVERALL TOTALS'),
('Account Trade History', None),
('Account Order History', None),
('Forex Statements', None),
('Futures Statements', None),
('Cash Balance', 'TOTAL')
]
for start, end in phrases:
lines = self.open_csv.get_lines(start, end)
print start, end
for line in lines:
print line
print '-' * 100
def test_make_dict(self):
"""
Test make dict using keys and values
"""
keys = ['symbol', 'description', 'pl_open', 'pl_pct', 'pl_day', 'pl_ytd', 'margin_req']
values = ['AA', 'ALCOA INC COM', 0.0, 0.0, 0.0, -271.98, 0.0]
print 'keys: %s' % keys
print 'values: %s' % values
result = self.open_csv.make_dict(keys, values)
print 'result: %s' % result
for key, value in result.items():
self.assertIn(key, keys)
self.assertIn(value, values)
index = keys.index(key)
self.assertEqual(key, keys[index])
self.assertEqual(value, values[index])
def test_del_empty_keys(self):
"""
Test delete empty key in dict
"""
items = {'': '', 'ggg': 'hhh', 'kkk': 'lll'}
result = self.open_csv.del_empty_keys(items)
print 'items: %s' % items
print 'result: %s' % result
self.assertGreater(len(items), len(result))
self.assertNotIn('', result.keys())
def test_fillna_dict(self):
"""
Test fill empty or none value in dict
"""
self.open_csv = OpenCSV(self.acc_data)
self.open_csv.trade_history = list()
trade_history_keys = [
'', 'execute_time', 'spread', 'side', 'quantity',
'pos_effect', 'symbol', 'expire_date', 'strike',
'contract', 'price', 'net_price', 'order_type'
]
self.open_csv.set_values(
start_phrase='Account Trade History',
end_phrase=None,
start_with=2,
end_until=-1,
prop_keys=trade_history_keys,
prop_name='trade_history'
)
before_fillna = self.open_csv.trade_history
after_fillna = self.open_csv.fillna_dict(self.open_csv.trade_history)
after_fillna = map(self.open_csv.del_empty_keys, after_fillna)
for tk1, tk2 in zip(before_fillna, after_fillna):
print tk1
print tk2
print ''
self.assertIn('', tk1.values())
self.assertNotIn('', tk2.values())
for value1, value2 in zip(tk1.values(), tk2.values()):
if value1 and (value1 != 'DEBIT' and value1 != 'CREDIT'):
self.assertIn(value1, tk2.values())
for key in tk1.keys():
if key:
self.assertIn(key, tk2.keys())
def test_fillna_dict_with_exists(self):
"""
Test fill empty or none value in dict
"""
self.acc_file = test_path + r'/2014-10-31/2014-10-31-AccountStatement.csv'
self.acc_data = open(self.acc_file, mode='r').read()
self.open_csv = OpenCSV(self.acc_data)
self.open_csv.trade_history = list()
trade_history_keys = [
'', 'execute_time', 'spread', 'side', 'quantity',
'pos_effect', 'symbol', 'expire_date', 'strike',
'contract', 'price', 'net_price', 'order_type'
]
self.open_csv.set_values(
start_phrase='Account Trade History',
end_phrase=None,
start_with=2,
end_until=-1,
prop_keys=trade_history_keys,
prop_name='trade_history'
)
before_fillna = deepcopy(self.open_csv.trade_history)
for t in before_fillna:
print t['execute_time']
self.open_csv.fillna_dict_with_exists(
self.open_csv.trade_history,
'execute_time',
('execute_time', 'spread', 'order_type')
)
after_fillna = self.open_csv.trade_history
after_fillna = map(self.open_csv.del_empty_keys, after_fillna)
for key, (tk1, tk2) in enumerate(zip(before_fillna, after_fillna)):
print tk1
print tk2
print ''
if not tk1['execute_time']:
self.assertEqual(tk1['execute_time'], '')
self.assertEqual(tk1['spread'], '')
self.assertEqual(tk1['order_type'], '')
self.assertEqual(tk2['execute_time'], after_fillna[key - 1]['execute_time'])
self.assertEqual(tk2['spread'], after_fillna[key - 1]['spread'])
self.assertEqual(tk2['order_type'], after_fillna[key - 1]['order_type'])
def test_convert_specific_type(self):
"""
Test convert specific type for dict in a list
"""
items = [
{'a': 0, 'ref_no': 793403165.0, 'c': 0},
{'a': 0, 'ref_no': 801854049.0, 'c': 0},
{'a': 0, 'ref_no': 802092476.0, 'c': 0},
{'a': 0, 'ref_no': 17.0, 'c': 0},
{'a': 0, 'ref_no': 123.0, 'c': 0},
]
self.open_csv.cash_balance = [dict(i) for i in items]
self.open_csv.convert_specific_type(self.open_csv.cash_balance, 'ref_no', int, 0)
for item, result in zip(items, self.open_csv.cash_balance):
print 'dict: %s, result: %s' % (item, result)
self.assertEqual(item['ref_no'], result['ref_no'])
self.assertNotEqual(type(item['ref_no']), type(result['ref_no']))
def test_set_values(self):
"""
Test open files, get lines section, format data,
make dict and finally set values in property
"""
self.open_csv = OpenCSV(self.acc_data)
self.open_csv.trade_history = list()
trade_history_keys = [
'', 'execute_time', 'spread', 'side', 'quantity',
'pos_effect', 'symbol', 'expire_date', 'strike',
'contract', 'price', 'net_price', 'order_type'
]
start_phrase = 'Account Trade History'
end_phrase = None
start_add = 2
end_reduce = -1
prop_name = 'trade_history'
print 'start phrase: %s, end phrase: %s' % (start_phrase, end_phrase)
print 'start add: %d, end reduce: %d' % (start_add, end_reduce)
print 'property name: %s' % prop_name
self.open_csv.set_values(
start_phrase=start_phrase,
end_phrase=end_phrase,
start_with=start_add,
end_until=end_reduce,
prop_keys=trade_history_keys,
prop_name=prop_name
)
trade_history = self.open_csv.trade_history
lines = self.open_csv.get_lines('Account Trade History')
print ''
for tk in trade_history:
print tk
self.assertEqual(type(tk), dict)
self.assertEqual(len(tk), 13)
self.assertEqual(len(lines), len(trade_history) + 2 + 1)
self.assertEqual(type(trade_history), list)
