def setUp(self):
"""
open three file and save it into table then get data
"""
TestSetUpDB.setUp(self)
self.ready_file2(real_date='2014-11-17', file_date='2014-11-18')
self.ready_file2(real_date='2014-11-18', file_date='2014-11-19')
self.ready_file2(real_date='2014-11-24', file_date='2014-11-25')
self.assertGreaterEqual(Statement.objects.count(), 1)
self.statement = Statement.objects.last()
# create instance now
self.save_day_stat = SaveStatDay(statement=self.statement)
self.expected_keys = [
'name',
'total_order_count',
'working_order_count',
'filled_order_count',
'cancelled_order_count',
'total_holding_count',
'profit_holding_count',
'loss_holding_count',
'pl_open_sum',
'profit_open_sum',
'loss_open_sum',
'pl_day_sum',
'profit_day_sum',
'loss_day_sum',
'bp_effect_sum'
]
def setUp(self):
"""
open three file and save it into table then get data
"""
TestSetUpDB.setUp(self)
self.ready_file2(real_date='2014-11-17', file_date='2014-11-18')
self.ready_file2(real_date='2014-11-18', file_date='2014-11-19')
self.ready_file2(real_date='2014-11-24', file_date='2014-11-25')
self.assertGreaterEqual(Statement.objects.count(), 1)
self.statement = Statement.objects.last()
# create instance now
self.save_day_stat = SaveStatDay(statement=self.statement)
self.expected_keys = [
'name', 'total_order_count', 'working_order_count',
'filled_order_count', 'cancelled_order_count',
'total_holding_count', 'profit_holding_count',
'loss_holding_count', 'pl_open_sum', 'profit_open_sum',
'loss_open_sum', 'pl_day_sum', 'profit_day_sum', 'loss_day_sum',
'bp_effect_sum'
]
class TestSaveDateStat(TestSetUpDB):
def setUp(self):
"""
open three file and save it into table then get data
"""
TestSetUpDB.setUp(self)
self.ready_file2(real_date='2014-11-17', file_date='2014-11-18')
self.ready_file2(real_date='2014-11-18', file_date='2014-11-19')
self.ready_file2(real_date='2014-11-24', file_date='2014-11-25')
self.assertGreaterEqual(Statement.objects.count(), 1)
self.statement = Statement.objects.last()
# create instance now
self.save_day_stat = SaveStatDay(statement=self.statement)
self.expected_keys = [
'name',
'total_order_count',
'working_order_count',
'filled_order_count',
'cancelled_order_count',
'total_holding_count',
'profit_holding_count',
'loss_holding_count',
'pl_open_sum',
'profit_open_sum',
'loss_open_sum',
'pl_day_sum',
'profit_day_sum',
'loss_day_sum',
'bp_effect_sum'
]
def ready_file(self, real_date, file_date):
if not User.objects.exists():
self.user = User.objects.create_superuser(
username='******',
email='*****@*****.**',
password='******'
)
self.date = real_date
self.account_statement_file = SimpleUploadedFile(
'%s-AccountStatement.csv' % file_date,
open(os.path.join(test_path, file_date, '%s-AccountStatement.csv' % file_date)).read()
)
self.position_statement_file = SimpleUploadedFile(
'%s-PositionStatement.csv' % file_date,
open(os.path.join(test_path, file_date, '%s-PositionStatement.csv' % file_date)).read()
)
self.trade_activity_file = SimpleUploadedFile(
'%s-TradeActivity.csv' % file_date,
open(os.path.join(test_path, file_date, '%s-TradeActivity.csv' % file_date)).read()
)
self.client.login(username='******', password='******')
self.client.post(
path=reverse('admin:statement_import'),
data=dict(
date=self.date,
account_statement=self.account_statement_file,
position_statement=self.position_statement_file,
trade_activity=self.trade_activity_file
)
)
def ready_file2(self, real_date, file_date):
acc_data = open(os.path.join(test_path, file_date, '%s-AccountStatement.csv' % file_date)).read()
pos_data = open(os.path.join(test_path, file_date, '%s-PositionStatement.csv' % file_date)).read()
ta_data = open(os.path.join(test_path, file_date, '%s-TradeActivity.csv' % file_date)).read()
statement = Statement()
statement.date = real_date
statement.account_statement = acc_data
statement.position_statement = pos_data
statement.trade_activity = ta_data
statement.save()
SaveAccountStatement(
date=file_date,
statement=statement,
file_data=acc_data
).save_all()
SavePositionStatement(
date=file_date,
statement=statement,
file_data=pos_data
).save_all()
SaveTradeActivity(
date=file_date,
statement=statement,
file_data=ta_data
).save_all()
def test_get_future(self):
"""
Test get holding future detail from table
using set foreign key reference
"""
future = self.save_day_stat.get_future()
self.assertEqual(type(future), dict)
self.assertEqual(future['name'], 'FUTURE')
for key in self.expected_keys:
self.assertIn(key, future.keys())
print '%s: %s' % (key, future[key])
self.assertNotEqual(future[key], 0)
def test_get_forex(self):
"""
Test get holding forex detail from table
using set foreign key reference
"""
forex = self.save_day_stat.get_forex()
self.assertEqual(type(forex), dict)
self.assertEqual(forex['name'], 'FOREX')
for key in self.expected_keys:
self.assertIn(key, forex.keys())
print '%s: %s' % (key, forex[key])
def test_get_equity(self):
"""
Test get holding stock detail from table
using set foreign key reference
"""
equity = self.save_day_stat.get_equity()
self.assertEqual(type(equity), dict)
self.assertEqual(equity['name'], 'EQUITY')
for key in self.expected_keys + ['option_greek']:
self.assertIn(key, equity.keys())
print '%s: %s' % (key, equity[key])
def test_get_option(self):
"""
Test get holding option detail from table
using set foreign key reference
"""
option = self.save_day_stat.get_option()
self.assertEqual(type(option), dict)
self.assertEqual(option['name'], 'OPTION')
for key in self.expected_keys + ['option_greek']:
self.assertIn(key, option.keys())
print '%s: %s' % (key, option[key])
def test_get_holding_spread(self):
"""
Test get holding option detail from table
using set foreign key reference
"""
spread = self.save_day_stat.get_spread()
self.assertEqual(type(spread), dict)
self.assertEqual(spread['name'], 'SPREAD')
for key in self.expected_keys + ['option_greek']:
self.assertIn(key, spread.keys())
print '%s: %s' % (key, spread[key])
def test_get_holding_hedge(self):
"""
Test get holding option detail from table
using set foreign key reference
"""
spread = self.save_day_stat.get_hedge()
self.assertEqual(type(spread), dict)
self.assertEqual(spread['name'], 'HEDGE')
for key in self.expected_keys + ['option_greek']:
self.assertIn(key, spread.keys())
print '%s: %s' % (key, spread[key])
def test_get_day_stat(self):
"""
Test get date stat
"""
expected_keys = [
'total_holding_count',
'total_order_count',
'working_order_count',
'filled_order_count',
'cancelled_order_count',
'account_pl_ytd',
'account_pl_day',
'holding_pl_day',
'holding_pl_open',
'commission_day',
'commission_ytd',
'option_bp_day',
'stock_bp_day'
]
day_stat = self.save_day_stat.get_day_stat()
self.assertEqual(type(day_stat), dict)
for key in expected_keys:
self.assertIn(key, day_stat.keys())
self.assertNotEqual(day_stat[key], 0.0)
print '%s: %s' % (key, day_stat[key])
def test_get_option_greek(self):
"""
Test get option greek for equity, option and spread
"""
position_instruments = [p for p in self.save_day_stat.position_instrument.all()]
print 'run get option greek using position instrument...'
option_greek = self.save_day_stat.get_option_greek(position_instruments)
print 'option greek result:'
pprint(option_greek)
self.assertNotEqual(option_greek['delta_sum'], 0.0)
self.assertNotEqual(option_greek['gamma_sum'], 0.0)
self.assertNotEqual(option_greek['theta_sum'], 0.0)
self.assertNotEqual(option_greek['vega_sum'], 0.0)
def test_start(self):
"""
Test save all data into day stat models
"""
day_stat_keys = [
'total_holding_count',
'total_order_count',
'working_order_count',
'filled_order_count',
'cancelled_order_count',
'account_pl_ytd',
'account_pl_day',
'holding_pl_day',
'holding_pl_open',
'commission_day',
'commission_ytd',
'option_bp_day',
'stock_bp_day'
]
investment_keys = [
'name',
'total_order_count',
'working_order_count',
'filled_order_count',
'cancelled_order_count',
'total_holding_count',
'profit_holding_count',
'loss_holding_count',
'pl_open_sum',
'profit_open_sum',
'loss_open_sum',
'pl_day_sum',
'profit_day_sum',
'loss_day_sum',
'bp_effect_sum'
]
stat_day_id = self.save_day_stat.save_all()
print 'Stat day id: %d' % stat_day_id
self.assertTrue(stat_day_id)
self.assertEqual(models.StatDay.objects.count(), 1)
self.assertEqual(models.StatDayHolding.objects.count(), 6)
self.assertEqual(models.StatDayOptionGreek.objects.count(), 4)
stat_day = models.StatDay.objects.first()
stat_day_holding = models.StatDayHolding.objects.all()
stat_day_option_greek = models.StatDayOptionGreek.objects.all()
print 'date stat:'
for key in day_stat_keys:
print '%s: %s' % (key, getattr(stat_day, key))
print '\n' + '-' * 80 + '\n\n' + 'investment stat:'
for investment in stat_day_holding:
for key in investment_keys:
print '%s: %s' % (key, getattr(investment, key))
print ''
print '\n' + '-' * 80 + '\n\n' + 'option greek:'
for option_greek in stat_day_option_greek:
print option_greek
for key in ['delta_sum', 'gamma_sum', 'theta_sum', 'vega_sum']:
print '%s: %s' % (key, getattr(option_greek, key))
print '\n'
def clean(self):
"""
Validate date for file date and all import file name date
"""
cleaned_data = super(PmsImportStatementsForm, self).clean()
if not len(self._errors):
# no error found for field
real_date = cleaned_data.get("date").strftime('%Y-%m-%d')
acc_date = datetime.strptime(
cleaned_data.get("account_statement").name[:10], '%Y-%m-%d'
)
pos_date = datetime.strptime(
cleaned_data.get("position_statement").name[:10], '%Y-%m-%d'
)
ta_date = datetime.strptime(
cleaned_data.get("trade_activity").name[:10], '%Y-%m-%d'
)
acc_date = acc_date - BDay(1)
acc_date = acc_date.strftime('%Y-%m-%d')
pos_date = pos_date - BDay(1)
pos_date = pos_date.strftime('%Y-%m-%d')
ta_date = ta_date - BDay(1)
ta_date = ta_date.strftime('%Y-%m-%d')
if acc_date == pos_date == ta_date:
if real_date != acc_date:
error_message = 'All date must have (-1 BDay, %s != %s).' % (real_date, acc_date)
self._errors['date'] = self.error_class([error_message])
else:
error_message = 'All file date must be same.'
self._errors['date'] = self.error_class([error_message])
if not len(self._errors):
# save here so django can track error location
acc_data = cleaned_data.get("account_statement")
acc_data = acc_data.read()
pos_data = cleaned_data.get("position_statement")
pos_data = pos_data.read()
ta_data = cleaned_data.get("trade_activity")
ta_data = ta_data.read()
statement = Statement()
statement.date = real_date
statement.account_statement = acc_data
statement.position_statement = pos_data
statement.trade_activity = ta_data
statement.save()
SaveAccountStatement(
date=real_date,
statement=statement,
file_data=acc_data
).save_all()
SavePositionStatement(
date=real_date,
statement=statement,
file_data=pos_data
).save_all()
trade_summary_id = SaveTradeActivity(
date=real_date,
statement=statement,
file_data=ta_data
).save_all()
SaveStatDay(statement).save_all()
# create or update position_set
filled_orders = FilledOrder.objects.filter(trade_summary__id=trade_summary_id)
controller = PositionSetController(filled_orders)
controller.close_position_sets(date=real_date)
controller.create_position_sets(date=real_date)
controller.batch_update_foreign_keys(date=real_date)
self.cleaned_data['statement_id'] = statement.id
self.cleaned_data['statement_name'] = statement.__unicode__()
return cleaned_data
def statement_import_all(request):
"""
import all statements in real_path folder
:param request: request
:return: render
"""
template = 'tos_import/statement_import_all.html'
real_files_folder = glob.glob('%s/*' % import_path)
imported_logs = list()
error_logs = list()
for folder in real_files_folder:
if os.path.isdir(folder):
real_date = os.path.basename(folder)
try:
datetime.strptime(real_date, '%Y-%m-%d')
except ValueError:
error_logs.append(
{
'path': folder,
'date': real_date.split(' ')[0],
'note': real_date.split(' ')[1],
'error': 'Invalid filename'
}
)
else:
# only import error free folder
# get file inside and get date
# skip date exist in db
if not Statement.objects.filter(date=real_date).exists():
statement = glob.glob('%s/*.csv' % folder)[0]
file_date = os.path.basename(statement)[0:10]
acc_data = pos_data = ta_data = ''
acc_file = os.path.join(folder, '%s-AccountStatement.csv' % file_date)
pos_file = os.path.join(folder, '%s-PositionStatement.csv' % file_date)
ta_file = os.path.join(folder, '%s-TradeActivity.csv' % file_date)
statement = Statement()
statement.date = real_date
if os.path.exists(acc_file):
acc_data = open(acc_file).read()
statement.account_statement = acc_data
if os.path.exists(pos_file):
pos_data = open(pos_file).read()
statement.position_statement = pos_data
if os.path.exists(ta_file):
ta_data = open(ta_file).read()
statement.trade_activity = ta_data
statement.save()
if acc_data:
account_summary_id = SaveAccountStatement(
date=real_date,
statement=statement,
file_data=acc_data
).save_all()
if pos_data:
position_summary_id = SavePositionStatement(
date=real_date,
statement=statement,
file_data=pos_data
).save_all()
trade_summary_id = 0
if ta_data:
trade_summary_id = SaveTradeActivity(
date=real_date,
statement=statement,
file_data=ta_data
).save_all()
# save stat day
SaveStatDay(statement).save_all()
# create or update position_set
filled_orders = FilledOrder.objects.filter(trade_summary__id=trade_summary_id)
controller = PositionSetController(filled_orders)
controller.close_position_sets(date=real_date)
controller.create_position_sets(date=real_date)
controller.batch_update_foreign_keys(date=real_date)
imported_logs.append({
'statement': {
'id': statement.id,
'change_url': reverse(
'admin:tos_import_statement_change',
args={statement.id}),
'delete_url': reverse(
'admin:tos_import_statement_delete',
args={statement.id}),
'date': statement.date
},
'account_statement': {
'id': account_summary_id,
'change_url': reverse(
'admin:statement_account_accountsummary_change',
args={account_summary_id}),
'delete_url': reverse(
'admin:statement_account_accountsummary_delete',
args={account_summary_id}),
},
'position_statement': {
'id': position_summary_id,
'change_url': reverse(
'admin:statement_position_positionsummary_change',
args={position_summary_id}),
'delete_url': reverse(
'admin:statement_position_positionsummary_delete',
args={position_summary_id}),
},
'trade_activity': {
'id': trade_summary_id,
'change_url': reverse(
'admin:statement_trade_tradesummary_change',
args={trade_summary_id}),
'delete_url': reverse(
'admin:statement_trade_tradesummary_delete',
args={trade_summary_id}),
},
})
# log entry
LogEntry.objects.log_action(
user_id=request.user.id,
content_type_id=ContentType.objects.get_for_model(Statement).id,
object_id=statement.id,
object_repr=statement.__unicode__(),
action_flag=ADDITION
)
parameters = dict(
imported_logs=imported_logs,
error_logs=error_logs
)
# testing page view, delete all after done...
# Statement.objects.all().delete()
return render(request, template, parameters)
class TestSaveDateStat(TestSetUpDB):
def setUp(self):
"""
open three file and save it into table then get data
"""
TestSetUpDB.setUp(self)
self.ready_file2(real_date='2014-11-17', file_date='2014-11-18')
self.ready_file2(real_date='2014-11-18', file_date='2014-11-19')
self.ready_file2(real_date='2014-11-24', file_date='2014-11-25')
self.assertGreaterEqual(Statement.objects.count(), 1)
self.statement = Statement.objects.last()
# create instance now
self.save_day_stat = SaveStatDay(statement=self.statement)
self.expected_keys = [
'name', 'total_order_count', 'working_order_count',
'filled_order_count', 'cancelled_order_count',
'total_holding_count', 'profit_holding_count',
'loss_holding_count', 'pl_open_sum', 'profit_open_sum',
'loss_open_sum', 'pl_day_sum', 'profit_day_sum', 'loss_day_sum',
'bp_effect_sum'
]
def ready_file(self, real_date, file_date):
if not User.objects.exists():
self.user = User.objects.create_superuser(username='******',
email='*****@*****.**',
password='******')
self.date = real_date
self.account_statement_file = SimpleUploadedFile(
'%s-AccountStatement.csv' % file_date,
open(
os.path.join(test_path, file_date,
'%s-AccountStatement.csv' % file_date)).read())
self.position_statement_file = SimpleUploadedFile(
'%s-PositionStatement.csv' % file_date,
open(
os.path.join(test_path, file_date,
'%s-PositionStatement.csv' % file_date)).read())
self.trade_activity_file = SimpleUploadedFile(
'%s-TradeActivity.csv' % file_date,
open(
os.path.join(test_path, file_date,
'%s-TradeActivity.csv' % file_date)).read())
self.client.login(username='******', password='******')
self.client.post(path=reverse('admin:statement_import'),
data=dict(
date=self.date,
account_statement=self.account_statement_file,
position_statement=self.position_statement_file,
trade_activity=self.trade_activity_file))
def ready_file2(self, real_date, file_date):
acc_data = open(
os.path.join(test_path, file_date,
'%s-AccountStatement.csv' % file_date)).read()
pos_data = open(
os.path.join(test_path, file_date,
'%s-PositionStatement.csv' % file_date)).read()
ta_data = open(
os.path.join(test_path, file_date,
'%s-TradeActivity.csv' % file_date)).read()
statement = Statement()
statement.date = real_date
statement.account_statement = acc_data
statement.position_statement = pos_data
statement.trade_activity = ta_data
statement.save()
SaveAccountStatement(date=file_date,
statement=statement,
file_data=acc_data).save_all()
SavePositionStatement(date=file_date,
statement=statement,
file_data=pos_data).save_all()
SaveTradeActivity(date=file_date,
statement=statement,
file_data=ta_data).save_all()
def test_get_future(self):
"""
Test get holding future detail from table
using set foreign key reference
"""
future = self.save_day_stat.get_future()
self.assertEqual(type(future), dict)
self.assertEqual(future['name'], 'FUTURE')
for key in self.expected_keys:
self.assertIn(key, future.keys())
print '%s: %s' % (key, future[key])
self.assertNotEqual(future[key], 0)
def test_get_forex(self):
"""
Test get holding forex detail from table
using set foreign key reference
"""
forex = self.save_day_stat.get_forex()
self.assertEqual(type(forex), dict)
self.assertEqual(forex['name'], 'FOREX')
for key in self.expected_keys:
self.assertIn(key, forex.keys())
print '%s: %s' % (key, forex[key])
def test_get_equity(self):
"""
Test get holding stock detail from table
using set foreign key reference
"""
equity = self.save_day_stat.get_equity()
self.assertEqual(type(equity), dict)
self.assertEqual(equity['name'], 'EQUITY')
for key in self.expected_keys + ['option_greek']:
self.assertIn(key, equity.keys())
print '%s: %s' % (key, equity[key])
def test_get_option(self):
"""
Test get holding option detail from table
using set foreign key reference
"""
option = self.save_day_stat.get_option()
self.assertEqual(type(option), dict)
self.assertEqual(option['name'], 'OPTION')
for key in self.expected_keys + ['option_greek']:
self.assertIn(key, option.keys())
print '%s: %s' % (key, option[key])
def test_get_holding_spread(self):
"""
Test get holding option detail from table
using set foreign key reference
"""
spread = self.save_day_stat.get_spread()
self.assertEqual(type(spread), dict)
self.assertEqual(spread['name'], 'SPREAD')
for key in self.expected_keys + ['option_greek']:
self.assertIn(key, spread.keys())
print '%s: %s' % (key, spread[key])
def test_get_holding_hedge(self):
"""
Test get holding option detail from table
using set foreign key reference
"""
spread = self.save_day_stat.get_hedge()
self.assertEqual(type(spread), dict)
self.assertEqual(spread['name'], 'HEDGE')
for key in self.expected_keys + ['option_greek']:
self.assertIn(key, spread.keys())
print '%s: %s' % (key, spread[key])
def test_get_day_stat(self):
"""
Test get date stat
"""
expected_keys = [
'total_holding_count', 'total_order_count', 'working_order_count',
'filled_order_count', 'cancelled_order_count', 'account_pl_ytd',
'account_pl_day', 'holding_pl_day', 'holding_pl_open',
'commission_day', 'commission_ytd', 'option_bp_day', 'stock_bp_day'
]
day_stat = self.save_day_stat.get_day_stat()
self.assertEqual(type(day_stat), dict)
for key in expected_keys:
self.assertIn(key, day_stat.keys())
self.assertNotEqual(day_stat[key], 0.0)
print '%s: %s' % (key, day_stat[key])
def test_get_option_greek(self):
"""
Test get option greek for equity, option and spread
"""
position_instruments = [
p for p in self.save_day_stat.position_instrument.all()
]
print 'run get option greek using position instrument...'
option_greek = self.save_day_stat.get_option_greek(
position_instruments)
print 'option greek result:'
pprint(option_greek)
self.assertNotEqual(option_greek['delta_sum'], 0.0)
self.assertNotEqual(option_greek['gamma_sum'], 0.0)
self.assertNotEqual(option_greek['theta_sum'], 0.0)
self.assertNotEqual(option_greek['vega_sum'], 0.0)
def test_start(self):
"""
Test save all data into day stat models
"""
day_stat_keys = [
'total_holding_count', 'total_order_count', 'working_order_count',
'filled_order_count', 'cancelled_order_count', 'account_pl_ytd',
'account_pl_day', 'holding_pl_day', 'holding_pl_open',
'commission_day', 'commission_ytd', 'option_bp_day', 'stock_bp_day'
]
investment_keys = [
'name', 'total_order_count', 'working_order_count',
'filled_order_count', 'cancelled_order_count',
'total_holding_count', 'profit_holding_count',
'loss_holding_count', 'pl_open_sum', 'profit_open_sum',
'loss_open_sum', 'pl_day_sum', 'profit_day_sum', 'loss_day_sum',
'bp_effect_sum'
]
stat_day_id = self.save_day_stat.save_all()
print 'Stat day id: %d' % stat_day_id
self.assertTrue(stat_day_id)
self.assertEqual(models.StatDay.objects.count(), 1)
self.assertEqual(models.StatDayHolding.objects.count(), 6)
self.assertEqual(models.StatDayOptionGreek.objects.count(), 4)
stat_day = models.StatDay.objects.first()
stat_day_holding = models.StatDayHolding.objects.all()
stat_day_option_greek = models.StatDayOptionGreek.objects.all()
print 'date stat:'
for key in day_stat_keys:
print '%s: %s' % (key, getattr(stat_day, key))
print '\n' + '-' * 80 + '\n\n' + 'investment stat:'
for investment in stat_day_holding:
for key in investment_keys:
print '%s: %s' % (key, getattr(investment, key))
print ''
print '\n' + '-' * 80 + '\n\n' + 'option greek:'
for option_greek in stat_day_option_greek:
print option_greek
for key in ['delta_sum', 'gamma_sum', 'theta_sum', 'vega_sum']:
print '%s: %s' % (key, getattr(option_greek, key))
print '\n'
