-
Notifications
You must be signed in to change notification settings - Fork 0
/
engine.py
463 lines (358 loc) · 17.8 KB
/
engine.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
# NOTES
# ON PARSING TRANSACTION DATA
#DONE step 1: parse when only the table is provided (strip off table heading if
# exist)
# step 2: parse even when more information is present(e.g. when the user does a
# ctrl+A and then pastes)
# step 3: see if some other browser/OS gives different text while copying info
# step 4: only take the transactions whose status is 'processed', and remove everything else
# step 5: see how to handle cases when user pastes second time, which has a transaction which is now in 'processed' state but was in a different state previously
#NOTE: we are using NAV values from only moneycontrol site, and not from amount/units. This should change in future
# TODO(rushiagr): get mutual fund IDs from internet, and not from a dict filled
# manually
# TODO(rushiagr): Use database's transaction object, and remove Txn
# TODO: auto-detect the company of transaction pasted
# TODO: Store NAV in Txn object
# TODO: while taking data from moneycontrol, also return the date of NAV
# TODO: show the users the profit they're earning on each of the fund
# TODO: when a user redeems some units, show how much profit he earned
# (assuming he is redeeming the units which were purchased first)
# TODO: give the user a choice to redeem units which were bought 'later', and
# not the ones he purchased first.
# Note : do not use NAV value from icici bank at all. it is usually outdated,
# and doesnt match with amount divided by units
# TODO: get value of NAV from 'units' and 'amount', and not from moneycontrol
# TODO: add these defects to github issues for this project
# fill al these information into database tables
# NOTE: for now, ignore that moneycontrol can have old values for today
# TODO check if deriving nav values from units and amount give more accurate final values
# TODOadd mechanism for giving the users updates regarding moneycontrol data (e.g.
# the data was last modified on moneycontrol server on yesterday, and generally
# it gets updated at so and so time so you can expect it to become updated again
# in blah hours
#from db import models
from db import api as db
import utils
import urllib2
import datetime
# Constants
NEW_PURCHASE = 1
ADDITIONAL_PURCHASE = 2
REDEMPTION = 3
PURCHASE = 101
def get_txns():
return db.get_all_transactions(1)
class Txn(object):
def __init__(self, fund_name=None, txn_type=None, amount=None, units=None,
date=None, status=None, remarks=None):
self.fund_name = fund_name
if txn_type.lower() in ['purchase', 'new purchase',
'additional purchase']:
self.txn_type = PURCHASE
elif txn_type.lower() in ['redemption']:
self.txn_type = REDEMPTION
else:
raise BaseException
self.amount = utils.get_float(amount)
self.units = utils.get_float(units)
self.date = utils.int_date(date)
self.status = status
self.remarks = remarks
self.fund_id = db.get_fund_id(fund_name)
self.nav = None # NAV on the day the transaction is performed
self.amc = None
# NOTE: The Txn object will also possess a list 'sold_units_nav_tuple_list'
# later in the processing, which will contain the sold units and the
# buying price of them
def get_detailed_stats(txn_list):
mf_dict = {}
# TODO: values of mf_dict should be objects, and not list. For those objects
# we will add additional attributes like total redemption, total profit in
# redemption, total units still held, total profit in current scenario, etc
# BUT make sure you study the implications of having all this information
# stored and retrieved from database
for txn in txn_list:
mf_dict[txn['fund_name']] = {}
mf_dict[txn['fund_name']]['txns'] = []
for txn in txn_list:
mf_dict[txn['fund_name']]['txns'].append(txn)
for mf in mf_dict:
mf_dict[mf]['txns'].sort(key=lambda x: x['date'])
for mf in mf_dict:
no_nav_txn = [txn for txn in mf_dict[mf]['txns'] if txn['nav'] is None]
if len(no_nav_txn) != 0:
# TODO: optimize here too
fill_all_navs_for_fund(mf_dict[mf]['txns'])
for mf in mf_dict:
fill_redemption_stats(mf_dict[mf]['txns'])
mf_curr_value_dict = get_curr_fund_value(mf_dict.keys())
for mf in mf_dict:
mf_dict[mf]['total_units'] = sum([txn['units'] for txn in mf_dict[mf]['txns'] if txn['txn_type'] == PURCHASE])
mf_dict[mf]['total_units'] -= sum([txn['units'] for txn in mf_dict[mf]['txns'] if txn['txn_type'] == REDEMPTION])
for mf in mf_dict.keys():
if mf_dict[mf]['total_units'] == 0.0:
del mf_dict[mf]
for mf in mf_dict:
mf_dict[mf]['total_amount_invested'] = sum([txn['left_units'] * txn['nav'] for txn in mf_dict[mf]['txns']])
mf_dict[mf]['total_amount_now'] = mf_curr_value_dict[mf] * mf_dict[mf]['total_units']
mf_dict[mf]['percentage_gains'] = mf_dict[mf]['total_amount_now']/mf_dict[mf]['total_amount_invested']*100.0 - 100.0 if mf_dict[mf]['total_amount_invested'] != 0.0 else 0.0
mf_dict[mf]['average_holding_period'] = sum([txn['left_units']*utils.diff_days(utils.today(), txn['date']) for txn in mf_dict[mf]['txns']])/sum(txn['left_units'] for txn in mf_dict[mf]['txns'])
mf_dict[mf]['annualized_gains'] = 365.25/mf_dict[mf]['average_holding_period']*mf_dict[mf]['percentage_gains']
stats = {}
stats['total_amount_invested'] = sum([mf_dict[mf]['total_amount_invested'] for mf in mf_dict])
stats['total_amount_now'] = sum([mf_dict[mf]['total_amount_now'] for mf in mf_dict])
stats['total_gains'] = stats['total_amount_now'] - stats['total_amount_invested']
print stats['total_gains']
stats['percentage_gains'] = stats['total_amount_now']/stats['total_amount_invested']*100.0 - 100.0 if stats['total_amount_invested'] != 0.0 else 0.0
return mf_dict, stats
def fill_redemption_stats(txn_list):
# Assumption: txn list is sane, i.e. a guy is not redeeming more units than he
# has purchased
# Assumption: list is sorted in order of date
txn_units = list([txn['units'] for txn in txn_list])
dup_list = list(txn_list)
for t in dup_list:
t['sold_units_nav_tuple_list'] = []
for i in range(len(dup_list)):
if dup_list[i]['txn_type'] in [REDEMPTION]:
units_left = dup_list[i]['units']
for j in range(i):
if units_left > 0.0:
if dup_list[j]['txn_type'] == PURCHASE:
if units_left < dup_list[j]['units']:
units_to_deduct = units_left
else:
units_to_deduct = dup_list[j]['units']
if units_to_deduct > 0.001:
dup_list[i]['sold_units_nav_tuple_list'].append(
(units_to_deduct, dup_list[j]['nav'], ))
dup_list[j]['units'] = dup_list[j]['units'] - units_to_deduct
units_left -= units_to_deduct
else:
break
for txn, units in zip(txn_list, txn_units):
txn['left_units'] = txn['units'] if txn['txn_type'] == PURCHASE else 0.0
txn['units'] = units
def fill_all_navs_for_fund(txn_list):
""" For the given list of Txn object, fetch the NAVs of all the
transactions for which this value is not present. All the transactions in txn_list
are of same fund"""
# Assumption: list is already sorted
max_date = txn_list[-1]['date']
min_date = txn_list[0]['date']
date_nav_dict, _blah = get_mf_data(txn_list[0]['fund_id'], min_date, max_date)
for txn in txn_list:
txn['nav'] = date_nav_dict[txn['date']]
def get_curr_fund_value(fund_name_list):
unit_values = {}
for fund in fund_name_list:
fund_id = db.get_fund_id(fund)
data_dict, last_date = get_mf_data(fund_id, utils.last_month(), utils.today())
unit_values[fund] = data_dict[max(data_dict)]
return unit_values
def txn_to_obj_list(txn_string, amc=None, user_id=None):
# TODO: in future, if amc is none, detect it from txn_string
txn_matrix = parse_txn(txn_string)
txn_obj_list = []
if amc.lower() == 'uti':
positions = [0, 1, 2, 3, 4]
elif amc.lower() == 'icici':
positions = [0, 1, 5, 3, 2]
else:
raise
for txn in txn_matrix:
obj = dict(fund_name=txn[positions[0]],
txn_type=txn[positions[1]],
amount=utils.get_float(txn[positions[2]]),
units=utils.get_float(txn[positions[3]]),
date=utils.int_date(txn[positions[4]]))
txn_obj_list.append(obj)
for txn in txn_obj_list:
if txn['txn_type'].lower() in ['purchase', 'new purchase',
'additional purchase']:
txn['txn_type'] = PURCHASE
elif txn['txn_type'].lower() in ['redemption']:
txn['txn_type'] = REDEMPTION
else:
raise BaseException
# Additional details contained in transactions
if amc.lower() == 'uti':
for obj, txn in zip(txn_obj_list, txn_matrix):
obj['remarks'] = txn[6].strip() if len(txn) >= 7 else ''
# DO NOT STORE ICICI NAV. IT IS SoMETIMES AN OLD VALUE
# elif amc.lower() == 'icici':
# for obj, txn in zip(txn_obj_list, txn_matrix):
# obj['nav'] = float(txn[4])
for obj in txn_obj_list:
obj['amc'] = amc.lower() if amc.lower() in ['icici', 'uti'] else ''
for obj in txn_obj_list:
obj['user_id'] = user_id
# TODO Do not make this much api calls to db!!
for obj in txn_obj_list:
obj['fund_id'] = db.get_fund_id(obj['fund_name'])
return txn_obj_list
def parse_txn(txn_string):
""" Takes transaction status from UTIMF or ICICI website, and converts
it to a usable matrix."""
txn_list = txn_string.strip().split('\n')
txn_matrix = [line.split(' ') if len(line.split(' ')) > 3 else line.split('\t') for line in txn_list ]
if txn_matrix[0][0].lower() in ['scheme', 'fund name']:
txn_matrix = txn_matrix[1:]
return txn_matrix
def get_mf_data(mf_code, from_date, to_date):
""" Returns a dict of date and mutual fund NAV for the given mutual fund,
between the given dates. Also returns the date of last day when NAV was updated
#Design principles
* Do not store last date's data if the NAV is exactly same as the previous
date
* If data in db is old (older than 4 hours), update it.
* ALSO RETURN WHAT IS THE LATEST DATE FOR WHICH CORRECT DATA IS RETURNED.
That is, if today is 4 jan, but in db, data is uptill only 2 jan.
If query is 1-2 jan, return 2 jan
but if query is 1-4 jan, return 2 jan and not 4 jan, as data for only 2 jan is present
#Workflow
if no data present for that mf in DB
fetch all data from moneycontrol and dump it in our DB.
Return appropriate data to user
if data present:
if data present in db for the specified dates
good! fetch data from db and return to user!
if data not present in db for all of the specified dates
if last-fetch-time is greater than 4 hours
get data from moneycontrol, and update the last_fetch time
if not, just return the data from db
"""
date_value_dict = db.get_navs(mf_code, from_date, to_date)
return_dict = {}
if not date_value_dict:
date_value_dict = update_fund(mf_code)
else:
db_dates = date_value_dict.keys()
db_dates.sort()
last_date = db_dates[-1]
if last_date != to_date:
now = datetime.datetime.utcnow()
last_update = db.last_updated(mf_code)
diff = now - last_update
if diff.days > 0 or (now - last_update).seconds > 4*60*60:
update_fund(mf_code, last_date+1)
date_value_dict = db.get_navs(mf_code, from_date, to_date)
for k, v in date_value_dict.iteritems():
if k >= from_date and k <= to_date:
return_dict[k] = v
return return_dict, max(return_dict.keys())
def update_fund(mf_code, last_date=None):
date_query_str = ''
if last_date is not None:
d = str(last_date)
date_query_str = '&dd='+d[6:]+'&mm='+d[4:6]+'&yy='+d[:4]
resp_str = urllib2.urlopen('http://moneycontrol.com/mf/mf_info/hist_tech_chart.php?im_id='+mf_code+date_query_str).read()
if len(resp_str) < 10:
return {}
date_value_dict, last_date = extract_moneycontrol_data(resp_str)
dates = date_value_dict.keys()
dates.sort()
# Moneycontrol.com quirk
# If moneycontrol returns data of only one day, it is the correct value,
# else it would have returned nothing
if len(dates) > 1 and date_value_dict[dates[-1]] == date_value_dict[dates[-2]]:
del date_value_dict[dates[-1]]
db.store_navs(mf_code, date_value_dict)
return date_value_dict
def extract_moneycontrol_data(data_str):
"""Takes moneycontrol data and converts it in slightly more usable form.
Converts '<date>,<value>,<value>,<value>,<value>' string returned by
the website into dict with date as key and value is value."""
# TODO: add check here if moneycontrol returns no data or bad data
lines = data_str.split('\n')
date_value_dict = {}
for line in lines:
l = line.split(',')
date_value_dict[utils.int_date(l[0])] = float(l[1])
return date_value_dict, utils.int_date(l[0])
#print extract_moneycontrol_data(get_mf_data('MPI110', intdate_last_month(), intdate_today()))
#print get_mf_data('MPI110', intdate_last_month(), intdate_today())
#print get_transaction_stats(txn_to_obj_list(utimf.txn_str, 'uti'))
#print get_transaction_stats(txn_to_obj_list(icicipru.txn_str, 'icici'))
# txns = txn_to_obj_list(utimf.txn_str, 'uti')
# mf_dict = get_detailed_stats(txns)
def amount_invested_from_list(txn_list):
"""Returns total amount invested for list of transactions of _same_ mutual
fund."""
curr_value = get_curr_fund_value([txn_list[0].fund_name]).values()[0]
invested_units = 0.0
for txn in txn_list:
if txn.txn_type == REDEMPTION:
invested_units -= txn.units
elif txn.txn_type == PURCHASE:
invested_units += txn.units
return invested_units * curr_value
################################################
#### All the user-related functions ###########
################################################
def store_transactions(txtbox_data, amc, user_id):
last_date = db.get_last_transaction_date(user_id, amc)
txns = txn_to_obj_list(txtbox_data, amc, user_id)
txns = [txn for txn in txns if txn['date'] > last_date]
db.store_transactions(txns)
def get_summary(user_id):
"""Get all the important data for the user"""
txns = db.get_all_transactions(user_id)
mf_dict, stats = get_detailed_stats(txns)
mf_dict, stats = prettify_data(mf_dict, stats)
return mf_dict, stats
def prettify_data(mf_dict, stats):
for mf in mf_dict:
for val in ['total_units', 'total_amount_invested',
'total_amount_now', 'percentage_gains',
'annualized_gains']:
mf_dict[mf][val] = utils.prettify_number(mf_dict[mf][val])
for val in ['total_amount_invested', 'percentage_gains',
'total_amount_now', 'total_gains']:
stats[val] = utils.prettify_number(stats[val])
return mf_dict, stats
################################################
################################################
################################################
def store_textbox_data_in_db(txtbox_data, amc=None, user_id=None):
"""Takes the transaction information pasted by user into textbox and
put it into DB. The AMC (Asset Management Company) might or might not be
provided by the user.
"""
txns = txn_to_obj_list(txtbox_data, amc)
user_id = 1 # For now, there is only one user
to_db_obj_list = db.get_db_objects_from_txn_list(txns, user_id)
# get transactions from database for that user
from_db_obj_list = db.get_txns_from_db(user_id, amc)
# validate if the data is correct. do all validation checks here
validate_existing_and_new_txn_data(to_db_obj_list,
from_db_obj_list)
# get only the new objects to be inserted in db
new_db_objs = get_new_db_objects(to_db_obj_list, from_db_obj_list)
# insert_objects_into_db(new_db_objs)
#def insert_objects_into_db(new_db_objs):
# for obj in new_db_objs:
# models.db.session.add(obj)
# models.db.session.commit()
def get_new_db_objects(to_db_obj_list, from_db_obj_list):
# NOTE(rushiagr): Assumption: when 2 transactions for say 14th Jan are
# stored into database, and if three transactions are found for 14th jan,
# reject the third one, or raise an exception. This is sane, as AMC
# websites generally update this information at once, for a day.
if not from_db_obj_list:
return to_db_obj_list
to_db_obj_list.sort(key=lambda x: x.date)
from_db_obj_list.sort(key=lambda x: x.date)
max_date_from_db = from_db_obj_list[-1]
to_db_obj_list = [obj for obj in to_db_obj_list if obj.date > max_date_from_db]
return to_db_obj_list
def validate_existing_and_new_txn_data(to_db_obj_list,
from_db_obj_list):
return
#store_textbox_data_in_db(utimf.txn_str, amc='uti', user_id=7777)
#store_textbox_data_in_db(icicipru.txn_str, amc='icici', user_id=6666)
# for fund in mf_dict:
# # not so good way of calculating this information, as it makes a lot of
# # calls to moneycontrol website
# print 'fund: %s value: %s' % (fund, amount_invested_from_list(mf_dict[fund]))
# print sum([amount_invested_from_list(mf_dict[fund]) for fund in mf_dict])