def mf_add_or_update_sip_kuvera(sips):
for sip in sips:
try:
print(f"name {sip['name']}")
folios = Folio.objects.filter(folio=sip['folio'])
folio = None
if len(folios) > 1:
for f in folios:
if f.fund.kuvera_name == sip['name']:
folio = f
else:
folio = folios[0]
if not folio:
description = 'Unable to decide on Folio with number ' + sip[
'folio'] + ' and KUVERA name ' + sip['name']
create_alert(summary='Folio:' + sip['folio'] +
' Failure to add a sip',
content=description,
severity=Severity.error,
alert_type="Action")
return
mf_add_or_update_sip(folio=folio,
amount=sip['amount'],
date=sip['date'])
except Folio.DoesNotExist:
description = 'Folio by that number doesnt exist'
create_alert(
summary='Folio:' + sip['folio'] +
' Failure to add a sip since no folio by that number found',
content=description,
severity=Severity.error,
alert_type="Action")
except Exception as ex:
print(f'failed while adding sip for {sip["folio"]}', ex)
def insert_trans_entry(symbol,
user,
trans_type,
quantity,
price,
date,
notes,
broker,
conversion_rate=1,
fees=0,
currency='USD',
trans_price=None):
try:
co = None
try:
co = Crypto.objects.get(symbol=symbol, user=user)
except Crypto.DoesNotExist:
print(f'creating crypto object {symbol} for user {user}')
co = Crypto.objects.create(symbol=symbol,
user=user,
units=0,
buy_price=0,
buy_value=0,
unrealised_gain=0,
realised_gain=0)
if not trans_price:
trans_price = price * quantity * conversion_rate
try:
Transaction.objects.create(crypto=co,
trans_date=date,
trans_type=trans_type,
price=price,
units=quantity,
conversion_rate=conversion_rate,
trans_price=trans_price,
broker=broker,
notes=notes,
fees=fees,
buy_currency=currency)
except IntegrityError:
print('Transaction exists')
except Exception as ex:
print(
f'failed to add transaction {symbol}, {user}, {trans_type}, {quantity}, {price}, {date}, {notes}, {broker}'
)
print(ex)
description = 'failed to add transaction for :' + symbol + ' for user ' + get_user_name_from_id(
user)
create_alert(summary=symbol + ' - Failed to add transaction',
content=description,
severity=Severity.warning,
alert_type="Action")
def get_transactions(self):
if isfile(self.filename):
ignored_folios = set()
with open(self.filename, mode='r',
encoding='utf-8-sig') as csv_file:
print("opened file as csv:", self.filename)
csv_reader = csv.DictReader(csv_file, delimiter=",")
for row in csv_reader:
trans_value = None
folio = None
#client_id isin scheme_name plan transaction_mode trade_date ordered_at folio_number amount units nav status remarks
for k, v in row.items():
if 'isin' in k:
isin = v.strip()
if 'folio_number' in k:
folio = v.strip()
elif 'trade_date' in k:
trans_date = get_datetime_or_none_from_string(
v.strip()) #2020-03-19
elif 'trade_type' in k:
trans_type = 'Buy' if 'buy' in v.lower(
) else 'Sell'
elif 'quantity' in k:
units = get_float_or_none_from_string(v.strip())
elif 'price' in k:
nav = get_float_or_none_from_string(v.strip())
elif 'amount' in k:
trans_value = get_float_or_none_from_string(
v.strip())
if not trans_value:
trans_value = nav * units
if not folio:
pass
fund, description = self._get_fund(isin, folio)
if fund:
yield {
'folio': folio,
'trans_date': trans_date,
'fund': fund,
'trans_type': trans_type,
'units': units,
'nav': nav,
'trans_value': trans_value
}
else:
create_alert(summary='Folio:' + folio +
' Failure to add transactions',
content=description,
severity=Severity.error,
alert_type="Action")
else:
print(f'{self.filename} is not a file or doesnt exist')
def download_bsestar_schemes():
url = 'https://www.bsestarmf.in/RptSchemeMaster.aspx'
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless")
path_to_media = get_path_to_media()
prefs = {'download.default_directory': path_to_media}
chrome_options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome(executable_path=get_path_to_chrome_driver(),
options=chrome_options)
driver.get(url)
timeout = 30
try:
WebDriverWait(driver, timeout).until(
EC.visibility_of_element_located((By.ID, "btnText")))
submit_element = driver.find_element_by_id("btnText")
submit_element.click()
time.sleep(10)
driver.close()
except TimeoutException:
driver.quit()
bse_star_file = None
for file in os.listdir(path_to_media):
if "schm" in file.lower():
bse_star_file = os.path.join(path_to_media, file)
break
if bse_star_file:
bse_star_obj = BSEStar()
bse_schemes = bse_star_obj.get_all_schemes(bse_star_file)
if len(bse_schemes) > 0:
update_bsestar_schemes(bse_schemes)
fs = FileSystemStorage()
fs.delete(bse_star_file)
else:
create_alert(summary='Failure to update BSE StAR schemes',
content='schemes parsed is 0',
severity=Severity.error,
alert_type="Application")
else:
print('no bse star file. not updating schemes')
description = 'no bse star file. not updating schemes'
create_alert(summary='Failure to update BSE StAR schemes',
content=description,
severity=Severity.error,
alert_type="Application")
'''
def reconcile_share(share_obj):
transactions = Transactions.objects.filter(
share=share_obj).order_by('trans_date')
try:
stock = Stock.objects.get(exchange=share_obj.exchange,
symbol=share_obj.symbol)
bonuses = Bonusv2.objects.filter(stock=stock)
splits = Splitv2.objects.filter(stock=stock)
round_qty_to_int = True if share_obj.exchange in [
'NSE', 'BSE', 'NSE/BSE'
] else False
qty, buy_value, buy_price, realised_gain, unrealised_gain = reconcile_event_based(
transactions,
bonuses,
splits,
round_qty_to_int=round_qty_to_int,
latest_price=share_obj.latest_price,
latest_conversion_rate=share_obj.conversion_rate)
share_obj.quantity = qty
share_obj.buy_value = buy_value
share_obj.buy_price = buy_price
share_obj.realised_gain = realised_gain
share_obj.gain = unrealised_gain
roi = 0
if qty > 0:
roi = get_roi(transactions, share_obj.latest_value)
share_obj.roi = roi
share_obj.save()
except Stock.DoesNotExist:
description = 'Stock not found in db. No splits/bonuses data available. This affects other calculations.'
create_alert(
summary=
f'{share_obj.exchange} {share_obj.symbol} not found in db.',
content=description,
severity=Severity.warning,
alert_type="Application")
except Exception as ex:
print(
f'exception {ex} when reconciling {share_obj.exchange} {share_obj.symbol}'
)
def get_transactions(self):
if isfile(self.filename):
ignored_folios = dict()
with open(self.filename, mode='r',
encoding='utf-8-sig') as csv_file:
print("opened file as csv:", self.filename)
csv_reader = csv.DictReader(csv_file, delimiter=",")
for row in csv_reader:
#OrderedDict([('Date', ' 2020-03-19'), (' Folio Number', '91011317175'), (' Name of the Fund', 'Motilal Oswal Focused 25 Growth Direct Plan'), (' Order', 'buy'), (' Units', '257.375'), (' NAV', '19.4269'), (' Current Nav', '20.3781'), (' Amount (INR)', '5000.0')])
for k, v in row.items():
if 'Folio Number' in k:
folio = v.strip()
elif 'Date' in k:
trans_date = get_datetime_or_none_from_string(
v.strip()) #2020-03-19
elif 'Name of the Fund' in k:
fund_name = v.strip()
elif 'Order' in k:
trans_type = 'Buy' if 'buy' in v else 'Sell'
elif 'Units' in k:
units = get_float_or_none_from_string(v.strip())
elif 'NAV' in k:
nav = get_float_or_none_from_string(v.strip())
elif 'Amount (INR)' in k:
trans_value = get_float_or_none_from_string(
v.strip())
fund = self._get_fund(fund_name)
if fund:
yield {
'folio': folio,
'trans_date': trans_date,
'fund': fund,
'trans_type': trans_type,
'units': units,
'nav': nav,
'trans_value': trans_value
}
'''
else:
fund = self._get_match_from_fund_nav(folio, fund_name, trans_date, nav)
if fund:
yield {'folio':folio,
'trans_date':trans_date,
'fund':fund,
'trans_type':trans_type,
'units':units,
'nav':nav,
'trans_value':trans_value}
'''
if not fund:
if not folio in ignored_folios:
ignored_folios[folio] = list()
ignored_folios[folio].append(fund_name)
else:
if fund_name not in ignored_folios[folio]:
ignored_folios[folio].append(fund_name)
for fol, fund_names in ignored_folios.items():
names = None
for fund_name in fund_names:
if not names:
names = fund_name
else:
names += ";" + fund_name
create_alert(
summary='Folio:' + fol + ' Failure to add transactions',
content=
f'Not able to find a matching entry between Kuvera names {names} and AMFII names.',
severity=Severity.error,
alert_type="Action")
def mf_add_or_update_sip_coin(sips, filenames):
mfs = list()
for filename in filenames:
if isfile(filename):
ignored_folios = set()
with open(filename, mode='r', encoding='utf-8-sig') as csv_file:
print("opened file as csv:", filename)
csv_reader = csv.DictReader(csv_file, delimiter=",")
for row in csv_reader:
#client_id isin scheme_name plan transaction_mode trade_date ordered_at folio_number amount units nav status remarks
for k, v in row.items():
if 'isin' in k:
isin = v.strip()
if 'folio_number' in k:
folio = v.strip()
if 'scheme_name' in k:
name = v.strip()
if 'plan' in k:
plan = v.strip()
if isin and folio and name:
if not name in mfs:
mf = dict()
mf['name'] = name
mf['isin'] = isin
mf['folio'] = folio
mf['plan'] = plan
mfs.append(mf)
print(mfs)
print(sips)
for sip in sips:
try:
folio = None
for mf in mfs:
if sip['name'] == mf['name'] and sip['plan'] == mf['plan']:
folio = mf['folio']
break
if not folio:
description = 'Unable to find Folio with COIN/ZERODHA name ' + sip[
'name'] + ' and plan ' + sip['plan']
create_alert(summary='Name: ' + sip['name'] +
' Failure to add a sip',
content=description,
severity=Severity.error,
alert_type="Action")
return
folios = Folio.objects.filter(folio=folio)
if len(folios) == 1:
folio = folios[0]
else:
description = 'Unable to decide on Folio for COIN/ZERODHA name ' + sip[
'name']
create_alert(summary='Name: ' + sip['name'] +
' Failure to add a sip',
content=description,
severity=Severity.error,
alert_type="Action")
return
mf_add_or_update_sip(folio=folio,
amount=sip['amount'],
date=sip['date'])
except Exception as ex:
print(f'failed while adding sip for {k}', ex)
def pull_ms(code, ignore_names, replaceAnd=False, token=None):
fund = None
try:
fund = MutualFund.objects.get(code=code)
except MutualFund.DoesNotExist:
create_alert(
summary='Code:' + code + ' Mutual fund not present',
content='Not able to find a matching Mutual Fund with the code.',
severity=Severity.error,
alert_type="Action")
return None, token
mf_name = None
if token and fund.ms_id:
data = use_api_to_get_vals(token, fund.ms_id)
if len(data.keys()):
return data, token
else:
print(f'couldnt get data using token {token}')
ms_code = fund.ms_id
if fund.ms_name:
mf_name = fund.ms_name
mf_name = mf_name.replace('&', '&')
else:
mf_name = fund.name
mf_name = mf_name.replace('-', '')
mf_name = mf_name.replace(' ', ' ')
mf_name = mf_name.replace('&', '&')
if replaceAnd:
if '&' in mf_name:
mf_name = mf_name.replace('&', 'and')
else:
mf_name = mf_name.replace('and', '&')
mf_name = mf_name.replace('And', '&')
url = decode(
b'\x03\x11\r\x07\x1cHKD\x12\x0e\x00A\x1f\x0b\x19\x0b\x10\x19\x08\x01\x10\n\x17W\x1e\x01'
)
capabilities = DesiredCapabilities.CHROME
capabilities["goog:loggingPrefs"] = {"performance": "ALL"}
driver = webdriver.Chrome(executable_path=get_path_to_chrome_driver(),
desired_capabilities=capabilities)
driver.get(url)
timeout = 30
if not ignore_names:
ignore_names = list()
try:
search_element_id = decode(
b'\x08\x11\x15G_-\x11\x08-\x1c\x16\x0b\x17\x164\x11\x01\x03>\x07\x0b\x1f\x00&\x03\x17\x06%\x1e\x11\x164\x00\x1f\x14\x07\x00\r\x12'
)
WebDriverWait(driver, timeout).until(
EC.visibility_of_element_located((By.ID, search_element_id)))
search_element = driver.find_element_by_id(search_element_id)
mf_name_parts = mf_name.split(' ')
for i, part in enumerate(mf_name_parts):
if i == len(mf_name_parts) - 1:
print('sending ', part)
search_element.send_keys(part)
else:
print('sending ', part + '<space>')
search_element.send_keys(part + ' ')
time.sleep(2)
#els = driver.find_elements_by_xpath("//*[@class='ui-autocomplete' and @class='quote-list']")
els = driver.find_elements_by_class_name('quote-list')
if len(els) == 0:
if not replaceAnd:
if mf_name_parts[i] == '&' or mf_name_parts[i].lower(
) == 'and':
driver.close()
return pull_ms(code, ignore_names, True)
print('len(els) is 0. closing driver')
driver.close()
return None, token
for el in els:
if el.tag_name == 'ul':
#print(el.id)
if 'more' in el.get_attribute('innerHTML'):
continue
li_elems = el.find_elements_by_xpath('.//li')
if len(li_elems) > 10:
print(f'too many results {len(li_elems)}')
continue
if len(li_elems):
qn_elems = el.find_elements_by_css_selector(
'div.quote-name')
if qn_elems:
map_elems = dict()
high_score = 0
high_element = -1
high_name = ''
for num, div in enumerate(qn_elems):
print("innerHtml",
div.get_attribute('innerHTML'))
temp = div.get_attribute('innerHTML').replace(
'<strong>', '')
temp = temp.replace('</strong>', '')
temp = temp.replace('&', '&')
if temp not in ignore_names:
map_elems[num] = match_score(mf_name, temp)
if high_score < map_elems[num]:
high_score = map_elems[num]
high_element = num
high_name = temp
if high_element != -1:
best_match_elem = qn_elems[high_element]
print('high_element:', high_element,
' high_name:', high_name,
' high_element:', high_element)
print('clicking')
best_match_elem.click()
print('done clicking')
time.sleep(5)
isin_elems = driver.find_elements_by_class_name(
'sal-mip-quote__symbol')
for isin_elem in isin_elems:
if isin_elem.text == fund.isin or (
fund.isin2
and isin_elem.text == fund.isin2):
if fund.isin2:
print(
f'matching fund found for {fund.isin} or {fund.isin2}'
)
else:
print(
f'matching fund found for {fund.isin}'
)
data = dict()
token, ms_code = get_token_and_ms_code(
driver)
if token != '' and ms_code != '':
set_ms_code_for_fund(
fund.id, ms_code)
data = use_api_to_get_vals(
token, ms_code)
else:
print(
f'not updated ms_code for {fund.name}'
)
is_elems = driver.find_elements_by_class_name(
'investment-style')
for is_elem in is_elems:
if is_elem.tag_name == 'svg':
print(
'is_elem',
is_elem.get_attribute(
'innerHTML'))
data[
'blend'] = is_elem.find_element_by_tag_name(
'title').text
blend_xpath = decode(
b'DJ\n\x16\x03_\x07\x04\x08\t\x18\x01\x17\n\x1f\x16T\x1a\x0b\x01I\x02\x06\x16\x1942\x05\x0f\x01\x10\x03\x06\x1d\n\n\tT\x14\x03\x13\x17\x18X[\x1e\x01\x04\x01\x18\x11\x14\x12\x01\x06I\x18\x11\x00\x1b\nP9'
)
blend_elems = driver.find_elements_by_xpath(
blend_xpath)
if len(blend_elems) == 1:
data['blend'] = blend_elems[
0].get_attribute("title")
'''
cat_elems = driver.find_elements_by_xpath("//div[contains(text(),'Category')]")
for cat_elem in cat_elems:
print('cat_elem class:', cat_elem.get_attribute('class'))
if 'sal-dp-name ng-binding' in cat_elem.get_attribute('class'):
parent_cat_elem = cat_elem.find_element_by_xpath('./..')
for elem in parent_cat_elem.find_elements_by_tag_name('div'):
if elem.text != 'Category':
data['categoryName'] = elem.text
'''
if data:
driver.close()
return data, token
grab_details(driver, data)
driver.close()
if not fund.ms_name or fund.ms_name != high_name.replace(
'&', '&'):
fund.ms_name = high_name.replace(
'&', '&')
fund.save()
return data, token
else:
print(isin_elem.text,
' not matched with isin:',
fund.isin, ' or isin2:',
fund.isin2)
time.sleep(10)
driver.close()
ignore_names.append(high_name)
return pull_ms(code, ignore_names)
else:
if not replaceAnd:
if mf_name_parts[i] == '&' or mf_name_parts[
i].lower() == 'and':
driver.close()
return pull_ms(code, ignore_names, True)
#driver.close()
#return None
else:
print('some other tag name', el.tag_name)
#submit_element.click()
driver.close()
return None, token
except TimeoutException:
driver.quit()
except ElementNotInteractableException:
driver.close()
return pull_ms(code, ignore_names, replaceAnd, token)
except Exception as ex:
print('Exception during processing', ex)
driver.close()
return None, token
def update_shares_latest_val():
start = datetime.date.today() + relativedelta(days=-5)
end = datetime.date.today()
share_objs = Share.objects.all()
for share_obj in share_objs:
if share_obj.quantity > 0:
latest_date = None
latest_val = None
if (share_obj.as_on_date and share_obj.as_on_date <
datetime.date.today()) or not share_obj.as_on_date:
vals = get_latest_vals(share_obj.symbol, share_obj.exchange,
start, end, share_obj.etf)
if vals:
for k, v in vals.items():
if k and v:
if not latest_date or k > latest_date:
latest_date = k
latest_val = v
else:
print(
f'Couldnt get latest value for {share_obj.symbol} {share_obj.exchange}'
)
if share_obj.exchange == 'BSE':
isin = None
for trans in Transactions.objects.filter(
share=share_obj):
isin = get_isin_from_bhav_copy(
share_obj.symbol, trans.trans_date)
if isin:
break
if isin:
nse_bse_data = get_nse_bse(None, None, isin)
if nse_bse_data and 'nse' in nse_bse_data:
print(
f'checking if share with symbol {nse_bse_data["nse"]} exists'
)
isin_objs = Share.objects.filter(
exchange='NSE/BSE',
symbol=nse_bse_data['nse'])
if len(isin_objs) == 1:
move_trans(share_obj, isin_objs[0], True)
continue
else:
nse_objs = Share.objects.filter(
exchange='NSE',
symbol=nse_bse_data['nse'])
if len(nse_objs) == 1:
move_trans(share_obj, nse_objs[0],
True)
nse_objs[0].exchange = 'NSE/BSE'
nse_objs[0].save()
continue
if 'bse' in nse_bse_data:
try:
bse_obj = Share.objects.get(
exchange='BSE',
symbol=nse_bse_data['bse'])
move_trans(share_obj, bse_obj, True)
except Share.DoesNotExist:
share_obj.symbol = nse_bse_data['bse']
share_obj.save()
continue
else:
print(
f'couldnt find isin for {share_obj.symbol} {share_obj.exchange} using transaction date bhav copy'
)
elif share_obj.exchange == 'NSE':
valid, _ = check_nse_valid(share_obj.symbol)
if not valid:
trans = Transactions.objects.filter(
share=share_obj).order_by('-trans_date')
if len(trans) > 0:
bhavcopy_symbol = share_obj.symbol
while True:
nh = NSEHistorical(bhavcopy_symbol,
trans[0].trans_date,
True)
isin = nh.get_isin_from_bhav_copy()
if isin:
n = NSE('')
symbol = n.get_symbol(isin)
if symbol:
nse_objs = Share.objects.filter(
exchange='NSE', symbol=symbol)
if len(nse_objs) == 1:
move_trans(
share_obj, nse_objs[0],
True)
else:
nse_objs = Share.objects.filter(
exchange='NSE/BSE',
symbol=symbol)
if len(nse_objs) == 1:
move_trans(
share_obj, nse_objs[0],
True)
else:
share_obj.symbol = symbol
share_obj.save()
break
else:
print(
f'Failed to get symbol from isin {isin}'
)
else:
print(
f'couldnt find isin for {share_obj.symbol} {share_obj.exchange} using transaction date bhav copy'
)
if '-' in bhavcopy_symbol:
bhavcopy_symbol = bhavcopy_symbol[
0:bhavcopy_symbol.rfind('-')]
else:
break
if latest_date and latest_val:
share_obj.as_on_date = latest_date
if share_obj.exchange == 'NASDAQ':
share_obj.conversion_rate = get_conversion_rate(
'USD', 'INR', k)
else:
share_obj.conversion_rate = 1
share_obj.latest_value = float(latest_val) * float(
share_obj.conversion_rate) * float(share_obj.quantity)
share_obj.latest_price = float(latest_val)
share_obj.save()
else:
create_alert(
summary=share_obj.exchange + ':' + share_obj.symbol +
' - Failed to get latest value',
content=share_obj.exchange + ':' + share_obj.symbol +
' - Failed to get latest value',
severity=Severity.warning,
alert_type="Action")
if share_obj.latest_value:
share_obj.gain = float(share_obj.latest_value) - float(
share_obj.buy_value)
share_obj.save()
elif share_obj.quantity == 0:
share_obj.latest_value = 0
share_obj.latest_price = 0
share_obj.gain = 0
share_obj.roi = 0
share_obj.as_on_date = datetime.date.today()
share_obj.save()
else:
print(f'ignoring {share_obj.symbol} with -ve quantity')
def insert_trans_entry(exchange,
symbol,
user,
trans_type,
quantity,
price,
date,
notes,
broker,
conversion_rate=1,
trans_price=None,
div_reinv=False):
try:
share_obj = None
try:
share_obj = Share.objects.get(exchange=exchange,
symbol=symbol,
user=user)
except Share.DoesNotExist:
if exchange == 'NSE' or exchange == 'BSE':
print("Couldnt find share object exchange:", exchange,
" symbol:", symbol)
nse_bse_data = None
if exchange == 'NSE':
nse_bse_data = get_nse_bse(symbol, None, None)
else:
nse_bse_data = get_nse_bse(None, symbol, None)
if not nse_bse_data:
isin = get_isin_from_bhav_copy(symbol, date)
if isin:
nse_bse_data = get_nse_bse(None, None, isin)
if nse_bse_data and 'nse' in nse_bse_data:
print(
f'checking if {symbol} with nse {nse_bse_data["nse"]} exists'
)
nse_objs = Share.objects.filter(exchange='NSE/BSE',
symbol=nse_bse_data['nse'],
user=user)
if len(nse_objs) == 1:
share_obj = nse_objs[0]
else:
nse_objs = Share.objects.filter(
exchange='NSE',
symbol=nse_bse_data['nse'],
user=user)
if len(nse_objs) == 1:
share_obj = nse_objs[0]
else:
print(
f'share with nse {nse_bse_data["nse"]} doesnt exist'
)
if not share_obj:
print(
f'creating share object {exchange} {symbol} for user {user}'
)
share_obj = Share.objects.create(exchange=exchange,
symbol=symbol,
user=user,
quantity=0,
buy_price=0,
buy_value=0,
gain=0,
realised_gain=0)
if not trans_price:
trans_price = price * quantity * conversion_rate
try:
Transactions.objects.create(share=share_obj,
trans_date=date,
trans_type=trans_type,
price=price,
quantity=quantity,
conversion_rate=conversion_rate,
trans_price=trans_price,
broker=broker,
notes=notes,
div_reinv=div_reinv)
reconcile_share(share_obj)
except IntegrityError:
print('Transaction exists')
except Exception as ex:
print(
f'failed to add transaction {exchange}, {symbol}, {user}, {trans_type}, {quantity}, {price}, {date}, {notes}, {broker}'
)
print(ex)
description = 'failed to add transaction for ' + exchange + ':' + symbol + ' for user ' + get_user_name_from_id(
user)
create_alert(summary=exchange + ':' + symbol +
' - Failed to add transaction',
content=description,
severity=Severity.warning,
alert_type="Action")
def pull_zerodha(userid, passwd, pin):
dload_files = list()
url = 'https://console.zerodha.com/reports/tradebook'
chrome_options = webdriver.ChromeOptions()
dload_path = pathlib.Path(__file__).parent.parent.absolute()
dload_path = os.path.join(dload_path, 'media')
prefs = {'download.default_directory': dload_path}
files_in_dload_dir = get_files_in_dir(dload_path)
chrome_options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome(executable_path=get_path_to_chrome_driver(),
chrome_options=chrome_options)
driver.get(url)
time.sleep(5)
print(driver.current_url)
try:
user_id_elem = driver.find_element_by_id('userid')
user_id_elem.send_keys(userid)
passwd_elem = driver.find_element_by_id('password')
passwd_elem.send_keys(passwd)
submit_button = driver.find_element_by_xpath(
'//button[text()="Login "]')
submit_button.click()
time.sleep(3)
pin_element = driver.find_element_by_id('pin')
pin_element.send_keys(pin)
submit_button = driver.find_element_by_xpath(
'//button[text()="Continue "]')
submit_button.click()
time.sleep(5)
if driver.current_url != url:
driver.get(url)
time.sleep(5)
except Exception as ex:
print(f'Exception {ex} while logging in')
driver.close()
return None
try:
time.sleep(5)
#from_date = datetime.date(year=2013,month=4,day=1)
yr = datetime.date.today().year
if datetime.date.today().month < 4:
yr -= 1
while True:
from_date = datetime.date(year=yr, month=4, day=1)
to_date = datetime.date(year=yr + 1, month=3, day=31)
if to_date > datetime.date.today():
to_date = datetime.date.today()
date_string = from_date.strftime(
"%Y-%m-%d") + " ~ " + to_date.strftime("%Y-%m-%d")
print(date_string)
#view_element = WebDriverWait(driver,20).until(EC.element_to_be_clickable((By.XPATH, '//button[text()[contains(.,"View")]]')))
view_element = WebDriverWait(driver, 20).until(
EC.element_to_be_clickable(
(By.XPATH,
'//button[@type="submit" and @class="btn-blue"]')))
date_range = driver.find_element_by_name('date')
date_range.clear()
time.sleep(5)
for _ in range(len(date_string)):
date_range.send_keys(Keys.BACKSPACE)
date_range.send_keys(date_string)
time.sleep(5)
view_element.click()
#WebDriverWait(driver,20).until(EC.element_to_be_clickable((By.XPATH, '//button[text()[contains(.,"View")]]')))
WebDriverWait(driver, 60).until(
EC.element_to_be_clickable(
(By.XPATH,
'//button[@type="submit" and @class="btn-blue"]')))
if len(
driver.find_elements_by_xpath(
"//a[text()[contains(.,'CSV')]]")) > 0:
dload_elem = driver.find_element_by_xpath(
"//a[text()[contains(.,'CSV')]]")
dload_elem.click()
dload_file = os.path.join(
dload_path,
userid + '_tradebook_' + from_date.strftime("%Y-%m-%d") +
"_to_" + to_date.strftime("%Y-%m-%d") + '.csv')
file_found = False
for _ in range(10):
if os.path.exists(dload_file):
print(
f'file found {dload_file}. Will parse for transactions.'
)
file_found = True
dload_files.append(dload_file)
break
time.sleep(1)
if not file_found:
new_file_list = get_new_files_added(
dload_path, files_in_dload_dir)
if len(new_file_list) == 1:
dload_files.append(new_file_list[0])
files_in_dload_dir.append(new_file_list[0])
print(
f'file found {new_file_list[0]}. Will parse for transactions.'
)
if len(new_file_list) > 1:
description = ''
for fil in new_file_list:
description = description + fil
create_alert(
summary='Failure to get transactions for ' +
userid + '. More than one file found',
content=description,
severity=Severity.error,
alert_type="Action")
else:
print(
f'couldnt download any transactions for period {date_string}'
)
if len(
driver.find_elements_by_xpath(
"//h3[@id='mainText' and @text='Something went wrong']"
)) > 0:
break
if len(driver.find_elements_by_xpath(
"//h3[@id='mainText']")) > 0:
h3_elem = driver.find_element_by_xpath(
"//h3[@id='mainText']")
print(f'h3 element found with text {h3_elem.text}')
src = driver.page_source
text_found = re.search(r'Something went wrong', src)
if text_found:
break
yr = yr - 1
userid_elem = driver.find_element_by_xpath(
"//a[@class='dropdown-label user-id']")
userid_elem.click()
logout_elem = driver.find_element_by_xpath(
"//a[text()[contains(.,'Logout')]]")
logout_elem.click()
time.sleep(5)
driver.close()
return dload_files
except TimeoutException as t:
print('timeout waiting', t)
driver.close()
return None
except Exception as ex:
print('Exception during processing', ex)
driver.close()
return None
def get_transactions(self):
if isfile(self.filename):
data = casparser.read_cas_pdf(self.filename, self.passwd)
print(data)
if data['cas_type'] != 'DETAILED':
print(
f'failed to add mutual fund transactions since document is not detailed'
)
return list()
for folio in data['folios']:
folio_num = folio['folio'].replace(' ', '')
if folio_num.endswith('/0'):
folio_num = folio_num.replace('/0', '')
for scheme in folio['schemes']:
if scheme['advisor'] == 'INA200005166' or scheme[
'advisor'] == '000000-0':
broker = 'KUVERA'
elif scheme['advisor'] == 'INZ000031633':
broker = 'COIN ZERODHA'
elif scheme['advisor'] == 'INA200011107':
broker = 'SCRIPBOX'
elif scheme['advisor'] == 'INA100009859':
broker = 'PAYTM MONEY'
elif scheme['advisor'] == 'INA000006651':
broker = 'NIYO MONEY'
elif scheme['advisor'] == 'INZ000208032':
broker = 'GROWW'
elif scheme['advisor'] == 'INA100006898':
broker = 'ET MONEY'
else:
broker = scheme['advisor']
isin = scheme['isin']
amfi_code = scheme['amfi']
fund, description = self._get_fund(isin, folio_num)
if not fund:
create_alert(summary='Folio:' + folio_num +
' Failure to add transactions',
content=description,
severity=Severity.error,
alert_type="Action")
continue
for trans in scheme['transactions']:
if 'tax' in trans['type'].lower():
continue
trans_date = trans['date']
if 'redemption' in trans['type'].lower():
trans_type = 'Sell'
elif 'purchase' in trans['type'].lower():
trans_type = 'Buy'
else:
print(
f"ignoring transaction of type {trans['type']}"
)
units = trans['units']
nav = trans['nav']
trans_value = trans['amount']
yield {
'folio': folio_num,
'trans_date': trans_date,
'fund': fund,
'trans_type': trans_type,
'units': float(units),
'nav': float(nav),
'trans_value': float(trans_value),
'broker': broker
}
else:
print(f'{self.filename} is not a file or doesnt exist')