def start_process():
cats = Scheme.get_fund_categorization()
for key in cats:
for row in cats[key]:
print(row['Text'], "xxx", row['Value'])
today = datetime.date.today()
download_data(row["Value"], today.month, today.year)
def process_ter_history():
cats = Scheme.get_fund_categorization()
for month in range(0, 24):
today = datetime.date.today() - datetime.timedelta(days=month * 30)
print(today)
for key in cats:
for row in cats[key]:
print(row['Text'], "xxx", row['Value'])
download_data(row["Value"], today.month, today.year)
def process_aum_history():
cats = Scheme.get_fund_categorization()
Scheme_Name_Mismatch.objects.all().delete()
for month in range(0, 24):
today = datetime.date.today() - datetime.timedelta(days=month * 30)
print(today)
for key in cats:
for row in cats[key]:
print(row['Text'], "xxx", row['Value'])
download_data(row["Value"], today, key, row["Text"])
def get_fund_categories(request):
cats = Scheme.get_fund_categorization()
types = []
subtypes = []
for key in cats:
types.append(key)
for row in cats[key]:
subtypes.append(row["Text"])
# ret = Scheme.objects.get_category_types()
# return Response(ret)
return Response(types)
def get_fund_subcategories(request, stype):
# ret = Scheme.objects.get_sub_category_types(type)
# return Response(ret)
cats = Scheme.get_fund_categorization()
subtypes = []
for key in cats:
if key == stype:
for row in cats[key]:
subtypes.append(row["Text"])
return Response(subtypes)
def start_process():
# fund = Scheme.find_fund_with_name("Tata India Pharma & Healthcare Fund")
# print(fund)
# return
cats = Scheme.get_fund_categorization()
Scheme_Name_Mismatch.objects.all().delete()
for key in cats:
for row in cats[key]:
#print(row['Text'], "xxx", row['Value'])
today = datetime.date.today() - datetime.timedelta(days=1)
#print(row["Value"], today, key, row["Text"])
download_data(row["Value"], today, key, row["Text"])
def get_funds_without_category_or_sub_category(request):
cats = Scheme.get_fund_categorization()
types = []
subtypes = []
for key in cats:
types.append(key)
for row in cats[key]:
subtypes.append(row["Text"])
# print(row['Text'], "xxx", row['Value'])
ret = Scheme.objects.exclude(
Q(scheme_type__in=types) or Q(scheme_sub_type__in=subtypes))
ser = SchemeSerializer(ret, many=True)
return Response(ser.data)
def download_data(cat_desc, month, year):
response = requests.post("https://www.amfiindia.com/modules/LoadTERData",
data={
"MonthTER": str(month) + "-" + str(year),
"MF_ID": "-1",
"NAV_ID": "1",
"SchemeCat_Desc": cat_desc
})
soup = BeautifulSoup(response.text, 'html.parser')
data = soup.find("table")
ter_data = []
for row in data.findAll('tr'):
col = row.findAll('td')
if len(col) > 1:
# there is an issue here, the current data which we get back
# has data for multiple date of same month. which is not needed.
# we just need to one data point for a month which is the latest
scheme_name = col[0].string
date = col[1].string
ter = col[-1].string
date = datetime.datetime.strptime(date, "%d-%b-%Y")
# print(scheme_name, "xxx", date, "xxx", ter)
ter_data.append([scheme_name, date, ter])
df = pd.DataFrame(ter_data, columns=['Scheme', 'Date', "Ter"])
df = df.drop_duplicates(subset="Scheme", keep="last")
print(df)
for row in df.itertuples():
scheme_name = row.Scheme
date = row.Date
ter = row.Ter
scheme = scheme = Scheme.find_fund_with_name(scheme_name)
if scheme:
print("scheme found ", scheme_name)
ters = Scheme_TER.objects.filter(scheme=scheme,
date__year=date.year,
date__month=date.month)
ters.delete()
tr_db = Scheme_TER(scheme=scheme, date=date, ter=ter)
tr_db.save()
else:
print(Fore.RED + "scheme not found with name ", scheme_name)
# break
# switch to amfi website as it has all amc's in a single location
# below code is all old not used any more.
# no point in parsing xml files when amfi has data already
return
# process_zip_file()
# process_file()
# move_files_from_folder_to_parent()
# Scheme_TER_Process
pass
def new_download_data(cat_desc, date, scheme_category, scheme_sub_category):
#print(date.strftime("%d-%b-%Y"))
date_det = date.strftime("%d-%b-%Y")
url = "https://www.valueresearchonline.com/amfi/fund-performance-data/?end-type=1&primary-category=" + str(
scheme_category) + "&category=" + str(
cat_desc) + "&amc=ALL&nav-date=" + str(date_det) + ""
response = requests.get(url=url)
soup = BeautifulSoup(response.text, 'html.parser')
data = soup.find("table")
fund_data = []
for row in data.findAll('tr'):
col = row.findAll('td')
if len(col) > 8:
amc_name = "NA"
scheme_name = col[0].text
benchmark = col[1].string
inception_date = datetime.date.today() - datetime.timedelta(days=1)
aum_direct = col[22].string
if "regular" in scheme_name.lower():
continue
fund_data.append([
amc_name, scheme_name, benchmark, inception_date, aum_direct,
date
])
df = pd.DataFrame(
fund_data,
columns=['AMC', 'Scheme', "Benchmark", "Inception", "AUM", "Date"])
#print(df)
for row in df.itertuples():
scheme_name = row.Scheme
date = row.Date
aum = row.AUM
amc = row.AMC
benchmark = row.Benchmark
inception = row.Inception
scheme = Scheme.find_fund_with_name(scheme_name)
if aum != "NA":
if scheme:
Scheme.objects.filter(pk=scheme.id).update(
scheme_type=scheme_category,
scheme_sub_type=scheme_sub_category)
#print("scheme found ", scheme_name)
ters = Scheme_AUM.objects.filter(scheme=scheme,
date__year=date.year,
date__month=date.month)
ters.delete()
tr_db = Scheme_AUM(scheme=scheme, date=date, aum=aum)
#print("saving scheme",scheme)
tr_db.save()
info = Scheme_Info.objects.filter(scheme=scheme)
info.delete()
info = Scheme_Info(scheme=scheme,
benchmark=benchmark,
inception=inception)
info.save()
info = Scheme_Name_Mismatch.objects.filter(name=scheme_name)
info.delete()
else:
print(Fore.RED + "scheme not found with name ", scheme_name)
info = Scheme_Name_Mismatch.objects.filter(name=scheme_name)
info.delete()
info = Scheme_Name_Mismatch(amc=amc,
name=scheme_name,
category=scheme_category,
subcategory=scheme_sub_category,
inception=inception,
aum=aum)
info.save()
else:
print(Fore.RED + "Aum not found with name ", scheme_name)
def download_data(cat_desc, date, scheme_category, scheme_sub_category):
#print(date.strftime("%d-%b-%Y"))
print({
"SchemeMonth": date.strftime("%d-%b-%Y"),
"MF_ID": "-1",
"NAV_ID": "1",
"SchemeCat_Desc": cat_desc
})
response = requests.post(
"https://www.amfiindia.com/modules/LoadMFPerformaceData",
data={
"SchemeMonth": date.strftime("%d-%b-%Y"),
"MF_ID": "-1",
"NAV_ID": "1",
"SchemeCat_Desc": cat_desc
})
#print(response.url)
soup = BeautifulSoup(response.text, 'html.parser')
data = soup.find("table")
fund_data = []
for row in data.findAll('tr'):
col = row.findAll('td')
if len(col) > 8:
# there is an issue here, the current data which we get back
# has data for multiple date of same month. which is not needed.
# we just need to one data point for a month which is the latest
amc_name = col[0].string
scheme_name = col[1].string
benchmark = col[2].string
inception_date = col[4].string
aum_direct = col[8].string
if "regular" in scheme_name.lower():
continue
if inception_date == "NA":
# this means plan doesn't existing for Direct investores
# maybe use this info later
continue
fund_data.append([
amc_name, scheme_name, benchmark, inception_date, aum_direct,
date
])
df = pd.DataFrame(
fund_data,
columns=['AMC', 'Scheme', "Benchmark", "Inception", "AUM", "Date"])
#print(df)
for row in df.itertuples():
scheme_name = row.Scheme
date = row.Date
aum = row.AUM
amc = row.AMC
benchmark = row.Benchmark
inception = row.Inception
scheme = Scheme.find_fund_with_name(scheme_name)
if scheme:
Scheme.objects.filter(pk=scheme.id).update(
scheme_type=scheme_category,
scheme_sub_type=scheme_sub_category)
#print("scheme found ", scheme_name)
ters = Scheme_AUM.objects.filter(scheme=scheme,
date__year=date.year,
date__month=date.month)
ters.delete()
#print("scheme data",scheme,date,aum)
tr_db = Scheme_AUM(scheme=scheme, date=date, aum=aum)
print("saving scheme", scheme)
tr_db.save()
info = Scheme_Info.objects.filter(scheme=scheme)
info.delete()
info = Scheme_Info(scheme=scheme,
benchmark=benchmark,
inception=datetime.datetime.strptime(
inception, "%d-%b-%Y"))
info.save()
else:
print(Fore.RED + "scheme not found with name ", scheme_name)
info = Scheme_Name_Mismatch.objects.filter(name=scheme_name)
info.delete()
info = Scheme_Name_Mismatch(amc=amc,
name=scheme_name,
category=scheme_category,
subcategory=scheme_sub_category,
inception=inception,
aum=aum)
info.save()