def fund_nav_str_view(request, p_fund_id, p_year):
try:
t_fund = FundClearInfoModel.get_fund(p_fund_id)
t_fdata = FundClearDataModel.get_by_key_name(
FundClearDataModel.compose_key_name(p_fund_id, p_year), t_fund)
nav_dict = t_fdata._get_nav_dict()
nav_dict = collections.OrderedDict(sorted(nav_dict.items()))
nav_info = ''
for t_key, t_entry in nav_dict.items():
nav_info += '{}:{}<br>\n'.format(t_key, t_entry)
next_year = int(p_year) - 1
t_fdata = FundClearDataModel.get_by_key_name(
FundClearDataModel.compose_key_name(p_fund_id, next_year), t_fund)
if t_fdata is None or len(t_fdata._get_nav_dict()) == 0:
next_year_link = 'NAV End'
else:
next_year_link = '<a href="/mf/fc/nav_str/{}/{}/">{}</a>'.format(
p_fund_id, next_year, 'next')
args = {
'tpl_section_title':
'fund {} year {}, {}'.format(p_fund_id, p_year, next_year_link),
'tpl_info':
nav_info,
}
return render_to_response('mf_simple_info.tpl.html', args)
except Exception, e:
err_msg = 'fund_nav_str_view ERROR: {}'.format(e)
logging.error(err_msg)
args = {
'tpl_info': err_msg,
}
return render_to_response('mf_simple_info.tpl.html', args)
def get(self, request, p_fund_id, p_year, *args, **kwargs):
t_fund = FundClearInfoModel.get_fund(p_fund_id)
t_fdata = FundClearDataModel.get_by_key_name(
FundClearDataModel.compose_key_name(p_fund_id, p_year), t_fund)
nav_dict = t_fdata._get_nav_dict()
nav_dict = collections.OrderedDict(sorted(nav_dict.items()))
json_nav = [[key, nav_dict[key][1]] for key in nav_dict]
return HttpResponse(json.dumps(json_nav, indent=2))
def update_funddata_taskhandler(request):
'''
function request 3 param: fund_id, download_year, update_type
if update_type == 'all', it will update fund nav before download_year
'''
p_fund_id = str(request.REQUEST['PARAM1'])
if 'PARAM2' in request.REQUEST.keys():
p_year = str(request.REQUEST['PARAM2'])
else:
p_year = date.today().year
if 'PARAM3' in request.REQUEST.keys():
p_type = str(request.REQUEST['PARAM3'])
else:
p_type = ''
logging.info(
'update_funddata_taskhandler: fund id:{fund_id}, year:{year}, type:{type}'
.format(fund_id=p_fund_id, year=p_year, type=p_type))
response = HttpResponse('update_funddata_taskhandler')
if FundClearDataModel._update_from_web(p_fund_id, p_year):
response.status_code = httplib.OK
logging.info('update_funddata_taskhandler success')
if p_type == 'all':
funddata = FundClearDataModel._get_funddata(p_fund_id, p_year)
nav_item_count = len(funddata._get_nav_dict())
logging.info(
'update_funddata_taskhandler: nav item count {count}'.format(
count=nav_item_count))
if nav_item_count == 0:
logging.info(
'update_funddata_taskhandler: year {year} nav_dict len 0, end update task.'
.format(year=p_year))
else:
logging.info(
'update_funddata_taskhandler: continue update with year {year} by adding new update task'
.format(year=str(int(p_year) - 1)))
taskqueue.add(method='GET',
url=os.path.dirname(request.get_full_path()) +
'/',
countdown=5,
params={
'PARAM1': p_fund_id,
'PARAM2': int(p_year) - 1,
'PARAM3': p_type,
})
return response
else:
response.status_code = httplib.INTERNAL_SERVER_ERROR
logging.warning('update_funddata_taskhandler fail!')
return response
def post(self, request, p_fund_id, p_year, *args, **kwargs):
csv_content = request.POST.get('csv_content', '').encode('utf-8')
fund_title = request.POST.get('fund_title', '')
if csv_content == '' or fund_title == '':
return HttpResponseBadRequest('Param Error\nfund_title length%d' %
(len(fund_title)))
FundClearDataModel.save_fund_csv_data(p_fund_id, p_year, fund_title,
csv_content)
return HttpResponse('OK')
def reload_funddata_taskhandler(request):
'''
reload task should delete all existing entity and then start a download all task
'''
try:
response = HttpResponse('reload_funddata_taskhandler')
p_fund_id = str(request.REQUEST['PARAM1'])
t_fund = FundClearInfoModel.get_fund(p_fund_id)
for t_data in FundClearDataModel.all().ancestor(t_fund):
logging.debug(
'reload_funddata_taskhandler: remove FundClearDataModel key {}'
.format(t_data.key().name()))
t_data.delete()
taskqueue.add(
method='GET',
url=get_update_funddata_taskhandler_url(p_fund_id=p_fund_id,
p_type='all'),
countdown=3,
)
logging.debug(
'reload_funddata_taskhandler: add update task for {}'.format(
p_fund_id))
response.status_code = httplib.OK
return response
except Exception, e:
err_msg = '{}'.format(e)
logging.error('reload_funddata_taskhandler ERROR: {}'.format(err_msg))
response.status_code = httplib.INTERNAL_SERVER_ERROR
return response
def datamodel_statistic_report_view(request):
#response = HttpResponse(content_type='text/plain')
response = HttpResponse()
content = '<table border="1px solid black">'
codename_list = FundCodeModel.get_codename_list()
for t_code, t_name in codename_list[:]:
ancestor = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_code))
if ancestor is None:
t_count = 0
t_years = ''
else:
t_funddata_query = FundClearDataModel.all().ancestor(
ancestor).order('year')
t_count = 0
t_years = ''
for t_funddata in t_funddata_query:
t_count += 1
if t_funddata.year is None:
t_years += '?,'
else:
t_years += t_funddata.year + ', '
t_report = '<tr><td width="10%" align="right">{}</td><td width="5%" align="right">{}</td><td width="50%">{}</td><td>{}</td></tr>\n'.format(
t_code,
t_count,
t_name,
t_years,
)
content += t_report
content += '</table>'
response.content = content
return response
def get_analysis(p_fund_code):
#TODO: return {key:value} dict
'''
not_in_fundclear_list
year_nav_count
has_discontinuous_year
'''
t_dict = {
'not_in_fundclear_list': False,
'year_nav_count': 0,
'zero_year_nav': False,
'has_discontinuous_year': False,
'not_update_yet': False,
'year_list': 'N/A'
}
#-> not_in_fundclear_list
fundclear_code_list = FundCodeModel.get_code_list()
if not p_fund_code in fundclear_code_list:
t_dict['not_in_fundclear_list'] = True
return t_dict
#->year_nav_count & has_discontinuous_year & year_list
t_fund = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(p_fund_code))
if t_fund is None:
t_dict['not_update_yet'] = True
return t_dict
data_query = FundClearDataModel.all().ancestor(t_fund).order('-year')
check_year = date.today().year
this_year = str(check_year)
t_year_list = ''
for t_data in data_query:
if t_data.year == this_year:
nav_dict = t_data._get_nav_dict()
t_dict['year_nav_count'] = len(nav_dict)
if t_dict['year_nav_count'] == 0:
t_dict['zero_year_nav'] = True
t_year_list += '{}, '.format(t_data.year)
if str(check_year) != t_data.year:
t_dict['has_discontinuous_year'] = True
else:
check_year -= 1
t_dict['year_list'] = t_year_list
return t_dict
def get_year_nav_fund_list():
#TODO: return a list of fund nav number for this year
codename_all = FundCodeModel.get_codename_list()
this_year = date.today().year
fund_list = []
for t_entry in codename_all[:10]:
t_fund_id = t_entry[0]
t_fundinfo = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_fund_id))
if t_fundinfo is None:
fund_list.append(t_entry + [0])
else:
t_funddata = FundClearDataModel.all().ancestor(t_fundinfo).filter(
'year', str(this_year)).get()
if t_funddata is None:
fund_list.append(t_entry + [0])
else:
t_nav_dict = t_funddata._get_nav_dict()
fund_list.append(t_entry + [len(t_nav_dict)])
return fund_list
def get_zero_nav_fund_list():
#TODO: return a list of fund which has no any nav for this year
codename_all = FundCodeModel.get_codename_list()
this_year = date.today().year
zero_fund_list = []
for t_entry in codename_all[:20]:
t_fund_id = t_entry[0]
t_fundinfo = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_fund_id))
if t_fundinfo is None:
zero_fund_list.append(t_entry)
else:
t_funddata = FundClearDataModel.all().ancestor(t_fundinfo).filter(
'year', str(this_year)).get()
if t_funddata is None:
zero_fund_list.append(t_entry)
else:
t_nav_dict = t_funddata._get_nav_dict()
if len(t_nav_dict) == 0:
zero_fund_list.append(t_entry)
return zero_fund_list
def db_scan_taskhandler(request):
'''
check each funddata in datastore for error, such as
> not_in_fundclear_list
> zero_year_nav
> has_discontinuous_year
> not_update_yet
'''
#TODO:
#response = HttpResponse(content_type='text/plain')
try:
response = HttpResponse('db_scan_taskhandler')
scan_dict_mem = memcache.get(KEY_FUND_SCAN)
if scan_dict_mem:
scan_dict = pickle.loads(scan_dict_mem)
all_fund_in_db = FundClearInfoModel.all()
fund_cursor = memcache.get(KEY_FUND_CURSOR)
if fund_cursor:
all_fund_in_db.with_cursor(start_cursor=fund_cursor)
logging.debug(
'db_scan_taskhandler: with cursor {}'.format(fund_cursor))
else:
scan_dict = {
'not_in_fundclear_list': [], #-> remove from db or fund closed
'zero_year_nav': [], #-> fund closed
'has_discontinuous_year': [], #-> need update
'not_update_yet': [], #-> need update
}
code_list_all = FundCodeModel.get_code_list()
t_content = ''
t_fetch_fund_list = all_fund_in_db.fetch(limit=FUND_DB_SCAN_SIZE)
for t_fund in t_fetch_fund_list:
t_code = t_fund.key().name()
t_content += '{} <br/>\n'.format(t_code)
#->not_in_fundclear_list
if not t_code in code_list_all:
if not t_code in scan_dict['not_in_fundclear_list']:
scan_dict['not_in_fundclear_list'].append(t_code)
logging.info(
'db_scan_taskhandler: {} not in fundclear list'.format(
t_code))
#-> zero_year_nav & has_discontinuous_year
data_query = FundClearDataModel.all().ancestor(t_fund).order(
'-year')
check_year = date.today().year
this_year = str(check_year)
t_year_list = ''
for t_data in data_query:
if t_data.year == this_year:
nav_dict = t_data._get_nav_dict()
if len(nav_dict) == 0:
if not t_code in scan_dict['zero_year_nav']:
scan_dict['zero_year_nav'].append(t_code)
logging.info(
'db_scan_taskhandler: {} has zero nav in year {}'
.format(t_code, this_year))
t_year_list += '{}, '.format(t_data.year)
if str(check_year) != t_data.year:
if not t_code in scan_dict['has_discontinuous_year']:
scan_dict['has_discontinuous_year'].append(t_code)
logging.info(
'db_scan_taskhandler: {} year {} with check {} not equal; {}'
.format(t_code, t_data.year, check_year,
t_year_list))
break
else:
check_year -= 1
if len(t_fetch_fund_list) == FUND_DB_SCAN_SIZE:
fund_cursor = all_fund_in_db.cursor()
memcache.set(KEY_FUND_CURSOR, fund_cursor)
t_content += 'cursor: {}<br/>\n'.format(fund_cursor)
t_content += '<a href="{}">next page</a>'.format(
get_db_scan_taskhandler_url())
taskqueue.add(
method='GET',
url=get_db_scan_taskhandler_url(),
countdown=3,
)
else:
memcache.delete(KEY_FUND_CURSOR)
#-> not_update_yet
for t_code in code_list_all:
t_fund = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_code))
if t_fund is None:
if not t_code in scan_dict['not_update_yet']:
scan_dict['not_update_yet'].append(t_code)
logging.info(
'db_scan_taskhandler: {} not update yet'.format(
t_code))
t_content += 'End of Scan<br/>\n'
logging.info(
'db_scan_taskhandler: end of scan, result:\n{}'.format(
str(scan_dict)))
memcache.set(KEY_FUND_SCAN, pickle.dumps(scan_dict))
response.content = t_content
response.status_code = httplib.OK
return response
except Exception, e:
err_msg = '{}'.format(e)
logging.error('db_scan_taskhandler: err {}'.format(err_msg))
response.status_code = httplib.OK
return response