def reload_funddata_taskhandler(request):
'''
reload task should delete all existing entity and then start a download all task
'''
try:
response = HttpResponse('reload_funddata_taskhandler')
p_fund_id = str(request.REQUEST['PARAM1'])
t_fund = FundClearInfoModel.get_fund(p_fund_id)
for t_data in FundClearDataModel.all().ancestor(t_fund):
logging.debug(
'reload_funddata_taskhandler: remove FundClearDataModel key {}'
.format(t_data.key().name()))
t_data.delete()
taskqueue.add(
method='GET',
url=get_update_funddata_taskhandler_url(p_fund_id=p_fund_id,
p_type='all'),
countdown=3,
)
logging.debug(
'reload_funddata_taskhandler: add update task for {}'.format(
p_fund_id))
response.status_code = httplib.OK
return response
except Exception, e:
err_msg = '{}'.format(e)
logging.error('reload_funddata_taskhandler ERROR: {}'.format(err_msg))
response.status_code = httplib.INTERNAL_SERVER_ERROR
return response
def datamodel_statistic_report_view(request):
#response = HttpResponse(content_type='text/plain')
response = HttpResponse()
content = '<table border="1px solid black">'
codename_list = FundCodeModel.get_codename_list()
for t_code, t_name in codename_list[:]:
ancestor = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_code))
if ancestor is None:
t_count = 0
t_years = ''
else:
t_funddata_query = FundClearDataModel.all().ancestor(
ancestor).order('year')
t_count = 0
t_years = ''
for t_funddata in t_funddata_query:
t_count += 1
if t_funddata.year is None:
t_years += '?,'
else:
t_years += t_funddata.year + ', '
t_report = '<tr><td width="10%" align="right">{}</td><td width="5%" align="right">{}</td><td width="50%">{}</td><td>{}</td></tr>\n'.format(
t_code,
t_count,
t_name,
t_years,
)
content += t_report
content += '</table>'
response.content = content
return response
def get_analysis(p_fund_code):
#TODO: return {key:value} dict
'''
not_in_fundclear_list
year_nav_count
has_discontinuous_year
'''
t_dict = {
'not_in_fundclear_list': False,
'year_nav_count': 0,
'zero_year_nav': False,
'has_discontinuous_year': False,
'not_update_yet': False,
'year_list': 'N/A'
}
#-> not_in_fundclear_list
fundclear_code_list = FundCodeModel.get_code_list()
if not p_fund_code in fundclear_code_list:
t_dict['not_in_fundclear_list'] = True
return t_dict
#->year_nav_count & has_discontinuous_year & year_list
t_fund = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(p_fund_code))
if t_fund is None:
t_dict['not_update_yet'] = True
return t_dict
data_query = FundClearDataModel.all().ancestor(t_fund).order('-year')
check_year = date.today().year
this_year = str(check_year)
t_year_list = ''
for t_data in data_query:
if t_data.year == this_year:
nav_dict = t_data._get_nav_dict()
t_dict['year_nav_count'] = len(nav_dict)
if t_dict['year_nav_count'] == 0:
t_dict['zero_year_nav'] = True
t_year_list += '{}, '.format(t_data.year)
if str(check_year) != t_data.year:
t_dict['has_discontinuous_year'] = True
else:
check_year -= 1
t_dict['year_list'] = t_year_list
return t_dict
def get_year_nav_fund_list():
#TODO: return a list of fund nav number for this year
codename_all = FundCodeModel.get_codename_list()
this_year = date.today().year
fund_list = []
for t_entry in codename_all[:10]:
t_fund_id = t_entry[0]
t_fundinfo = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_fund_id))
if t_fundinfo is None:
fund_list.append(t_entry + [0])
else:
t_funddata = FundClearDataModel.all().ancestor(t_fundinfo).filter(
'year', str(this_year)).get()
if t_funddata is None:
fund_list.append(t_entry + [0])
else:
t_nav_dict = t_funddata._get_nav_dict()
fund_list.append(t_entry + [len(t_nav_dict)])
return fund_list
def get_zero_nav_fund_list():
#TODO: return a list of fund which has no any nav for this year
codename_all = FundCodeModel.get_codename_list()
this_year = date.today().year
zero_fund_list = []
for t_entry in codename_all[:20]:
t_fund_id = t_entry[0]
t_fundinfo = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_fund_id))
if t_fundinfo is None:
zero_fund_list.append(t_entry)
else:
t_funddata = FundClearDataModel.all().ancestor(t_fundinfo).filter(
'year', str(this_year)).get()
if t_funddata is None:
zero_fund_list.append(t_entry)
else:
t_nav_dict = t_funddata._get_nav_dict()
if len(t_nav_dict) == 0:
zero_fund_list.append(t_entry)
return zero_fund_list
def db_scan_taskhandler(request):
'''
check each funddata in datastore for error, such as
> not_in_fundclear_list
> zero_year_nav
> has_discontinuous_year
> not_update_yet
'''
#TODO:
#response = HttpResponse(content_type='text/plain')
try:
response = HttpResponse('db_scan_taskhandler')
scan_dict_mem = memcache.get(KEY_FUND_SCAN)
if scan_dict_mem:
scan_dict = pickle.loads(scan_dict_mem)
all_fund_in_db = FundClearInfoModel.all()
fund_cursor = memcache.get(KEY_FUND_CURSOR)
if fund_cursor:
all_fund_in_db.with_cursor(start_cursor=fund_cursor)
logging.debug(
'db_scan_taskhandler: with cursor {}'.format(fund_cursor))
else:
scan_dict = {
'not_in_fundclear_list': [], #-> remove from db or fund closed
'zero_year_nav': [], #-> fund closed
'has_discontinuous_year': [], #-> need update
'not_update_yet': [], #-> need update
}
code_list_all = FundCodeModel.get_code_list()
t_content = ''
t_fetch_fund_list = all_fund_in_db.fetch(limit=FUND_DB_SCAN_SIZE)
for t_fund in t_fetch_fund_list:
t_code = t_fund.key().name()
t_content += '{} <br/>\n'.format(t_code)
#->not_in_fundclear_list
if not t_code in code_list_all:
if not t_code in scan_dict['not_in_fundclear_list']:
scan_dict['not_in_fundclear_list'].append(t_code)
logging.info(
'db_scan_taskhandler: {} not in fundclear list'.format(
t_code))
#-> zero_year_nav & has_discontinuous_year
data_query = FundClearDataModel.all().ancestor(t_fund).order(
'-year')
check_year = date.today().year
this_year = str(check_year)
t_year_list = ''
for t_data in data_query:
if t_data.year == this_year:
nav_dict = t_data._get_nav_dict()
if len(nav_dict) == 0:
if not t_code in scan_dict['zero_year_nav']:
scan_dict['zero_year_nav'].append(t_code)
logging.info(
'db_scan_taskhandler: {} has zero nav in year {}'
.format(t_code, this_year))
t_year_list += '{}, '.format(t_data.year)
if str(check_year) != t_data.year:
if not t_code in scan_dict['has_discontinuous_year']:
scan_dict['has_discontinuous_year'].append(t_code)
logging.info(
'db_scan_taskhandler: {} year {} with check {} not equal; {}'
.format(t_code, t_data.year, check_year,
t_year_list))
break
else:
check_year -= 1
if len(t_fetch_fund_list) == FUND_DB_SCAN_SIZE:
fund_cursor = all_fund_in_db.cursor()
memcache.set(KEY_FUND_CURSOR, fund_cursor)
t_content += 'cursor: {}<br/>\n'.format(fund_cursor)
t_content += '<a href="{}">next page</a>'.format(
get_db_scan_taskhandler_url())
taskqueue.add(
method='GET',
url=get_db_scan_taskhandler_url(),
countdown=3,
)
else:
memcache.delete(KEY_FUND_CURSOR)
#-> not_update_yet
for t_code in code_list_all:
t_fund = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_code))
if t_fund is None:
if not t_code in scan_dict['not_update_yet']:
scan_dict['not_update_yet'].append(t_code)
logging.info(
'db_scan_taskhandler: {} not update yet'.format(
t_code))
t_content += 'End of Scan<br/>\n'
logging.info(
'db_scan_taskhandler: end of scan, result:\n{}'.format(
str(scan_dict)))
memcache.set(KEY_FUND_SCAN, pickle.dumps(scan_dict))
response.content = t_content
response.status_code = httplib.OK
return response
except Exception, e:
err_msg = '{}'.format(e)
logging.error('db_scan_taskhandler: err {}'.format(err_msg))
response.status_code = httplib.OK
return response