def get_db_err_fund_id_list():
fund_id_list = []
fundclear_id_list = FundCodeModel.get_code_list()
fund_query = FundClearInfoModel.all()
for t_fund in fund_query:
t_id = t_fund.key().name()
if not t_id in fundclear_id_list:
fund_id_list.append(t_id)
return fund_id_list
def db_scan_taskhandler(request):
'''
check each funddata in datastore for error, such as
> not_in_fundclear_list
> zero_year_nav
> has_discontinuous_year
> not_update_yet
'''
#TODO:
#response = HttpResponse(content_type='text/plain')
try:
response = HttpResponse('db_scan_taskhandler')
scan_dict_mem = memcache.get(KEY_FUND_SCAN)
if scan_dict_mem:
scan_dict = pickle.loads(scan_dict_mem)
all_fund_in_db = FundClearInfoModel.all()
fund_cursor = memcache.get(KEY_FUND_CURSOR)
if fund_cursor:
all_fund_in_db.with_cursor(start_cursor=fund_cursor)
logging.debug(
'db_scan_taskhandler: with cursor {}'.format(fund_cursor))
else:
scan_dict = {
'not_in_fundclear_list': [], #-> remove from db or fund closed
'zero_year_nav': [], #-> fund closed
'has_discontinuous_year': [], #-> need update
'not_update_yet': [], #-> need update
}
code_list_all = FundCodeModel.get_code_list()
t_content = ''
t_fetch_fund_list = all_fund_in_db.fetch(limit=FUND_DB_SCAN_SIZE)
for t_fund in t_fetch_fund_list:
t_code = t_fund.key().name()
t_content += '{} <br/>\n'.format(t_code)
#->not_in_fundclear_list
if not t_code in code_list_all:
if not t_code in scan_dict['not_in_fundclear_list']:
scan_dict['not_in_fundclear_list'].append(t_code)
logging.info(
'db_scan_taskhandler: {} not in fundclear list'.format(
t_code))
#-> zero_year_nav & has_discontinuous_year
data_query = FundClearDataModel.all().ancestor(t_fund).order(
'-year')
check_year = date.today().year
this_year = str(check_year)
t_year_list = ''
for t_data in data_query:
if t_data.year == this_year:
nav_dict = t_data._get_nav_dict()
if len(nav_dict) == 0:
if not t_code in scan_dict['zero_year_nav']:
scan_dict['zero_year_nav'].append(t_code)
logging.info(
'db_scan_taskhandler: {} has zero nav in year {}'
.format(t_code, this_year))
t_year_list += '{}, '.format(t_data.year)
if str(check_year) != t_data.year:
if not t_code in scan_dict['has_discontinuous_year']:
scan_dict['has_discontinuous_year'].append(t_code)
logging.info(
'db_scan_taskhandler: {} year {} with check {} not equal; {}'
.format(t_code, t_data.year, check_year,
t_year_list))
break
else:
check_year -= 1
if len(t_fetch_fund_list) == FUND_DB_SCAN_SIZE:
fund_cursor = all_fund_in_db.cursor()
memcache.set(KEY_FUND_CURSOR, fund_cursor)
t_content += 'cursor: {}<br/>\n'.format(fund_cursor)
t_content += '<a href="{}">next page</a>'.format(
get_db_scan_taskhandler_url())
taskqueue.add(
method='GET',
url=get_db_scan_taskhandler_url(),
countdown=3,
)
else:
memcache.delete(KEY_FUND_CURSOR)
#-> not_update_yet
for t_code in code_list_all:
t_fund = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_code))
if t_fund is None:
if not t_code in scan_dict['not_update_yet']:
scan_dict['not_update_yet'].append(t_code)
logging.info(
'db_scan_taskhandler: {} not update yet'.format(
t_code))
t_content += 'End of Scan<br/>\n'
logging.info(
'db_scan_taskhandler: end of scan, result:\n{}'.format(
str(scan_dict)))
memcache.set(KEY_FUND_SCAN, pickle.dumps(scan_dict))
response.content = t_content
response.status_code = httplib.OK
return response
except Exception, e:
err_msg = '{}'.format(e)
logging.error('db_scan_taskhandler: err {}'.format(err_msg))
response.status_code = httplib.OK
return response