def datamodel_statistic_report_view(request): #response = HttpResponse(content_type='text/plain') response = HttpResponse() content = '<table border="1px solid black">' codename_list = FundCodeModel.get_codename_list() for t_code, t_name in codename_list[:]: ancestor = FundClearInfoModel.get_by_key_name( FundClearInfoModel.compose_key_name(t_code)) if ancestor is None: t_count = 0 t_years = '' else: t_funddata_query = FundClearDataModel.all().ancestor( ancestor).order('year') t_count = 0 t_years = '' for t_funddata in t_funddata_query: t_count += 1 if t_funddata.year is None: t_years += '?,' else: t_years += t_funddata.year + ', ' t_report = '<tr><td width="10%" align="right">{}</td><td width="5%" align="right">{}</td><td width="50%">{}</td><td>{}</td></tr>\n'.format( t_code, t_count, t_name, t_years, ) content += t_report content += '</table>' response.content = content return response
def get_analysis(p_fund_code): #TODO: return {key:value} dict ''' not_in_fundclear_list year_nav_count has_discontinuous_year ''' t_dict = { 'not_in_fundclear_list': False, 'year_nav_count': 0, 'zero_year_nav': False, 'has_discontinuous_year': False, 'not_update_yet': False, 'year_list': 'N/A' } #-> not_in_fundclear_list fundclear_code_list = FundCodeModel.get_code_list() if not p_fund_code in fundclear_code_list: t_dict['not_in_fundclear_list'] = True return t_dict #->year_nav_count & has_discontinuous_year & year_list t_fund = FundClearInfoModel.get_by_key_name( FundClearInfoModel.compose_key_name(p_fund_code)) if t_fund is None: t_dict['not_update_yet'] = True return t_dict data_query = FundClearDataModel.all().ancestor(t_fund).order('-year') check_year = date.today().year this_year = str(check_year) t_year_list = '' for t_data in data_query: if t_data.year == this_year: nav_dict = t_data._get_nav_dict() t_dict['year_nav_count'] = len(nav_dict) if t_dict['year_nav_count'] == 0: t_dict['zero_year_nav'] = True t_year_list += '{}, '.format(t_data.year) if str(check_year) != t_data.year: t_dict['has_discontinuous_year'] = True else: check_year -= 1 t_dict['year_list'] = t_year_list return t_dict
def get_year_nav_fund_list(): #TODO: return a list of fund nav number for this year codename_all = FundCodeModel.get_codename_list() this_year = date.today().year fund_list = [] for t_entry in codename_all[:10]: t_fund_id = t_entry[0] t_fundinfo = FundClearInfoModel.get_by_key_name( FundClearInfoModel.compose_key_name(t_fund_id)) if t_fundinfo is None: fund_list.append(t_entry + [0]) else: t_funddata = FundClearDataModel.all().ancestor(t_fundinfo).filter( 'year', str(this_year)).get() if t_funddata is None: fund_list.append(t_entry + [0]) else: t_nav_dict = t_funddata._get_nav_dict() fund_list.append(t_entry + [len(t_nav_dict)]) return fund_list
def get_zero_nav_fund_list(): #TODO: return a list of fund which has no any nav for this year codename_all = FundCodeModel.get_codename_list() this_year = date.today().year zero_fund_list = [] for t_entry in codename_all[:20]: t_fund_id = t_entry[0] t_fundinfo = FundClearInfoModel.get_by_key_name( FundClearInfoModel.compose_key_name(t_fund_id)) if t_fundinfo is None: zero_fund_list.append(t_entry) else: t_funddata = FundClearDataModel.all().ancestor(t_fundinfo).filter( 'year', str(this_year)).get() if t_funddata is None: zero_fund_list.append(t_entry) else: t_nav_dict = t_funddata._get_nav_dict() if len(t_nav_dict) == 0: zero_fund_list.append(t_entry) return zero_fund_list
def db_scan_taskhandler(request): ''' check each funddata in datastore for error, such as > not_in_fundclear_list > zero_year_nav > has_discontinuous_year > not_update_yet ''' #TODO: #response = HttpResponse(content_type='text/plain') try: response = HttpResponse('db_scan_taskhandler') scan_dict_mem = memcache.get(KEY_FUND_SCAN) if scan_dict_mem: scan_dict = pickle.loads(scan_dict_mem) all_fund_in_db = FundClearInfoModel.all() fund_cursor = memcache.get(KEY_FUND_CURSOR) if fund_cursor: all_fund_in_db.with_cursor(start_cursor=fund_cursor) logging.debug( 'db_scan_taskhandler: with cursor {}'.format(fund_cursor)) else: scan_dict = { 'not_in_fundclear_list': [], #-> remove from db or fund closed 'zero_year_nav': [], #-> fund closed 'has_discontinuous_year': [], #-> need update 'not_update_yet': [], #-> need update } code_list_all = FundCodeModel.get_code_list() t_content = '' t_fetch_fund_list = all_fund_in_db.fetch(limit=FUND_DB_SCAN_SIZE) for t_fund in t_fetch_fund_list: t_code = t_fund.key().name() t_content += '{} <br/>\n'.format(t_code) #->not_in_fundclear_list if not t_code in code_list_all: if not t_code in scan_dict['not_in_fundclear_list']: scan_dict['not_in_fundclear_list'].append(t_code) logging.info( 'db_scan_taskhandler: {} not in fundclear list'.format( t_code)) #-> zero_year_nav & has_discontinuous_year data_query = FundClearDataModel.all().ancestor(t_fund).order( '-year') check_year = date.today().year this_year = str(check_year) t_year_list = '' for t_data in data_query: if t_data.year == this_year: nav_dict = t_data._get_nav_dict() if len(nav_dict) == 0: if not t_code in scan_dict['zero_year_nav']: scan_dict['zero_year_nav'].append(t_code) logging.info( 'db_scan_taskhandler: {} has zero nav in year {}' .format(t_code, this_year)) t_year_list += '{}, '.format(t_data.year) if str(check_year) != t_data.year: if not t_code in scan_dict['has_discontinuous_year']: scan_dict['has_discontinuous_year'].append(t_code) logging.info( 'db_scan_taskhandler: {} year {} with check {} not equal; {}' .format(t_code, t_data.year, check_year, t_year_list)) break else: check_year -= 1 if len(t_fetch_fund_list) == FUND_DB_SCAN_SIZE: fund_cursor = all_fund_in_db.cursor() memcache.set(KEY_FUND_CURSOR, fund_cursor) t_content += 'cursor: {}<br/>\n'.format(fund_cursor) t_content += '<a href="{}">next page</a>'.format( get_db_scan_taskhandler_url()) taskqueue.add( method='GET', url=get_db_scan_taskhandler_url(), countdown=3, ) else: memcache.delete(KEY_FUND_CURSOR) #-> not_update_yet for t_code in code_list_all: t_fund = FundClearInfoModel.get_by_key_name( FundClearInfoModel.compose_key_name(t_code)) if t_fund is None: if not t_code in scan_dict['not_update_yet']: scan_dict['not_update_yet'].append(t_code) logging.info( 'db_scan_taskhandler: {} not update yet'.format( t_code)) t_content += 'End of Scan<br/>\n' logging.info( 'db_scan_taskhandler: end of scan, result:\n{}'.format( str(scan_dict))) memcache.set(KEY_FUND_SCAN, pickle.dumps(scan_dict)) response.content = t_content response.status_code = httplib.OK return response except Exception, e: err_msg = '{}'.format(e) logging.error('db_scan_taskhandler: err {}'.format(err_msg)) response.status_code = httplib.OK return response