def fund_nav_str_view(request, p_fund_id, p_year): try: t_fund = FundClearInfoModel.get_fund(p_fund_id) t_fdata = FundClearDataModel.get_by_key_name( FundClearDataModel.compose_key_name(p_fund_id, p_year), t_fund) nav_dict = t_fdata._get_nav_dict() nav_dict = collections.OrderedDict(sorted(nav_dict.items())) nav_info = '' for t_key, t_entry in nav_dict.items(): nav_info += '{}:{}<br>\n'.format(t_key, t_entry) next_year = int(p_year) - 1 t_fdata = FundClearDataModel.get_by_key_name( FundClearDataModel.compose_key_name(p_fund_id, next_year), t_fund) if t_fdata is None or len(t_fdata._get_nav_dict()) == 0: next_year_link = 'NAV End' else: next_year_link = '<a href="/mf/fc/nav_str/{}/{}/">{}</a>'.format( p_fund_id, next_year, 'next') args = { 'tpl_section_title': 'fund {} year {}, {}'.format(p_fund_id, p_year, next_year_link), 'tpl_info': nav_info, } return render_to_response('mf_simple_info.tpl.html', args) except Exception, e: err_msg = 'fund_nav_str_view ERROR: {}'.format(e) logging.error(err_msg) args = { 'tpl_info': err_msg, } return render_to_response('mf_simple_info.tpl.html', args)
def get(self, request, p_fund_id, p_year, *args, **kwargs): t_fund = FundClearInfoModel.get_fund(p_fund_id) t_fdata = FundClearDataModel.get_by_key_name( FundClearDataModel.compose_key_name(p_fund_id, p_year), t_fund) nav_dict = t_fdata._get_nav_dict() nav_dict = collections.OrderedDict(sorted(nav_dict.items())) json_nav = [[key, nav_dict[key][1]] for key in nav_dict] return HttpResponse(json.dumps(json_nav, indent=2))
def update_funddata_taskhandler(request): ''' function request 3 param: fund_id, download_year, update_type if update_type == 'all', it will update fund nav before download_year ''' p_fund_id = str(request.REQUEST['PARAM1']) if 'PARAM2' in request.REQUEST.keys(): p_year = str(request.REQUEST['PARAM2']) else: p_year = date.today().year if 'PARAM3' in request.REQUEST.keys(): p_type = str(request.REQUEST['PARAM3']) else: p_type = '' logging.info( 'update_funddata_taskhandler: fund id:{fund_id}, year:{year}, type:{type}' .format(fund_id=p_fund_id, year=p_year, type=p_type)) response = HttpResponse('update_funddata_taskhandler') if FundClearDataModel._update_from_web(p_fund_id, p_year): response.status_code = httplib.OK logging.info('update_funddata_taskhandler success') if p_type == 'all': funddata = FundClearDataModel._get_funddata(p_fund_id, p_year) nav_item_count = len(funddata._get_nav_dict()) logging.info( 'update_funddata_taskhandler: nav item count {count}'.format( count=nav_item_count)) if nav_item_count == 0: logging.info( 'update_funddata_taskhandler: year {year} nav_dict len 0, end update task.' .format(year=p_year)) else: logging.info( 'update_funddata_taskhandler: continue update with year {year} by adding new update task' .format(year=str(int(p_year) - 1))) taskqueue.add(method='GET', url=os.path.dirname(request.get_full_path()) + '/', countdown=5, params={ 'PARAM1': p_fund_id, 'PARAM2': int(p_year) - 1, 'PARAM3': p_type, }) return response else: response.status_code = httplib.INTERNAL_SERVER_ERROR logging.warning('update_funddata_taskhandler fail!') return response
def post(self, request, p_fund_id, p_year, *args, **kwargs): csv_content = request.POST.get('csv_content', '').encode('utf-8') fund_title = request.POST.get('fund_title', '') if csv_content == '' or fund_title == '': return HttpResponseBadRequest('Param Error\nfund_title length%d' % (len(fund_title))) FundClearDataModel.save_fund_csv_data(p_fund_id, p_year, fund_title, csv_content) return HttpResponse('OK')
def reload_funddata_taskhandler(request): ''' reload task should delete all existing entity and then start a download all task ''' try: response = HttpResponse('reload_funddata_taskhandler') p_fund_id = str(request.REQUEST['PARAM1']) t_fund = FundClearInfoModel.get_fund(p_fund_id) for t_data in FundClearDataModel.all().ancestor(t_fund): logging.debug( 'reload_funddata_taskhandler: remove FundClearDataModel key {}' .format(t_data.key().name())) t_data.delete() taskqueue.add( method='GET', url=get_update_funddata_taskhandler_url(p_fund_id=p_fund_id, p_type='all'), countdown=3, ) logging.debug( 'reload_funddata_taskhandler: add update task for {}'.format( p_fund_id)) response.status_code = httplib.OK return response except Exception, e: err_msg = '{}'.format(e) logging.error('reload_funddata_taskhandler ERROR: {}'.format(err_msg)) response.status_code = httplib.INTERNAL_SERVER_ERROR return response
def datamodel_statistic_report_view(request): #response = HttpResponse(content_type='text/plain') response = HttpResponse() content = '<table border="1px solid black">' codename_list = FundCodeModel.get_codename_list() for t_code, t_name in codename_list[:]: ancestor = FundClearInfoModel.get_by_key_name( FundClearInfoModel.compose_key_name(t_code)) if ancestor is None: t_count = 0 t_years = '' else: t_funddata_query = FundClearDataModel.all().ancestor( ancestor).order('year') t_count = 0 t_years = '' for t_funddata in t_funddata_query: t_count += 1 if t_funddata.year is None: t_years += '?,' else: t_years += t_funddata.year + ', ' t_report = '<tr><td width="10%" align="right">{}</td><td width="5%" align="right">{}</td><td width="50%">{}</td><td>{}</td></tr>\n'.format( t_code, t_count, t_name, t_years, ) content += t_report content += '</table>' response.content = content return response
def get_analysis(p_fund_code): #TODO: return {key:value} dict ''' not_in_fundclear_list year_nav_count has_discontinuous_year ''' t_dict = { 'not_in_fundclear_list': False, 'year_nav_count': 0, 'zero_year_nav': False, 'has_discontinuous_year': False, 'not_update_yet': False, 'year_list': 'N/A' } #-> not_in_fundclear_list fundclear_code_list = FundCodeModel.get_code_list() if not p_fund_code in fundclear_code_list: t_dict['not_in_fundclear_list'] = True return t_dict #->year_nav_count & has_discontinuous_year & year_list t_fund = FundClearInfoModel.get_by_key_name( FundClearInfoModel.compose_key_name(p_fund_code)) if t_fund is None: t_dict['not_update_yet'] = True return t_dict data_query = FundClearDataModel.all().ancestor(t_fund).order('-year') check_year = date.today().year this_year = str(check_year) t_year_list = '' for t_data in data_query: if t_data.year == this_year: nav_dict = t_data._get_nav_dict() t_dict['year_nav_count'] = len(nav_dict) if t_dict['year_nav_count'] == 0: t_dict['zero_year_nav'] = True t_year_list += '{}, '.format(t_data.year) if str(check_year) != t_data.year: t_dict['has_discontinuous_year'] = True else: check_year -= 1 t_dict['year_list'] = t_year_list return t_dict
def get_year_nav_fund_list(): #TODO: return a list of fund nav number for this year codename_all = FundCodeModel.get_codename_list() this_year = date.today().year fund_list = [] for t_entry in codename_all[:10]: t_fund_id = t_entry[0] t_fundinfo = FundClearInfoModel.get_by_key_name( FundClearInfoModel.compose_key_name(t_fund_id)) if t_fundinfo is None: fund_list.append(t_entry + [0]) else: t_funddata = FundClearDataModel.all().ancestor(t_fundinfo).filter( 'year', str(this_year)).get() if t_funddata is None: fund_list.append(t_entry + [0]) else: t_nav_dict = t_funddata._get_nav_dict() fund_list.append(t_entry + [len(t_nav_dict)]) return fund_list
def get_zero_nav_fund_list(): #TODO: return a list of fund which has no any nav for this year codename_all = FundCodeModel.get_codename_list() this_year = date.today().year zero_fund_list = [] for t_entry in codename_all[:20]: t_fund_id = t_entry[0] t_fundinfo = FundClearInfoModel.get_by_key_name( FundClearInfoModel.compose_key_name(t_fund_id)) if t_fundinfo is None: zero_fund_list.append(t_entry) else: t_funddata = FundClearDataModel.all().ancestor(t_fundinfo).filter( 'year', str(this_year)).get() if t_funddata is None: zero_fund_list.append(t_entry) else: t_nav_dict = t_funddata._get_nav_dict() if len(t_nav_dict) == 0: zero_fund_list.append(t_entry) return zero_fund_list
def db_scan_taskhandler(request): ''' check each funddata in datastore for error, such as > not_in_fundclear_list > zero_year_nav > has_discontinuous_year > not_update_yet ''' #TODO: #response = HttpResponse(content_type='text/plain') try: response = HttpResponse('db_scan_taskhandler') scan_dict_mem = memcache.get(KEY_FUND_SCAN) if scan_dict_mem: scan_dict = pickle.loads(scan_dict_mem) all_fund_in_db = FundClearInfoModel.all() fund_cursor = memcache.get(KEY_FUND_CURSOR) if fund_cursor: all_fund_in_db.with_cursor(start_cursor=fund_cursor) logging.debug( 'db_scan_taskhandler: with cursor {}'.format(fund_cursor)) else: scan_dict = { 'not_in_fundclear_list': [], #-> remove from db or fund closed 'zero_year_nav': [], #-> fund closed 'has_discontinuous_year': [], #-> need update 'not_update_yet': [], #-> need update } code_list_all = FundCodeModel.get_code_list() t_content = '' t_fetch_fund_list = all_fund_in_db.fetch(limit=FUND_DB_SCAN_SIZE) for t_fund in t_fetch_fund_list: t_code = t_fund.key().name() t_content += '{} <br/>\n'.format(t_code) #->not_in_fundclear_list if not t_code in code_list_all: if not t_code in scan_dict['not_in_fundclear_list']: scan_dict['not_in_fundclear_list'].append(t_code) logging.info( 'db_scan_taskhandler: {} not in fundclear list'.format( t_code)) #-> zero_year_nav & has_discontinuous_year data_query = FundClearDataModel.all().ancestor(t_fund).order( '-year') check_year = date.today().year this_year = str(check_year) t_year_list = '' for t_data in data_query: if t_data.year == this_year: nav_dict = t_data._get_nav_dict() if len(nav_dict) == 0: if not t_code in scan_dict['zero_year_nav']: scan_dict['zero_year_nav'].append(t_code) logging.info( 'db_scan_taskhandler: {} has zero nav in year {}' .format(t_code, this_year)) t_year_list += '{}, '.format(t_data.year) if str(check_year) != t_data.year: if not t_code in scan_dict['has_discontinuous_year']: scan_dict['has_discontinuous_year'].append(t_code) logging.info( 'db_scan_taskhandler: {} year {} with check {} not equal; {}' .format(t_code, t_data.year, check_year, t_year_list)) break else: check_year -= 1 if len(t_fetch_fund_list) == FUND_DB_SCAN_SIZE: fund_cursor = all_fund_in_db.cursor() memcache.set(KEY_FUND_CURSOR, fund_cursor) t_content += 'cursor: {}<br/>\n'.format(fund_cursor) t_content += '<a href="{}">next page</a>'.format( get_db_scan_taskhandler_url()) taskqueue.add( method='GET', url=get_db_scan_taskhandler_url(), countdown=3, ) else: memcache.delete(KEY_FUND_CURSOR) #-> not_update_yet for t_code in code_list_all: t_fund = FundClearInfoModel.get_by_key_name( FundClearInfoModel.compose_key_name(t_code)) if t_fund is None: if not t_code in scan_dict['not_update_yet']: scan_dict['not_update_yet'].append(t_code) logging.info( 'db_scan_taskhandler: {} not update yet'.format( t_code)) t_content += 'End of Scan<br/>\n' logging.info( 'db_scan_taskhandler: end of scan, result:\n{}'.format( str(scan_dict))) memcache.set(KEY_FUND_SCAN, pickle.dumps(scan_dict)) response.content = t_content response.status_code = httplib.OK return response except Exception, e: err_msg = '{}'.format(e) logging.error('db_scan_taskhandler: err {}'.format(err_msg)) response.status_code = httplib.OK return response