def datamodel_statistic_report_view(request):
#response = HttpResponse(content_type='text/plain')
response = HttpResponse()
content = '<table border="1px solid black">'
codename_list = FundCodeModel.get_codename_list()
for t_code, t_name in codename_list[:]:
ancestor = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_code))
if ancestor is None:
t_count = 0
t_years = ''
else:
t_funddata_query = FundClearDataModel.all().ancestor(
ancestor).order('year')
t_count = 0
t_years = ''
for t_funddata in t_funddata_query:
t_count += 1
if t_funddata.year is None:
t_years += '?,'
else:
t_years += t_funddata.year + ', '
t_report = '<tr><td width="10%" align="right">{}</td><td width="5%" align="right">{}</td><td width="50%">{}</td><td>{}</td></tr>\n'.format(
t_code,
t_count,
t_name,
t_years,
)
content += t_report
content += '</table>'
response.content = content
return response
def get_db_err_fund_id_list():
fund_id_list = []
fundclear_id_list = FundCodeModel.get_code_list()
fund_query = FundClearInfoModel.all()
for t_fund in fund_query:
t_id = t_fund.key().name()
if not t_id in fundclear_id_list:
fund_id_list.append(t_id)
return fund_id_list
def update_fundcode_taskhandler(request):
fname = '{} {}'.format(__name__, 'download_fundcode_taskhandler')
logging.info('{}: task start'.format(fname))
response = HttpResponse('download_fundcode_taskhandler')
if FundCodeModel._update_from_web():
response.status_code = httplib.OK
logging.info('download_fundcode success')
else:
response.status_code = httplib.INTERNAL_SERVER_ERROR
logging.warning('download_fundcode fail!')
return response
def fc2_review_cupdate_taskhandler(request):
response = HttpResponse('fc2_review_cupdate_taskhandler')
UPDATE_SIZE = 5
try:
fundcode_list = FundCodeModel.get_codename_list()
#logging.debug('fc2_review_cupdate_taskhandler: debug mode!!!')
#fundcode_list = fundcode_list[:]
fund_index = int(request.REQUEST['PARAM1'])
logging.info(
'fc2_review_cupdate_taskhandler: start with index {ind}'.format(
ind=fund_index))
for i in range(UPDATE_SIZE):
next_index = fund_index + i
if next_index < len(fundcode_list):
fund_code = fundcode_list[next_index][0]
logging.info(
'fc2_review_cupdate_taskhandler: index {ind} get fund code {fund_code}'
.format(ind=next_index, fund_code=fund_code))
taskqueue.add(method='GET',
url=get_fc_update_taskhandler_url(),
countdown=2,
params={
'PARAM1': fund_code,
})
else:
logging.info(
'fc2_review_cupdate_taskhandler: end of chain update task.'
)
break
if (fund_index + UPDATE_SIZE) < len(fundcode_list):
logging.debug(
'fc2_review_cupdate_taskhandler: add next chain update task with index {index}'
.format(index=fund_index + UPDATE_SIZE))
taskqueue.add(method='GET',
url=get_fc_cupdate_taskhandler_url(),
countdown=5,
params={
'PARAM1': str(fund_index + UPDATE_SIZE),
})
response.status_code = httplib.OK
return response
except Exception, e:
response.status_code = httplib.INTERNAL_SERVER_ERROR
logging.error('fc2_review_update_taskhandler exception: {msg}'.format(
msg=str(e)))
def get_analysis(p_fund_code):
#TODO: return {key:value} dict
'''
not_in_fundclear_list
year_nav_count
has_discontinuous_year
'''
t_dict = {
'not_in_fundclear_list': False,
'year_nav_count': 0,
'zero_year_nav': False,
'has_discontinuous_year': False,
'not_update_yet': False,
'year_list': 'N/A'
}
#-> not_in_fundclear_list
fundclear_code_list = FundCodeModel.get_code_list()
if not p_fund_code in fundclear_code_list:
t_dict['not_in_fundclear_list'] = True
return t_dict
#->year_nav_count & has_discontinuous_year & year_list
t_fund = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(p_fund_code))
if t_fund is None:
t_dict['not_update_yet'] = True
return t_dict
data_query = FundClearDataModel.all().ancestor(t_fund).order('-year')
check_year = date.today().year
this_year = str(check_year)
t_year_list = ''
for t_data in data_query:
if t_data.year == this_year:
nav_dict = t_data._get_nav_dict()
t_dict['year_nav_count'] = len(nav_dict)
if t_dict['year_nav_count'] == 0:
t_dict['zero_year_nav'] = True
t_year_list += '{}, '.format(t_data.year)
if str(check_year) != t_data.year:
t_dict['has_discontinuous_year'] = True
else:
check_year -= 1
t_dict['year_list'] = t_year_list
return t_dict
def fund_analysis_view(request, p_key=None):
response = HttpResponse()
codename_all = FundCodeModel.get_codename_list()
t_count = 1
t_table = '<table border="1px solid black">{}{}</table>\n'
t_thead = ''
t_tbody = ''
t_keys = []
for (t_code, t_name) in codename_all[:]:
t_reviews = fundtask.get_analysis(t_code)
if t_thead == '':
t_thead = '<tr><td>No</td><td>ID</td><td>Name</td>\n'
if p_key is None:
for t_key in sorted(t_reviews.keys()):
t_thead += '<td>{}</td>'.format(t_key)
t_keys.append(t_key)
else:
if p_key in t_reviews.keys():
t_thead += '<td>{}</td>'.format(p_key)
t_keys.append(p_key)
else:
response.content = 'key {} not exist!!!'.format(p_key)
return response
t_thead += '</tr>\n'
if p_key is None:
t_row = '<tr><td>{}</td><td>{}</td><td>{}</td>'.format(
t_count, t_code, t_name)
for t_key in t_keys:
t_row += '<td>{}</td>'.format(t_reviews[t_key])
t_row += '</tr>\n'
t_tbody += t_row
else:
if t_reviews[p_key]:
t_row = '<tr><td>{}</td><td>{}</td><td>{}</td>'.format(
t_count, t_code, t_name)
for t_key in t_keys:
t_row += '<td>{}</td>'.format(t_reviews[t_key])
t_row += '</tr>\n'
t_tbody += t_row
t_count += 1
response.content = t_table.format(t_thead, t_tbody)
return response
def list_all_fund_view(request):
t_fundcode_list = FundCodeModel.get_codename_list()
content_head_list = [
'Code', 'Name', 'BB View', 'NAV View', 'Update', 'Year Detail',
'SRC View'
]
content_rows = []
this_year = date.today().year
t_begin_date = date(this_year - 2, 1, 1).strftime("%Y/%m/%d")
t_end_date = date.today().strftime("%Y/%m/%d")
for t_entry in t_fundcode_list:
#t_entry[2] = '<a href="/mf/bb/'+t_entry[1]+'/">' + t_entry[2] + "</a>"
#t_entry[2] = '<a href="/fc/flot/'+t_entry[1]+'/">' + t_entry[2] + "</a>"
src_url = fc2.URL_TEMPLATE.format(fund_id=t_entry[0],
begin_date=t_begin_date,
end_date=t_end_date)
content_rows.append([
t_entry[0],
t_entry[1],
'<a href="/mf/fc/bb/' + t_entry[0] + '/">BB View</a>',
'<a href="/mf/fc/nav/' + t_entry[0] + '/">NAV View</a>',
'<a target="_blank" href="{}">Reload</a>'.format(
fc2task.get_reload_funddata_taskhandler_url(
p_fund_id=t_entry[0])),
'<a href="/mf/fc/nav_str/{}/{}/">{}</a>'.format(
t_entry[0],
date.today().year,
date.today().year),
'<a target="_blank" href="{}">View</a>'.format(src_url),
])
tbl_content = {
'heads': content_head_list,
'rows': content_rows,
}
args = {
'tpl_section_title': _("HEAD_FUND_REVIEW"),
'tbl_content': tbl_content,
}
return render_to_response('mf_simple_table.tpl.html', args)
def get_year_nav_fund_list():
#TODO: return a list of fund nav number for this year
codename_all = FundCodeModel.get_codename_list()
this_year = date.today().year
fund_list = []
for t_entry in codename_all[:10]:
t_fund_id = t_entry[0]
t_fundinfo = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_fund_id))
if t_fundinfo is None:
fund_list.append(t_entry + [0])
else:
t_funddata = FundClearDataModel.all().ancestor(t_fundinfo).filter(
'year', str(this_year)).get()
if t_funddata is None:
fund_list.append(t_entry + [0])
else:
t_nav_dict = t_funddata._get_nav_dict()
fund_list.append(t_entry + [len(t_nav_dict)])
return fund_list
def get_zero_nav_fund_list():
#TODO: return a list of fund which has no any nav for this year
codename_all = FundCodeModel.get_codename_list()
this_year = date.today().year
zero_fund_list = []
for t_entry in codename_all[:20]:
t_fund_id = t_entry[0]
t_fundinfo = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_fund_id))
if t_fundinfo is None:
zero_fund_list.append(t_entry)
else:
t_funddata = FundClearDataModel.all().ancestor(t_fundinfo).filter(
'year', str(this_year)).get()
if t_funddata is None:
zero_fund_list.append(t_entry)
else:
t_nav_dict = t_funddata._get_nav_dict()
if len(t_nav_dict) == 0:
zero_fund_list.append(t_entry)
return zero_fund_list
def nav_view(request, p_fund_id):
f_fund_id = p_fund_id
t_fund = FundClearInfoModel.get_fund(f_fund_id)
year_since = date.today().year - 3
t_fund.get_value_list(year_since)
IndexReviewModel._update_review(t_fund)
t_review = IndexReviewModel.get_index_review(t_fund)
if t_review is None:
args = {
'tpl_img_header': FundCodeModel.get_fundname(f_fund_id),
}
return render_to_response('mf_my_japan.tpl.html', args)
t_content_heads = []
t_content_rows = {}
t_nav_list = list(t_review.index_list())
for t_entry in t_nav_list:
t_content_rows[t_entry[0].strftime('%Y%m%d')] = (
t_entry[0].strftime('%Y/%m/%d'), )
t_content_rows[t_entry[0].strftime('%Y%m%d')] += (t_entry[1], )
t_entry[0] = calendar.timegm((t_entry[0]).timetuple()) * 1000
t_content_heads.append('Date')
t_content_heads.append('NAV')
t_yoy_1_list = list(t_review.yoy_list(1))
for t_entry in t_yoy_1_list:
t_content_rows[t_entry[0].strftime('%Y%m%d')] += ('{:.2f}%'.format(
t_entry[1]), )
t_entry[0] = calendar.timegm((t_entry[0]).timetuple()) * 1000
t_content_heads.append('YoY_1')
t_yoy_2_list = list(t_review.yoy_list(2))
for t_entry in t_yoy_2_list:
t_content_rows[t_entry[0].strftime('%Y%m%d')] += ('{:.2f}%'.format(
t_entry[1]), )
t_entry[0] = calendar.timegm((t_entry[0]).timetuple()) * 1000
t_content_heads.append('YoY_2')
t_data_str = ''
t_data_str += '{data: ' + str(t_nav_list).replace(
'L', '') + ', label:"NAV", yaxis: 4},'
t_data_str += '{data: ' + str(t_yoy_1_list).replace(
'L', '') + ', label:"YoY_1", yaxis: 5},'
t_data_str += '{data: ' + str(t_yoy_2_list).replace(
'L', '') + ', label:"YoY_2", yaxis: 5},'
plot = {
'data': t_data_str,
}
t_content_rows = collections.OrderedDict(sorted(t_content_rows.items()))
tbl_content = {
'heads': t_content_heads,
'rows': t_content_rows.values(),
}
args = {
'tpl_img_header': FundCodeModel.get_fundname(f_fund_id),
'tpl_section_title':
_("HEAD_FUND_REVIEW"), #_("TITLE_NAV_REVIEW_DETAIL"),
'plot': plot,
'tbl_content': tbl_content,
}
return render_to_response('mf_my_japan.tpl.html', args)
def chain_update_taskhandler(request):
'''
function request 3 param:
code_index(mondatory): the start index of FundCodeModel.get_codename_list(),
type(optional): fund data update type; <all> means update all years data
year(optional): which year nav to download
'''
p_code_index = int(request.REQUEST['PARAM1'])
p_type = ''
p_year = date.today().year - 1
if 'PARAM2' in request.REQUEST.keys():
p_type = str(request.REQUEST['PARAM2'])
if 'PARAM3' in request.REQUEST.keys():
p_year = str(request.REQUEST['PARAM3'])
codename_list = FundCodeModel.get_codename_list()
#codename_list = codename_list[10:15]
response = HttpResponse(
'chain_update_taskhandler with code_index {code_index}'.format(
code_index=p_code_index))
if p_code_index < len(codename_list):
#-> add update_funddata_task for p_code_index
p_code = codename_list[p_code_index][0]
taskhandler_url = os.path.dirname(
os.path.dirname(request.get_full_path())) + '/dupdate/'
taskqueue.add(method='GET',
url=taskhandler_url,
countdown=5,
params={
'PARAM1': p_code,
'PARAM2': p_year,
'PARAM3': p_type,
})
logging.info(
'chain_update_taskhandler: add update task for fund_id {fund_id}'.
format(fund_id=p_code))
#-> add chain_update_task for (p_code_index+1)
p_next_index = p_code_index + 1
if p_next_index < len(codename_list):
taskqueue.add(method='GET',
url=os.path.dirname(request.get_full_path()) + '/',
countdown=50,
params={
'PARAM1': p_next_index,
'PARAM2': p_type,
'PARAM3': p_year,
})
logging.info(
'chain_update_taskhandler: add chain_update task for index {index}'
.format(index=p_next_index))
else:
logging.info(
'chain_update_taskhandler: end chain_update task with index {index}'
.format(index=p_next_index))
'''
taskqueue.add(method = 'GET',
url = reviewtask.get_fc_init_url(),
countdown = 10,
)
logging.info('chain_update_taskhandler: review chain update task added.')
'''
else:
logging.warning(
'chain_update_taskhandler: code index {code_index} param error'.
format(code_index=p_code_index))
response.status_code = httplib.OK
return response
def db_scan_taskhandler(request):
'''
check each funddata in datastore for error, such as
> not_in_fundclear_list
> zero_year_nav
> has_discontinuous_year
> not_update_yet
'''
#TODO:
#response = HttpResponse(content_type='text/plain')
try:
response = HttpResponse('db_scan_taskhandler')
scan_dict_mem = memcache.get(KEY_FUND_SCAN)
if scan_dict_mem:
scan_dict = pickle.loads(scan_dict_mem)
all_fund_in_db = FundClearInfoModel.all()
fund_cursor = memcache.get(KEY_FUND_CURSOR)
if fund_cursor:
all_fund_in_db.with_cursor(start_cursor=fund_cursor)
logging.debug(
'db_scan_taskhandler: with cursor {}'.format(fund_cursor))
else:
scan_dict = {
'not_in_fundclear_list': [], #-> remove from db or fund closed
'zero_year_nav': [], #-> fund closed
'has_discontinuous_year': [], #-> need update
'not_update_yet': [], #-> need update
}
code_list_all = FundCodeModel.get_code_list()
t_content = ''
t_fetch_fund_list = all_fund_in_db.fetch(limit=FUND_DB_SCAN_SIZE)
for t_fund in t_fetch_fund_list:
t_code = t_fund.key().name()
t_content += '{} <br/>\n'.format(t_code)
#->not_in_fundclear_list
if not t_code in code_list_all:
if not t_code in scan_dict['not_in_fundclear_list']:
scan_dict['not_in_fundclear_list'].append(t_code)
logging.info(
'db_scan_taskhandler: {} not in fundclear list'.format(
t_code))
#-> zero_year_nav & has_discontinuous_year
data_query = FundClearDataModel.all().ancestor(t_fund).order(
'-year')
check_year = date.today().year
this_year = str(check_year)
t_year_list = ''
for t_data in data_query:
if t_data.year == this_year:
nav_dict = t_data._get_nav_dict()
if len(nav_dict) == 0:
if not t_code in scan_dict['zero_year_nav']:
scan_dict['zero_year_nav'].append(t_code)
logging.info(
'db_scan_taskhandler: {} has zero nav in year {}'
.format(t_code, this_year))
t_year_list += '{}, '.format(t_data.year)
if str(check_year) != t_data.year:
if not t_code in scan_dict['has_discontinuous_year']:
scan_dict['has_discontinuous_year'].append(t_code)
logging.info(
'db_scan_taskhandler: {} year {} with check {} not equal; {}'
.format(t_code, t_data.year, check_year,
t_year_list))
break
else:
check_year -= 1
if len(t_fetch_fund_list) == FUND_DB_SCAN_SIZE:
fund_cursor = all_fund_in_db.cursor()
memcache.set(KEY_FUND_CURSOR, fund_cursor)
t_content += 'cursor: {}<br/>\n'.format(fund_cursor)
t_content += '<a href="{}">next page</a>'.format(
get_db_scan_taskhandler_url())
taskqueue.add(
method='GET',
url=get_db_scan_taskhandler_url(),
countdown=3,
)
else:
memcache.delete(KEY_FUND_CURSOR)
#-> not_update_yet
for t_code in code_list_all:
t_fund = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_code))
if t_fund is None:
if not t_code in scan_dict['not_update_yet']:
scan_dict['not_update_yet'].append(t_code)
logging.info(
'db_scan_taskhandler: {} not update yet'.format(
t_code))
t_content += 'End of Scan<br/>\n'
logging.info(
'db_scan_taskhandler: end of scan, result:\n{}'.format(
str(scan_dict)))
memcache.set(KEY_FUND_SCAN, pickle.dumps(scan_dict))
response.content = t_content
response.status_code = httplib.OK
return response
except Exception, e:
err_msg = '{}'.format(e)
logging.error('db_scan_taskhandler: err {}'.format(err_msg))
response.status_code = httplib.OK
return response