def datamodel_statistic_report_view(request):
#response = HttpResponse(content_type='text/plain')
response = HttpResponse()
content = '<table border="1px solid black">'
codename_list = FundCodeModel.get_codename_list()
for t_code, t_name in codename_list[:]:
ancestor = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_code))
if ancestor is None:
t_count = 0
t_years = ''
else:
t_funddata_query = FundClearDataModel.all().ancestor(
ancestor).order('year')
t_count = 0
t_years = ''
for t_funddata in t_funddata_query:
t_count += 1
if t_funddata.year is None:
t_years += '?,'
else:
t_years += t_funddata.year + ', '
t_report = '<tr><td width="10%" align="right">{}</td><td width="5%" align="right">{}</td><td width="50%">{}</td><td>{}</td></tr>\n'.format(
t_code,
t_count,
t_name,
t_years,
)
content += t_report
content += '</table>'
response.content = content
return response
def fund_nav_str_view(request, p_fund_id, p_year):
try:
t_fund = FundClearInfoModel.get_fund(p_fund_id)
t_fdata = FundClearDataModel.get_by_key_name(
FundClearDataModel.compose_key_name(p_fund_id, p_year), t_fund)
nav_dict = t_fdata._get_nav_dict()
nav_dict = collections.OrderedDict(sorted(nav_dict.items()))
nav_info = ''
for t_key, t_entry in nav_dict.items():
nav_info += '{}:{}<br>\n'.format(t_key, t_entry)
next_year = int(p_year) - 1
t_fdata = FundClearDataModel.get_by_key_name(
FundClearDataModel.compose_key_name(p_fund_id, next_year), t_fund)
if t_fdata is None or len(t_fdata._get_nav_dict()) == 0:
next_year_link = 'NAV End'
else:
next_year_link = '<a href="/mf/fc/nav_str/{}/{}/">{}</a>'.format(
p_fund_id, next_year, 'next')
args = {
'tpl_section_title':
'fund {} year {}, {}'.format(p_fund_id, p_year, next_year_link),
'tpl_info':
nav_info,
}
return render_to_response('mf_simple_info.tpl.html', args)
except Exception, e:
err_msg = 'fund_nav_str_view ERROR: {}'.format(e)
logging.error(err_msg)
args = {
'tpl_info': err_msg,
}
return render_to_response('mf_simple_info.tpl.html', args)
def reload_funddata_taskhandler(request):
'''
reload task should delete all existing entity and then start a download all task
'''
try:
response = HttpResponse('reload_funddata_taskhandler')
p_fund_id = str(request.REQUEST['PARAM1'])
t_fund = FundClearInfoModel.get_fund(p_fund_id)
for t_data in FundClearDataModel.all().ancestor(t_fund):
logging.debug(
'reload_funddata_taskhandler: remove FundClearDataModel key {}'
.format(t_data.key().name()))
t_data.delete()
taskqueue.add(
method='GET',
url=get_update_funddata_taskhandler_url(p_fund_id=p_fund_id,
p_type='all'),
countdown=3,
)
logging.debug(
'reload_funddata_taskhandler: add update task for {}'.format(
p_fund_id))
response.status_code = httplib.OK
return response
except Exception, e:
err_msg = '{}'.format(e)
logging.error('reload_funddata_taskhandler ERROR: {}'.format(err_msg))
response.status_code = httplib.INTERNAL_SERVER_ERROR
return response
def current_nav(request, p_fund_id):
'''
get func latest NAV with id p_fund_id
'''
t_fund = FundClearInfoModel.get_fund(p_fund_id)
t_list = t_fund.get_value_list()
return HttpResponse(t_list[-1][1])
def get_analysis(p_fund_code):
#TODO: return {key:value} dict
'''
not_in_fundclear_list
year_nav_count
has_discontinuous_year
'''
t_dict = {
'not_in_fundclear_list': False,
'year_nav_count': 0,
'zero_year_nav': False,
'has_discontinuous_year': False,
'not_update_yet': False,
'year_list': 'N/A'
}
#-> not_in_fundclear_list
fundclear_code_list = FundCodeModel.get_code_list()
if not p_fund_code in fundclear_code_list:
t_dict['not_in_fundclear_list'] = True
return t_dict
#->year_nav_count & has_discontinuous_year & year_list
t_fund = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(p_fund_code))
if t_fund is None:
t_dict['not_update_yet'] = True
return t_dict
data_query = FundClearDataModel.all().ancestor(t_fund).order('-year')
check_year = date.today().year
this_year = str(check_year)
t_year_list = ''
for t_data in data_query:
if t_data.year == this_year:
nav_dict = t_data._get_nav_dict()
t_dict['year_nav_count'] = len(nav_dict)
if t_dict['year_nav_count'] == 0:
t_dict['zero_year_nav'] = True
t_year_list += '{}, '.format(t_data.year)
if str(check_year) != t_data.year:
t_dict['has_discontinuous_year'] = True
else:
check_year -= 1
t_dict['year_list'] = t_year_list
return t_dict
def get_db_err_fund_id_list():
fund_id_list = []
fundclear_id_list = FundCodeModel.get_code_list()
fund_query = FundClearInfoModel.all()
for t_fund in fund_query:
t_id = t_fund.key().name()
if not t_id in fundclear_id_list:
fund_id_list.append(t_id)
return fund_id_list
def get(self, request, p_fund_id, p_year, *args, **kwargs):
t_fund = FundClearInfoModel.get_fund(p_fund_id)
t_fdata = FundClearDataModel.get_by_key_name(
FundClearDataModel.compose_key_name(p_fund_id, p_year), t_fund)
nav_dict = t_fdata._get_nav_dict()
nav_dict = collections.OrderedDict(sorted(nav_dict.items()))
json_nav = [[key, nav_dict[key][1]] for key in nav_dict]
return HttpResponse(json.dumps(json_nav, indent=2))
def get_year_nav_fund_list():
#TODO: return a list of fund nav number for this year
codename_all = FundCodeModel.get_codename_list()
this_year = date.today().year
fund_list = []
for t_entry in codename_all[:10]:
t_fund_id = t_entry[0]
t_fundinfo = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_fund_id))
if t_fundinfo is None:
fund_list.append(t_entry + [0])
else:
t_funddata = FundClearDataModel.all().ancestor(t_fundinfo).filter(
'year', str(this_year)).get()
if t_funddata is None:
fund_list.append(t_entry + [0])
else:
t_nav_dict = t_funddata._get_nav_dict()
fund_list.append(t_entry + [len(t_nav_dict)])
return fund_list
def fc2_review_update_taskhandler(request):
response = HttpResponse('fc2_review_update_taskhandler')
try:
f_fund_id = request.REQUEST['PARAM1']
t_fund = FundClearInfoModel.get_fund(f_fund_id)
t_fund.load_all_nav()
IndexReviewModel._update_review(t_fund)
response.status_code = httplib.OK
except Exception, e:
response.status_code = httplib.INTERNAL_SERVER_ERROR
logging.error('fc2_review_update_taskhandler exception: {msg}'.format(
msg=str(e)))
def get_zero_nav_fund_list():
#TODO: return a list of fund which has no any nav for this year
codename_all = FundCodeModel.get_codename_list()
this_year = date.today().year
zero_fund_list = []
for t_entry in codename_all[:20]:
t_fund_id = t_entry[0]
t_fundinfo = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_fund_id))
if t_fundinfo is None:
zero_fund_list.append(t_entry)
else:
t_funddata = FundClearDataModel.all().ancestor(t_fundinfo).filter(
'year', str(this_year)).get()
if t_funddata is None:
zero_fund_list.append(t_entry)
else:
t_nav_dict = t_funddata._get_nav_dict()
if len(t_nav_dict) == 0:
zero_fund_list.append(t_entry)
return zero_fund_list
def bb_view(request,
p_fund_id,
p_b_type=BB_TYPE_DAILY,
p_timeframe=None,
p_sdw=None,
p_month=18):
t_fund = FundClearInfoModel.get_fund(p_fund_id)
if p_b_type == BB_TYPE_DAILY:
if p_timeframe is None:
p_timeframe = 130
p_sdw = 100
p_month = 6
t_date_since = date.today() + relativedelta(months=-(p_month * 2))
year_since = t_date_since.year
t_fund.get_value_list(year_since)
else:
if p_timeframe is None:
p_timeframe = 26
p_sdw = 100
p_month = 12
return bb_tool._bb_view(t_fund, t_fund.title, p_b_type, p_timeframe, p_sdw,
p_month)
def get_mfreport_by_id(cls, p_fund_id, p_currency_type):
report = MFReport(fund_id=p_fund_id, currency_type=p_currency_type)
report.m_fund = FundClearInfoModel.get_fund(
p_fund_id) # FundClearModel.get_fund(report.fund_id)
report.m_exchange = bot_ex.BotExchangeInfoModel.get_bot_exchange(
bot_ex.CURRENCY_JPY)
if report.m_fund is None or report.m_exchange is None:
logging.warning(
__name__ +
', get_mfreport_by_id: fund or exchange data download fail!!!')
return None
report.report_nav = report.m_fund.get_sample_value_list(
report._sample_date_list)
report.report_exchange = report.m_exchange.get_sample_value_list(
report._sample_date_list)
report._get_history_cost_n_share_report()
report._get_history_market_value_report()
return report
def _bb_view(p_fund_id, p_b_type, p_timeframe, p_sdw):
func = '{} {}'.format(__name__, '_bb_view')
t_fund = FundClearInfoModel.get_fund(p_fund_id)
if p_b_type == BB_TYPE_DAILY:
BB_VIEW_MONTHS = 7
t_date_since = date.today() + relativedelta(
months=-(BB_VIEW_MONTHS * 2))
year_since = t_date_since.year
t_value_list = t_fund.get_value_list(year_since)
else: #-> BB_TYPE_WEEKLY
BB_VIEW_MONTHS = 14
t_date_since = date.today() + relativedelta(
months=-(2 * BB_VIEW_MONTHS))
t_offset = 2 - t_date_since.weekday()
t_date_since += relativedelta(days=t_offset)
t_date_list = []
while t_date_since <= date.today():
t_date_list.append(t_date_since)
t_date_since += relativedelta(days=+7)
t_value_list = t_fund.get_sample_value_list(t_date_list)
sma, tb1, tb2, bb1, bb2 = get_bollingerbands(t_value_list, p_timeframe,
float(p_sdw) / 100)
t_content_heads = ['Date', 'NAV', 'BB2', 'BB1', 'SMA', 'TB1', 'TB2']
t_content_rows = {}
t_lastdate = t_value_list[-1][0]
t_view_date_since = date.today() + relativedelta(months=-BB_VIEW_MONTHS)
t_ndx = 0
for ndx2, t_list in enumerate([t_value_list, bb2, bb1, sma, tb1, tb2]):
for ndx, t_entry in enumerate(t_list):
if t_entry[0] < t_view_date_since:
t_ndx = ndx
else:
t_key = t_entry[0].strftime('%Y%m%d')
if t_key in t_content_rows.keys():
t_content_rows[t_entry[0].strftime("%Y%m%d")] += [
'{:.2f}'.format(t_entry[1])
]
else:
t_content_rows[t_entry[0].strftime("%Y%m%d")] = [
t_entry[0].strftime("%Y/%m/%d"), t_entry[1]
]
t_entry[0] = calendar.timegm((t_entry[0]).timetuple()) * 1000
del t_list[:(t_ndx + 1)]
t_content_rows = collections.OrderedDict(sorted(t_content_rows.items()))
t_keys = sorted(t_content_rows.keys())
for i in range(5):
t_key_1, t_key_2 = t_keys[-i - 1], t_keys[-i - 2]
#-> add % for NAV
t_value_1 = float(t_content_rows[t_key_1][1])
t_value_2 = float(t_content_rows[t_key_2][1])
if t_value_2 != 0:
t_content_rows[t_key_1][1] = '{} ({:.2%})'.format(
t_value_1, ((t_value_1 / t_value_2) - 1))
#-> add % for SMA
t_value_1 = float(t_content_rows[t_key_1][4])
t_value_2 = float(t_content_rows[t_key_2][4])
if t_value_2 != 0:
t_content_rows[t_key_1][4] = '{} ({:.2%})'.format(
t_value_1, ((t_value_1 / t_value_2) - 1))
#-> add bb width
t_value_1 = float(t_content_rows[t_key_1][2])
t_value_2 = float(t_content_rows[t_key_1][6])
t_content_rows[t_key_1][6] = '{} (BW:{})'.format(
t_value_2, (t_value_2 - t_value_1))
tbl_content = {
'heads': t_content_heads,
'rows': reversed(t_content_rows.values()),
}
t_label = ' lastDate:{}, {},TF:{},SDW:{}'.format(str(t_lastdate), p_b_type,
p_timeframe, p_sdw)
plot = {
'data': '{data: ' + str(sma).replace('L', '') + \
', label: "' + t_label + '", color: "black", lines: {show: true}, yaxis: 4},' + \
'{data: ' + str(t_value_list).replace('L', '') + \
', color: "blue", lines: {show: true}, yaxis: 4},' + \
'{data: ' + str(tb1).replace('L', '') + \
', color: "red", lines: {show: true}, yaxis: 4},' + \
'{data: ' + str(tb2).replace('L', '') + \
', color: "purple", lines: {show: true}, yaxis: 4},' + \
'{data: ' + str(bb1).replace('L', '') + \
', color: "red", lines: {show: true}, yaxis: 4},' + \
'{data: ' + str(bb2).replace('L', '') + \
', color: "purple", lines: {show: true}, yaxis: 4},'
}
args = {
'tpl_img_header':
t_fund.title, # FundCodeModel.get_fundname(p_fund_id),
'plot':
plot,
'tpl_section_title':
'Fund {} View ;{}; TF:{}; SD_W: {}'.format(p_b_type, p_fund_id,
p_timeframe, p_sdw),
'tbl_content':
tbl_content,
}
return render_to_response('mf_simple_flot.tpl.html', args)
def nav_view(request, p_fund_id):
f_fund_id = p_fund_id
t_fund = FundClearInfoModel.get_fund(f_fund_id)
year_since = date.today().year - 3
t_fund.get_value_list(year_since)
IndexReviewModel._update_review(t_fund)
t_review = IndexReviewModel.get_index_review(t_fund)
if t_review is None:
args = {
'tpl_img_header': FundCodeModel.get_fundname(f_fund_id),
}
return render_to_response('mf_my_japan.tpl.html', args)
t_content_heads = []
t_content_rows = {}
t_nav_list = list(t_review.index_list())
for t_entry in t_nav_list:
t_content_rows[t_entry[0].strftime('%Y%m%d')] = (
t_entry[0].strftime('%Y/%m/%d'), )
t_content_rows[t_entry[0].strftime('%Y%m%d')] += (t_entry[1], )
t_entry[0] = calendar.timegm((t_entry[0]).timetuple()) * 1000
t_content_heads.append('Date')
t_content_heads.append('NAV')
t_yoy_1_list = list(t_review.yoy_list(1))
for t_entry in t_yoy_1_list:
t_content_rows[t_entry[0].strftime('%Y%m%d')] += ('{:.2f}%'.format(
t_entry[1]), )
t_entry[0] = calendar.timegm((t_entry[0]).timetuple()) * 1000
t_content_heads.append('YoY_1')
t_yoy_2_list = list(t_review.yoy_list(2))
for t_entry in t_yoy_2_list:
t_content_rows[t_entry[0].strftime('%Y%m%d')] += ('{:.2f}%'.format(
t_entry[1]), )
t_entry[0] = calendar.timegm((t_entry[0]).timetuple()) * 1000
t_content_heads.append('YoY_2')
t_data_str = ''
t_data_str += '{data: ' + str(t_nav_list).replace(
'L', '') + ', label:"NAV", yaxis: 4},'
t_data_str += '{data: ' + str(t_yoy_1_list).replace(
'L', '') + ', label:"YoY_1", yaxis: 5},'
t_data_str += '{data: ' + str(t_yoy_2_list).replace(
'L', '') + ', label:"YoY_2", yaxis: 5},'
plot = {
'data': t_data_str,
}
t_content_rows = collections.OrderedDict(sorted(t_content_rows.items()))
tbl_content = {
'heads': t_content_heads,
'rows': t_content_rows.values(),
}
args = {
'tpl_img_header': FundCodeModel.get_fundname(f_fund_id),
'tpl_section_title':
_("HEAD_FUND_REVIEW"), #_("TITLE_NAV_REVIEW_DETAIL"),
'plot': plot,
'tbl_content': tbl_content,
}
return render_to_response('mf_my_japan.tpl.html', args)
def db_scan_taskhandler(request):
'''
check each funddata in datastore for error, such as
> not_in_fundclear_list
> zero_year_nav
> has_discontinuous_year
> not_update_yet
'''
#TODO:
#response = HttpResponse(content_type='text/plain')
try:
response = HttpResponse('db_scan_taskhandler')
scan_dict_mem = memcache.get(KEY_FUND_SCAN)
if scan_dict_mem:
scan_dict = pickle.loads(scan_dict_mem)
all_fund_in_db = FundClearInfoModel.all()
fund_cursor = memcache.get(KEY_FUND_CURSOR)
if fund_cursor:
all_fund_in_db.with_cursor(start_cursor=fund_cursor)
logging.debug(
'db_scan_taskhandler: with cursor {}'.format(fund_cursor))
else:
scan_dict = {
'not_in_fundclear_list': [], #-> remove from db or fund closed
'zero_year_nav': [], #-> fund closed
'has_discontinuous_year': [], #-> need update
'not_update_yet': [], #-> need update
}
code_list_all = FundCodeModel.get_code_list()
t_content = ''
t_fetch_fund_list = all_fund_in_db.fetch(limit=FUND_DB_SCAN_SIZE)
for t_fund in t_fetch_fund_list:
t_code = t_fund.key().name()
t_content += '{} <br/>\n'.format(t_code)
#->not_in_fundclear_list
if not t_code in code_list_all:
if not t_code in scan_dict['not_in_fundclear_list']:
scan_dict['not_in_fundclear_list'].append(t_code)
logging.info(
'db_scan_taskhandler: {} not in fundclear list'.format(
t_code))
#-> zero_year_nav & has_discontinuous_year
data_query = FundClearDataModel.all().ancestor(t_fund).order(
'-year')
check_year = date.today().year
this_year = str(check_year)
t_year_list = ''
for t_data in data_query:
if t_data.year == this_year:
nav_dict = t_data._get_nav_dict()
if len(nav_dict) == 0:
if not t_code in scan_dict['zero_year_nav']:
scan_dict['zero_year_nav'].append(t_code)
logging.info(
'db_scan_taskhandler: {} has zero nav in year {}'
.format(t_code, this_year))
t_year_list += '{}, '.format(t_data.year)
if str(check_year) != t_data.year:
if not t_code in scan_dict['has_discontinuous_year']:
scan_dict['has_discontinuous_year'].append(t_code)
logging.info(
'db_scan_taskhandler: {} year {} with check {} not equal; {}'
.format(t_code, t_data.year, check_year,
t_year_list))
break
else:
check_year -= 1
if len(t_fetch_fund_list) == FUND_DB_SCAN_SIZE:
fund_cursor = all_fund_in_db.cursor()
memcache.set(KEY_FUND_CURSOR, fund_cursor)
t_content += 'cursor: {}<br/>\n'.format(fund_cursor)
t_content += '<a href="{}">next page</a>'.format(
get_db_scan_taskhandler_url())
taskqueue.add(
method='GET',
url=get_db_scan_taskhandler_url(),
countdown=3,
)
else:
memcache.delete(KEY_FUND_CURSOR)
#-> not_update_yet
for t_code in code_list_all:
t_fund = FundClearInfoModel.get_by_key_name(
FundClearInfoModel.compose_key_name(t_code))
if t_fund is None:
if not t_code in scan_dict['not_update_yet']:
scan_dict['not_update_yet'].append(t_code)
logging.info(
'db_scan_taskhandler: {} not update yet'.format(
t_code))
t_content += 'End of Scan<br/>\n'
logging.info(
'db_scan_taskhandler: end of scan, result:\n{}'.format(
str(scan_dict)))
memcache.set(KEY_FUND_SCAN, pickle.dumps(scan_dict))
response.content = t_content
response.status_code = httplib.OK
return response
except Exception, e:
err_msg = '{}'.format(e)
logging.error('db_scan_taskhandler: err {}'.format(err_msg))
response.status_code = httplib.OK
return response