def main():
f = common.load_filter_results()
todo_stocks = set()
for _k, v in f.items():
for s_no in v['stocks']:
todo_stocks.add(s_no)
sorted_stocks = sorted(todo_stocks)
results = {}
for s_no in sorted_stocks:
try:
s_info = common.load_stock(s_no)
meta = s_info[common.META]
s_name = meta[common.META_NAME]
annuals = meta[common.ANNUALS]
last_2y = annuals[:2] if len(annuals) >= 2 else annuals
finance = s_info[common.FINANCE]
extracted_info = {'name': s_name}
for y in last_2y:
extracted_info[y] = _gather_info(finance, y)
results[s_no] = extracted_info
except:
print s_no, s_name, y, last_2y
raise
# print json.dumps(results)
_ugly_output(results)
def main():
stocks = _gather_stocks()
share_data = {'categories': common.load_categories()}
indicators = {}
for no in stocks:
indicators[no] = {}
stock_data = common.load_stock(no)
for n, func in MAPPINGS.items():
indicators[no][n] = func(stock_data, share_data)
common.save_indicator_results(indicators)
def _prepare(stock_no):
'''
calculate the missing annual cash_flow_operating from quarter report,
calculate last 4Q data
'''
stock_data = common.load_stock(stock_no)
finance = stock_data[common.FINANCE]
annuals = sorted([k for k in finance.keys() if k.isdigit()],
key=_to_number,
reverse=True)
quarters = sorted(
[k for k in finance.keys() if k[-1] == 'Q' and '~' not in k],
key=_to_number,
reverse=True)
last_year = None
for year in annuals:
field_name = _field_name(u'來自營運之現金流量')
if field_name in finance[year]:
if last_year is None:
last_year = year
continue
year_q = [year + q for q in ('.1Q', '.2Q', '.3Q', '.4Q')]
# not calculate if there is data missing
if all([yq in quarters and field_name in finance[yq]
for yq in year_q]):
cash_flow_operating = sum(
(finance[yq][field_name] for yq in year_q))
finance[year][field_name] = cash_flow_operating
if last_year is None:
last_year = year
last4Q = quarters[:4]
last_4q_year = {}
if last4Q and _continous_q(last4Q):
fields = finance[last4Q[0]].keys()
# not calculate if there is data missing
if all((f in finance[q] for q in last4Q for f in fields)):
for field in fields:
value = sum((finance[q][field] for q in last4Q))
if common.field_unit(field) == common.PERCENTAGE:
value = value / len(last4Q)
last_4q_year[field] = value
finance[common.LAST_4Q_YEAR] = last_4q_year
stock_data[common.META].update({
common.LAST_YEAR: last_year,
common.LAST_4Q: last4Q,
common.ANNUALS: annuals,
common.QUARTERS: quarters,
})
return stock_data
def output(fn, out_format):
filter_results = common.load_filter_results()
indicators = common.load_indicator_results()
categories = common.load_categories()
stocks = _gather_stocks(filter_results)
results = []
for s in stocks:
sd = common.load_stock(s)
ind = indicators[s]
for n, v in filter_results.items():
ind[n] = s in v['stocks']
results.append(format_report.format(sd, ind, categories))
if out_format == 'text':
format_text(results, fn)
else:
format_html(results, fn)
def calculate_latest_day(stock_no, average_data):
'''
calculate p/e ratio for last year and last 4Q
'''
stock_data = common.load_stock(stock_no)
finance = stock_data.get(common.FINANCE, None)
if not finance:
common.report_error('%s does not have finance report!!' % stock_no)
# XXX new stock no, trigger parse finance report and calculate
return
daily_prices = stock_data[common.DAILY]
# day format exampe: 101/10/28
latest_day = sorted((k for k in daily_prices.keys() if k[0].isdigit()),
reverse=True)[0]
calculate_day(latest_day, stock_data, average_data, stock_no=stock_no)
common.save_stock(stock_no, stock_data)
return latest_day
share_data={'categories': categories},
data=data)
return data
MAPPINGS = {
config.KAZUYO_KATSUMA: kazuyo_katsuma,
config.OLD_BROTHER: old_brother,
config.PER_LOW: per_low,
config.PER_HIGH: per_high,
config.AMOUNT_LOW: amount_low,
config.AMOUNT_HIGH: amount_high,
}
if __name__ == '__main__':
""" for test """
import sys
no = sys.argv[1]
sd = common.load_stock(no)
data = format_basic(sd)
format_by_indicator(
{
config.KAZUYO_KATSUMA: True,
config.OLD_BROTHER: True,
config.AMOUNT_LOW: True
},
sd,
data=data)
import pprint
pprint.pprint(data.output())
def main():
catalog = {}
curr_data_date = None
# Add some more to prevent error when new stocks found
total = _total_stocks() + 10
widgets = [
FormatLabel(
'Processed: %(value)d / {0} (in: %(elapsed)s)'.format(total))
]
pbar = ProgressBar(widgets=widgets, maxval=total)
count = 0
pbar.start()
state = common.load_state()
for catalog_key, url in CATELOG.items():
data_date, result = get_category_stock_info(url)
if not result:
raise Exception('Empty parsing result, key: {}, url: {}'.foramt(
catalog_key, url))
if curr_data_date is None:
curr_data_date = data_date
elif curr_data_date != data_date:
msg = 'Data date is not the same!'\
' curr_data_date: %s, data_date: %s, url: %s'\
% (curr_data_date, data_date, url)
common.report_error(msg)
raise Exception(msg)
stype, category = catalog_key
for stock_no, data in result.items():
stock_data = common.load_stock(stock_no)
daily_report = stock_data.setdefault(common.DAILY, {})
meta = stock_data.setdefault(common.META, {})
daily_report[data_date] = data
category_key = SEPARATOR.join(catalog_key)
meta.update({
common.META_STOCK_NO:
stock_no,
common.META_COMPANY_TYPE:
stype,
common.META_COMPANY_CATEGORY:
category,
common.META_CATEGORY_KEY:
category_key,
common.META_NAME:
data.pop('name'),
common.META_DAYS:
sorted(daily_report.keys(), reverse=True),
})
stock_data.setdefault(common.META, {}).update(meta)
common.save_stock(stock_no, stock_data)
catalog.setdefault(category_key, []).append(stock_no)
pbar.update(count)
count += 1
if not catalog.setdefault(SEPARATOR.join(catalog_key), []):
common.report_error('NO STOCK FOUND!!!! %s, %s' %
(catalog_key, url))
common.save_catalog(catalog)
state[common.CURRENT_DATA_DATE] = curr_data_date
common.save_state(state)
pbar.finish()
def fix_it(stock_no):
data = common.load_stock(stock_no)
for k, f in data[common.FINANCE].items():
if 'cash_flow_operatinh' in f:
f['cash_flow_operating'] = f.pop('cash_flow_operatinh')
common.save_stock(stock_no, data)
def transfer(read_from, save_to):
click.echo('%s --> %s' % (read_from, save_to))
if read_from not in OPTIONS or save_to not in OPTIONS:
print 'Should be %s or %s' % (LOCAL, FIREBASE)
sys.exit(-1)
if read_from == save_to:
print 'Saving data to where it is from does not make sense.'
sys.exit(-2)
click.echo('This will OVERWRITE data in "%s". Are you sure? [y/N]' %
save_to)
confirm = sys.stdin.readline()
if confirm.strip() != 'y':
print 'byebye~'
return
common.READ_FROM = common.LOCAL if read_from == LOCAL else common.FIREBASE
common.SAVE_TO = (common.LOCAL,)\
if save_to == LOCAL else (common.FIREBASE,)
print 'Transfering catalog...'
catalog = common.load_catalog()
common.save_catalog(catalog)
print 'Transfering categories...'
catalog = common.load_catalog()
categories = common.load_categories()
common.save_categories(categories)
print 'Transfering filter results...'
f_results = common.load_filter_results()
common.save_filter_results(f_results)
print 'Transfering indicator results...'
i_results = common.load_indicator_results()
common.save_indicator_results(i_results)
print 'Transfering config...'
config = common.load_config()
common.save_config(config)
todo = []
for stocks in catalog.values():
todo.extend(stocks)
total = len(todo)
print 'Transfering sotcks...'
widgets = [
FormatLabel(
'Processed: %(value)d / {0} (in: %(elapsed)s)'.format(total))
]
pbar = ProgressBar(widgets=widgets, maxval=total)
count = 0
pbar.start()
for s in todo:
data = common.load_stock(s)
common.save_stock(s, data)
pbar.update(count)
count += 1
pbar.finish()
print 'Transfering state...'
catalog = common.load_catalog()
state = common.load_state()
common.save_state(state)
#!/usr/bin/env python
import common
def _show(info):
return '%s %s' % info
f = common.load_filter_results()
for k, v in f.items():
print 'filter:', k
results = {}
for s_no in sorted(v['stocks']):
s_info = common.load_stock(s_no)
meta = s_info[common.META]
s_name = meta[common.META_NAME]
last_q = meta[common.QUARTERS][0]
last_y = meta[common.LAST_YEAR]
info = (s_no, s_name)
fin_info = 'Y:%s Q:%s' % (last_y, last_q)
if fin_info not in results:
results[fin_info] = []
results[fin_info].append(info)
for fin_key in sorted(results.keys(), reverse=True):
print ' %s' % fin_key
print ' %s' % ', '.join([_show(info) for info in results[fin_key]])