def main(argv):
exit_code = 0
argv0 = sys.argv[0]
parser = argparse.ArgumentParser(description='Test the broker by using the poker to fire messages at it.')
parser.add_argument('exe_path', help='The path of the pstore binaries')
parser.add_argument('--timeout', help='Process timeout in seconds', type=float,
default=timed_process.DEFAULT_PROCESS_TIMEOUT)
args = parser.parse_args(args=argv)
# Before starting the broker and the broker-poker I need to check that both binaries are available.
# This is to minimize the risk that the broker is started, but the poker cannot be found. If this
# happens, the broker will never receive the command telling it to quit, and the test will hang.
broker_path = os.path.join(args.exe_path, common.executable('pstore-brokerd'))
poker_path = os.path.join(args.exe_path, common.executable('pstore-broker-poker'))
if not os.path.exists(broker_path):
print('did not find broker executable at "%s"' % broker_path, file=sys.stderr)
return 1
if not os.path.exists(poker_path):
print('did not find broker-poker executable at "%s"' % poker_path, file=sys.stderr)
return 1
# Start both processes and wait for them to exit.
cmd = [broker_path] + BROKER_ARGS
print("Running broker: %s" % (cmd,), file=sys.stderr)
broker = timed_process.TimedProcess(args=cmd, timeout=args.timeout, name='broker')
broker.start()
cmd = [poker_path] + POKER_ARGS
print("Running poker: %s" % (cmd,), file=sys.stderr)
poker = timed_process.TimedProcess(args=cmd, timeout=args.timeout, name='poker')
poker.start()
print("%s: waiting for poker to exit" % (argv0,), file=sys.stderr)
poker.join(args.timeout)
# If the poker failed for some reason then kill the broker early.
if poker.did_timeout() or poker.exception():
print("%s: poker failed so killing broker" % (argv0,), file=sys.stderr)
broker.timeout()
print("%s: waiting for broker to exit" % (argv0,), file=sys.stderr)
broker.join(args.timeout)
print("%s: done" % (argv0,), file=sys.stderr)
if not common.report_error(argv0, "broker", broker):
exit_code = 1
else:
print("broker exited successfully")
if not common.report_error(argv0, "poker", poker):
exit_code = 1
else:
print("poker exited successfully")
# Note that the output is sorted to guarantee stability: the order in
# which the broker processes incoming messages may not match the order in
# which they were sent.
print('\n'.join(sorted(broker.output().splitlines())))
return exit_code
def main(argv):
exit_code = 0
argv0 = sys.argv[0]
parser = argparse.ArgumentParser(description='Test the broker by using the poker to fire messages at it.')
parser.add_argument('exe_path', help='The path of the pstore binaries')
parser.add_argument('--timeout', help='Process timeout in seconds', type=float,
default=timed_process.DEFAULT_PROCESS_TIMEOUT)
args = parser.parse_args(args=argv)
paths = get_tool_paths(args.exe_path)
pipe_path = os.path.join(common.pipe_root_dir(), 'pstore_broker_kill')
broker_command = [paths.broker, '--pipe-path', pipe_path, '--disable-http']
print("Popen: ", ' '.join(broker_command), file=sys.stderr)
broker_process = timed_process.TimedProcess(args=broker_command,
timeout=args.timeout,
name='broker',
creation_flags=subprocess.CREATE_NEW_PROCESS_GROUP if common.IS_WINDOWS else 0)
broker_process.start()
# TODO: this is a crude way to know whether the broker is up: we don't know how long the system
# will take to start the process before it actually gets to do any work.
time.sleep(2) # Wait until the broker is alive.
broker_process.send_signal(signal.CTRL_BREAK_EVENT if common.IS_WINDOWS else signal.SIGTERM)
print("Sent SIGTERM. Waiting for broker to exit.", file=sys.stderr)
broker_process.join()
print("Broker exited. Done.", file=sys.stderr)
common.report_error(argv0, 'broker', broker_process)
print(broker_process.output())
return exit_code
def per_high(stock_data, share_data):
try:
per, avg_per = _prepare_per(stock_data, share_data)
except KeyError as err:
meta = stock_data['meta']
common.report_error(u'ERROR: {} {} KeyError: {}'.format(
meta['stock_no'], meta['name'], err))
else:
return per > avg_per * 2.0
def main():
common.save_errors([])
catalog = common.load_catalog()
categories = common.load_categories()
for category, stocks in catalog.items():
average_data = categories.setdefault(category, {})
for stock_no in stocks:
try:
latest_day = calculate_latest_day(stock_no, average_data)
except:
common.report_error('%s calculation failed' % stock_no)
raise
calculate_average_per(average_data[latest_day], len(stocks))
common.save_categories(categories)
def calculate_day(day, stock_data, average_data, stock_no=None):
average_day = average_data.setdefault(day, {})
finance = stock_data[common.FINANCE]
daily_prices = stock_data[common.DAILY]
daily = daily_prices[day]
price = daily[common.field_var(u'股價')]
last_year = stock_data[common.META].get(common.LAST_YEAR, None)
for y, field in ((common.LAST_4Q_YEAR, u'4Q本益比'), (last_year, u'本益比')):
f = finance.get(y, None)
if not f:
continue
try:
# per 本益比 = 股價 / 每股盈餘(元)
eps = f.get(common.field_var(u'每股盈餘(元)'), 0)
if eps > 0:
per = price / eps
else:
per = 0
field_name = common.field_var(field)
daily[field_name] = per
# data for average per
for postfix in (common.AVG_SUM, common.AVG_COUNT):
k = field_name + postfix
if k not in average_day:
average_day[k] = 0
if per > 0:
average_day[field_name + common.AVG_SUM] += per
average_day[field_name + common.AVG_COUNT] += 1
except Exception as e:
msg = '%s: %s, per failed: %s %s' % (stock_no, y, type(e),
e.message)
common.report_error(msg)
# 殖利率 = 股價 / 最近一年股利
f = finance.get(last_year, None)
if f and price:
dividend = f.get(common.field_var(u'股利'), 0)
if dividend:
yield_rate = dividend / price
daily[common.field_var(u'殖利率')] = yield_rate
def calculate(stock_no):
'''
calculate accrual for last year and last 4Q
calculate roa for every finance report
'''
stock_data = _prepare(stock_no)
finance = stock_data[common.FINANCE]
key_net_income_after_tax = _field_name(u'本期稅後淨利')
key_cash_flow_of_investment = _field_name(u'投資活動之現金流量')
key_net_income_before_tax = _field_name(u'稅前淨利')
key_total_assets = _field_name(u'資產總額')
meta = stock_data[common.META]
for y in (common.LAST_4Q_YEAR, meta[common.LAST_YEAR]):
try:
f = None
if y:
f = finance[y]
if f and len(meta[common.LAST_4Q]) == 4:
# accrual 權責發生額 = 本期稅後淨利 - 來自營運之現金流量
accrual = f[_field_name(u'本期稅後淨利')] - \
f[_field_name(u'來自營運之現金流量')]
f[_field_name(u'權責發生額')] = accrual
except Exception as e:
msg = '%s: %s, accrual failed: %s %s' % (stock_no, y, type(e),
e.message)
common.report_error(msg)
for period, f in finance.items():
try:
# ROA ＝ 稅前純利 / 資產總額
var1 = _field_name(u'稅前淨利')
var2 = _field_name(u'資產總額')
if var1 in f and var2 in f:
roa = f[var1] / f[var2]
f[_field_name(u'總資產報酬率')] = roa
except Exception as e:
msg = '%s: %s, ROA failed: %s %s' % (stock_no, period, type(e),
e.message)
common.report_error(msg)
common.save_stock(stock_no, stock_data)
def calculate_latest_day(stock_no, average_data):
'''
calculate p/e ratio for last year and last 4Q
'''
stock_data = common.load_stock(stock_no)
finance = stock_data.get(common.FINANCE, None)
if not finance:
common.report_error('%s does not have finance report!!' % stock_no)
# XXX new stock no, trigger parse finance report and calculate
return
daily_prices = stock_data[common.DAILY]
# day format exampe: 101/10/28
latest_day = sorted((k for k in daily_prices.keys() if k[0].isdigit()),
reverse=True)[0]
calculate_day(latest_day, stock_data, average_data, stock_no=stock_no)
common.save_stock(stock_no, stock_data)
return latest_day
def _gather_stocks():
s = []
filter_results = common.load_filter_results()
for n, f in filter_results.items():
s.extend(f[common.KEY_STOCKS])
for n, c in config.preferences.items():
s.extend(c['stocks'])
return set(s)
def main():
stocks = _gather_stocks()
share_data = {'categories': common.load_categories()}
indicators = {}
for no in stocks:
indicators[no] = {}
stock_data = common.load_stock(no)
for n, func in MAPPINGS.items():
indicators[no][n] = func(stock_data, share_data)
common.save_indicator_results(indicators)
if __name__ == '__main__':
try:
main()
except Exception as e:
common.report_error(traceback.format_exc(e))
raise
def main():
catalog = {}
curr_data_date = None
# Add some more to prevent error when new stocks found
total = _total_stocks() + 10
widgets = [
FormatLabel(
'Processed: %(value)d / {0} (in: %(elapsed)s)'.format(total))
]
pbar = ProgressBar(widgets=widgets, maxval=total)
count = 0
pbar.start()
state = common.load_state()
for catalog_key, url in CATELOG.items():
data_date, result = get_category_stock_info(url)
if not result:
raise Exception('Empty parsing result, key: {}, url: {}'.foramt(
catalog_key, url))
if curr_data_date is None:
curr_data_date = data_date
elif curr_data_date != data_date:
msg = 'Data date is not the same!'\
' curr_data_date: %s, data_date: %s, url: %s'\
% (curr_data_date, data_date, url)
common.report_error(msg)
raise Exception(msg)
stype, category = catalog_key
for stock_no, data in result.items():
stock_data = common.load_stock(stock_no)
daily_report = stock_data.setdefault(common.DAILY, {})
meta = stock_data.setdefault(common.META, {})
daily_report[data_date] = data
category_key = SEPARATOR.join(catalog_key)
meta.update({
common.META_STOCK_NO:
stock_no,
common.META_COMPANY_TYPE:
stype,
common.META_COMPANY_CATEGORY:
category,
common.META_CATEGORY_KEY:
category_key,
common.META_NAME:
data.pop('name'),
common.META_DAYS:
sorted(daily_report.keys(), reverse=True),
})
stock_data.setdefault(common.META, {}).update(meta)
common.save_stock(stock_no, stock_data)
catalog.setdefault(category_key, []).append(stock_no)
pbar.update(count)
count += 1
if not catalog.setdefault(SEPARATOR.join(catalog_key), []):
common.report_error('NO STOCK FOUND!!!! %s, %s' %
(catalog_key, url))
common.save_catalog(catalog)
state[common.CURRENT_DATA_DATE] = curr_data_date
common.save_state(state)
pbar.finish()
