def main(argv): exit_code = 0 argv0 = sys.argv[0] parser = argparse.ArgumentParser(description='Test the broker by using the poker to fire messages at it.') parser.add_argument('exe_path', help='The path of the pstore binaries') parser.add_argument('--timeout', help='Process timeout in seconds', type=float, default=timed_process.DEFAULT_PROCESS_TIMEOUT) args = parser.parse_args(args=argv) # Before starting the broker and the broker-poker I need to check that both binaries are available. # This is to minimize the risk that the broker is started, but the poker cannot be found. If this # happens, the broker will never receive the command telling it to quit, and the test will hang. broker_path = os.path.join(args.exe_path, common.executable('pstore-brokerd')) poker_path = os.path.join(args.exe_path, common.executable('pstore-broker-poker')) if not os.path.exists(broker_path): print('did not find broker executable at "%s"' % broker_path, file=sys.stderr) return 1 if not os.path.exists(poker_path): print('did not find broker-poker executable at "%s"' % poker_path, file=sys.stderr) return 1 # Start both processes and wait for them to exit. cmd = [broker_path] + BROKER_ARGS print("Running broker: %s" % (cmd,), file=sys.stderr) broker = timed_process.TimedProcess(args=cmd, timeout=args.timeout, name='broker') broker.start() cmd = [poker_path] + POKER_ARGS print("Running poker: %s" % (cmd,), file=sys.stderr) poker = timed_process.TimedProcess(args=cmd, timeout=args.timeout, name='poker') poker.start() print("%s: waiting for poker to exit" % (argv0,), file=sys.stderr) poker.join(args.timeout) # If the poker failed for some reason then kill the broker early. if poker.did_timeout() or poker.exception(): print("%s: poker failed so killing broker" % (argv0,), file=sys.stderr) broker.timeout() print("%s: waiting for broker to exit" % (argv0,), file=sys.stderr) broker.join(args.timeout) print("%s: done" % (argv0,), file=sys.stderr) if not common.report_error(argv0, "broker", broker): exit_code = 1 else: print("broker exited successfully") if not common.report_error(argv0, "poker", poker): exit_code = 1 else: print("poker exited successfully") # Note that the output is sorted to guarantee stability: the order in # which the broker processes incoming messages may not match the order in # which they were sent. print('\n'.join(sorted(broker.output().splitlines()))) return exit_code
def main(argv): exit_code = 0 argv0 = sys.argv[0] parser = argparse.ArgumentParser(description='Test the broker by using the poker to fire messages at it.') parser.add_argument('exe_path', help='The path of the pstore binaries') parser.add_argument('--timeout', help='Process timeout in seconds', type=float, default=timed_process.DEFAULT_PROCESS_TIMEOUT) args = parser.parse_args(args=argv) paths = get_tool_paths(args.exe_path) pipe_path = os.path.join(common.pipe_root_dir(), 'pstore_broker_kill') broker_command = [paths.broker, '--pipe-path', pipe_path, '--disable-http'] print("Popen: ", ' '.join(broker_command), file=sys.stderr) broker_process = timed_process.TimedProcess(args=broker_command, timeout=args.timeout, name='broker', creation_flags=subprocess.CREATE_NEW_PROCESS_GROUP if common.IS_WINDOWS else 0) broker_process.start() # TODO: this is a crude way to know whether the broker is up: we don't know how long the system # will take to start the process before it actually gets to do any work. time.sleep(2) # Wait until the broker is alive. broker_process.send_signal(signal.CTRL_BREAK_EVENT if common.IS_WINDOWS else signal.SIGTERM) print("Sent SIGTERM. Waiting for broker to exit.", file=sys.stderr) broker_process.join() print("Broker exited. Done.", file=sys.stderr) common.report_error(argv0, 'broker', broker_process) print(broker_process.output()) return exit_code
def per_high(stock_data, share_data): try: per, avg_per = _prepare_per(stock_data, share_data) except KeyError as err: meta = stock_data['meta'] common.report_error(u'ERROR: {} {} KeyError: {}'.format( meta['stock_no'], meta['name'], err)) else: return per > avg_per * 2.0
def main(): common.save_errors([]) catalog = common.load_catalog() categories = common.load_categories() for category, stocks in catalog.items(): average_data = categories.setdefault(category, {}) for stock_no in stocks: try: latest_day = calculate_latest_day(stock_no, average_data) except: common.report_error('%s calculation failed' % stock_no) raise calculate_average_per(average_data[latest_day], len(stocks)) common.save_categories(categories)
def calculate_day(day, stock_data, average_data, stock_no=None): average_day = average_data.setdefault(day, {}) finance = stock_data[common.FINANCE] daily_prices = stock_data[common.DAILY] daily = daily_prices[day] price = daily[common.field_var(u'股價')] last_year = stock_data[common.META].get(common.LAST_YEAR, None) for y, field in ((common.LAST_4Q_YEAR, u'4Q本益比'), (last_year, u'本益比')): f = finance.get(y, None) if not f: continue try: # per 本益比 = 股價 / 每股盈餘(元) eps = f.get(common.field_var(u'每股盈餘(元)'), 0) if eps > 0: per = price / eps else: per = 0 field_name = common.field_var(field) daily[field_name] = per # data for average per for postfix in (common.AVG_SUM, common.AVG_COUNT): k = field_name + postfix if k not in average_day: average_day[k] = 0 if per > 0: average_day[field_name + common.AVG_SUM] += per average_day[field_name + common.AVG_COUNT] += 1 except Exception as e: msg = '%s: %s, per failed: %s %s' % (stock_no, y, type(e), e.message) common.report_error(msg) # 殖利率 = 股價 / 最近一年股利 f = finance.get(last_year, None) if f and price: dividend = f.get(common.field_var(u'股利'), 0) if dividend: yield_rate = dividend / price daily[common.field_var(u'殖利率')] = yield_rate
def calculate(stock_no): ''' calculate accrual for last year and last 4Q calculate roa for every finance report ''' stock_data = _prepare(stock_no) finance = stock_data[common.FINANCE] key_net_income_after_tax = _field_name(u'本期稅後淨利') key_cash_flow_of_investment = _field_name(u'投資活動之現金流量') key_net_income_before_tax = _field_name(u'稅前淨利') key_total_assets = _field_name(u'資產總額') meta = stock_data[common.META] for y in (common.LAST_4Q_YEAR, meta[common.LAST_YEAR]): try: f = None if y: f = finance[y] if f and len(meta[common.LAST_4Q]) == 4: # accrual 權責發生額 = 本期稅後淨利 - 來自營運之現金流量 accrual = f[_field_name(u'本期稅後淨利')] - \ f[_field_name(u'來自營運之現金流量')] f[_field_name(u'權責發生額')] = accrual except Exception as e: msg = '%s: %s, accrual failed: %s %s' % (stock_no, y, type(e), e.message) common.report_error(msg) for period, f in finance.items(): try: # ROA = 稅前純利 / 資產總額 var1 = _field_name(u'稅前淨利') var2 = _field_name(u'資產總額') if var1 in f and var2 in f: roa = f[var1] / f[var2] f[_field_name(u'總資產報酬率')] = roa except Exception as e: msg = '%s: %s, ROA failed: %s %s' % (stock_no, period, type(e), e.message) common.report_error(msg) common.save_stock(stock_no, stock_data)
def calculate_latest_day(stock_no, average_data): ''' calculate p/e ratio for last year and last 4Q ''' stock_data = common.load_stock(stock_no) finance = stock_data.get(common.FINANCE, None) if not finance: common.report_error('%s does not have finance report!!' % stock_no) # XXX new stock no, trigger parse finance report and calculate return daily_prices = stock_data[common.DAILY] # day format exampe: 101/10/28 latest_day = sorted((k for k in daily_prices.keys() if k[0].isdigit()), reverse=True)[0] calculate_day(latest_day, stock_data, average_data, stock_no=stock_no) common.save_stock(stock_no, stock_data) return latest_day
def _gather_stocks(): s = [] filter_results = common.load_filter_results() for n, f in filter_results.items(): s.extend(f[common.KEY_STOCKS]) for n, c in config.preferences.items(): s.extend(c['stocks']) return set(s) def main(): stocks = _gather_stocks() share_data = {'categories': common.load_categories()} indicators = {} for no in stocks: indicators[no] = {} stock_data = common.load_stock(no) for n, func in MAPPINGS.items(): indicators[no][n] = func(stock_data, share_data) common.save_indicator_results(indicators) if __name__ == '__main__': try: main() except Exception as e: common.report_error(traceback.format_exc(e)) raise
def main(): catalog = {} curr_data_date = None # Add some more to prevent error when new stocks found total = _total_stocks() + 10 widgets = [ FormatLabel( 'Processed: %(value)d / {0} (in: %(elapsed)s)'.format(total)) ] pbar = ProgressBar(widgets=widgets, maxval=total) count = 0 pbar.start() state = common.load_state() for catalog_key, url in CATELOG.items(): data_date, result = get_category_stock_info(url) if not result: raise Exception('Empty parsing result, key: {}, url: {}'.foramt( catalog_key, url)) if curr_data_date is None: curr_data_date = data_date elif curr_data_date != data_date: msg = 'Data date is not the same!'\ ' curr_data_date: %s, data_date: %s, url: %s'\ % (curr_data_date, data_date, url) common.report_error(msg) raise Exception(msg) stype, category = catalog_key for stock_no, data in result.items(): stock_data = common.load_stock(stock_no) daily_report = stock_data.setdefault(common.DAILY, {}) meta = stock_data.setdefault(common.META, {}) daily_report[data_date] = data category_key = SEPARATOR.join(catalog_key) meta.update({ common.META_STOCK_NO: stock_no, common.META_COMPANY_TYPE: stype, common.META_COMPANY_CATEGORY: category, common.META_CATEGORY_KEY: category_key, common.META_NAME: data.pop('name'), common.META_DAYS: sorted(daily_report.keys(), reverse=True), }) stock_data.setdefault(common.META, {}).update(meta) common.save_stock(stock_no, stock_data) catalog.setdefault(category_key, []).append(stock_no) pbar.update(count) count += 1 if not catalog.setdefault(SEPARATOR.join(catalog_key), []): common.report_error('NO STOCK FOUND!!!! %s, %s' % (catalog_key, url)) common.save_catalog(catalog) state[common.CURRENT_DATA_DATE] = curr_data_date common.save_state(state) pbar.finish()