def fetch_new_stock_datasets(): """fetch_new_stock_datasets Collect datasets for a ticker from IEX Cloud or Tradier .. warning: IEX Cloud charges per request. Here are example commands to help you monitor your usage while handling first time users and automation (intraday, daily, and weekly options are supported). **Setup** :: export IEX_TOKEN=YOUR_IEX_CLOUD_TOKEN export TD_TOKEN=YOUR_TRADIER_TOKEN **Pull Data for a Ticker from IEX and Tradier** :: fetch -t TICKER **Pull from All Supported IEX Feeds** :: fetch -t TICKER -g iex-all **Pull from All Supported Tradier Feeds** :: fetch -t TICKER -g td **Intraday IEX and Tradier Feeds (only minute and news to reduce costs)** :: fetch -t TICKER -g intra # or manually: # fetch -t TICKER -g td,iex_min,iex_news **Daily IEX Feeds (daily and news)** :: fetch -t TICKER -g daily # or manually: # fetch -t TICKER -g iex_day,iex_news **Weekly IEX Feeds (company, financials, earnings, dividends, and peers)** :: fetch -t TICKER -g weekly # or manually: # fetch -t TICKER -g iex_fin,iex_earn,iex_div,iex_peers,iex_news, # iex_comp **IEX Minute** :: fetch -t TICKER -g iex_min **IEX News** :: fetch -t TICKER -g iex_news **IEX Daily** :: fetch -t TICKER -g iex_day **IEX Stats** :: fetch -t TICKER -g iex_stats **IEX Peers** :: fetch -t TICKER -g iex_peers **IEX Financials** :: fetch -t TICKER -g iex_fin **IEX Earnings** :: fetch -t TICKER -g iex_earn **IEX Dividends** :: fetch -t TICKER -g iex_div **IEX Quote** :: fetch -t TICKER -g iex_quote **IEX Company** :: fetch -t TICKER -g iex_comp .. note:: This requires the following services are listening on: - redis ``localhost:6379`` - minio ``localhost:9000`` """ log.info('start - fetch_new_stock_datasets') parser = argparse.ArgumentParser( description=('Download and store the latest stock pricing, ' 'news, and options chain data ' 'and store it in Minio (S3) and Redis. ' 'Also includes support for getting FinViz ' 'screener tickers')) parser.add_argument('-t', help=('ticker'), required=False, dest='ticker') parser.add_argument( '-g', help=('optional - fetch mode: ' 'initial = default fetch from initial data feeds ' '(IEX and Tradier), ' 'intra = fetch intraday from IEX and Tradier, ' 'daily or day = fetch daily from IEX, ' 'weekly = fetch weekly from IEX, ' 'all = fetch from all data feeds, ' 'td = fetch from Tradier feeds only, ' 'iex = fetch from IEX Cloud feeds only, ' 'min or minute or iex_min = fetch IEX Cloud intraday ' 'per-minute feed ' 'https://iexcloud.io/docs/api/#historical-prices, ' 'day or daily or iex_day = fetch IEX Cloud daily feed ' 'https://iexcloud.io/docs/api/#historical-prices, ' 'quote or iex_quote = fetch IEX Cloud quotes feed ' 'https://iexcloud.io/docs/api/#quote, ' 'stats or iex_stats = fetch IEX Cloud key stats feed ' 'https://iexcloud.io/docs/api/#key-stats, ' 'peers or iex_peers = fetch from just IEX Cloud peers feed ' 'https://iexcloud.io/docs/api/#peers, ' 'news or iex_news = fetch IEX Cloud news feed ' 'https://iexcloud.io/docs/api/#news, ' 'fin or iex_fin = fetch IEX Cloud financials feed' 'https://iexcloud.io/docs/api/#financials, ' 'earn or iex_earn = fetch from just IEX Cloud earnings feeed ' 'https://iexcloud.io/docs/api/#earnings, ' 'div or iex_div = fetch from just IEX Cloud dividends feed' 'https://iexcloud.io/docs/api/#dividends, ' 'iex_comp = fetch from just IEX Cloud company feed ' 'https://iexcloud.io/docs/api/#company'), required=False, dest='fetch_mode') parser.add_argument('-i', help=('optional - ticker id ' 'not used without a database'), required=False, dest='ticker_id') parser.add_argument('-e', help=('optional - options expiration date'), required=False, dest='exp_date_str') parser.add_argument('-l', help=('optional - path to the log config file'), required=False, dest='log_config_path') parser.add_argument('-b', help=('optional - broker url for Celery'), required=False, dest='broker_url') parser.add_argument('-B', help=('optional - backend url for Celery'), required=False, dest='backend_url') parser.add_argument('-k', help=('optional - s3 access key'), required=False, dest='s3_access_key') parser.add_argument('-s', help=('optional - s3 secret key'), required=False, dest='s3_secret_key') parser.add_argument('-a', help=('optional - s3 address format: <host:port>'), required=False, dest='s3_address') parser.add_argument('-S', help=('optional - s3 ssl or not'), required=False, dest='s3_secure') parser.add_argument('-u', help=('optional - s3 bucket name'), required=False, dest='s3_bucket_name') parser.add_argument('-G', help=('optional - s3 region name'), required=False, dest='s3_region_name') parser.add_argument('-p', help=('optional - redis_password'), required=False, dest='redis_password') parser.add_argument('-r', help=('optional - redis_address format: <host:port>'), required=False, dest='redis_address') parser.add_argument('-n', help=('optional - redis and s3 key name'), required=False, dest='keyname') parser.add_argument( '-m', help=('optional - redis database number (0 by default)'), required=False, dest='redis_db') parser.add_argument('-x', help=('optional - redis expiration in seconds'), required=False, dest='redis_expire') parser.add_argument('-z', help=('optional - strike price'), required=False, dest='strike') parser.add_argument( '-c', help=('optional - contract type "C" for calls "P" for puts'), required=False, dest='contract_type') parser.add_argument( '-P', help=('optional - get pricing data if "1" or "0" disabled'), required=False, dest='get_pricing') parser.add_argument( '-N', help=('optional - get news data if "1" or "0" disabled'), required=False, dest='get_news') parser.add_argument( '-O', help=('optional - get options data if "1" or "0" disabled'), required=False, dest='get_options') parser.add_argument('-U', help=('optional - s3 enabled for publishing if "1" or ' '"0" is disabled'), required=False, dest='s3_enabled') parser.add_argument( '-R', help=('optional - redis enabled for publishing if "1" or ' '"0" is disabled'), required=False, dest='redis_enabled') parser.add_argument('-A', help=('optional - run an analysis ' 'supported modes: scn'), required=False, dest='analysis_type') parser.add_argument('-L', help=('optional - screener urls to pull ' 'tickers for analysis'), required=False, dest='urls') parser.add_argument( '-Z', help=('disable run without an engine for local testing and demos'), required=False, dest='celery_enabled', action='store_true') parser.add_argument('-F', help=('optional - backfill date for filling in ' 'gaps for the IEX Cloud minute dataset ' 'format is YYYY-MM-DD'), required=False, dest='backfill_date') parser.add_argument('-d', help=('debug'), required=False, dest='debug', action='store_true') args = parser.parse_args() run_offline = True ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID fetch_mode = 'initial' exp_date_str = ae_consts.NEXT_EXP_STR ssl_options = ae_consts.SSL_OPTIONS transport_options = ae_consts.TRANSPORT_OPTIONS broker_url = ae_consts.WORKER_BROKER_URL backend_url = ae_consts.WORKER_BACKEND_URL celery_config_module = ae_consts.WORKER_CELERY_CONFIG_MODULE include_tasks = ae_consts.INCLUDE_TASKS s3_access_key = ae_consts.S3_ACCESS_KEY s3_secret_key = ae_consts.S3_SECRET_KEY s3_region_name = ae_consts.S3_REGION_NAME s3_address = ae_consts.S3_ADDRESS s3_secure = ae_consts.S3_SECURE s3_bucket_name = ae_consts.S3_BUCKET s3_key = ae_consts.S3_KEY redis_address = ae_consts.REDIS_ADDRESS redis_key = ae_consts.REDIS_KEY redis_password = ae_consts.REDIS_PASSWORD redis_db = ae_consts.REDIS_DB redis_expire = ae_consts.REDIS_EXPIRE strike = None contract_type = None get_pricing = True get_news = True get_options = True s3_enabled = True redis_enabled = True analysis_type = None backfill_date = None debug = False if args.ticker: ticker = args.ticker.upper() if args.ticker_id: ticker_id = args.ticker_id if args.exp_date_str: exp_date_str = ae_consts.NEXT_EXP_STR if args.broker_url: broker_url = args.broker_url if args.backend_url: backend_url = args.backend_url if args.s3_access_key: s3_access_key = args.s3_access_key if args.s3_secret_key: s3_secret_key = args.s3_secret_key if args.s3_region_name: s3_region_name = args.s3_region_name if args.s3_address: s3_address = args.s3_address if args.s3_secure: s3_secure = args.s3_secure if args.s3_bucket_name: s3_bucket_name = args.s3_bucket_name if args.keyname: s3_key = args.keyname redis_key = args.keyname if args.redis_address: redis_address = args.redis_address if args.redis_password: redis_password = args.redis_password if args.redis_db: redis_db = args.redis_db if args.redis_expire: redis_expire = args.redis_expire if args.strike: strike = args.strike if args.contract_type: contract_type = args.contract_type if args.get_pricing: get_pricing = args.get_pricing == '1' if args.get_news: get_news = args.get_news == '1' if args.get_options: get_options = args.get_options == '1' if args.s3_enabled: s3_enabled = args.s3_enabled == '1' if args.redis_enabled: redis_enabled = args.redis_enabled == '1' if args.fetch_mode: fetch_mode = str(args.fetch_mode).lower() if args.analysis_type: analysis_type = str(args.analysis_type).lower() if args.celery_enabled: run_offline = False if args.backfill_date: backfill_date = args.backfill_date if args.debug: debug = True work = api_requests.build_get_new_pricing_request() work['ticker'] = ticker work['ticker_id'] = ticker_id work['s3_bucket'] = s3_bucket_name work['s3_key'] = s3_key work['redis_key'] = redis_key work['strike'] = strike work['contract'] = contract_type work['exp_date'] = exp_date_str work['s3_access_key'] = s3_access_key work['s3_secret_key'] = s3_secret_key work['s3_region_name'] = s3_region_name work['s3_address'] = s3_address work['s3_secure'] = s3_secure work['redis_address'] = redis_address work['redis_password'] = redis_password work['redis_db'] = redis_db work['redis_expire'] = redis_expire work['get_pricing'] = get_pricing work['get_news'] = get_news work['get_options'] = get_options work['s3_enabled'] = s3_enabled work['redis_enabled'] = redis_enabled work['fetch_mode'] = fetch_mode work['analysis_type'] = analysis_type work['iex_datasets'] = iex_consts.DEFAULT_FETCH_DATASETS work['backfill_date'] = backfill_date work['debug'] = debug work['label'] = f'ticker={ticker}' if analysis_type == 'scn': label = f'screener={work["ticker"]}' fv_urls = [] if args.urls: fv_urls = str(args.urls).split('|') if len(fv_urls) == 0: fv_urls = os.getenv('SCREENER_URLS', []).split('|') screener_req = api_requests.build_screener_analysis_request( ticker=ticker, fv_urls=fv_urls, label=label) work.update(screener_req) start_screener_analysis(req=work) # end of analysis_type else: last_close_date = ae_utils.last_close() last_close_str = last_close_date.strftime(ae_consts.COMMON_DATE_FORMAT) cache_base_key = f'{ticker}_{last_close_str}' if not args.keyname: work['s3_key'] = cache_base_key work['redis_key'] = cache_base_key path_to_tasks = 'analysis_engine.work_tasks' task_name = (f'{path_to_tasks}' f'.get_new_pricing_data.get_new_pricing_data') task_res = None if ae_consts.is_celery_disabled() or run_offline: work['celery_disabled'] = True work['verbose'] = debug log.debug(f'starting without celery work={ae_consts.ppj(work)} ' f'offline={run_offline}') task_res = task_pricing.get_new_pricing_data(work) status_str = ae_consts.get_status(status=task_res['status']) cur_date = backfill_date if not backfill_date: cur_date = ae_utils.get_last_close_str() redis_arr = work["redis_address"].split(':') include_results = '' if debug: include_results = task_res['rec'] if task_res['status'] == ae_consts.SUCCESS: if task_res['rec']['num_success'] == 0: log.error(f'failed fetching ticker={work["ticker"]} ' f'from {fetch_mode} - please check the ' 'environment variables') else: log.info(f'done fetching ticker={work["ticker"]} ' f'mode={fetch_mode} ' f'status={status_str} ' f'err={task_res["err"]} {include_results}') print('View keys in redis with:\n' f'redis-cli -h {redis_arr[0]} ' 'keys ' f'"{work["ticker"]}_{cur_date}*"') elif task_res['status'] == ae_consts.MISSING_TOKEN: print('Set an IEX or Tradier token: ' '\n' ' export IEX_TOKEN=YOUR_IEX_TOKEN\n' ' export TD_TOKEN=YOUR_TD_TOKEN\n') else: log.error(f'done fetching ticker={work["ticker"]} ' f'mode={fetch_mode} ' f'status={status_str} ' f'err={task_res["err"]}') # if/else debug else: log.debug(f'connecting to broker={broker_url} ' f'backend={backend_url}') # Get the Celery app app = get_celery_app.get_celery_app( name=__name__, auth_url=broker_url, backend_url=backend_url, path_to_config_module=celery_config_module, ssl_options=ssl_options, transport_options=transport_options, include_tasks=include_tasks) log.debug(f'calling task={task_name} - work={ae_consts.ppj(work)}') job_id = app.send_task(task_name, (work, )) log.debug(f'task={task_name} - job_id={job_id}')
def fetch_new_stock_datasets(): """fetch_new_stock_datasets Collect all datasets for the ticker **SPY**: :: fetch_new_stock_datasets.py -t SPY .. note:: This requires the following services are listening on: - redis ``localhost:6379`` - minio ``localhost:9000`` """ log.info('start - fetch_new_stock_datasets') parser = argparse.ArgumentParser( description=('Download and store the latest stock pricing, ' 'news, and options chain data ' 'and store it in Minio (S3) and Redis. ' 'Also includes support for getting FinViz ' 'screener tickers')) parser.add_argument('-t', help=('ticker'), required=False, dest='ticker') parser.add_argument('-g', help=('optional - fetch mode: ' 'all = fetch from all data sources (default), ' 'td = fetch from just Tradier sources, ' 'iex = fetch from just IEX sources'), required=False, dest='fetch_mode') parser.add_argument('-i', help=('optional - ticker id ' 'not used without a database'), required=False, dest='ticker_id') parser.add_argument('-e', help=('optional - options expiration date'), required=False, dest='exp_date_str') parser.add_argument('-l', help=('optional - path to the log config file'), required=False, dest='log_config_path') parser.add_argument('-b', help=('optional - broker url for Celery'), required=False, dest='broker_url') parser.add_argument('-B', help=('optional - backend url for Celery'), required=False, dest='backend_url') parser.add_argument('-k', help=('optional - s3 access key'), required=False, dest='s3_access_key') parser.add_argument('-s', help=('optional - s3 secret key'), required=False, dest='s3_secret_key') parser.add_argument('-a', help=('optional - s3 address format: <host:port>'), required=False, dest='s3_address') parser.add_argument('-S', help=('optional - s3 ssl or not'), required=False, dest='s3_secure') parser.add_argument('-u', help=('optional - s3 bucket name'), required=False, dest='s3_bucket_name') parser.add_argument('-G', help=('optional - s3 region name'), required=False, dest='s3_region_name') parser.add_argument('-p', help=('optional - redis_password'), required=False, dest='redis_password') parser.add_argument('-r', help=('optional - redis_address format: <host:port>'), required=False, dest='redis_address') parser.add_argument('-n', help=('optional - redis and s3 key name'), required=False, dest='keyname') parser.add_argument( '-m', help=('optional - redis database number (0 by default)'), required=False, dest='redis_db') parser.add_argument('-x', help=('optional - redis expiration in seconds'), required=False, dest='redis_expire') parser.add_argument('-z', help=('optional - strike price'), required=False, dest='strike') parser.add_argument( '-c', help=('optional - contract type "C" for calls "P" for puts'), required=False, dest='contract_type') parser.add_argument( '-P', help=('optional - get pricing data if "1" or "0" disabled'), required=False, dest='get_pricing') parser.add_argument( '-N', help=('optional - get news data if "1" or "0" disabled'), required=False, dest='get_news') parser.add_argument( '-O', help=('optional - get options data if "1" or "0" disabled'), required=False, dest='get_options') parser.add_argument('-U', help=('optional - s3 enabled for publishing if "1" or ' '"0" is disabled'), required=False, dest='s3_enabled') parser.add_argument( '-R', help=('optional - redis enabled for publishing if "1" or ' '"0" is disabled'), required=False, dest='redis_enabled') parser.add_argument('-A', help=('optional - run an analysis ' 'supported modes: scn'), required=False, dest='analysis_type') parser.add_argument('-L', help=('optional - screener urls to pull ' 'tickers for analysis'), required=False, dest='urls') parser.add_argument( '-Z', help=('disable run without an engine for local testing and demos'), required=False, dest='celery_enabled', action='store_true') parser.add_argument('-d', help=('debug'), required=False, dest='debug', action='store_true') args = parser.parse_args() run_offline = True ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID fetch_mode = 'all' exp_date_str = ae_consts.NEXT_EXP_STR ssl_options = ae_consts.SSL_OPTIONS transport_options = ae_consts.TRANSPORT_OPTIONS broker_url = ae_consts.WORKER_BROKER_URL backend_url = ae_consts.WORKER_BACKEND_URL celery_config_module = ae_consts.WORKER_CELERY_CONFIG_MODULE include_tasks = ae_consts.INCLUDE_TASKS s3_access_key = ae_consts.S3_ACCESS_KEY s3_secret_key = ae_consts.S3_SECRET_KEY s3_region_name = ae_consts.S3_REGION_NAME s3_address = ae_consts.S3_ADDRESS s3_secure = ae_consts.S3_SECURE s3_bucket_name = ae_consts.S3_BUCKET s3_key = ae_consts.S3_KEY redis_address = ae_consts.REDIS_ADDRESS redis_key = ae_consts.REDIS_KEY redis_password = ae_consts.REDIS_PASSWORD redis_db = ae_consts.REDIS_DB redis_expire = ae_consts.REDIS_EXPIRE strike = None contract_type = None get_pricing = True get_news = True get_options = True s3_enabled = True redis_enabled = True analysis_type = None debug = False if args.ticker: ticker = args.ticker.upper() if args.ticker_id: ticker_id = args.ticker_id if args.exp_date_str: exp_date_str = ae_consts.NEXT_EXP_STR if args.broker_url: broker_url = args.broker_url if args.backend_url: backend_url = args.backend_url if args.s3_access_key: s3_access_key = args.s3_access_key if args.s3_secret_key: s3_secret_key = args.s3_secret_key if args.s3_region_name: s3_region_name = args.s3_region_name if args.s3_address: s3_address = args.s3_address if args.s3_secure: s3_secure = args.s3_secure if args.s3_bucket_name: s3_bucket_name = args.s3_bucket_name if args.keyname: s3_key = args.keyname redis_key = args.keyname if args.redis_address: redis_address = args.redis_address if args.redis_password: redis_password = args.redis_password if args.redis_db: redis_db = args.redis_db if args.redis_expire: redis_expire = args.redis_expire if args.strike: strike = args.strike if args.contract_type: contract_type = args.contract_type if args.get_pricing: get_pricing = args.get_pricing == '1' if args.get_news: get_news = args.get_news == '1' if args.get_options: get_options = args.get_options == '1' if args.s3_enabled: s3_enabled = args.s3_enabled == '1' if args.redis_enabled: redis_enabled = args.redis_enabled == '1' if args.fetch_mode: fetch_mode = str(args.fetch_mode).lower() if args.analysis_type: analysis_type = str(args.analysis_type).lower() if args.celery_enabled: run_offline = False if args.debug: debug = True work = api_requests.build_get_new_pricing_request() work['ticker'] = ticker work['ticker_id'] = ticker_id work['s3_bucket'] = s3_bucket_name work['s3_key'] = s3_key work['redis_key'] = redis_key work['strike'] = strike work['contract'] = contract_type work['exp_date'] = exp_date_str work['s3_access_key'] = s3_access_key work['s3_secret_key'] = s3_secret_key work['s3_region_name'] = s3_region_name work['s3_address'] = s3_address work['s3_secure'] = s3_secure work['redis_address'] = redis_address work['redis_password'] = redis_password work['redis_db'] = redis_db work['redis_expire'] = redis_expire work['get_pricing'] = get_pricing work['get_news'] = get_news work['get_options'] = get_options work['s3_enabled'] = s3_enabled work['redis_enabled'] = redis_enabled work['fetch_mode'] = fetch_mode work['analysis_type'] = analysis_type work['iex_datasets'] = iex_consts.DEFAULT_FETCH_DATASETS work['debug'] = debug work['label'] = 'ticker={}'.format(ticker) if analysis_type == 'scn': label = 'screener={}'.format(work['ticker']) fv_urls = [] if args.urls: fv_urls = str(args.urls).split('|') if len(fv_urls) == 0: fv_urls = os.getenv('SCREENER_URLS', []).split('|') screener_req = api_requests.build_screener_analysis_request( ticker=ticker, fv_urls=fv_urls, label=label) work.update(screener_req) start_screener_analysis(req=work) # end of analysis_type else: if not args.keyname: last_close_date = ae_utils.last_close() work['s3_key'] = '{}_{}'.format( work['ticker'], last_close_date.strftime(ae_consts.COMMON_DATE_FORMAT)) work['redis_key'] = '{}_{}'.format( work['ticker'], last_close_date.strftime(ae_consts.COMMON_DATE_FORMAT)) path_to_tasks = 'analysis_engine.work_tasks' task_name = ('{}.get_new_pricing_data.get_new_pricing_data'.format( path_to_tasks)) task_res = None if ae_consts.is_celery_disabled() or run_offline: work['celery_disabled'] = True log.debug('starting without celery work={} offline={}'.format( ae_consts.ppj(work), run_offline)) task_res = task_pricing.get_new_pricing_data(work) if debug: log.info('done - result={} ' 'task={} status={} ' 'err={} label={}'.format( ae_consts.ppj(task_res), task_name, ae_consts.get_status(status=task_res['status']), task_res['err'], work['label'])) else: log.info('done - result ' 'task={} status={} ' 'err={} label={}'.format( task_name, ae_consts.get_status(status=task_res['status']), task_res['err'], work['label'])) # if/else debug else: log.info('connecting to broker={} backend={}'.format( broker_url, backend_url)) # Get the Celery app app = get_celery_app.get_celery_app( name=__name__, auth_url=broker_url, backend_url=backend_url, path_to_config_module=celery_config_module, ssl_options=ssl_options, transport_options=transport_options, include_tasks=include_tasks) log.info('calling task={} - work={}'.format( task_name, ae_consts.ppj(work))) job_id = app.send_task(task_name, (work, )) log.info('calling task={} - success job_id={}'.format( task_name, job_id))
def run_aws_backup(): """run_aws_backup Run buy and sell analysis on a stock to send alerts to subscribed users """ log.debug('start - aws-backup') parser = argparse.ArgumentParser(description=('stock analysis tool')) parser.add_argument('-t', help=('ticker'), required=True, dest='ticker') parser.add_argument('-e', help=('file path to extract an ' 'algorithm-ready datasets from redis'), required=False, dest='algo_extract_loc') parser.add_argument('-l', help=('show dataset in this file'), required=False, dest='show_from_file') parser.add_argument('-H', help=('show trading history dataset in this file'), required=False, dest='show_history_from_file') parser.add_argument( '-E', help=('show trading performance report dataset in this file'), required=False, dest='show_report_from_file') parser.add_argument( '-L', help=('restore an algorithm-ready dataset file back into redis'), required=False, dest='restore_algo_file') parser.add_argument('-f', help=('run in mode: prepare dataset from ' 'redis key or s3 key'), required=False, dest='prepare_mode', action='store_true') parser.add_argument( '-J', help=('plot action - after preparing you can use: ' '-J show to open the image (good for debugging)'), required=False, dest='plot_action') parser.add_argument( '-b', help=('run a backtest using the dataset in ' 'a file path/s3 key/redis key formats: ' 'file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' 's3://algoready/SPY-latest.json or ' 'redis:SPY-latest'), required=False, dest='backtest_loc') parser.add_argument('-B', help=('optional - broker url for Celery'), required=False, dest='broker_url') parser.add_argument('-C', help=('optional - broker url for Celery'), required=False, dest='backend_url') parser.add_argument( '-w', help=('optional - flag for publishing an algorithm job ' 'using Celery to the ae workers'), required=False, dest='run_on_engine', action='store_true') parser.add_argument('-k', help=('optional - s3 access key'), required=False, dest='s3_access_key') parser.add_argument('-K', help=('optional - s3 secret key'), required=False, dest='s3_secret_key') parser.add_argument('-a', help=('optional - s3 address format: <host:port>'), required=False, dest='s3_address') parser.add_argument('-Z', help=('optional - s3 secure: default False'), required=False, dest='s3_secure') parser.add_argument('-s', help=('optional - start date: YYYY-MM-DD'), required=False, dest='start_date') parser.add_argument('-n', help=('optional - end date: YYYY-MM-DD'), required=False, dest='end_date') parser.add_argument('-u', help=('optional - s3 bucket name'), required=False, dest='s3_bucket_name') parser.add_argument('-G', help=('optional - s3 region name'), required=False, dest='s3_region_name') parser.add_argument( '-g', help=('Path to a custom algorithm module file ' 'on disk. This module must have a single ' 'class that inherits from: ' 'https://github.com/AlgoTraders/stock-analysis-engine/' 'blob/master/' 'analysis_engine/algo.py Additionally you ' 'can find the Example-Minute-Algorithm here: ' 'https://github.com/AlgoTraders/stock-analysis-engine/' 'blob/master/analysis_engine/mocks/' 'example_algo_minute.py'), required=False, dest='run_algo_in_file') parser.add_argument('-p', help=('optional - s3 bucket/file for trading history'), required=False, dest='algo_history_loc') parser.add_argument( '-o', help=('optional - s3 bucket/file for trading performance report'), required=False, dest='algo_report_loc') parser.add_argument('-r', help=('optional - redis_address format: <host:port>'), required=False, dest='redis_address') parser.add_argument('-R', help=('optional - redis and s3 key name'), required=False, dest='keyname') parser.add_argument( '-m', help=('optional - redis database number (0 by default)'), required=False, dest='redis_db') parser.add_argument('-x', help=('optional - redis expiration in seconds'), required=False, dest='redis_expire') parser.add_argument('-z', help=('optional - strike price'), required=False, dest='strike') parser.add_argument( '-c', help=('optional - algorithm config_file path for setting ' 'up internal algorithm trading strategies and ' 'indicators'), required=False, dest='config_file') parser.add_argument( '-P', help=('optional - get pricing data if "1" or "0" disabled'), required=False, dest='get_pricing') parser.add_argument( '-N', help=('optional - get news data if "1" or "0" disabled'), required=False, dest='get_news') parser.add_argument( '-O', help=('optional - get options data if "1" or "0" disabled'), required=False, dest='get_options') parser.add_argument( '-i', help=('optional - ignore column names (comma separated)'), required=False, dest='ignore_columns') parser.add_argument('-d', help=('debug'), required=False, dest='debug', action='store_true') args = parser.parse_args() mode = 'prepare' plot_action = ae_consts.PLOT_ACTION_SHOW ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID ssl_options = ae_consts.SSL_OPTIONS transport_options = ae_consts.TRANSPORT_OPTIONS broker_url = ae_consts.WORKER_BROKER_URL backend_url = ae_consts.WORKER_BACKEND_URL path_to_config_module = ae_consts.WORKER_CELERY_CONFIG_MODULE include_tasks = ae_consts.INCLUDE_TASKS s3_access_key = ae_consts.S3_ACCESS_KEY s3_secret_key = ae_consts.S3_SECRET_KEY s3_region_name = ae_consts.S3_REGION_NAME s3_address = ae_consts.S3_ADDRESS s3_secure = ae_consts.S3_SECURE s3_bucket_name = ae_consts.S3_BUCKET s3_key = ae_consts.S3_KEY redis_address = ae_consts.REDIS_ADDRESS redis_key = ae_consts.REDIS_KEY redis_password = ae_consts.REDIS_PASSWORD redis_db = ae_consts.REDIS_DB redis_expire = ae_consts.REDIS_EXPIRE dataset_type = ae_consts.SA_DATASET_TYPE_ALGO_READY serialize_datasets = ae_consts.DEFAULT_SERIALIZED_DATASETS output_redis_key = None output_s3_bucket = None output_s3_key = None ignore_columns = None compress = False encoding = 'utf-8' slack_enabled = False slack_code_block = False slack_full_width = False verbose = False debug = False redis_serializer = 'json' redis_encoding = 'utf-8' output_redis_key = None output_s3_bucket = None output_s3_key = None s3_enabled = True redis_enabled = True ignore_columns = None debug = False run_on_engine = False show_from_file = None show_history_from_file = None show_report_from_file = None restore_algo_file = None backtest_loc = None use_custom_algo = False algo_history_loc = 's3://algohistory' algo_report_loc = 's3://algoreport' algo_extract_loc = 's3://algoready' use_balance = 5000.0 use_commission = 6.0 auto_fill = True use_start_date = '2018-11-01 00:00:00' use_end_date = None use_config_file = None use_name = 'myalgo' if args.ticker: ticker = args.ticker.upper() if args.broker_url: broker_url = args.broker_url if args.backend_url: backend_url = args.backend_url if args.s3_access_key: s3_access_key = args.s3_access_key if args.s3_secret_key: s3_secret_key = args.s3_secret_key if args.s3_region_name: s3_region_name = args.s3_region_name if args.s3_address: s3_address = args.s3_address s3_enabled = True if args.s3_secure: s3_secure = args.s3_secure if args.s3_bucket_name: s3_bucket_name = args.s3_bucket_name if args.keyname: s3_key = args.keyname redis_key = args.keyname if args.redis_address: redis_address = args.redis_address if args.redis_db: redis_db = args.redis_db if args.redis_expire: redis_expire = args.redis_expire if args.prepare_mode: mode = ae_consts.SA_MODE_PREPARE if args.ignore_columns: ignore_columns_org = args.ignore_columns ignore_columns = ignore_columns_org.split(",") if args.plot_action: if str(args.plot_action).lower() == 'show': plot_action = ae_consts.PLOT_ACTION_SHOW elif str(args.plot_action).lower() == 's3': plot_action = ae_consts.PLOT_ACTION_SAVE_TO_S3 elif str(args.plot_action).lower() == 'save': plot_action = ae_consts.PLOT_ACTION_SAVE_AS_FILE else: plot_action = ae_consts.PLOT_ACTION_SHOW log.warning(f'unsupported plot_action: {args.plot_action}') if args.debug: debug = True if args.algo_extract_loc: mode = ae_consts.SA_MODE_EXTRACT if args.show_from_file: show_from_file = args.show_from_file mode = ae_consts.SA_MODE_SHOW_DATASET if args.show_history_from_file: show_history_from_file = args.show_history_from_file mode = ae_consts.SA_MODE_SHOW_HISTORY_DATASET if args.show_report_from_file: show_report_from_file = args.show_report_from_file mode = ae_consts.SA_MODE_SHOW_REPORT_DATASET if args.restore_algo_file: restore_algo_file = args.restore_algo_file mode = ae_consts.SA_MODE_RESTORE_REDIS_DATASET if args.run_algo_in_file: mode = ae_consts.SA_MODE_RUN_ALGO if args.backtest_loc: mode = ae_consts.SA_MODE_RUN_ALGO if args.start_date: try: use_start_date = f'{str(args.start_date)} 00:00:00' datetime.datetime.strptime(args.start_date, ae_consts.COMMON_DATE_FORMAT) except Exception as e: msg = ('please use a start date formatted as: ' f'{ae_consts.COMMON_DATE_FORMAT}\n' f'error was: {e}') log.error(msg) sys.exit(1) # end of testing for a valid date # end of args.start_date if args.end_date: try: use_end_date = f'{str(args.end_date)} 00:00:00' datetime.datetime.strptime(args.end_date, ae_consts.COMMON_DATE_FORMAT) except Exception as e: msg = ('please use an end date formatted as: ' f'{ae_consts.COMMON_DATE_FORMAT}\n' f'error was: {e}') log.error(msg) sys.exit(1) # end of testing for a valid date # end of args.end_date if args.config_file: use_config_file = args.config_file if not os.path.exists(use_config_file): log.error( f'Failed: unable to find config file: -c {use_config_file}') sys.exit(1) config_dict = None load_from_s3_bucket = None load_from_s3_key = None load_from_redis_key = None load_from_file = None load_compress = False load_publish = True load_config = None report_redis_key = None report_s3_bucket = None report_s3_key = None report_file = None report_compress = False report_publish = False report_config = None history_redis_key = None history_s3_bucket = None history_s3_key = None history_file = None history_compress = False history_publish = False history_config = None extract_redis_key = None extract_s3_bucket = None extract_s3_key = None extract_file = None extract_save_dir = None extract_compress = False extract_publish = True extract_config = None publish_to_slack = False publish_to_s3 = True publish_to_redis = False use_timeseries = 'day' use_trade_strategy = 'count' valid = False required_task = False work = None task_name = None work = {} path_to_tasks = 'analysis_engine.work_tasks' if mode == ae_consts.SA_MODE_PREPARE: task_name = (f'{path_to_tasks}.' 'prepare_pricing_dataset.prepare_pricing_dataset') work = api_requests.build_prepare_dataset_request() if output_s3_key: work['prepared_s3_key'] = output_s3_key if output_s3_bucket: work['prepared_s3_bucket'] = output_s3_bucket if output_redis_key: work['prepared_redis_key'] = output_redis_key work['ignore_columns'] = ignore_columns valid = True required_task = True elif mode == ae_consts.SA_MODE_EXTRACT: if args.algo_extract_loc: algo_extract_loc = args.algo_extract_loc if ('file:/' not in algo_extract_loc and 's3://' not in algo_extract_loc and 'redis://' not in algo_extract_loc): log.error( 'invalid -e <extract_to_file_or_s3_key_or_redis_key> ' 'specified. please use either: ' '-e file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-e s3://algoready/SPY-latest.json or ' '-e redis://SPY-latest') sys.exit(1) if 's3://' in algo_extract_loc: extract_s3_bucket = algo_extract_loc.split('/')[-2] extract_s3_key = algo_extract_loc.split('/')[-1] elif 'redis://' in algo_extract_loc: extract_redis_key = algo_extract_loc.split('/')[-1] elif 'file:/' in algo_extract_loc: extract_file = algo_extract_loc.split(':')[-1] # end of parsing supported transport for loading use_custom_algo = True elif mode == ae_consts.SA_MODE_SHOW_DATASET: examine_dataset_in_file(ticker=ticker, path_to_file=show_from_file) log.info(f'done showing {ticker} dataset from file={show_from_file}') sys.exit(0) elif mode == ae_consts.SA_MODE_SHOW_HISTORY_DATASET: examine_dataset_in_file( ticker=ticker, dataset_type=ae_consts.SA_DATASET_TYPE_TRADING_HISTORY, path_to_file=show_history_from_file) log.info(f'done showing trading history {ticker} dataset from ' f'file={show_from_file}') sys.exit(0) elif mode == ae_consts.SA_MODE_SHOW_REPORT_DATASET: examine_dataset_in_file( ticker=ticker, dataset_type=ae_consts.SA_DATASET_TYPE_TRADING_REPORT, path_to_file=show_report_from_file) log.info( f'done showing trading performance report {ticker} dataset from ' f'file={show_from_file}') sys.exit(0) elif mode == ae_consts.SA_MODE_RESTORE_REDIS_DATASET: restore_missing_dataset_values_from_algo_ready_file( ticker=ticker, path_to_file=restore_algo_file, redis_address=redis_address, redis_password=redis_password, redis_db=redis_db, output_redis_db=redis_db, dataset_type=ae_consts.SA_DATASET_TYPE_ALGO_READY, serialize_datasets=ae_consts.DEFAULT_SERIALIZED_DATASETS) log.info( f'done restoring {ticker} dataset from file={restore_algo_file} ' f'into redis_db={redis_db}') sys.exit(0) elif mode == ae_consts.SA_MODE_RUN_ALGO: if args.run_algo_in_file: if not os.path.exists(args.run_algo_in_file): log.error( f'missing algorithm module file: {args.run_algo_in_file}') sys.exit(1) if args.backtest_loc: backtest_loc = args.backtest_loc if ('file:/' not in backtest_loc and 's3://' not in backtest_loc and 'redis://' not in backtest_loc): log.error( 'invalid -b <backtest dataset file> specified. ' f'{backtest_loc} ' 'please use either: ' '-b file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-b s3://algoready/SPY-latest.json or ' '-b redis://SPY-latest') sys.exit(1) if 's3://' in backtest_loc: load_from_s3_bucket = backtest_loc.split('/')[-2] load_from_s3_key = backtest_loc.split('/')[-1] elif 'redis://' in backtest_loc: load_from_redis_key = backtest_loc.split('/')[-1] elif 'file:/' in backtest_loc: load_from_file = backtest_loc.split(':')[-1] load_publish = True # end of parsing supported transport - loading an algo-ready if args.algo_history_loc: algo_history_loc = args.algo_history_loc if ('file:/' not in algo_history_loc and 's3://' not in algo_history_loc and 'redis://' not in algo_history_loc): log.error( 'invalid -p <backtest dataset file> specified. ' f'{algo_history_loc} ' 'please use either: ' '-p file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-p s3://algoready/SPY-latest.json or ' '-p redis://SPY-latest') sys.exit(1) if 's3://' in algo_history_loc: history_s3_bucket = algo_history_loc.split('/')[-2] history_s3_key = algo_history_loc.split('/')[-1] elif 'redis://' in algo_history_loc: history_redis_key = algo_history_loc.split('/')[-1] elif 'file:/' in algo_history_loc: history_file = algo_history_loc.split(':')[-1] history_publish = True # end of parsing supported transport - trading history if args.algo_report_loc: algo_report_loc = args.algo_report_loc if ('file:/' not in algo_report_loc and 's3://' not in algo_report_loc and 'redis://' not in algo_report_loc): log.error( 'invalid -o <backtest dataset file> specified. ' f'{algo_report_loc} ' 'please use either: ' '-o file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-o s3://algoready/SPY-latest.json or ' '-o redis://SPY-latest') sys.exit(1) if 's3://' in algo_report_loc: report_s3_bucket = algo_report_loc.split('/')[-2] report_s3_key = algo_report_loc.split('/')[-1] elif 'redis://' in algo_report_loc: report_redis_key = algo_report_loc.split('/')[-1] elif 'file:/' in algo_report_loc: report_file = algo_report_loc.split(':')[-1] report_publish = True # end of parsing supported transport - trading performance report if args.algo_extract_loc: algo_extract_loc = args.algo_extract_loc if ('file:/' not in algo_extract_loc and 's3://' not in algo_extract_loc and 'redis://' not in algo_extract_loc): log.error( 'invalid -e <backtest dataset file> specified. ' f'{algo_extract_loc} ' 'please use either: ' '-e file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-e s3://algoready/SPY-latest.json or ' '-e redis://SPY-latest') sys.exit(1) if 's3://' in algo_extract_loc: extract_s3_bucket = algo_extract_loc.split('/')[-2] extract_s3_key = algo_extract_loc.split('/')[-1] elif 'redis://' in algo_extract_loc: extract_redis_key = algo_extract_loc.split('/')[-1] elif 'file:/' in algo_extract_loc: extract_file = algo_extract_loc.split(':')[-1] extract_publish = True # end of parsing supported transport - extract algorithm-ready use_custom_algo = True # end of set up for backtest if use_custom_algo: if args.run_on_engine: run_on_engine = True log.info('starting algo on the engine') else: log.info('starting algo') algo_res = run_custom_algo.run_custom_algo( mod_path=args.run_algo_in_file, ticker=ticker, balance=use_balance, commission=use_commission, start_date=use_start_date, end_date=use_end_date, config_file=use_config_file, name=use_name, auto_fill=auto_fill, config_dict=config_dict, load_from_s3_bucket=load_from_s3_bucket, load_from_s3_key=load_from_s3_key, load_from_redis_key=load_from_redis_key, load_from_file=load_from_file, load_compress=load_compress, load_publish=load_publish, load_config=load_config, report_redis_key=report_redis_key, report_s3_bucket=report_s3_bucket, report_s3_key=report_s3_key, report_file=report_file, report_compress=report_compress, report_publish=report_publish, report_config=report_config, history_redis_key=history_redis_key, history_s3_bucket=history_s3_bucket, history_s3_key=history_s3_key, history_file=history_file, history_compress=history_compress, history_publish=history_publish, history_config=history_config, extract_redis_key=extract_redis_key, extract_s3_bucket=extract_s3_bucket, extract_s3_key=extract_s3_key, extract_file=extract_file, extract_save_dir=extract_save_dir, extract_compress=extract_compress, extract_publish=extract_publish, extract_config=extract_config, publish_to_slack=publish_to_slack, publish_to_s3=publish_to_s3, publish_to_redis=publish_to_redis, dataset_type=dataset_type, serialize_datasets=serialize_datasets, compress=compress, encoding=encoding, redis_enabled=redis_enabled, redis_key=redis_key, redis_address=redis_address, redis_db=redis_db, redis_password=redis_password, redis_expire=redis_expire, redis_serializer=redis_serializer, redis_encoding=redis_encoding, s3_enabled=s3_enabled, s3_key=s3_key, s3_address=s3_address, s3_bucket=s3_bucket_name, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, slack_enabled=slack_enabled, slack_code_block=slack_code_block, slack_full_width=slack_full_width, dataset_publish_extract=extract_publish, dataset_publish_history=history_publish, dataset_publish_report=report_publish, run_on_engine=run_on_engine, auth_url=broker_url, backend_url=backend_url, include_tasks=include_tasks, ssl_options=ssl_options, transport_options=transport_options, path_to_config_module=path_to_config_module, timeseries=use_timeseries, trade_strategy=use_trade_strategy, verbose=verbose) show_label = f'algo.name={use_name}' show_extract = f'{algo_extract_loc}' show_history = f'{algo_history_loc}' show_report = f'{algo_report_loc}' base_label = (f'load={args.run_algo_in_file} extract={show_extract} ' f'history={show_history} report={show_report}') show_label = (f'{ticker} running in engine ' f'''task_id={algo_res["rec"].get( "task_id", "missing-task-id")} {base_label}''') if not run_on_engine: algo_trade_history_recs = algo_res['rec'].get('history', []) show_label = (f'{ticker} algo.name={use_name} {base_label} ' f'trade_history_len={len(algo_trade_history_recs)}') if args.debug: log.info(f'algo_res={algo_res}') if algo_res['status'] == ae_consts.SUCCESS: log.info( f'{ae_consts.get_status(status=algo_res["status"])} - ' f'done running {show_label}') else: log.error( f'{ae_consts.get_status(status=algo_res["status"])} - ' f'done running {show_label}') else: if algo_res['status'] == ae_consts.SUCCESS: log.info( f'{ae_consts.get_status(status=algo_res["status"])} - ' f'done running {show_label}') else: log.error(f'run_custom_algo returned error: {algo_res["err"]}') sys.exit(1) # end of running the custom algo handler if mode == ae_consts.SA_MODE_EXTRACT: log.info(f'done extracting dataset - {ticker}') elif mode == ae_consts.SA_MODE_RUN_ALGO: log.info(f'done running algo - {ticker}') sys.exit(0) # end of handling mode-specific arg assignments # sanity checking the work and task are valid if not valid: log.error('usage error: missing a supported mode: ' '-f (for prepare a dataset) ') sys.exit(1) if required_task and not task_name: log.error('usage error: missing a supported task_name') sys.exit(1) # end of sanity checks work['ticker'] = ticker work['ticker_id'] = ticker_id work['s3_bucket'] = s3_bucket_name work['s3_key'] = s3_key work['redis_key'] = redis_key work['s3_access_key'] = s3_access_key work['s3_secret_key'] = s3_secret_key work['s3_region_name'] = s3_region_name work['s3_address'] = s3_address work['s3_secure'] = s3_secure work['redis_address'] = redis_address work['redis_password'] = redis_password work['redis_db'] = redis_db work['redis_expire'] = redis_expire work['s3_enabled'] = s3_enabled work['redis_enabled'] = redis_enabled work['debug'] = debug work['label'] = f'ticker={ticker}' task_res = None if ae_consts.is_celery_disabled(): work['celery_disabled'] = True log.debug(f'starting without celery work={ae_consts.ppj(work)}') if mode == ae_consts.SA_MODE_PREPARE: task_res = prep_dataset.prepare_pricing_dataset(work) if debug: log.info( f'done - result={ae_consts.ppj(task_res)} task={task_name} ' f'status={ae_consts.get_status(status=task_res["status"])} ' f'err={task_res["err"]} label={work["label"]}') else: log.info( f'done - result task={task_name} ' f'status={ae_consts.get_status(status=task_res["status"])} ' f'err={task_res["err"]} label={work["label"]}') if task_res['status'] == ae_consts.SUCCESS: image_res = None label = work['label'] ticker = work['ticker'] if plot_action == ae_consts.PLOT_ACTION_SHOW: log.info('showing plot') """ minute_key = f'{redis_key}_minute' minute_df_res = build_df.build_df_from_redis( label=label', address=redis_address, db=redis_db, key=minute_key) minute_df = None if ( minute_df_res['status'] == SUCCESS and minute_df_res['rec']['valid_df']): minute_df = minute_df_res['rec']['data'] print(minute_df.columns.values) column_list = [ 'close', 'date' ] """ today_str = datetime.datetime.now().strftime('%Y-%m-%d') extract_req = work extract_req['redis_key'] = f'{work["redis_key"]}_minute' extract_status, minute_df = \ extract_utils.extract_minute_dataset( work_dict=work) if extract_status == ae_consts.SUCCESS: log.info(f'{label} - ticker={ticker} ' f'creating chart date={today_str}') """ Plot Pricing with the Volume Overlay: """ image_res = ae_charts.plot_overlay_pricing_and_volume( log_label=label, ticker=ticker, date_format=ae_consts.IEX_MINUTE_DATE_FORMAT, df=minute_df, show_plot=True) """ Plot the High-Low-Open-Close Pricing: """ """ image_res = ae_charts.plot_hloc_pricing( log_label=label, ticker=ticker, title=f'{ticker} - Minute Pricing - {today_str}', df=minute_df, show_plot=True) """ """ Plot by custom columns in the DataFrame """ """ column_list = minute_df.columns.values column_list = [ 'date', 'close', 'high', 'low', 'open' ] image_res = ae_charts.plot_df( log_label=label, title='Pricing Title', column_list=column_list, df=minute_df, xcol='date', xlabel='Date', ylabel='Pricing', show_plot=True) """ elif plot_action == ae_consts.PLOT_ACTION_SAVE_TO_S3: log.info('coming soon - support to save to s3') elif plot_action == ae_consts.PLOT_ACTION_SAVE_AS_FILE: log.info('coming soon - support to save as file') if image_res: log.info(f'{label} show plot - ' f'status={ae_consts.get_status(image_res["status"])} ' f'err={image_res["err"]}') else: log.info(f'connecting to broker={broker_url} backend={backend_url}') # Get the Celery app app = get_celery_app.get_celery_app( name=__name__, auth_url=broker_url, backend_url=backend_url, path_to_config_module=path_to_config_module, ssl_options=ssl_options, transport_options=transport_options, include_tasks=include_tasks) log.info(f'calling task={task_name} - work={ae_consts.ppj(work)}') job_id = app.send_task(task_name, (work, )) log.info(f'calling task={task_name} - success job_id={job_id}')
def publish_ticker_aggregate_from_s3(): """publish_ticker_aggregate_from_s3 Download all ticker data from S3 and publish it's contents to Redis and back to S3 """ log.info('start - publish_ticker_aggregate_from_s3') parser = argparse.ArgumentParser( description=('Download and aggregated all ticker data, ' 'and store it in S3 and Redis. ')) parser.add_argument('-t', help=('ticker'), required=True, dest='ticker') parser.add_argument('-i', help=('optional - ticker id ' 'not used without a database'), required=False, dest='ticker_id') parser.add_argument('-l', help=('optional - path to the log config file'), required=False, dest='log_config_path') parser.add_argument('-b', help=('optional - broker url for Celery'), required=False, dest='broker_url') parser.add_argument('-B', help=('optional - backend url for Celery'), required=False, dest='backend_url') parser.add_argument('-k', help=('optional - s3 access key'), required=False, dest='s3_access_key') parser.add_argument('-s', help=('optional - s3 secret key'), required=False, dest='s3_secret_key') parser.add_argument('-a', help=('optional - s3 address format: <host:port>'), required=False, dest='s3_address') parser.add_argument('-S', help=('optional - s3 ssl or not'), required=False, dest='s3_secure') parser.add_argument('-u', help=('optional - s3 bucket name'), required=False, dest='s3_bucket_name') parser.add_argument('-c', help=('optional - s3 compiled bucket name'), required=False, dest='s3_compiled_bucket_name') parser.add_argument('-g', help=('optional - s3 region name'), required=False, dest='s3_region_name') parser.add_argument('-p', help=('optional - redis_password'), required=False, dest='redis_password') parser.add_argument('-r', help=('optional - redis_address format: <host:port>'), required=False, dest='redis_address') parser.add_argument('-n', help=('optional - redis and s3 key name'), required=False, dest='keyname') parser.add_argument( '-m', help=('optional - redis database number (0 by default)'), required=False, dest='redis_db') parser.add_argument('-x', help=('optional - redis expiration in seconds'), required=False, dest='redis_expire') parser.add_argument('-d', help=('debug'), required=False, dest='debug', action='store_true') args = parser.parse_args() ticker = TICKER ticker_id = TICKER_ID ssl_options = SSL_OPTIONS transport_options = TRANSPORT_OPTIONS broker_url = WORKER_BROKER_URL backend_url = WORKER_BACKEND_URL celery_config_module = WORKER_CELERY_CONFIG_MODULE include_tasks = INCLUDE_TASKS s3_access_key = S3_ACCESS_KEY s3_secret_key = S3_SECRET_KEY s3_region_name = S3_REGION_NAME s3_address = S3_ADDRESS s3_secure = S3_SECURE s3_bucket_name = S3_BUCKET s3_compiled_bucket_name = S3_COMPILED_BUCKET s3_key = S3_KEY redis_address = REDIS_ADDRESS redis_key = REDIS_KEY redis_password = REDIS_PASSWORD redis_db = REDIS_DB redis_expire = REDIS_EXPIRE debug = False if args.ticker: ticker = args.ticker.upper() if args.ticker_id: ticker = args.ticker_id if args.broker_url: broker_url = args.broker_url if args.backend_url: backend_url = args.backend_url if args.s3_access_key: s3_access_key = args.s3_access_key if args.s3_secret_key: s3_secret_key = args.s3_secret_key if args.s3_region_name: s3_region_name = args.s3_region_name if args.s3_address: s3_address = args.s3_address if args.s3_secure: s3_secure = args.s3_secure if args.s3_bucket_name: s3_bucket_name = args.s3_bucket_name if args.s3_compiled_bucket_name: s3_compiled_bucket_name = args.s3_compiled_bucket_name if args.keyname: s3_key = args.keyname redis_key = args.keyname if args.redis_address: redis_address = args.redis_address if args.redis_password: redis_password = args.redis_password if args.redis_db: redis_db = args.redis_db if args.redis_expire: redis_expire = args.redis_expire if args.debug: debug = True work = build_publish_ticker_aggregate_from_s3_request() work['ticker'] = ticker work['ticker_id'] = ticker_id work['s3_bucket'] = s3_bucket_name work['s3_compiled_bucket'] = s3_compiled_bucket_name if args.keyname: work['s3_key'] = s3_key work['redis_key'] = redis_key work['s3_access_key'] = s3_access_key work['s3_secret_key'] = s3_secret_key work['s3_region_name'] = s3_region_name work['s3_address'] = s3_address work['s3_secure'] = s3_secure work['redis_address'] = redis_address work['redis_password'] = redis_password work['redis_db'] = redis_db work['redis_expire'] = redis_expire work['debug'] = debug work['label'] = 'ticker={}'.format(ticker) path_to_tasks = 'analysis_engine.work_tasks' task_name = ('{}.publish_ticker_aggregate_from_s3.' 'publish_ticker_aggregate_from_s3'.format(path_to_tasks)) task_res = None if is_celery_disabled(): work['celery_disabled'] = True log.debug('starting without celery work={}'.format(ppj(work))) task_res = task_publisher.publish_ticker_aggregate_from_s3( work_dict=work) if debug: log.info('done - result={} ' 'task={} status={} ' 'err={} label={}'.format( ppj(task_res), task_name, get_status(status=task_res['status']), task_res['err'], work['label'])) else: log.info('done - result ' 'task={} status={} ' 'err={} label={}'.format( task_name, get_status(status=task_res['status']), task_res['err'], work['label'])) # if/else debug else: log.info('connecting to broker={} backend={}'.format( broker_url, backend_url)) # Get the Celery app app = get_celery_app(name=__name__, auth_url=broker_url, backend_url=backend_url, path_to_config_module=celery_config_module, ssl_options=ssl_options, transport_options=transport_options, include_tasks=include_tasks) log.info('calling task={} - work={}'.format(task_name, ppj(work))) job_id = app.send_task(task_name, (work, )) log.info('calling task={} - success job_id={}'.format( task_name, job_id))
# Disable celery log hijacking # https://github.com/celery/celery/issues/2509 @celery.signals.setup_logging.connect def setup_celery_logging(**kwargs): pass log = log_utils.build_colorized_logger(name=consts.APP_NAME, log_config_path=consts.LOG_CONFIG_PATH) log.info('start - {}'.format(consts.APP_NAME)) log.info('broker={} backend={} ' 'config={} include_tasks={}'.format( consts.WORKER_BROKER_URL, consts.WORKER_BACKEND_URL, consts.WORKER_CELERY_CONFIG_MODULE, consts.WORKER_TASKS)) # Get the Celery app from the project's get_celery_app module app = get_celery_app.get_celery_app( name=consts.APP_NAME, path_to_config_module=consts.WORKER_CELERY_CONFIG_MODULE, auth_url=consts.WORKER_BROKER_URL, backend_url=consts.WORKER_BACKEND_URL, include_tasks=consts.INCLUDE_TASKS) log.info('starting celery') app.start() log.info('end - {}'.format(consts.APP_NAME))
def run_custom_algo( mod_path, ticker='SPY', balance=50000, commission=6.0, start_date=None, end_date=None, name='myalgo', auto_fill=True, config_file=None, config_dict=None, load_from_s3_bucket=None, load_from_s3_key=None, load_from_redis_key=None, load_from_file=None, load_compress=False, load_publish=True, load_config=None, report_redis_key=None, report_s3_bucket=None, report_s3_key=None, report_file=None, report_compress=False, report_publish=True, report_config=None, history_redis_key=None, history_s3_bucket=None, history_s3_key=None, history_file=None, history_compress=False, history_publish=True, history_config=None, extract_redis_key=None, extract_s3_bucket=None, extract_s3_key=None, extract_file=None, extract_save_dir=None, extract_compress=False, extract_publish=True, extract_config=None, publish_to_s3=True, publish_to_redis=True, publish_to_slack=True, dataset_type=ae_consts.SA_DATASET_TYPE_ALGO_READY, serialize_datasets=ae_consts.DEFAULT_SERIALIZED_DATASETS, compress=False, encoding='utf-8', redis_enabled=True, redis_key=None, redis_address=None, redis_db=None, redis_password=None, redis_expire=None, redis_serializer='json', redis_encoding='utf-8', s3_enabled=True, s3_key=None, s3_address=None, s3_bucket=None, s3_access_key=None, s3_secret_key=None, s3_region_name=None, s3_secure=False, slack_enabled=False, slack_code_block=False, slack_full_width=False, timeseries=None, trade_strategy=None, verbose=False, debug=False, dataset_publish_extract=False, dataset_publish_history=False, dataset_publish_report=False, run_on_engine=False, auth_url=ae_consts.WORKER_BROKER_URL, backend_url=ae_consts.WORKER_BACKEND_URL, include_tasks=ae_consts.INCLUDE_TASKS, ssl_options=ae_consts.SSL_OPTIONS, transport_options=ae_consts.TRANSPORT_OPTIONS, path_to_config_module=ae_consts.WORKER_CELERY_CONFIG_MODULE, raise_on_err=True): """run_custom_algo Run a custom algorithm that derives the ``analysis_engine.algo.BaseAlgo`` class .. note:: Make sure to only have **1** class defined in an algo module. Imports from other modules should work just fine. **Algorithm arguments** :param mod_path: file path to custom algorithm class module :param ticker: ticker symbol :param balance: float - starting balance capital for creating buys and sells :param commission: float - cost pet buy or sell :param name: string - name for tracking algorithm in the logs :param start_date: string - start date for backtest with format ``YYYY-MM-DD HH:MM:SS`` :param end_date: end date for backtest with format ``YYYY-MM-DD HH:MM:SS`` :param auto_fill: optional - boolean for auto filling buy and sell orders for backtesting (default is ``True``) :param config_file: path to a json file containing custom algorithm object member values (like indicator configuration and predict future date units ahead for a backtest) :param config_dict: optional - dictionary that can be passed to derived class implementations of: ``def load_from_config(config_dict=config_dict)`` **Timeseries** :param timeseries: optional - string to set ``day`` or ``minute`` backtesting or live trading (default is ``minute``) **Trading Strategy** :param trade_strategy: optional - string to set the type of ``Trading Strategy`` for backtesting or live trading (default is ``count``) **Running Distributed Algorithms on the Engine Workers** :param run_on_engine: optional - boolean flag for publishing custom algorithms to Celery ae workers for distributing algorithm workloads (default is ``False`` which will run algos locally) this is required for distributing algorithms :param auth_url: Celery broker address (default is ``redis://localhost:6379/11`` or ``analysis_engine.consts.WORKER_BROKER_URL`` environment variable) this is required for distributing algorithms :param backend_url: Celery backend address (default is ``redis://localhost:6379/12`` or ``analysis_engine.consts.WORKER_BACKEND_URL`` environment variable) this is required for distributing algorithms :param include_tasks: list of modules containing tasks to add (default is ``analysis_engine.consts.INCLUDE_TASKS``) :param ssl_options: security options dictionary (default is ``analysis_engine.consts.SSL_OPTIONS``) :param trasport_options: transport options dictionary (default is ``analysis_engine.consts.TRANSPORT_OPTIONS``) :param path_to_config_module: config module for advanced Celery worker connectivity requirements (default is ``analysis_engine.work_tasks.celery_config`` or ``analysis_engine.consts.WORKER_CELERY_CONFIG_MODULE``) **Load Algorithm-Ready Dataset From Source** Use these arguments to load algorithm-ready datasets from supported sources (file, s3 or redis) :param load_from_s3_bucket: optional - string load the algo from an a previously-created s3 bucket holding an s3 key with an algorithm-ready dataset for use with: ``handle_data`` :param load_from_s3_key: optional - string load the algo from an a previously-created s3 key holding an algorithm-ready dataset for use with: ``handle_data`` :param load_from_redis_key: optional - string load the algo from a a previously-created redis key holding an algorithm-ready dataset for use with: ``handle_data`` :param load_from_file: optional - string path to a previously-created local file holding an algorithm-ready dataset for use with: ``handle_data`` :param load_compress: optional - boolean flag for toggling to decompress or not when loading an algorithm-ready dataset (``True`` means the dataset must be decompressed to load correctly inside an algorithm to run a backtest) :param load_publish: boolean - toggle publishing the load progress to slack, s3, redis or a file (default is ``True``) :param load_config: optional - dictionary for setting member variables to load an agorithm-ready dataset from a file, s3 or redis **Publishing Control Bool Flags** :param publish_to_s3: optional - boolean for toggling publishing to s3 on/off (default is ``True``) :param publish_to_redis: optional - boolean for publishing to redis on/off (default is ``True``) :param publish_to_slack: optional - boolean for publishing to slack (default is ``True``) **Algorithm Trade History Arguments** :param history_redis_key: optional - string where the algorithm trading history will be stored in an redis key :param history_s3_bucket: optional - string where the algorithm trading history will be stored in an s3 bucket :param history_s3_key: optional - string where the algorithm trading history will be stored in an s3 key :param history_file: optional - string key where the algorithm trading history will be stored in a file serialized as a json-string :param history_compress: optional - boolean flag for toggling to decompress or not when loading an algorithm-ready dataset (``True`` means the dataset will be compressed on publish) :param history_publish: boolean - toggle publishing the history to s3, redis or a file (default is ``True``) :param history_config: optional - dictionary for setting member variables to publish an algo ``trade history`` to s3, redis, a file or slack **Algorithm Trade Performance Report Arguments (Output Dataset)** :param report_redis_key: optional - string where the algorithm ``trading performance report`` (report) will be stored in an redis key :param report_s3_bucket: optional - string where the algorithm report will be stored in an s3 bucket :param report_s3_key: optional - string where the algorithm report will be stored in an s3 key :param report_file: optional - string key where the algorithm report will be stored in a file serialized as a json-string :param report_compress: optional - boolean flag for toggling to decompress or not when loading an algorithm-ready dataset (``True`` means the dataset will be compressed on publish) :param report_publish: boolean - toggle publishing the ``trading performance report`` s3, redis or a file (default is ``True``) :param report_config: optional - dictionary for setting member variables to publish an algo ``trading performance report`` to s3, redis, a file or slack **Extract an Algorithm-Ready Dataset Arguments** :param extract_redis_key: optional - string where the algorithm report will be stored in an redis key :param extract_s3_bucket: optional - string where the algorithm report will be stored in an s3 bucket :param extract_s3_key: optional - string where the algorithm report will be stored in an s3 key :param extract_file: optional - string key where the algorithm report will be stored in a file serialized as a json-string :param extract_save_dir: optional - string path to auto-generated files from the algo :param extract_compress: optional - boolean flag for toggling to decompress or not when loading an algorithm-ready dataset (``True`` means the dataset will be compressed on publish) :param extract_publish: boolean - toggle publishing the used ``algorithm-ready dataset`` to s3, redis or a file (default is ``True``) :param extract_config: optional - dictionary for setting member variables to publish an algo ``trading performance report`` to s3, redis, a file or slack **Dataset Arguments** :param dataset_type: optional - dataset type (default is ``SA_DATASET_TYPE_ALGO_READY``) :param serialize_datasets: optional - list of dataset names to deserialize in the dataset (default is ``DEFAULT_SERIALIZED_DATASETS``) :param encoding: optional - string for data encoding **Publish Algorithm Datasets to S3, Redis or a File** :param dataset_publish_extract: optional - bool for publishing the algorithm's ``algorithm-ready`` dataset to: s3, redis or file :param dataset_publish_history: optional - bool for publishing the algorithm's ``trading history`` dataset to: s3, redis or file :param dataset_publish_report: optional - bool for publishing the algorithm's ``trading performance report`` dataset to: s3, redis or file **Redis connectivity arguments** :param redis_enabled: bool - toggle for auto-caching all datasets in Redis (default is ``True``) :param redis_key: string - key to save the data in redis (default is ``None``) :param redis_address: Redis connection string format: ``host:port`` (default is ``localhost:6379``) :param redis_db: Redis db to use (default is ``0``) :param redis_password: optional - Redis password (default is ``None``) :param redis_expire: optional - Redis expire value (default is ``None``) :param redis_serializer: not used yet - support for future pickle objects in redis :param redis_encoding: format of the encoded key in redis **Minio (S3) connectivity arguments** :param s3_enabled: bool - toggle for auto-archiving on Minio (S3) (default is ``True``) :param s3_key: string - key to save the data in redis (default is ``None``) :param s3_address: Minio S3 connection string format: ``host:port`` (default is ``localhost:9000``) :param s3_bucket: S3 Bucket for storing the artifacts (default is ``dev``) which should be viewable on a browser: http://localhost:9000/minio/dev/ :param s3_access_key: S3 Access key (default is ``trexaccesskey``) :param s3_secret_key: S3 Secret key (default is ``trex123321``) :param s3_region_name: S3 region name (default is ``us-east-1``) :param s3_secure: Transmit using tls encryption (default is ``False``) **Slack arguments** :param slack_enabled: optional - boolean for publishing to slack :param slack_code_block: optional - boolean for publishing as a code black in slack :param slack_full_width: optional - boolean for publishing as a to slack using the full width allowed **Debugging arguments** :param debug: optional - bool for debug tracking :param verbose: optional - bool for increasing logging :param raise_on_err: boolean - set this to ``False`` on prod to ensure exceptions do not interrupt services. With the default (``True``) any exceptions from the library and your own algorithm are sent back out immediately exiting the backtest. """ module_name = 'BaseAlgo' custom_algo_module = None new_algo_object = None use_custom_algo = False found_algo_module = True should_publish_extract_dataset = False should_publish_history_dataset = False should_publish_report_dataset = False use_config_file = None use_config_dict = config_dict if config_file: if os.path.exists(config_file): use_config_file = config_file if not config_dict: try: use_config_dict = json.loads(open(config_file, 'r').read()) except Exception as e: msg = (f'failed parsing json config_file={config_file} ' f'with ex={e}') log.error(msg) raise Exception(msg) # end of loading the config_file err = None if mod_path: module_name = mod_path.split('/')[-1] loader = importlib.machinery.SourceFileLoader(module_name, mod_path) custom_algo_module = types.ModuleType(loader.name) loader.exec_module(custom_algo_module) use_custom_algo = True for member in inspect.getmembers(custom_algo_module): if module_name in str(member): found_algo_module = True break # for all members in this custom module file # if loading a custom algorithm module from a file on disk if not found_algo_module: err = (f'unable to find custom algorithm module={custom_algo_module}') if mod_path: err = ( 'analysis_engine.run_custom_algo.run_custom_algo was unable ' f'to find custom algorithm module={custom_algo_module} with ' f'provided path to \n file: {mod_path} \n' '\n' 'Please confirm ' 'that the class inherits from the BaseAlgo class like:\n' '\n' 'import analysis_engine.algo\n' 'class MyAlgo(analysis_engine.algo.BaseAlgo):\n ' '\n' 'If it is then please file an issue on github:\n ' 'https://github.com/AlgoTraders/stock-analysis-engine/' 'issues/new \n\nFor now this error results in a shutdown' '\n') # if mod_path set if verbose or debug: log.error(err) return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) # if not found_algo_module use_start_date = start_date use_end_date = end_date if not use_end_date: end_date = datetime.datetime.utcnow() use_end_date = end_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT) if not use_start_date: start_date = end_date - datetime.timedelta(days=75) use_start_date = start_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT) if verbose: log.info( f'{name} {ticker} setting default start_date={use_start_date}') # Load an algorithm-ready dataset from: # file, s3, or redis if not load_config: load_config = build_publish_request.build_publish_request( ticker=ticker, output_file=None, s3_bucket=None, s3_key=None, redis_key=None, compress=load_compress, redis_enabled=publish_to_redis, redis_address=redis_address, redis_db=redis_db, redis_password=redis_password, redis_expire=redis_expire, redis_serializer=redis_serializer, redis_encoding=redis_encoding, s3_enabled=publish_to_s3, s3_address=s3_address, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, slack_enabled=publish_to_slack, slack_code_block=slack_code_block, slack_full_width=slack_full_width, verbose=verbose, label=f'load-{name}') if load_from_file: load_config['output_file'] = load_from_file if load_from_redis_key: load_config['redis_key'] = load_from_redis_key load_config['redis_enabled'] = True if load_from_s3_bucket and load_from_s3_key: load_config['s3_bucket'] = load_from_s3_bucket load_config['s3_key'] = load_from_s3_key load_config['s3_enabled'] = True # end of building load_config dictionary if not already set # Automatically save all datasets to an algorithm-ready: # file, s3, or redis if not extract_config: extract_config = build_publish_request.build_publish_request( ticker=ticker, output_file=None, s3_bucket=None, s3_key=None, redis_key=None, compress=extract_compress, redis_enabled=publish_to_redis, redis_address=redis_address, redis_db=redis_db, redis_password=redis_password, redis_expire=redis_expire, redis_serializer=redis_serializer, redis_encoding=redis_encoding, s3_enabled=publish_to_s3, s3_address=s3_address, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, slack_enabled=publish_to_slack, slack_code_block=slack_code_block, slack_full_width=slack_full_width, verbose=verbose, label=f'extract-{name}') should_publish_extract_dataset = False if extract_file: extract_config['output_file'] = extract_file should_publish_extract_dataset = True if extract_redis_key and publish_to_redis: extract_config['redis_key'] = extract_redis_key extract_config['redis_enabled'] = True should_publish_extract_dataset = True if extract_s3_bucket and extract_s3_key and publish_to_s3: extract_config['s3_bucket'] = extract_s3_bucket extract_config['s3_key'] = extract_s3_key extract_config['s3_enabled'] = True should_publish_extract_dataset = True else: extract_config['s3_enabled'] = False # end of building extract_config dictionary if not already set # Automatically save the trading performance report: # file, s3, or redis if not report_config: report_config = build_publish_request.build_publish_request( ticker=ticker, output_file=None, s3_bucket=None, s3_key=None, redis_key=None, compress=report_compress, redis_enabled=publish_to_redis, redis_address=redis_address, redis_db=redis_db, redis_password=redis_password, redis_expire=redis_expire, redis_serializer=redis_serializer, redis_encoding=redis_encoding, s3_enabled=publish_to_s3, s3_address=s3_address, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, slack_enabled=publish_to_slack, slack_code_block=slack_code_block, slack_full_width=slack_full_width, verbose=verbose, label=f'report-{name}') should_publish_report_dataset = False if report_file: report_config['output_file'] = report_file should_publish_report_dataset = True if report_redis_key and publish_to_redis: report_config['redis_key'] = report_redis_key report_config['redis_enabled'] = True should_publish_report_dataset = True if report_s3_bucket and report_s3_key and publish_to_s3: report_config['s3_bucket'] = report_s3_bucket report_config['s3_key'] = report_s3_key report_config['s3_enabled'] = True should_publish_report_dataset = True # end of building report_config dictionary if not already set # Automatically save the trade history: # file, s3, or redis if not history_config: history_config = build_publish_request.build_publish_request( ticker=ticker, output_file=None, s3_bucket=None, s3_key=None, redis_key=None, compress=report_compress, redis_enabled=publish_to_redis, redis_address=redis_address, redis_db=redis_db, redis_password=redis_password, redis_expire=redis_expire, redis_serializer=redis_serializer, redis_encoding=redis_encoding, s3_enabled=publish_to_s3, s3_address=s3_address, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, slack_enabled=publish_to_slack, slack_code_block=slack_code_block, slack_full_width=slack_full_width, verbose=verbose, label=f'history-{name}') should_publish_history_dataset = False if history_file: history_config['output_file'] = history_file should_publish_history_dataset = True if history_redis_key and publish_to_redis: history_config['redis_key'] = history_redis_key history_config['redis_enabled'] = True should_publish_history_dataset = True if history_s3_bucket and history_s3_key and publish_to_s3: history_config['s3_bucket'] = history_s3_bucket history_config['s3_key'] = history_s3_key history_config['s3_enabled'] = True should_publish_history_dataset = True # end of building history_config dictionary if not already set if verbose: remove_vals = ['s3_access_key', 's3_secret_key', 'redis_password'] debug_extract_config = {} for k in extract_config: if k not in remove_vals: debug_extract_config[k] = extract_config[k] debug_report_config = {} for k in report_config: if k not in remove_vals: debug_report_config[k] = report_config[k] debug_history_config = {} for k in history_config: if k not in remove_vals: debug_history_config[k] = history_config[k] debug_load_config = {} for k in load_config: if k not in remove_vals: debug_load_config[k] = load_config[k] log.info(f'{name} {ticker} using extract config ' f'{ae_consts.ppj(debug_extract_config)}') log.info(f'{name} {ticker} using report config ' f'{ae_consts.ppj(debug_report_config)}') log.info(f'{name} {ticker} using trade history config ' f'{ae_consts.ppj(debug_history_config)}') log.info(f'{name} {ticker} using load config ' f'{ae_consts.ppj(debug_load_config)}') log.info(f'{name} {ticker} - building algo request') # end of verbose algo_req = build_algo_request.build_algo_request( ticker=ticker, balance=balance, commission=commission, start_date=use_start_date, end_date=use_end_date, timeseries=timeseries, trade_strategy=trade_strategy, config_file=use_config_file, config_dict=use_config_dict, load_config=load_config, history_config=history_config, report_config=report_config, extract_config=extract_config, label=name) algo_req['name'] = name algo_req['should_publish_extract_dataset'] = should_publish_extract_dataset algo_req['should_publish_history_dataset'] = should_publish_history_dataset algo_req['should_publish_report_dataset'] = should_publish_report_dataset algo_res = build_result.build_result(status=ae_consts.NOT_RUN, err=None, rec=None) if run_on_engine: rec = {'algo_req': algo_req, 'task_id': None} task_name = ('analysis_engine.work_tasks.' 'task_run_algo.task_run_algo') if verbose: log.info(f'starting distributed algo task={task_name}') elif debug: log.info( 'starting distributed algo by publishing to ' f'task={task_name} broker={auth_url} backend={backend_url}') # Get the Celery app app = get_celery_app.get_celery_app( name=__name__, auth_url=auth_url, backend_url=backend_url, path_to_config_module=path_to_config_module, ssl_options=ssl_options, transport_options=transport_options, include_tasks=include_tasks) if debug: log.info(f'calling distributed algo task={task_name} ' f'request={ae_consts.ppj(algo_req)}') elif verbose: log.info(f'calling distributed algo task={task_name}') job_id = app.send_task(task_name, (algo_req, )) if verbose: log.info(f'calling task={task_name} - success job_id={job_id}') rec['task_id'] = job_id algo_res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) return algo_res # end of run_on_engine if use_custom_algo: if verbose: log.info( f'inspecting {custom_algo_module} for class {module_name}') use_class_member_object = None for member in inspect.getmembers(custom_algo_module): if module_name in str(member): if verbose: log.info(f'start {name} with {member[1]}') use_class_member_object = member break # end of looking over the class definition but did not find it if use_class_member_object: new_algo_object = member[1](**algo_req) else: err = ('did not find a derived analysis_engine.algo.BaseAlgo ' f'class in the module file={mod_path} ' f'for ticker={ticker} algo_name={name}') if verbose or debug: log.error(err) return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) # end of finding a valid algorithm object else: new_algo_object = ae_algo.BaseAlgo(**algo_req) # if using a custom module path or the BaseAlgo if new_algo_object: # heads up - logging this might have passwords in the algo_req # log.debug( # f'{name} algorithm request: {algo_req}') if verbose: log.info(f'{name} - run ticker={ticker} from {use_start_date} ' f'to {use_end_date}') algo_res = run_algo.run_algo(algo=new_algo_object, raise_on_err=raise_on_err, **algo_req) algo_res['algo'] = new_algo_object if verbose: log.info(f'{name} - run ticker={ticker} from {use_start_date} ' f'to {use_end_date}') if custom_algo_module: if verbose: log.info(f'{name} - done run_algo ' f'custom_algo_module={custom_algo_module} ' f'module_name={module_name} ticker={ticker} ' f'from {use_start_date} to {use_end_date}') else: if verbose: log.info(f'{name} - done run_algo BaseAlgo ticker={ticker} ' f'from {use_start_date} to {use_end_date}') else: err = ('missing a derived analysis_engine.algo.BaseAlgo ' f'class in the module file={mod_path} for ticker={ticker} ' f'algo_name={name}') return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) # end of finding a valid algorithm object algo = algo_res.get('algo', None) if not algo: err = (f'failed creating algorithm object - ticker={ticker} ' f'status={ae_consts.get_status(status=algo_res["status"])} ' f'error={algo_res["err"]} algo name={name} ' f'custom_algo_module={custom_algo_module} ' f'module_name={module_name} ' f'from {use_start_date} to {use_end_date}') return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) if should_publish_extract_dataset or dataset_publish_extract: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (extract_config['redis_address'] and extract_config['redis_db'] >= 0 and extract_config['redis_key']): redis_log = ( f'redis://{extract_config["redis_address"]}' f'@{extract_config["redis_db"]}/{extract_config["redis_key"]}') use_log += f' {redis_log}' else: extract_config['redis_enabled'] = False if (extract_config['s3_address'] and extract_config['s3_bucket'] and extract_config['s3_key']): s3_log = ( f's3://{extract_config["s3_address"]}' f'/{extract_config["s3_bucket"]}/{extract_config["s3_key"]}') use_log += f' {s3_log}' else: extract_config['s3_enabled'] = False if extract_config['output_file']: file_log = f'file:{extract_config["output_file"]}' use_log += f' {file_log}' if verbose: log.info(f'{name} - publish - start ticker={ticker} ' f'algorithm-ready {use_log}') publish_status = algo.publish_input_dataset(**extract_config) if publish_status != ae_consts.SUCCESS: msg = ( 'failed to publish algorithm-ready datasets ' f'with status {ae_consts.get_status(status=publish_status)} ' f'attempted to {use_log}') log.error(msg) return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) if verbose: log.info(f'{name} - publish - done ticker={ticker} ' f'algorithm-ready {use_log}') # if publish the algorithm-ready dataset if should_publish_history_dataset or dataset_publish_history: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (history_config['redis_address'] and history_config['redis_db'] >= 0 and history_config['redis_key']): redis_log = ( f'redis://{history_config["redis_address"]}' f'@{history_config["redis_db"]}/{history_config["redis_key"]}') use_log += f' {redis_log}' else: history_config['redis_enabled'] = False if (history_config['s3_address'] and history_config['s3_bucket'] and history_config['s3_key']): s3_log = ( f's3://{history_config["s3_address"]}' f'/{history_config["s3_bucket"]}/{history_config["s3_key"]}') use_log += f' {s3_log}' else: history_config['s3_enabled'] = False if history_config['output_file']: file_log = f'file:{history_config["output_file"]}' use_log += f' {file_log}' if verbose: log.info(f'{name} - publish - start ticker={ticker} trading ' f'history {use_log}') publish_status = algo.publish_trade_history_dataset(**history_config) if publish_status != ae_consts.SUCCESS: msg = ( 'failed to publish trading history datasets ' f'with status {ae_consts.get_status(status=publish_status)} ' f'attempted to {use_log}') log.error(msg) return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) if verbose: log.info(f'{name} - publish - done ticker={ticker} trading ' f'history {use_log}') # if publish an trading history dataset if should_publish_report_dataset or dataset_publish_report: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (report_config['redis_address'] and report_config['redis_db'] >= 0 and report_config['redis_key']): redis_log = ( f'redis://{report_config["redis_address"]}' f'@{report_config["redis_db"]}/{report_config["redis_key"]}') use_log += f' {redis_log}' else: report_config['redis_enabled'] = False if (report_config['s3_address'] and report_config['s3_bucket'] and report_config['s3_key']): s3_log = ( f's3://{report_config["s3_address"]}' f'/{report_config["s3_bucket"]}/{report_config["s3_key"]}') use_log += f' {s3_log}' else: report_config['s3_enabled'] = False if report_config['output_file']: file_log = f'file:{report_config["output_file"]}' use_log += f' {file_log}' if verbose: log.info( f'{name} - publishing ticker={ticker} trading performance ' f'report {use_log}') publish_status = algo.publish_report_dataset(**report_config) if publish_status != ae_consts.SUCCESS: msg = ( 'failed to publish trading performance report datasets ' f'with status {ae_consts.get_status(status=publish_status)} ' f'attempted to {use_log}') log.error(msg) return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) if verbose: log.info( f'{name} - publish - done ticker={ticker} trading performance ' f'report {use_log}') # if publish an trading performance report dataset if verbose: log.info(f'{name} - done publishing datasets for ticker={ticker} ' f'from {use_start_date} to {use_end_date}') return algo_res