def test_run_daily_indicator_with_algo_config_sell(self): """test_run_daily_indicator_with_algo_config_sell""" algo = base_algo.BaseAlgo( ticker=self.ticker, balance=self.balance, start_date_str=self.start_date_str, end_date_str=self.end_date_str, config_dict=self.algo_config_dict) self.assertEqual( algo.name, self.algo_config_dict['name']) self.assertEqual( algo.tickers, [self.ticker]) algo.handle_data( data=self.data) res = algo.get_result() print(ae_consts.ppj(res)) self.assertEqual( res['history'][0]['total_sells'], 1) self.assertEqual( res['history'][0]['total_buys'], 0)
def test_integration_daily_indicator_with_algo_config(self): """test_integration_daily_indicator_with_algo_config""" if ae_consts.ev('INT_TESTS', '0') == '0': return algo = base_algo.BaseAlgo( ticker=self.ticker, balance=self.balance, start_date_str=self.start_date_str, end_date_str=self.end_date_str, config_dict=self.algo_config_dict) self.assertEqual( algo.name, self.algo_config_dict['name']) self.assertEqual( algo.tickers, [self.ticker]) algo.handle_data( data=self.data) res = algo.get_result() print(ae_consts.ppj(res)) self.assertTrue( res['history'][0]['total_sells'] >= 1) self.assertTrue( res['history'][0]['total_buys'] == 0)
def run_screener_analysis(work_dict): """run_screener_analysis Celery wrapper for running without celery :param work_dict: task data """ fn_name = 'run_screener_analysis' label = '{} - {}'.format(fn_name, work_dict.get('label', '')) log.info('{} - start'.format(label)) response = build_result.build_result(status=ae_consts.NOT_RUN, err=None, rec={}) task_res = {} # allow running without celery if ae_consts.is_celery_disabled(work_dict=work_dict): work_dict['celery_disabled'] = True task_res = task_screener_analysis(work_dict) if task_res: response = task_res.get('result', task_res) if ae_consts.ev('DEBUG_RESULTS', '0') == '1': response_details = response try: response_details = ae_consts.ppj(response) except Exception: response_details = response log.info('{} task result={}'.format(label, response_details)) else: log.error('{} celery was disabled but the task={} ' 'did not return anything'.format(label, response)) # end of if response else: task_res = task_screener_analysis.delay(work_dict=work_dict) rec = {'task_id': task_res} response = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) # if celery enabled if response: if ae_consts.ev('DEBUG_RESULTS', '0') == '1': log.info('{} - done ' 'status={} err={} rec={}'.format( label, ae_consts.get_status(response['status']), response['err'], response['rec'])) else: log.info('{} - done ' 'status={} err={}'.format( label, ae_consts.get_status(response['status']), response['err'])) else: log.info('{} - done ' 'no response'.format(label)) # end of if/else response return response
def run_screener_analysis(work_dict): """run_screener_analysis Celery wrapper for running without celery :param work_dict: task data """ fn_name = 'run_screener_analysis' label = f'''{fn_name} - {work_dict.get( 'label', '')}''' log.info(f'{label} - start') response = build_result.build_result(status=ae_consts.NOT_RUN, err=None, rec={}) task_res = {} # allow running without celery if ae_consts.is_celery_disabled(work_dict=work_dict): work_dict['celery_disabled'] = True task_res = task_screener_analysis(work_dict) if task_res: response = task_res.get('result', task_res) if ae_consts.ev('DEBUG_RESULTS', '0') == '1': response_details = response try: response_details = ae_consts.ppj(response) except Exception: response_details = response log.info(f'{label} task result={response_details}') else: log.error(f'{label} celery was disabled but the task={response} ' 'did not return anything') # end of if response else: task_res = task_screener_analysis.delay(work_dict=work_dict) rec = {'task_id': task_res} response = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) # if celery enabled if response: if ae_consts.ev('DEBUG_RESULTS', '0') == '1': log.info(f'{label} - done ' f'status={ae_consts.get_status(response["status"])} ' f'err={response["err"]} rec={response["rec"]}') else: log.info(f'{label} - done ' f'status={ae_consts.get_status(response["status"])} ' f'err={response["err"]}') else: log.info(f'{label} - done no response') # end of if/else response return response
def run_get_new_pricing_data(work_dict): """run_get_new_pricing_data Celery wrapper for running without celery :param work_dict: task data """ label = work_dict.get('label', '') log.debug(f'run_get_new_pricing_data - {label} - start') response = build_result.build_result(status=ae_consts.NOT_RUN, err=None, rec={}) task_res = {} # allow running without celery if ae_consts.is_celery_disabled(work_dict=work_dict): work_dict['celery_disabled'] = True task_res = get_new_pricing_data(work_dict) if task_res: response = task_res.get('result', task_res) if ae_consts.ev('DEBUG_RESULTS', '0') == '1': response_details = response try: response_details = ae_consts.ppj(response) except Exception: response_details = response log.debug(f'{label} task result={response_details}') else: log.error(f'{label} celery was disabled but the task={response} ' 'did not return anything') # end of if response else: task_res = get_new_pricing_data.delay(work_dict=work_dict) rec = {'task_id': task_res} response = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) # if celery enabled if response: status_str = ae_consts.get_status(response['status']) if ae_consts.ev('DEBUG_RESULTS', '0') == '1': log.debug(f'run_get_new_pricing_data - {label} - done ' f'status={status_str} ' f'err={response["err"]} ' f'rec={response["rec"]}') else: log.debug(f'run_get_new_pricing_data - {label} - done ' f'status={status_str} ' f'rec={response["rec"]}') else: log.debug(f'run_get_new_pricing_data - {label} - done ' 'no response') # end of if/else response return response
def run_publish_ticker_aggregate_from_s3(work_dict): """run_publish_ticker_aggregate_from_s3 Celery wrapper for running without celery :param work_dict: task data """ label = work_dict.get('label', '') log.info('run_publish_ticker_aggregate_from_s3 - {} - start'.format(label)) response = build_result.build_result(status=NOT_RUN, err=None, rec={}) task_res = {} # allow running without celery if is_celery_disabled(work_dict=work_dict): work_dict['celery_disabled'] = True task_res = publish_ticker_aggregate_from_s3(work_dict=work_dict) if task_res: response = task_res.get('result', task_res) if ev('DEBUG_RESULTS', '0') == '1': response_details = response try: response_details = ppj(response) except Exception: response_details = response log.info('{} task result={}'.format(label, response_details)) else: log.error('{} celery was disabled but the task={} ' 'did not return anything'.format(label, response)) # end of if response else: task_res = publish_ticker_aggregate_from_s3.delay(work_dict=work_dict) rec = {'task_id': task_res} response = build_result.build_result(status=SUCCESS, err=None, rec=rec) # if celery enabled if response: if ev('DEBUG_RESULTS', '0') == '1': log.info('run_publish_ticker_aggregate_from_s3 - {} - done ' 'status={} err={} rec={}'.format( label, get_status(response['status']), response['err'], response['rec'])) else: log.info('run_publish_ticker_aggregate_from_s3 - {} - done ' 'status={} err={}'.format(label, get_status(response['status']), response['err'])) else: log.info('run_publish_ticker_aggregate_from_s3 - {} - done ' 'no response'.format(label)) # end of if/else response return response
def build_entry_call_spread_details( ticker, close, num_contracts, low_strike, low_ask, low_bid, high_strike, high_ask, high_bid): """build_entry_call_spread_details Calculate pricing information for buying into ``Vertical Bull Call Option Spread`` contracts :param ticker: string ticker symbol :param num_contracts: integer number of contracts :param low_strike: float - strike for the low leg of the spread :param low_ask: float - ask price for the low leg of the spread :param low_bid: float - bid price for the low leg of the spread :param high_strike: float - strike for the high leg of the spread :param high_ask: float - ask price for the high leg of the spread :param high_bid: float - bid price for the high leg of the spread """ spread_details = spread_utils.build_option_spread_details( trade_type=TRADE_ENTRY, spread_type=SPREAD_VERTICAL_BULL, option_type=OPTION_CALL, close=close, num_contracts=num_contracts, low_strike=low_strike, low_ask=low_ask, low_bid=low_bid, high_strike=high_strike, high_ask=high_ask, high_bid=high_bid) log.debug( '{} type={} spread={} option={} close={} spread={}'.format( ticker, get_status(status=spread_details['trade_type']), get_status(status=spread_details['spread_type']), get_status(status=spread_details['option_type']), close, ppj(spread_details))) return spread_details
def get_configurables(self, **kwargs): """get_configurables **Derive this in your indicators** This is used as a helper for setting up algorithm configs for this indicator and to programmatically set the values based off the domain rules :param kwargs: optional keyword args """ self.ind_confs = [] self.lg('configurables={} for class={}'.format( ae_consts.ppj(self.ind_confs), self.__class__.__name__)) return self.ind_confs
def build_exit_put_spread_details(ticker, close, num_contracts, low_strike, low_ask, low_bid, high_strike, high_ask, high_bid): """build_exit_put_spread_details Calculate pricing information for selling (closing-out) ``Vertical Bear Put Option Spread`` contracts :param ticker: string ticker name :param num_contracts: integer number of contracts :param low_strike: float - strike for the low leg of the spread :param low_ask: float - ask price for the low leg of the spread :param low_bid: float - bid price for the low leg of the spread :param high_strike: float - strike for the high leg of the spread :param high_ask: float - ask price for the high leg of the spread :param high_bid: float - bid price for the high leg of the spread """ spread_details = spread_utils.build_option_spread_details( trade_type=ae_consts.TRADE_EXIT, spread_type=ae_consts.SPREAD_VERTICAL_BEAR, option_type=ae_consts.OPTION_PUT, close=close, num_contracts=num_contracts, low_strike=low_strike, low_ask=low_ask, low_bid=low_bid, high_strike=high_strike, high_ask=high_ask, high_bid=high_bid) log.debug('{} type={} spread={} option={} close={} spread={}'.format( ticker, ae_consts.get_status(status=spread_details['trade_type']), ae_consts.get_status(status=spread_details['spread_type']), ae_consts.get_status(status=spread_details['option_type']), close, ae_consts.ppj(spread_details))) return spread_details
def test_integration_extract_option_calls(self): """test_integration_extract_option_calls""" if ae_consts.ev('INT_TESTS', '0') == '0': return # build dataset cache dictionary work = api_requests.get_ds_dict( ticker='SPY', base_key='SPY_2018-12-31', label='test_integration_extract_option_calls') status, df = yahoo_extract.extract_option_calls_dataset(work_dict=work) if status == ae_consts.SUCCESS: self.assertIsNotNone(df) self.debug_df(df=df) self.assertTrue(ae_consts.is_df(df=df)) for i, r in df.iterrows(): print(ae_consts.ppj(json.loads(r.to_json()))) log.info('done printing option call data') else: log.critical('Yahoo Option Calls are missing in redis ' f'for ticker={work["ticker"]} ' f'status={ae_consts.get_status(status=status)}')
def get_data_from_yahoo(work_dict): """get_data_from_yahoo Get data from yahoo :param work_dict: request dictionary """ label = 'get_data_from_yahoo' log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict)) num_news_rec = 0 num_option_calls = 0 num_option_puts = 0 cur_high = -1 cur_low = -1 cur_open = -1 cur_close = -1 cur_volume = -1 rec = { 'pricing': None, 'options': None, 'calls': None, 'puts': None, 'news': None, 'exp_date': None, 'publish_pricing_update': None, 'date': None, 'updated': None } res = {'status': NOT_RUN, 'err': None, 'rec': rec} try: ticker = work_dict.get('ticker', TICKER) exp_date = work_dict.get('exp_date', None) cur_strike = work_dict.get('strike', None) contract_type = str(work_dict.get('contract', 'C')).upper() get_pricing = work_dict.get('get_pricing', True) get_news = work_dict.get('get_news', True) get_options = work_dict.get('get_options', True) orient = work_dict.get('orient', 'records') label = work_dict.get('label', label) ticker_results = pinance.Pinance(ticker) num_news_rec = 0 use_date = exp_date if not exp_date: exp_date = opt_dates.option_expiration(date=exp_date) use_date = exp_date.strftime('%Y-%m-%d') """ Debug control flags Quickly turn specific fetches off: get_news = False get_pricing = False get_options = False """ if get_pricing: log.info('{} getting ticker={} pricing'.format(label, ticker)) ticker_results.get_quotes() if ticker_results.quotes_data: pricing_dict = ticker_results.quotes_data cur_high = pricing_dict.get('regularMarketDayHigh', None) cur_low = pricing_dict.get('regularMarketDayLow', None) cur_open = pricing_dict.get('regularMarketOpen', None) cur_close = pricing_dict.get('regularMarketPreviousClose', None) cur_volume = pricing_dict.get('regularMarketVolume', None) pricing_dict['high'] = cur_high pricing_dict['low'] = cur_low pricing_dict['open'] = cur_open pricing_dict['close'] = cur_close pricing_dict['volume'] = cur_volume pricing_dict['date'] = get_last_close_str() if 'regularMarketTime' in pricing_dict: pricing_dict['market_time'] = \ datetime.datetime.fromtimestamp( pricing_dict['regularMarketTime']).strftime( COMMON_TICK_DATE_FORMAT) if 'postMarketTime' in pricing_dict: pricing_dict['post_market_time'] = \ datetime.datetime.fromtimestamp( pricing_dict['postMarketTime']).strftime( COMMON_TICK_DATE_FORMAT) log.info('{} ticker={} converting pricing to ' 'df orient={}'.format(label, ticker, orient)) try: rec['pricing'] = pricing_dict except Exception as f: rec['pricing'] = '{}' log.info('{} ticker={} failed converting pricing ' 'data={} to df ex={}'.format( label, ticker, ppj(pricing_dict), f)) # try/ex log.info('{} ticker={} done converting pricing to ' 'df orient={}'.format(label, ticker, orient)) else: log.error('{} ticker={} missing quotes_data={}'.format( label, ticker, ticker_results.quotes_data)) # end of if ticker_results.quotes_data log.info('{} ticker={} close={} vol={}'.format( label, ticker, cur_close, cur_volume)) else: log.info('{} skip - getting ticker={} pricing'.format( label, ticker, get_pricing)) # if get_pricing if get_news: log.info('{} getting ticker={} news'.format(label, ticker)) ticker_results.get_news() if ticker_results.news_data: news_list = None try: news_list = ticker_results.news_data log.info('{} ticker={} converting news to ' 'df orient={}'.format(label, ticker, orient)) num_news_rec = len(news_list) rec['news'] = news_list except Exception as f: rec['news'] = '{}' log.info('{} ticker={} failed converting news ' 'data={} to df ex={}'.format( label, ticker, news_list, f)) # try/ex log.info('{} ticker={} done converting news to ' 'df orient={}'.format(label, ticker, orient)) else: log.info('{} ticker={} Yahoo NO news={}'.format( label, ticker, ticker_results.news_data)) # end of if ticker_results.news_data else: log.info('{} skip - getting ticker={} news'.format(label, ticker)) # end if get_news if get_options: get_all_strikes = True if get_all_strikes: cur_strike = None else: if cur_close: cur_strike = int(cur_close) if not cur_strike: cur_strike = 287 log.info('{} ticker={} num_news={} get options close={} ' 'exp_date={} contract={} strike={}'.format( label, ticker, num_news_rec, cur_close, use_date, contract_type, cur_strike)) options_dict = \ yahoo_get_pricing.get_options( ticker=ticker, exp_date_str=use_date, contract_type=contract_type, strike=cur_strike) rec['options'] = '{}' try: log.info('{} ticker={} converting options to ' 'df orient={}'.format(label, ticker, orient)) num_option_calls = options_dict.get('num_calls', None) num_option_puts = options_dict.get('num_puts', None) rec['options'] = { 'exp_date': options_dict.get('exp_date', None), 'calls': options_dict.get('calls', None), 'puts': options_dict.get('puts', None), 'num_calls': num_option_calls, 'num_puts': num_option_puts } rec['calls'] = rec['options'].get('calls', EMPTY_DF_STR) rec['puts'] = rec['options'].get('puts', EMPTY_DF_STR) except Exception as f: rec['options'] = '{}' log.info('{} ticker={} failed converting options ' 'data={} to df ex={}'.format(label, ticker, options_dict, f)) # try/ex log.info('{} ticker={} done converting options to ' 'df orient={} num_calls={} num_puts={}'.format( label, ticker, orient, num_option_calls, num_option_puts)) else: log.info('{} skip - getting ticker={} options'.format( label, ticker)) # end of if get_options log.info('{} yahoo pricing for ticker={} close={} ' 'num_calls={} num_puts={} news={}'.format( label, ticker, cur_close, num_option_calls, num_option_puts, num_news_rec)) fields_to_upload = ['pricing', 'options', 'calls', 'puts', 'news'] for field_name in fields_to_upload: upload_and_cache_req = copy.deepcopy(work_dict) upload_and_cache_req['celery_disabled'] = True upload_and_cache_req['data'] = rec[field_name] if not upload_and_cache_req['data']: upload_and_cache_req['data'] = '{}' if 'redis_key' in work_dict: upload_and_cache_req['redis_key'] = '{}_{}'.format( work_dict.get('redis_key', '{}_{}'.format(ticker, field_name)), field_name) if 's3_key' in work_dict: upload_and_cache_req['s3_key'] = '{}_{}'.format( work_dict.get('s3_key', '{}_{}'.format(ticker, field_name)), field_name) try: update_res = publisher.run_publish_pricing_update( work_dict=upload_and_cache_req) update_status = update_res.get('status', NOT_SET) log.info('{} publish update status={} data={}'.format( label, get_status(status=update_status), update_res)) except Exception as f: err = ('{} - failed to upload YAHOO data={} to ' 'to s3_key={} and redis_key={}'.format( label, upload_and_cache_req, upload_and_cache_req['s3_key'], upload_and_cache_req['redis_key'])) log.error(err) # end of try/ex to upload and cache if not rec[field_name]: log.debug('{} - ticker={} no data from YAHOO for ' 'field_name={}'.format(label, ticker, field_name)) # end of for all fields res = build_result.build_result(status=SUCCESS, err=None, rec=rec) except Exception as e: res = build_result.build_result(status=ERR, err=('failed - get_data_from_yahoo ' 'dict={} with ex={}').format( work_dict, e), rec=rec) log.error('{} - {}'.format(label, res['err'])) # end of try/ex log.info('task - get_data_from_yahoo done - ' '{} - status={}'.format(label, get_status(res['status']))) return res
def get_data_from_redis_key(label=None, client=None, host=None, port=None, password=None, db=None, key=None, expire=None, serializer='json', encoding='utf-8'): """get_data_from_redis_key :param label: log tracking label :param client: initialized redis client :param host: not used yet - redis host :param port: not used yet - redis port :param password: not used yet - redis password :param db: not used yet - redis db :param key: not used yet - redis key :param expire: not used yet - redis expire :param serializer: not used yet - support for future pickle objects in redis :param encoding: format of the encoded key in redis """ decoded_data = None data = None rec = {'data': data} res = build_result.build_result(status=NOT_RUN, err=None, rec=rec) log_id = label if label else 'get-data' try: use_client = client if not use_client: log.debug('{} get key={} new client={}:{}@{}'.format( log_id, key, host, port, db)) use_client = redis.Redis(host=host, port=port, password=password, db=db) else: log.debug('{} get key={} client'.format(log_id, key)) # create Redis client if not set # https://redis-py.readthedocs.io/en/latest/index.html#redis.StrictRedis.get # noqa raw_data = use_client.get(name=key) if raw_data: log.debug('{} decoding key={} encoding={}'.format( log_id, key, encoding)) decoded_data = raw_data.decode(encoding) log.debug('{} deserial key={} serializer={}'.format( log_id, key, serializer)) if serializer == 'json': data = json.loads(decoded_data) elif serializer == 'df': data = decoded_data else: data = decoded_data if data: if ev('DEBUG_REDIS', '0') == '1': log.info('{} - found key={} data={}'.format( log_id, key, ppj(data))) else: log.debug('{} - found key={}'.format(log_id, key)) # log snippet - if data rec['data'] = data return build_result.build_result(status=SUCCESS, err=None, rec=rec) else: log.debug('{} no data key={}'.format(log_id, key)) return build_result.build_result(status=SUCCESS, err=None, rec=rec) except Exception as e: err = ('{} failed - redis get from decoded={} data={} ' 'key={} ex={}'.format(log_id, decoded_data, data, key, e)) log.error(err) res = build_result.build_result(status=ERR, err=err, rec=rec) # end of try/ex for getting redis data return res
def publish_ticker_aggregate_from_s3(): """publish_ticker_aggregate_from_s3 Download all ticker data from S3 and publish it's contents to Redis and back to S3 """ log.info('start - publish_ticker_aggregate_from_s3') parser = argparse.ArgumentParser( description=('Download and aggregated all ticker data, ' 'and store it in S3 and Redis. ')) parser.add_argument('-t', help=('ticker'), required=True, dest='ticker') parser.add_argument('-i', help=('optional - ticker id ' 'not used without a database'), required=False, dest='ticker_id') parser.add_argument('-l', help=('optional - path to the log config file'), required=False, dest='log_config_path') parser.add_argument('-b', help=('optional - broker url for Celery'), required=False, dest='broker_url') parser.add_argument('-B', help=('optional - backend url for Celery'), required=False, dest='backend_url') parser.add_argument('-k', help=('optional - s3 access key'), required=False, dest='s3_access_key') parser.add_argument('-s', help=('optional - s3 secret key'), required=False, dest='s3_secret_key') parser.add_argument('-a', help=('optional - s3 address format: <host:port>'), required=False, dest='s3_address') parser.add_argument('-S', help=('optional - s3 ssl or not'), required=False, dest='s3_secure') parser.add_argument('-u', help=('optional - s3 bucket name'), required=False, dest='s3_bucket_name') parser.add_argument('-c', help=('optional - s3 compiled bucket name'), required=False, dest='s3_compiled_bucket_name') parser.add_argument('-g', help=('optional - s3 region name'), required=False, dest='s3_region_name') parser.add_argument('-p', help=('optional - redis_password'), required=False, dest='redis_password') parser.add_argument('-r', help=('optional - redis_address format: <host:port>'), required=False, dest='redis_address') parser.add_argument('-n', help=('optional - redis and s3 key name'), required=False, dest='keyname') parser.add_argument( '-m', help=('optional - redis database number (0 by default)'), required=False, dest='redis_db') parser.add_argument('-x', help=('optional - redis expiration in seconds'), required=False, dest='redis_expire') parser.add_argument('-d', help=('debug'), required=False, dest='debug', action='store_true') args = parser.parse_args() ticker = TICKER ticker_id = TICKER_ID ssl_options = SSL_OPTIONS transport_options = TRANSPORT_OPTIONS broker_url = WORKER_BROKER_URL backend_url = WORKER_BACKEND_URL celery_config_module = WORKER_CELERY_CONFIG_MODULE include_tasks = INCLUDE_TASKS s3_access_key = S3_ACCESS_KEY s3_secret_key = S3_SECRET_KEY s3_region_name = S3_REGION_NAME s3_address = S3_ADDRESS s3_secure = S3_SECURE s3_bucket_name = S3_BUCKET s3_compiled_bucket_name = S3_COMPILED_BUCKET s3_key = S3_KEY redis_address = REDIS_ADDRESS redis_key = REDIS_KEY redis_password = REDIS_PASSWORD redis_db = REDIS_DB redis_expire = REDIS_EXPIRE debug = False if args.ticker: ticker = args.ticker.upper() if args.ticker_id: ticker = args.ticker_id if args.broker_url: broker_url = args.broker_url if args.backend_url: backend_url = args.backend_url if args.s3_access_key: s3_access_key = args.s3_access_key if args.s3_secret_key: s3_secret_key = args.s3_secret_key if args.s3_region_name: s3_region_name = args.s3_region_name if args.s3_address: s3_address = args.s3_address if args.s3_secure: s3_secure = args.s3_secure if args.s3_bucket_name: s3_bucket_name = args.s3_bucket_name if args.s3_compiled_bucket_name: s3_compiled_bucket_name = args.s3_compiled_bucket_name if args.keyname: s3_key = args.keyname redis_key = args.keyname if args.redis_address: redis_address = args.redis_address if args.redis_password: redis_password = args.redis_password if args.redis_db: redis_db = args.redis_db if args.redis_expire: redis_expire = args.redis_expire if args.debug: debug = True work = build_publish_ticker_aggregate_from_s3_request() work['ticker'] = ticker work['ticker_id'] = ticker_id work['s3_bucket'] = s3_bucket_name work['s3_compiled_bucket'] = s3_compiled_bucket_name if args.keyname: work['s3_key'] = s3_key work['redis_key'] = redis_key work['s3_access_key'] = s3_access_key work['s3_secret_key'] = s3_secret_key work['s3_region_name'] = s3_region_name work['s3_address'] = s3_address work['s3_secure'] = s3_secure work['redis_address'] = redis_address work['redis_password'] = redis_password work['redis_db'] = redis_db work['redis_expire'] = redis_expire work['debug'] = debug work['label'] = 'ticker={}'.format(ticker) path_to_tasks = 'analysis_engine.work_tasks' task_name = ('{}.publish_ticker_aggregate_from_s3.' 'publish_ticker_aggregate_from_s3'.format(path_to_tasks)) task_res = None if is_celery_disabled(): work['celery_disabled'] = True log.debug('starting without celery work={}'.format(ppj(work))) task_res = task_publisher.publish_ticker_aggregate_from_s3( work_dict=work) if debug: log.info('done - result={} ' 'task={} status={} ' 'err={} label={}'.format( ppj(task_res), task_name, get_status(status=task_res['status']), task_res['err'], work['label'])) else: log.info('done - result ' 'task={} status={} ' 'err={} label={}'.format( task_name, get_status(status=task_res['status']), task_res['err'], work['label'])) # if/else debug else: log.info('connecting to broker={} backend={}'.format( broker_url, backend_url)) # Get the Celery app app = get_celery_app(name=__name__, auth_url=broker_url, backend_url=backend_url, path_to_config_module=celery_config_module, ssl_options=ssl_options, transport_options=transport_options, include_tasks=include_tasks) log.info('calling task={} - work={}'.format(task_name, ppj(work))) job_id = app.send_task(task_name, (work, )) log.info('calling task={} - success job_id={}'.format( task_name, job_id))
def build_publish_request( ticker=None, tickers=None, convert_to_json=False, output_file=None, compress=False, redis_enabled=ae_consts.ENABLED_REDIS_PUBLISH, redis_key=None, redis_address=ae_consts.REDIS_ADDRESS, redis_db=ae_consts.REDIS_DB, redis_password=ae_consts.REDIS_PASSWORD, redis_expire=ae_consts.REDIS_EXPIRE, redis_serializer='json', redis_encoding='utf-8', s3_enabled=ae_consts.ENABLED_S3_UPLOAD, s3_key=None, s3_address=ae_consts.S3_ADDRESS, s3_bucket=ae_consts.S3_BUCKET, s3_access_key=ae_consts.S3_ACCESS_KEY, s3_secret_key=ae_consts.S3_SECRET_KEY, s3_region_name=ae_consts.S3_REGION_NAME, s3_secure=ae_consts.S3_SECURE, slack_enabled=False, slack_code_block=False, slack_full_width=False, verbose=False, label='publisher'): """build_publish_request Build a dictionary for helping to quickly publish to multiple optional endpoints: - a local file path (``output_file``) - minio (``s3_bucket`` and ``s3_key``) - redis (``redis_key``) - slack :param ticker: ticker :param tickers: optional - list of tickers :param label: optional - algo log tracking name :param output_file: path to save the data to a file :param compress: optional - compress before publishing :param verbose: optional - boolean to log output :param kwargs: optional - future argument support **(Optional) Redis connectivity arguments** :param redis_enabled: bool - toggle for auto-caching all datasets in Redis (default is ``ENABLED_REDIS_PUBLISH``) :param redis_key: string - key to save the data in redis (default is ``None``) :param redis_address: Redis connection string format: ``host:port`` (default is ``REDIS_ADDRESS``) :param redis_db: Redis db to use (default is ``REDIS_DB``) :param redis_password: optional - Redis password (default is ``REDIS_PASSWORD``) :param redis_expire: optional - Redis expire value (default is ``REDIS_EXPIRE``) :param redis_serializer: not used yet - support for future pickle objects in redis (default is ``json``) :param redis_encoding: format of the encoded key in redis (default is ``utf-8``) **(Optional) Minio (S3) connectivity arguments** :param s3_enabled: bool - toggle for auto-archiving on Minio (S3) (default is ``ENABLED_S3_UPLOAD``) :param s3_key: string - key to save the data in redis (default is ``None``) :param s3_address: Minio S3 connection string format: ``host:port`` (default is ``S3_ADDRESS``) :param s3_bucket: S3 Bucket for storing the artifacts (default is ``S3_BUCKET``) which should be viewable on a browser: http://localhost:9000/minio/dev/ :param s3_access_key: S3 Access key (default is ``S3_ACCESS_KEY``) :param s3_secret_key: S3 Secret key (default is ``S3_SECRET_KEY``) :param s3_region_name: S3 region name (default is ``S3_REGION_NAME``) :param s3_secure: Transmit using tls encryption (default is ``S3_SECURE``) **(Optional) Slack arguments** :param slack_enabled: optional - boolean for publishing to slack :param slack_code_block: optional - boolean for publishing as a code black in slack :param slack_full_width: optional - boolean for publishing as a to slack using the full width allowed """ use_tickers = [] if ticker: use_tickers = [ ticker.upper() ] if tickers: for t in tickers: if t not in use_tickers: use_tickers.append(t.upper()) work = { 'tickers': use_tickers, 'label': label, 'convert_to_json': convert_to_json, 'output_file': output_file, 'compress': compress, 'redis_enabled': redis_enabled, 'redis_key': redis_key, 'redis_address': redis_address, 'redis_db': redis_db, 'redis_password': redis_password, 'redis_expire': redis_expire, 'redis_serializer': redis_serializer, 'redis_encoding': redis_encoding, 's3_enabled': s3_enabled, 's3_key': s3_key, 's3_address': s3_address, 's3_bucket': s3_bucket, 's3_access_key': s3_access_key, 's3_secret_key': s3_secret_key, 's3_region_name': s3_region_name, 's3_secure': s3_secure, 'slack_enabled': slack_enabled, 'slack_code_block': slack_code_block, 'slack_full_width': slack_full_width, 'verbose': verbose, 'version': 1, 'label': label } log.debug( 'created publish_request={}'.format( ae_consts.ppj(work))) return work
def get_task_results(work_dict=None, result=None, **kwargs): """get_task_results If celery is disabled by the environment key ``export CELERY_DISABLED=1`` or requested in the ``work_dict['celery_disabled'] = True`` then return the task result dictionary, otherwise return ``None``. This method is useful for allowing tests to override the returned payloads during task chaining using ``@mock.patch``. :param work_dict: task work dictionary :param result: task result dictionary :param kwargs: keyword arguments """ send_results_back = None cel_disabled = False if work_dict: if ae_consts.is_celery_disabled(work_dict=work_dict): send_results_back = result cel_disabled = True # end of sending back results if told to do so if ae_consts.ev('DEBUG_TASK', '0') == '1': status = ae_consts.NOT_SET err = None record = None label = None if result: status = result.get('status', ae_consts.NOT_SET) err = result.get('err', None) record = result.get('rec', None) if work_dict: label = work_dict.get('label', None) log_id = 'get_task_results' if label: log_id = '{} - get_task_results'.format(label) result_details = record if record: result_details = ae_consts.ppj(record) status_details = status if status: status_details = ae_consts.get_status(status=status) work_details = work_dict if work_dict: work_details = ae_consts.ppj(work_dict) if status == ae_consts.SUCCESS: log.info('{} celery_disabled={} ' 'status={} err={} work_dict={} result={}'.format( log_id, cel_disabled, status_details, err, work_details, result_details)) else: if cel_disabled: log.error('{} celery_disabled={} ' 'status={} err={} work_dict={} result={}'.format( log_id, cel_disabled, status_details, err, work_details, result_details)) else: log.info('{} celery_disabled={} ' 'status={} err={} work_dict={} result={}'.format( log_id, cel_disabled, status_details, err, work_details, result_details)) # end of if debugging the task results return send_results_back
def build_df_from_redis(label=None, client=None, address=None, host=None, port=None, password=None, db=None, key=None, expire=None, serializer='json', encoding='utf-8', orient='records'): """build_df_from_redis :param label: log tracking label :param client: initialized redis client :param address: redis address: <host:port> :param host: redis host :param port: redis port :param password: redis password :param db: redis db :param key: redis key :param expire: not used yet - redis expire :param serializer: support for future pickle objects in redis :param encoding: format of the encoded key in redis :param orient: use the same orient value as the ``to_json(orient='records')`` used to deserialize the DataFrame correctly. """ data = None valid_df = False df = None rec = {'valid_df': valid_df, 'data': data} res = build_result.build_result(status=NOT_RUN, err=None, rec=rec) log_id = label if label else 'build-df' try: log.debug('{} calling get redis key={}'.format(log_id, key)) use_host = host use_port = port if not use_host and not use_port: if address: use_host = address.split(':')[0] use_port = int(address.split(':')[1]) use_client = client if not use_client: log.debug('{} connecting to redis={}:{}@{}'.format( log_id, use_host, use_port, db)) use_client = redis.Redis(host=use_host, port=use_port, password=password, db=db) redis_res = redis_get.get_data_from_redis_key(label=log_id, client=use_client, host=use_host, port=use_port, password=password, db=db, key=key, expire=expire, serializer='json', encoding=encoding) valid_df = False if redis_res['status'] == SUCCESS: data = redis_res['rec'].get('data', None) if data: if ev('DEBUG_REDIS', '0') == '1': log.info('{} - found key={} data={}'.format( log_id, key, ppj(data))) else: log.debug('{} - loading df from key={}'.format( log_id, key)) df = pd.read_json(data, orient='records') valid_df = True else: log.debug('{} key={} no data'.format(log_id, key)) # if data rec['data'] = df rec['valid_df'] = valid_df res = build_result.build_result(status=SUCCESS, err=None, rec=rec) return res else: log.debug('{} no data key={}'.format(log_id, key)) res = build_result.build_result(status=SUCCESS, err=None, rec=rec) return res except Exception as e: err = ('{} failed - build_df_from_redis data={} ' 'key={} ex={}'.format(log_id, (data == '0'), key, e)) log.error(err) res = build_result.build_result(status=ERR, err=err, rec=rec) # end of try/ex for getting redis data return res
def run_algo( ticker=None, tickers=None, algo=None, # optional derived ``analysis_engine.algo.Algo`` instance balance=None, # float starting base capital commission=None, # float for single trade commission for buy or sell start_date=None, # string YYYY-MM-DD HH:MM:SS end_date=None, # string YYYY-MM-DD HH:MM:SS datasets=None, # string list of identifiers num_owned_dict=None, # not supported cache_freq='daily', # 'minute' not supported auto_fill=True, load_config=None, report_config=None, history_config=None, extract_config=None, use_key=None, extract_mode='all', iex_datasets=None, redis_enabled=True, redis_address=None, redis_db=None, redis_password=None, redis_expire=None, redis_key=None, s3_enabled=True, s3_address=None, s3_bucket=None, s3_access_key=None, s3_secret_key=None, s3_region_name=None, s3_secure=False, s3_key=None, celery_disabled=True, broker_url=None, result_backend=None, label=None, name=None, timeseries=None, trade_strategy=None, verbose=False, publish_to_slack=True, publish_to_s3=True, publish_to_redis=True, extract_datasets=None, config_file=None, config_dict=None, version=1, raise_on_err=True, **kwargs): """run_algo Run an algorithm with steps: 1) Extract redis keys between dates 2) Compile a data pipeline dictionary (call it ``data``) 3) Call algorithm's ``myalgo.handle_data(data=data)`` .. note:: If no ``algo`` is set, the ``analysis_engine.algo.BaseAlgo`` algorithm is used. .. note:: Please ensure Redis and Minio are running before trying to extract tickers **Stock tickers to extract** :param ticker: single stock ticker/symbol/ETF to extract :param tickers: optional - list of tickers to extract :param use_key: optional - extract historical key from Redis **Algo Configuration** :param algo: derived instance of ``analysis_engine.algo.Algo`` object :param balance: optional - float balance parameter can also be set on the ``algo`` object if not set on the args :param commission: float for single trade commission for buy or sell. can also be set on the ``algo`` objet :param start_date: string ``YYYY-MM-DD_HH:MM:SS`` cache value :param end_date: string ``YYYY-MM-DD_HH:MM:SS`` cache value :param dataset_types: list of strings that are ``iex`` or ``yahoo`` datasets that are cached. :param cache_freq: optional - depending on if you are running data feeds on a ``daily`` cron (default) vs every ``minute`` (or faster) :param num_owned_dict: not supported yet :param auto_fill: optional - boolean for auto filling buy/sell orders for backtesting (default is ``True``) :param trading_calendar: ``trading_calendar.TradingCalendar`` object, by default ``analysis_engine.calendars. always_open.AlwaysOpen`` trading calendar # TradingCalendar by ``TFSExchangeCalendar`` :param config_file: path to a json file containing custom algorithm object member values (like indicator configuration and predict future date units ahead for a backtest) :param config_dict: optional - dictionary that can be passed to derived class implementations of: ``def load_from_config(config_dict=config_dict)`` **Timeseries** :param timeseries: optional - string to set ``day`` or ``minute`` backtesting or live trading (default is ``minute``) **Trading Strategy** :param trade_strategy: optional - string to set the type of ``Trading Strategy`` for backtesting or live trading (default is ``count``) **Algorithm Dataset Loading, Extracting, Reporting and Trading History arguments** :param load_config: optional - dictionary for setting member variables to load an agorithm-ready dataset from a file, s3 or redis :param report_config: optional - dictionary for setting member variables to publish an algo ``trading performance report`` to s3, redis, a file or slack :param history_config: optional - dictionary for setting member variables to publish an algo ``trade history`` to s3, redis, a file or slack :param extract_config: optional - dictionary for setting member variables to publish an algo ``trading performance report`` to s3, redis, a file or slack **(Optional) Data sources, datafeeds and datasets to gather** :param iex_datasets: list of strings for gathering specific `IEX datasets <https://iexcloud.io/>`__ which are set as consts: ``analysis_engine.iex.consts.FETCH_*``. **(Optional) Redis connectivity arguments** :param redis_enabled: bool - toggle for auto-caching all datasets in Redis (default is ``True``) :param redis_address: Redis connection string format is ``host:port`` (default is ``localhost:6379``) :param redis_db: Redis db to use (default is ``0``) :param redis_password: optional - Redis password (default is ``None``) :param redis_expire: optional - Redis expire value (default is ``None``) :param redis_key: optional - redis key not used (default is ``None``) **(Optional) Minio (S3) connectivity arguments** :param s3_enabled: bool - toggle for auto-archiving on Minio (S3) (default is ``True``) :param s3_address: Minio S3 connection string format ``host:port`` (default is ``localhost:9000``) :param s3_bucket: S3 Bucket for storing the artifacts (default is ``dev``) which should be viewable on a browser: http://localhost:9000/minio/dev/ :param s3_access_key: S3 Access key (default is ``trexaccesskey``) :param s3_secret_key: S3 Secret key (default is ``trex123321``) :param s3_region_name: S3 region name (default is ``us-east-1``) :param s3_secure: Transmit using tls encryption (default is ``False``) :param s3_key: optional s3 key not used (default is ``None``) **(Optional) Celery worker broker connectivity arguments** :param celery_disabled: bool - toggle synchronous mode or publish to an engine connected to the `Celery broker and backend <https://github.com/celery/celery#transports-and-backends>`__ (default is ``True`` - synchronous mode without an engine or need for a broker or backend for Celery) :param broker_url: Celery broker url (default is ``redis://0.0.0.0:6379/13``) :param result_backend: Celery backend url (default is ``redis://0.0.0.0:6379/14``) :param label: tracking log label :param publish_to_slack: optional - boolean for publishing to slack (coming soon) :param publish_to_s3: optional - boolean for publishing to s3 (coming soon) :param publish_to_redis: optional - boolean for publishing to redis (coming soon) **(Optional) Debugging** :param verbose: bool - show extract warnings and other debug logging (default is False) :param raise_on_err: optional - boolean for unittests and developing algorithms with the ``analysis_engine.run_algo.run_algo`` helper. When set to ``True`` exceptions will are raised to the calling functions :param kwargs: keyword arguments dictionary """ # dictionary structure with a list sorted on: ascending dates # algo_data_req[ticker][list][dataset] = pd.DataFrame algo_data_req = {} extract_requests = [] return_algo = False # return created algo objects for use by caller rec = {} msg = None use_tickers = tickers use_balance = balance use_commission = commission if ticker: use_tickers = [ticker] else: if not use_tickers: use_tickers = [] # if these are not set as args, but the algo object # has them, use them instead: if algo: if len(use_tickers) == 0: use_tickers = algo.get_tickers() if not use_balance: use_balance = algo.get_balance() if not use_commission: use_commission = algo.get_commission() default_iex_datasets = [ 'daily', 'minute', 'quote', 'stats', 'peers', 'news', 'financials', 'earnings', 'dividends', 'company' ] if not iex_datasets: iex_datasets = default_iex_datasets if redis_enabled: if not redis_address: redis_address = os.getenv('REDIS_ADDRESS', 'localhost:6379') if not redis_password: redis_password = os.getenv('REDIS_PASSWORD', None) if not redis_db: redis_db = int(os.getenv('REDIS_DB', '0')) if not redis_expire: redis_expire = os.getenv('REDIS_EXPIRE', None) if s3_enabled: if not s3_address: s3_address = os.getenv('S3_ADDRESS', 'localhost:9000') if not s3_access_key: s3_access_key = os.getenv('AWS_ACCESS_KEY_ID', 'trexaccesskey') if not s3_secret_key: s3_secret_key = os.getenv('AWS_SECRET_ACCESS_KEY', 'trex123321') if not s3_region_name: s3_region_name = os.getenv('AWS_DEFAULT_REGION', 'us-east-1') if not s3_secure: s3_secure = os.getenv('S3_SECURE', '0') == '1' if not s3_bucket: s3_bucket = os.getenv('S3_BUCKET', 'dev') if not broker_url: broker_url = os.getenv('WORKER_BROKER_URL', 'redis://0.0.0.0:6379/11') if not result_backend: result_backend = os.getenv('WORKER_BACKEND_URL', 'redis://0.0.0.0:6379/12') if not label: label = 'run-algo' num_tickers = len(use_tickers) last_close_str = ae_utils.get_last_close_str() if iex_datasets: if verbose: log.info(f'{label} - tickers={num_tickers} ' f'iex={json.dumps(iex_datasets)}') else: if verbose: log.info(f'{label} - tickers={num_tickers}') ticker_key = use_key if not ticker_key: ticker_key = f'{ticker}_{last_close_str}' if not algo: algo = base_algo.BaseAlgo(ticker=None, tickers=use_tickers, balance=use_balance, commission=use_commission, config_dict=config_dict, name=label, auto_fill=auto_fill, timeseries=timeseries, trade_strategy=trade_strategy, publish_to_slack=publish_to_slack, publish_to_s3=publish_to_s3, publish_to_redis=publish_to_redis, raise_on_err=raise_on_err) return_algo = True # the algo object is stored # in the result at: res['rec']['algo'] if not algo: msg = f'{label} - missing algo object' log.error(msg) return build_result.build_result(status=ae_consts.EMPTY, err=msg, rec=rec) if raise_on_err: log.debug(f'{label} - enabling algo exception raises') algo.raise_on_err = True indicator_datasets = algo.get_indicator_datasets() if len(indicator_datasets) == 0: indicator_datasets = ae_consts.BACKUP_DATASETS log.info(f'using all datasets={indicator_datasets}') verbose_extract = False if config_dict: verbose_extract = config_dict.get('verbose_extract', False) common_vals = {} common_vals['base_key'] = ticker_key common_vals['celery_disabled'] = celery_disabled common_vals['ticker'] = ticker common_vals['label'] = label common_vals['iex_datasets'] = iex_datasets common_vals['s3_enabled'] = s3_enabled common_vals['s3_bucket'] = s3_bucket common_vals['s3_address'] = s3_address common_vals['s3_secure'] = s3_secure common_vals['s3_region_name'] = s3_region_name common_vals['s3_access_key'] = s3_access_key common_vals['s3_secret_key'] = s3_secret_key common_vals['s3_key'] = ticker_key common_vals['redis_enabled'] = redis_enabled common_vals['redis_address'] = redis_address common_vals['redis_password'] = redis_password common_vals['redis_db'] = redis_db common_vals['redis_key'] = ticker_key common_vals['redis_expire'] = redis_expire use_start_date_str = start_date use_end_date_str = end_date last_close_date = ae_utils.last_close() end_date_val = None cache_freq_fmt = ae_consts.COMMON_TICK_DATE_FORMAT if not use_end_date_str: use_end_date_str = last_close_date.strftime(cache_freq_fmt) end_date_val = ae_utils.get_date_from_str(date_str=use_end_date_str, fmt=cache_freq_fmt) start_date_val = None if not use_start_date_str: start_date_val = end_date_val - datetime.timedelta(days=60) use_start_date_str = start_date_val.strftime(cache_freq_fmt) else: start_date_val = datetime.datetime.strptime( use_start_date_str, ae_consts.COMMON_TICK_DATE_FORMAT) total_dates = (end_date_val - start_date_val).days if end_date_val < start_date_val: msg = ( f'{label} - invalid dates - start_date={start_date_val} is after ' f'end_date={end_date_val}') raise Exception(msg) if verbose: log.info(f'{label} - days={total_dates} ' f'start={use_start_date_str} ' f'end={use_end_date_str} ' f'datasets={indicator_datasets}') for ticker in use_tickers: req = algo_utils.build_algo_request(ticker=ticker, use_key=use_key, start_date=use_start_date_str, end_date=use_end_date_str, datasets=datasets, balance=use_balance, cache_freq=cache_freq, timeseries=timeseries, trade_strategy=trade_strategy, label=label) ticker_key = f'{ticker}_{last_close_str}' common_vals['ticker'] = ticker common_vals['base_key'] = ticker_key common_vals['redis_key'] = ticker_key common_vals['s3_key'] = ticker_key for date_key in req['extract_datasets']: date_req = api_requests.get_ds_dict(ticker=ticker, base_key=date_key, ds_id=label, service_dict=common_vals) node_date_key = date_key.replace(f'{ticker}_', '') extract_requests.append({ 'id': date_key, 'ticker': ticker, 'date_key': date_key, 'date': node_date_key, 'req': date_req }) # end of for all ticker in use_tickers first_extract_date = None last_extract_date = None total_extract_requests = len(extract_requests) cur_idx = 1 for idx, extract_node in enumerate(extract_requests): extract_ticker = extract_node['ticker'] extract_date = extract_node['date'] ds_node_id = extract_node['id'] if not first_extract_date: first_extract_date = extract_date last_extract_date = extract_date perc_progress = ae_consts.get_percent_done( progress=cur_idx, total=total_extract_requests) percent_label = (f'{label} ' f'ticker={extract_ticker} ' f'date={extract_date} ' f'{perc_progress} ' f'{idx}/{total_extract_requests} ' f'{indicator_datasets}') if verbose: log.info(f'extracting - {percent_label}') ticker_bt_data = build_ds_node.build_dataset_node( ticker=extract_ticker, date=extract_date, service_dict=common_vals, datasets=indicator_datasets, log_label=label, verbose=verbose_extract) if ticker not in algo_data_req: algo_data_req[ticker] = [] algo_data_req[ticker].append({ 'id': ds_node_id, # id is currently the cache key in redis 'date': extract_date, # used to confirm dates in asc order 'data': ticker_bt_data }) if verbose: log.info(f'extract - {percent_label} ' f'dataset={len(algo_data_req[ticker])}') cur_idx += 1 # end of for service_dict in extract_requests # this could be a separate celery task status = ae_consts.NOT_RUN if len(algo_data_req) == 0: msg = (f'{label} - nothing to test - no data found for ' f'tickers={use_tickers} ' f'between {first_extract_date} and {last_extract_date}') log.info(msg) return build_result.build_result(status=ae_consts.EMPTY, err=msg, rec=rec) # this could be a separate celery task try: if verbose: log.info(f'handle_data START - {percent_label} from ' f'{first_extract_date} to {last_extract_date}') algo.handle_data(data=algo_data_req) if verbose: log.info(f'handle_data END - {percent_label} from ' f'{first_extract_date} to {last_extract_date}') except Exception as e: a_name = algo.get_name() a_debug_msg = algo.get_debug_msg() if not a_debug_msg: a_debug_msg = 'debug message not set' a_config_dict = ae_consts.ppj(algo.config_dict) msg = (f'{percent_label} - algo={a_name} ' f'encountered exception in handle_data tickers={use_tickers} ' f'from {first_extract_date} to {last_extract_date} ex={e} ' f'and failed during operation: {a_debug_msg}') if raise_on_err: if algo: try: ind_obj = \ algo.get_indicator_process_last_indicator() if ind_obj: ind_obj_path = ind_obj.get_path_to_module() ind_obj_config = ae_consts.ppj(ind_obj.get_config()) found_error_hint = False if hasattr(ind_obj.use_df, 'to_json'): if len(ind_obj.use_df.index) == 0: log.critical( f'indicator failure report for ' f'last module: ' f'{ind_obj_path} ' f'indicator={ind_obj.get_name()} ' f'config={ind_obj_config} ' f'dataset={ind_obj.use_df.head(5)} ' f'name_of_dataset={ind_obj.uses_data}') log.critical( '--------------------------------------' '--------------------------------------') log.critical('Please check if this indicator: ' f'{ind_obj_path} ' 'supports Empty Dataframes') log.critical( '--------------------------------------' '--------------------------------------') found_error_hint = True # indicator error hints if not found_error_hint: log.critical( f'indicator failure report for last module: ' f'{ind_obj_path} ' f'indicator={ind_obj.get_name()} ' f'config={ind_obj_config} ' f'dataset={ind_obj.use_df.head(5)} ' f'name_of_dataset={ind_obj.uses_data}') except Exception as f: log.critical(f'failed to pull indicator processor ' f'last indicator for debugging ' f'from ex={e} with parsing ex={f}') # end of ignoring non-supported ways of creating # indicator processors log.error(msg) log.error(f'algo failure report: ' f'algo={a_name} handle_data() ' f'config={a_config_dict} ') log.critical(f'algo failed during operation: {a_debug_msg}') raise e else: log.error(msg) return build_result.build_result(status=ae_consts.ERR, err=msg, rec=rec) # end of try/ex # this could be a separate celery task try: if verbose: log.info(f'get_result START - {percent_label} from ' f'{first_extract_date} to {last_extract_date}') rec = algo.get_result() status = ae_consts.SUCCESS if verbose: log.info(f'get_result END - {percent_label} from ' f'{first_extract_date} to {last_extract_date}') except Exception as e: msg = ( f'{percent_label} - algo={algo.get_name()} encountered exception ' f'in get_result tickers={use_tickers} from ' f'{first_extract_date} to {last_extract_date} ex={e}') if raise_on_err: if algo: log.error(f'algo={algo.get_name()} failed in get_result with ' f'debug_msg={algo.get_debug_msg()}') log.error(msg) raise e else: log.error(msg) return build_result.build_result(status=ae_consts.ERR, err=msg, rec=rec) # end of try/ex if return_algo: rec['algo'] = algo return build_result.build_result(status=status, err=msg, rec=rec)
def get_new_pricing_data(self, work_dict): """get_new_pricing_data Get Ticker information on: - prices - turn off with ``work_dict.get_pricing = False`` - news - turn off with ``work_dict.get_news = False`` - options - turn off with ``work_dict.get_options = False`` :param work_dict: dictionary for key/values """ label = 'get_new_pricing_data' log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict)) ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID rec = { 'pricing': None, 'options': None, 'calls': None, 'puts': None, 'news': None, 'daily': None, 'minute': None, 'quote': None, 'stats': None, 'peers': None, 'iex_news': None, 'financials': None, 'earnings': None, 'dividends': None, 'company': None, 'exp_date': None, 'publish_pricing_update': None, 'date': ae_utils.utc_now_str(), 'updated': None, 'version': ae_consts.DATASET_COLLECTION_VERSION } res = {'status': ae_consts.NOT_RUN, 'err': None, 'rec': rec} try: ticker = work_dict.get('ticker', ticker) ticker_id = work_dict.get('ticker_id', ae_consts.TICKER_ID) s3_bucket = work_dict.get('s3_bucket', ae_consts.S3_BUCKET) s3_key = work_dict.get('s3_key', ae_consts.S3_KEY) redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY) exp_date = work_dict.get('exp_date', None) cur_date = datetime.datetime.utcnow() cur_strike = work_dict.get('strike', None) contract_type = str(work_dict.get('contract', 'C')).upper() label = work_dict.get('label', label) iex_datasets = work_dict.get('iex_datasets', iex_consts.DEFAULT_FETCH_DATASETS) td_datasets = work_dict.get('td_datasets', td_consts.DEFAULT_FETCH_DATASETS_TD) fetch_mode = work_dict.get('fetch_mode', ae_consts.FETCH_MODE_ALL) # control flags to deal with feed issues: get_iex_data = True get_td_data = True if (fetch_mode == ae_consts.FETCH_MODE_ALL or str(fetch_mode).lower() == 'all'): get_iex_data = True get_td_data = True elif (fetch_mode == ae_consts.FETCH_MODE_YHO or str(fetch_mode).lower() == 'yahoo'): get_iex_data = False get_td_data = False elif (fetch_mode == ae_consts.FETCH_MODE_IEX or str(fetch_mode).lower() == 'iex'): get_iex_data = True get_td_data = False elif (fetch_mode == ae_consts.FETCH_MODE_TD or str(fetch_mode).lower() == 'td'): get_iex_data = False get_td_data = True else: log.debug('{} - unsupported fetch_mode={} value'.format( label, fetch_mode)) """ as of Thursday, Jan. 3, 2019: https://developer.yahoo.com/yql/ Important EOL Notice: As of Thursday, Jan. 3, 2019 the YQL service at query.yahooapis.com will be retired """ get_yahoo_data = False if not exp_date: exp_date = opt_dates.option_expiration(date=exp_date) else: exp_date = datetime.datetime.strptime(exp_date, '%Y-%m-%d') rec['updated'] = cur_date.strftime('%Y-%m-%d %H:%M:%S') log.info('{} getting pricing for ticker={} ' 'cur_date={} exp_date={} ' 'yahoo={} iex={}'.format(label, ticker, cur_date, exp_date, get_yahoo_data, get_iex_data)) yahoo_rec = { 'ticker': ticker, 'pricing': None, 'options': None, 'calls': None, 'puts': None, 'news': None, 'exp_date': None, 'publish_pricing_update': None, 'date': None, 'updated': None } # disabled on 2019-01-03 if get_yahoo_data: log.info('{} yahoo ticker={}'.format(label, ticker)) yahoo_res = yahoo_data.get_data_from_yahoo(work_dict=work_dict) if yahoo_res['status'] == ae_consts.SUCCESS: yahoo_rec = yahoo_res['rec'] log.info('{} yahoo ticker={} ' 'status={} err={}'.format( label, ticker, ae_consts.get_status(status=yahoo_res['status']), yahoo_res['err'])) rec['pricing'] = yahoo_rec.get('pricing', '{}') rec['news'] = yahoo_rec.get('news', '{}') rec['options'] = yahoo_rec.get('options', '{}') rec['calls'] = rec['options'].get('calls', ae_consts.EMPTY_DF_STR) rec['puts'] = rec['options'].get('puts', ae_consts.EMPTY_DF_STR) else: log.error('{} failed YAHOO ticker={} ' 'status={} err={}'.format( label, ticker, ae_consts.get_status(status=yahoo_res['status']), yahoo_res['err'])) # end of get from yahoo if get_iex_data: num_iex_ds = len(iex_datasets) log.debug('{} iex datasets={}'.format(label, num_iex_ds)) for idx, ft_type in enumerate(iex_datasets): dataset_field = iex_consts.get_ft_str(ft_type=ft_type) log.info('{} iex={}/{} field={} ticker={}'.format( label, idx, num_iex_ds, dataset_field, ticker)) iex_label = '{}-{}'.format(label, dataset_field) iex_req = copy.deepcopy(work_dict) iex_req['label'] = iex_label iex_req['ft_type'] = ft_type iex_req['field'] = dataset_field iex_req['ticker'] = ticker iex_res = iex_data.get_data_from_iex(work_dict=iex_req) if iex_res['status'] == ae_consts.SUCCESS: iex_rec = iex_res['rec'] log.info( '{} iex ticker={} field={} ' 'status={} err={}'.format( label, ticker, dataset_field, ae_consts.get_status(status=iex_res['status']), iex_res['err'])) if dataset_field == 'news': rec['iex_news'] = iex_rec['data'] else: rec[dataset_field] = iex_rec['data'] else: log.debug( '{} failed IEX ticker={} field={} ' 'status={} err={}'.format( label, ticker, dataset_field, ae_consts.get_status(status=iex_res['status']), iex_res['err'])) # end of if/else succcess # end idx, ft_type in enumerate(iex_datasets): # end of if get_iex_data if get_td_data: num_td_ds = len(td_datasets) log.debug('{} td datasets={}'.format(label, num_td_ds)) for idx, ft_type in enumerate(td_datasets): dataset_field = td_consts.get_ft_str_td(ft_type=ft_type) log.info('{} td={}/{} field={} ticker={}'.format( label, idx, num_td_ds, dataset_field, ticker)) td_label = '{}-{}'.format(label, dataset_field) td_req = copy.deepcopy(work_dict) td_req['label'] = td_label td_req['ft_type'] = ft_type td_req['field'] = dataset_field td_req['ticker'] = ticker td_res = td_data.get_data_from_td(work_dict=td_req) if td_res['status'] == ae_consts.SUCCESS: td_rec = td_res['rec'] log.info('{} td ticker={} field={} ' 'status={} err={}'.format( label, ticker, dataset_field, ae_consts.get_status(status=td_res['status']), td_res['err'])) if dataset_field == 'tdcalls': rec['tdcalls'] = td_rec['data'] if dataset_field == 'tdputs': rec['tdputs'] = td_rec['data'] else: rec[dataset_field] = td_rec['data'] else: log.critical( '{} failed TD ticker={} field={} ' 'status={} err={}'.format( label, ticker, dataset_field, ae_consts.get_status(status=td_res['status']), td_res['err'])) # end of if/else succcess # end idx, ft_type in enumerate(td_datasets): # end of if get_td_data update_req = {'data': rec} update_req['ticker'] = ticker update_req['ticker_id'] = ticker_id update_req['strike'] = cur_strike update_req['contract'] = contract_type update_req['s3_enabled'] = work_dict.get('s3_enabled', ae_consts.ENABLED_S3_UPLOAD) update_req['redis_enabled'] = work_dict.get( 'redis_enabled', ae_consts.ENABLED_REDIS_PUBLISH) update_req['s3_bucket'] = s3_bucket update_req['s3_key'] = s3_key update_req['s3_access_key'] = work_dict.get('s3_access_key', ae_consts.S3_ACCESS_KEY) update_req['s3_secret_key'] = work_dict.get('s3_secret_key', ae_consts.S3_SECRET_KEY) update_req['s3_region_name'] = work_dict.get('s3_region_name', ae_consts.S3_REGION_NAME) update_req['s3_address'] = work_dict.get('s3_address', ae_consts.S3_ADDRESS) update_req['s3_secure'] = work_dict.get('s3_secure', ae_consts.S3_SECURE) update_req['redis_key'] = redis_key update_req['redis_address'] = work_dict.get('redis_address', ae_consts.REDIS_ADDRESS) update_req['redis_password'] = work_dict.get('redis_password', ae_consts.REDIS_PASSWORD) update_req['redis_db'] = int( work_dict.get('redis_db', ae_consts.REDIS_DB)) update_req['redis_expire'] = work_dict.get('redis_expire', ae_consts.REDIS_EXPIRE) update_req['updated'] = rec['updated'] update_req['label'] = label update_req['celery_disabled'] = True update_status = ae_consts.NOT_SET try: update_res = publisher.run_publish_pricing_update( work_dict=update_req) update_status = update_res.get('status', ae_consts.NOT_SET) if ae_consts.ev('DEBUG_RESULTS', '0') == '1': log.info('{} update_res status={} data={}'.format( label, ae_consts.get_status(status=update_status), ae_consts.ppj(update_res))) else: log.info('{} run_publish_pricing_update status={}'.format( label, ae_consts.get_status(status=update_status))) # end of if/else rec['publish_pricing_update'] = update_res res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) except Exception as f: err = ('{} publisher.run_publish_pricing_update failed ' 'with ex={}'.format(label, f)) log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=rec) # end of trying to publish results to connected services except Exception as e: res = build_result.build_result(status=ae_consts.ERR, err=('failed - get_new_pricing_data ' 'dict={} with ex={}').format( work_dict, e), rec=rec) log.error('{} - {}'.format(label, res['err'])) # end of try/ex if ae_consts.ev('DATASET_COLLECTION_SLACK_ALERTS', '0') == '1': env_name = 'DEV' if ae_consts.ev('PROD_SLACK_ALERTS', '1') == '1': env_name = 'PROD' done_msg = ('Dataset collected ticker=*{}* on env=*{}* ' 'redis_key={} s3_key={} iex={} yahoo={}'.format( ticker, env_name, redis_key, s3_key, get_iex_data, get_yahoo_data)) log.debug('{} sending slack msg={}'.format(label, done_msg)) if res['status'] == ae_consts.SUCCESS: slack_utils.post_success(msg=done_msg, block=False, jupyter=True) else: slack_utils.post_failure(msg=done_msg, block=False, jupyter=True) # end of if/else success # end of publishing to slack log.info('task - get_new_pricing_data done - ' '{} - status={}'.format(label, ae_consts.get_status(res['status']))) return get_task_results.get_task_results(work_dict=work_dict, result=res)
daily_df = res['SPY']['daily'] minute_df = res['SPY']['minute'] out_dir = '/opt/sa/tests/datasets' if not os.path.exists(out_dir): print(f'missing output dir: {out_dir}') sys.exit(1) daily_file = f'{out_dir}/{ticker.lower()}-daily.json' minute_file = f'{out_dir}/{ticker.lower()}-minute.json' print('converting dates') print(f'converting to pretty printed json file={daily_file}') daily_out_json = ppj( json.loads(daily_df.iloc[-100:-1].to_json(orient='records', date_format='iso'))) print(f'writing to file daily_file={daily_file}') with open(daily_file, 'w') as f: f.write(daily_out_json) if not os.path.exists(daily_file): print(f'failed creating daily ticker={ticker} daily_file={daily_file}') print(f'converting to pretty printed json file={minute_file}') minute_out_json = ppj( json.loads(minute_df.iloc[-100:-1].to_json(orient='records', date_format='iso'))) print(f'writing to file minute_file={minute_file}') with open(minute_file, 'w') as f: f.write(minute_out_json)
def publish_from_s3_to_redis(self, work_dict): """publish_from_s3_to_redis Publish Ticker Data from S3 to Redis :param work_dict: dictionary for key/values """ label = 'pub-s3-to-redis' log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict)) ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID rec = { 'ticker': None, 'ticker_id': None, 's3_enabled': True, 'redis_enabled': True, 's3_bucket': None, 's3_key': None, 'redis_key': None, 'updated': None } res = build_result.build_result(status=ae_consts.NOT_RUN, err=None, rec=rec) try: ticker = work_dict.get('ticker', ae_consts.TICKER) ticker_id = int(work_dict.get('ticker_id', ae_consts.TICKER_ID)) if not ticker: res = build_result.build_result(status=ae_consts.ERR, err='missing ticker', rec=rec) return res s3_key = work_dict.get('s3_key', None) s3_bucket_name = work_dict.get('s3_bucket', 'pricing') redis_key = work_dict.get('redis_key', None) updated = work_dict.get('updated', None) serializer = work_dict.get('serializer', 'json') encoding = work_dict.get('encoding', 'utf-8') label = work_dict.get('label', label) enable_s3_read = True enable_redis_publish = True rec['ticker'] = ticker rec['ticker_id'] = ticker_id rec['s3_bucket'] = s3_bucket_name rec['s3_key'] = s3_key rec['redis_key'] = redis_key rec['updated'] = updated rec['s3_enabled'] = enable_s3_read rec['redis_enabled'] = enable_redis_publish data = None if enable_s3_read: log.info('{} parsing s3 values'.format(label)) access_key = work_dict.get('s3_access_key', ae_consts.S3_ACCESS_KEY) secret_key = work_dict.get('s3_secret_key', ae_consts.S3_SECRET_KEY) region_name = work_dict.get('s3_region_name', ae_consts.S3_REGION_NAME) service_address = work_dict.get('s3_address', ae_consts.S3_ADDRESS) secure = work_dict.get('s3_secure', ae_consts.S3_SECURE) == '1' endpoint_url = 'http://{}'.format(service_address) if secure: endpoint_url = 'https://{}'.format(service_address) log.info('{} building s3 endpoint_url={} ' 'region={}'.format(label, endpoint_url, region_name)) s3 = boto3.resource( 's3', endpoint_url=endpoint_url, aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region_name, config=boto3.session.Config(signature_version='s3v4')) try: log.info('{} checking bucket={} exists'.format( label, s3_bucket_name)) if s3.Bucket(s3_bucket_name) not in s3.buckets.all(): log.info('{} creating bucket={}'.format( label, s3_bucket_name)) s3.create_bucket(Bucket=s3_bucket_name) except Exception as e: log.info('{} failed creating bucket={} ' 'with ex={}'.format(label, s3_bucket_name, e)) # end of try/ex for creating bucket try: log.info('{} reading to s3={}/{} ' 'updated={}'.format(label, s3_bucket_name, s3_key, updated)) data = s3_read_contents_from_key.s3_read_contents_from_key( s3=s3, s3_bucket_name=s3_bucket_name, s3_key=s3_key, encoding=encoding, convert_as_json=True) initial_size_value = \ len(str(data)) / 1024000 initial_size_str = ae_consts.to_f(initial_size_value) if ae_consts.ev('DEBUG_S3', '0') == '1': log.info('{} read s3={}/{} data={}'.format( label, s3_bucket_name, s3_key, ae_consts.ppj(data))) else: log.info('{} read s3={}/{} data size={} MB'.format( label, s3_bucket_name, s3_key, initial_size_str)) except Exception as e: err = ('{} failed reading bucket={} ' 'key={} ex={}').format(label, s3_bucket_name, s3_key, e) log.error(err) res = build_result.build_result(status=ae_consts.NOT_RUN, err=err, rec=rec) # end of try/ex for creating bucket else: log.info('{} SKIP S3 read bucket={} ' 'key={}'.format(label, s3_bucket_name, s3_key)) # end of if enable_s3_read if enable_redis_publish: redis_address = work_dict.get('redis_address', ae_consts.REDIS_ADDRESS) redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY) redis_password = work_dict.get('redis_password', ae_consts.REDIS_PASSWORD) redis_db = work_dict.get('redis_db', None) if not redis_db: redis_db = ae_consts.REDIS_DB redis_expire = None if 'redis_expire' in work_dict: redis_expire = work_dict.get('redis_expire', ae_consts.REDIS_EXPIRE) log.info('redis enabled address={}@{} ' 'key={}'.format(redis_address, redis_db, redis_key)) redis_host = redis_address.split(':')[0] redis_port = redis_address.split(':')[1] try: if ae_consts.ev('DEBUG_REDIS', '0') == '1': log.info('{} publishing redis={}:{} ' 'db={} key={} ' 'updated={} expire={} ' 'data={}'.format(label, redis_host, redis_port, redis_db, redis_key, updated, redis_expire, ae_consts.ppj(data))) else: log.info('{} publishing redis={}:{} ' 'db={} key={} ' 'updated={} expire={}'.format( label, redis_host, redis_port, redis_db, redis_key, updated, redis_expire)) # end of if/else rc = redis.Redis(host=redis_host, port=redis_port, password=redis_password, db=redis_db) redis_set_res = redis_set.set_data_in_redis_key( label=label, client=rc, key=redis_key, data=data, serializer=serializer, encoding=encoding, expire=redis_expire, px=None, nx=False, xx=False) log.info('{} redis_set status={} err={}'.format( label, ae_consts.get_status(redis_set_res['status']), redis_set_res['err'])) except Exception as e: log.error('{} failed - redis publish to ' 'key={} ex={}'.format(label, redis_key, e)) # end of try/ex for creating bucket else: log.info('{} SKIP REDIS publish ' 'key={}'.format(label, redis_key)) # end of if enable_redis_publish res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) except Exception as e: res = build_result.build_result( status=ae_consts.ERR, err=('failed - publish_from_s3_to_redis ' 'dict={} with ex={}').format(work_dict, e), rec=rec) log.error('{} - {}'.format(label, res['err'])) # end of try/ex log.info('task - publish_from_s3_to_redis done - ' '{} - status={}'.format(label, ae_consts.get_status(res['status']))) return get_task_results.get_task_results(work_dict=work_dict, result=res)
def build_algo_request(ticker=None, tickers=None, use_key=None, start_date=None, end_date=None, datasets=None, balance=None, commission=None, num_shares=None, config_file=None, config_dict=None, load_config=None, history_config=None, report_config=None, extract_config=None, timeseries=None, trade_strategy=None, cache_freq='daily', label='algo'): """build_algo_request Create a dictionary for building an algorithm. This is opinionated to how the underlying date-based caching strategy is running per day. Each business day becomes a possible dataset to process with an algorithm. :param ticker: ticker :param tickers: optional - list of tickers :param use_key: redis and s3 to store the algo result :param start_date: string date format ``YYYY-MM-DD HH:MM:SS`` :param end_date: string date format ``YYYY-MM-DD HH:MM:SS`` :param datasets: list of string dataset types :param balance: starting capital balance :param commission: commission for buy or sell :param num_shares: optional - integer number of starting shares :param cache_freq: optional - cache frequency (``daily`` is default) :param label: optional - algo log tracking name :param config_file: path to a json file containing custom algorithm object member values (like indicator configuration and predict future date units ahead for a backtest) :param config_dict: optional - dictionary that can be passed to derived class implementations of: ``def load_from_config(config_dict=config_dict)`` **Timeseries** :param timeseries: optional - string to set ``day`` or ``minute`` backtesting or live trading (default is ``minute``) **Trading Strategy** :param trade_strategy: optional - string to set the type of ``Trading Strategy`` for backtesting or live trading (default is ``count``) **Algorithm Dataset Extraction, Loading and Publishing arguments** :param load_config: optional - dictionary for setting member variables to load an agorithm-ready dataset from a file, s3 or redis :param history_config: optional - dictionary for setting member variables to publish an algo ``trade history`` to s3, redis, a file or slack :param report_config: optional - dictionary for setting member variables to publish an algo ``trading performance report`` to s3, redis, a file or slack :param extract_config: optional - dictionary for setting member variables to publish an algo ``trading performance report`` to s3, redis, a file or slack """ use_tickers = [] if ticker: use_tickers = [ticker.upper()] if tickers: for t in tickers: if t not in use_tickers: use_tickers.append(t.upper()) s3_bucket_name = ae_consts.ALGO_RESULT_S3_BUCKET_NAME s3_key = use_key redis_key = use_key s3_enabled = True redis_enabled = True work = { 'tickers': use_tickers, 's3_bucket': s3_bucket_name, 's3_key': s3_key, 'redis_key': redis_key, 's3_enabled': s3_enabled, 'redis_enabled': redis_enabled, 'extract_datasets': [], 'cache_freq': cache_freq, 'config_file': config_file, 'config_dict': config_dict, 'balance': balance, 'commission': commission, 'load_config': load_config, 'history_config': history_config, 'report_config': report_config, 'extract_config': extract_config, 'start_date': None, 'end_date': None, 'timeseries': timeseries, 'trade_strategy': trade_strategy, 'version': 1, 'label': label } start_date_val = ae_utils.get_date_from_str(start_date) end_date_val = ae_utils.get_date_from_str(end_date) if start_date_val > end_date_val: raise Exception( 'Invalid start_date={} must be less than end_date={}'.format( start_date, end_date)) use_dates = [] new_dataset = None cur_date = start_date_val if not work['start_date']: work['start_date'] = start_date_val.strftime( ae_consts.COMMON_TICK_DATE_FORMAT) if not work['end_date']: work['end_date'] = end_date_val.strftime( ae_consts.COMMON_TICK_DATE_FORMAT) while cur_date <= end_date_val: if cur_date.weekday() < 5: for t in use_tickers: if cache_freq == 'daily': new_dataset = '{}_{}'.format( t, cur_date.strftime(ae_consts.COMMON_DATE_FORMAT)) else: new_dataset = '{}_{}'.format( t, cur_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT)) if new_dataset: use_dates.append(new_dataset) new_dataset = None # end for all tickers # end of valid days M-F if cache_freq == 'daily': cur_date += datetime.timedelta(days=1) else: cur_date += datetime.timedelta(minute=1) # end of walking all dates to add if len(use_dates) > 0: work['extract_datasets'] = use_dates log.debug('tickers={} balance={} start={} end={} ' 'cache_freq={} request={}'.format( work['tickers'], work['balance'], work['extract_datasets'][0], work['extract_datasets'][-1], cache_freq, ae_consts.ppj(work))) else: log.error('there are not enough dates to test between start={} end={} ' 'tickers={} request={}'.format(start_date_val, end_date_val, work['tickers'], cache_freq, ae_consts.ppj(work))) return work
def show_dataset(algo_dataset=None, dataset_type=ae_consts.SA_DATASET_TYPE_ALGO_READY, serialize_datasets=ae_consts.DEFAULT_SERIALIZED_DATASETS, path_to_file=None, compress=False, encoding='utf-8', redis_enabled=True, redis_key=None, redis_address=None, redis_db=None, redis_password=None, redis_expire=None, redis_serializer='json', redis_encoding='utf-8', s3_enabled=True, s3_key=None, s3_address=None, s3_bucket=None, s3_access_key=None, s3_secret_key=None, s3_region_name=None, s3_secure=False, slack_enabled=False, slack_code_block=False, slack_full_width=False, verbose=False): """show_dataset Show a supported dataset's internal structure and preview some of the values to debug mapping, serialization issues :param algo_dataset: optional - already loaded algorithm-ready dataset :param dataset_type: optional - dataset type (default is ``SA_DATASET_TYPE_ALGO_READY``) :param serialize_datasets: optional - list of dataset names to deserialize in the dataset :param path_to_file: optional - path to an algorithm-ready dataset in a file :param compress: optional - boolean flag for decompressing the contents of the ``path_to_file`` if necessary (default is ``False`` and algorithms use ``zlib`` for compression) :param encoding: optional - string for data encoding **(Optional) Redis connectivity arguments** :param redis_enabled: bool - toggle for auto-caching all datasets in Redis (default is ``True``) :param redis_key: string - key to save the data in redis (default is ``None``) :param redis_address: Redis connection string format: ``host:port`` (default is ``localhost:6379``) :param redis_db: Redis db to use (default is ``0``) :param redis_password: optional - Redis password (default is ``None``) :param redis_expire: optional - Redis expire value (default is ``None``) :param redis_serializer: not used yet - support for future pickle objects in redis :param redis_encoding: format of the encoded key in redis **(Optional) Minio (S3) connectivity arguments** :param s3_enabled: bool - toggle for auto-archiving on Minio (S3) (default is ``True``) :param s3_key: string - key to save the data in redis (default is ``None``) :param s3_address: Minio S3 connection string format: ``host:port`` (default is ``localhost:9000``) :param s3_bucket: S3 Bucket for storing the artifacts (default is ``dev``) which should be viewable on a browser: http://localhost:9000/minio/dev/ :param s3_access_key: S3 Access key (default is ``trexaccesskey``) :param s3_secret_key: S3 Secret key (default is ``trex123321``) :param s3_region_name: S3 region name (default is ``us-east-1``) :param s3_secure: Transmit using tls encryption (default is ``False``) **(Optional) Slack arguments** :param slack_enabled: optional - boolean for publishing to slack :param slack_code_block: optional - boolean for publishing as a code black in slack :param slack_full_width: optional - boolean for publishing as a to slack using the full width allowed Additonal arguments :param verbose: optional - bool for increasing logging """ use_ds = algo_dataset if not use_ds: log.info('loading from file={} s3={} redis={}'.format( path_to_file, s3_key, redis_key)) use_ds = load_dataset.load_dataset( dataset_type=dataset_type, compress=compress, encoding=redis_encoding, path_to_file=path_to_file, s3_key=s3_key, s3_address=s3_address, s3_bucket=s3_bucket, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, redis_key=redis_key, redis_address=redis_address, redis_db=redis_db, redis_password=redis_password, redis_expire=redis_expire, redis_serializer=redis_serializer, serialize_datasets=serialize_datasets) if not use_ds: log.error('unable to load a dataset from file={} ' 's3={} redis={}'.format(path_to_file, s3_key, redis_key)) return None # load if not created if dataset_type == ae_consts.SA_DATASET_TYPE_ALGO_READY: print('-----------------------------------') print('dates found in dataset') root_keys = [] for root_key in use_ds: print(root_key) root_keys.append(root_key) all_dates = [] all_ids = [] for root_key in root_keys: second_layer = use_ds[root_key] for ds in second_layer: if 'date' in ds: if len(all_dates) == 0: print('') print('dates found in dataset') cur_date = ds.get('date', None) if cur_date: print(cur_date) all_dates.append(cur_date) first_node = None last_node = None end_nodes = [] for root_key in root_keys: second_layer = use_ds[root_key] for ds in second_layer: if not first_node: first_node = ds end_nodes.append(ds) last_node = ds if 'id' in ds: if len(all_ids) == 0: print('') print('ids in the file') cur_id = ds.get('id', None) if cur_id: print(cur_id) all_ids.append(cur_id) if first_node and last_node: show_first = {} for ds_key in first_node: if ds_key == 'data': show_first[ds_key] = {} for ds_name in first_node[ds_key]: print('first_node has dataset with name: {}'.format( ds_name)) show_first[ds_key][ds_name] = 'EMPTY_DF' if hasattr(first_node[ds_key][ds_name], 'index'): show_first[ds_key][ds_name] = ( 'pd.DataFrame() rows={}'.format( len(first_node[ds_key][ds_name].index))) else: show_first[ds_key] = first_node[ds_key] print('') print('first node:') print(ae_consts.ppj(show_first)) print('') num_records = len(all_ids) cur_cell = num_records - 4 for cur_node in end_nodes[-5:]: show_node = {} for ds_key in cur_node: if ds_key == 'data': show_node[ds_key] = {} for ds_name in cur_node[ds_key]: show_node[ds_key][ds_name] = 'EMPTY_DF' if hasattr(cur_node[ds_key][ds_name], 'index'): show_node[ds_key][ds_name] = ( 'pd.DataFrame() rows={}'.format( len(cur_node[ds_key][ds_name].index))) else: show_node[ds_key] = cur_node[ds_key] # end of show cur_node print('node={}/{} values:'.format(cur_cell, num_records)) print(ae_consts.ppj(show_node)) print('') cur_cell += 1 # end of end_nodes else: if not first_node: print('missing first node in dataset') if not last_node: print('missing last node in dataset') if len(all_dates) > 0: print('root_keys={} from {} to {}'.format(root_keys, all_dates[0], all_dates[-1])) else: print('root_keys={} missing dates'.format(root_keys)) print('-----------------------------------') return use_ds
def fetch_new_stock_datasets(): """fetch_new_stock_datasets Collect all datasets for the ticker **SPY**: :: fetch_new_stock_datasets.py -t SPY .. note:: This requires the following services are listening on: - redis ``localhost:6379`` - minio ``localhost:9000`` """ log.info('start - fetch_new_stock_datasets') parser = argparse.ArgumentParser( description=('Download and store the latest stock pricing, ' 'news, and options chain data ' 'and store it in Minio (S3) and Redis. ' 'Also includes support for getting FinViz ' 'screener tickers')) parser.add_argument('-t', help=('ticker'), required=False, dest='ticker') parser.add_argument('-g', help=('optional - fetch mode: ' 'all = fetch from all data sources (default), ' 'td = fetch from just Tradier sources, ' 'iex = fetch from just IEX sources'), required=False, dest='fetch_mode') parser.add_argument('-i', help=('optional - ticker id ' 'not used without a database'), required=False, dest='ticker_id') parser.add_argument('-e', help=('optional - options expiration date'), required=False, dest='exp_date_str') parser.add_argument('-l', help=('optional - path to the log config file'), required=False, dest='log_config_path') parser.add_argument('-b', help=('optional - broker url for Celery'), required=False, dest='broker_url') parser.add_argument('-B', help=('optional - backend url for Celery'), required=False, dest='backend_url') parser.add_argument('-k', help=('optional - s3 access key'), required=False, dest='s3_access_key') parser.add_argument('-s', help=('optional - s3 secret key'), required=False, dest='s3_secret_key') parser.add_argument('-a', help=('optional - s3 address format: <host:port>'), required=False, dest='s3_address') parser.add_argument('-S', help=('optional - s3 ssl or not'), required=False, dest='s3_secure') parser.add_argument('-u', help=('optional - s3 bucket name'), required=False, dest='s3_bucket_name') parser.add_argument('-G', help=('optional - s3 region name'), required=False, dest='s3_region_name') parser.add_argument('-p', help=('optional - redis_password'), required=False, dest='redis_password') parser.add_argument('-r', help=('optional - redis_address format: <host:port>'), required=False, dest='redis_address') parser.add_argument('-n', help=('optional - redis and s3 key name'), required=False, dest='keyname') parser.add_argument( '-m', help=('optional - redis database number (0 by default)'), required=False, dest='redis_db') parser.add_argument('-x', help=('optional - redis expiration in seconds'), required=False, dest='redis_expire') parser.add_argument('-z', help=('optional - strike price'), required=False, dest='strike') parser.add_argument( '-c', help=('optional - contract type "C" for calls "P" for puts'), required=False, dest='contract_type') parser.add_argument( '-P', help=('optional - get pricing data if "1" or "0" disabled'), required=False, dest='get_pricing') parser.add_argument( '-N', help=('optional - get news data if "1" or "0" disabled'), required=False, dest='get_news') parser.add_argument( '-O', help=('optional - get options data if "1" or "0" disabled'), required=False, dest='get_options') parser.add_argument('-U', help=('optional - s3 enabled for publishing if "1" or ' '"0" is disabled'), required=False, dest='s3_enabled') parser.add_argument( '-R', help=('optional - redis enabled for publishing if "1" or ' '"0" is disabled'), required=False, dest='redis_enabled') parser.add_argument('-A', help=('optional - run an analysis ' 'supported modes: scn'), required=False, dest='analysis_type') parser.add_argument('-L', help=('optional - screener urls to pull ' 'tickers for analysis'), required=False, dest='urls') parser.add_argument( '-Z', help=('disable run without an engine for local testing and demos'), required=False, dest='celery_enabled', action='store_true') parser.add_argument('-d', help=('debug'), required=False, dest='debug', action='store_true') args = parser.parse_args() run_offline = True ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID fetch_mode = 'all' exp_date_str = ae_consts.NEXT_EXP_STR ssl_options = ae_consts.SSL_OPTIONS transport_options = ae_consts.TRANSPORT_OPTIONS broker_url = ae_consts.WORKER_BROKER_URL backend_url = ae_consts.WORKER_BACKEND_URL celery_config_module = ae_consts.WORKER_CELERY_CONFIG_MODULE include_tasks = ae_consts.INCLUDE_TASKS s3_access_key = ae_consts.S3_ACCESS_KEY s3_secret_key = ae_consts.S3_SECRET_KEY s3_region_name = ae_consts.S3_REGION_NAME s3_address = ae_consts.S3_ADDRESS s3_secure = ae_consts.S3_SECURE s3_bucket_name = ae_consts.S3_BUCKET s3_key = ae_consts.S3_KEY redis_address = ae_consts.REDIS_ADDRESS redis_key = ae_consts.REDIS_KEY redis_password = ae_consts.REDIS_PASSWORD redis_db = ae_consts.REDIS_DB redis_expire = ae_consts.REDIS_EXPIRE strike = None contract_type = None get_pricing = True get_news = True get_options = True s3_enabled = True redis_enabled = True analysis_type = None debug = False if args.ticker: ticker = args.ticker.upper() if args.ticker_id: ticker_id = args.ticker_id if args.exp_date_str: exp_date_str = ae_consts.NEXT_EXP_STR if args.broker_url: broker_url = args.broker_url if args.backend_url: backend_url = args.backend_url if args.s3_access_key: s3_access_key = args.s3_access_key if args.s3_secret_key: s3_secret_key = args.s3_secret_key if args.s3_region_name: s3_region_name = args.s3_region_name if args.s3_address: s3_address = args.s3_address if args.s3_secure: s3_secure = args.s3_secure if args.s3_bucket_name: s3_bucket_name = args.s3_bucket_name if args.keyname: s3_key = args.keyname redis_key = args.keyname if args.redis_address: redis_address = args.redis_address if args.redis_password: redis_password = args.redis_password if args.redis_db: redis_db = args.redis_db if args.redis_expire: redis_expire = args.redis_expire if args.strike: strike = args.strike if args.contract_type: contract_type = args.contract_type if args.get_pricing: get_pricing = args.get_pricing == '1' if args.get_news: get_news = args.get_news == '1' if args.get_options: get_options = args.get_options == '1' if args.s3_enabled: s3_enabled = args.s3_enabled == '1' if args.redis_enabled: redis_enabled = args.redis_enabled == '1' if args.fetch_mode: fetch_mode = str(args.fetch_mode).lower() if args.analysis_type: analysis_type = str(args.analysis_type).lower() if args.celery_enabled: run_offline = False if args.debug: debug = True work = api_requests.build_get_new_pricing_request() work['ticker'] = ticker work['ticker_id'] = ticker_id work['s3_bucket'] = s3_bucket_name work['s3_key'] = s3_key work['redis_key'] = redis_key work['strike'] = strike work['contract'] = contract_type work['exp_date'] = exp_date_str work['s3_access_key'] = s3_access_key work['s3_secret_key'] = s3_secret_key work['s3_region_name'] = s3_region_name work['s3_address'] = s3_address work['s3_secure'] = s3_secure work['redis_address'] = redis_address work['redis_password'] = redis_password work['redis_db'] = redis_db work['redis_expire'] = redis_expire work['get_pricing'] = get_pricing work['get_news'] = get_news work['get_options'] = get_options work['s3_enabled'] = s3_enabled work['redis_enabled'] = redis_enabled work['fetch_mode'] = fetch_mode work['analysis_type'] = analysis_type work['iex_datasets'] = iex_consts.DEFAULT_FETCH_DATASETS work['debug'] = debug work['label'] = 'ticker={}'.format(ticker) if analysis_type == 'scn': label = 'screener={}'.format(work['ticker']) fv_urls = [] if args.urls: fv_urls = str(args.urls).split('|') if len(fv_urls) == 0: fv_urls = os.getenv('SCREENER_URLS', []).split('|') screener_req = api_requests.build_screener_analysis_request( ticker=ticker, fv_urls=fv_urls, label=label) work.update(screener_req) start_screener_analysis(req=work) # end of analysis_type else: if not args.keyname: last_close_date = ae_utils.last_close() work['s3_key'] = '{}_{}'.format( work['ticker'], last_close_date.strftime(ae_consts.COMMON_DATE_FORMAT)) work['redis_key'] = '{}_{}'.format( work['ticker'], last_close_date.strftime(ae_consts.COMMON_DATE_FORMAT)) path_to_tasks = 'analysis_engine.work_tasks' task_name = ('{}.get_new_pricing_data.get_new_pricing_data'.format( path_to_tasks)) task_res = None if ae_consts.is_celery_disabled() or run_offline: work['celery_disabled'] = True log.debug('starting without celery work={} offline={}'.format( ae_consts.ppj(work), run_offline)) task_res = task_pricing.get_new_pricing_data(work) if debug: log.info('done - result={} ' 'task={} status={} ' 'err={} label={}'.format( ae_consts.ppj(task_res), task_name, ae_consts.get_status(status=task_res['status']), task_res['err'], work['label'])) else: log.info('done - result ' 'task={} status={} ' 'err={} label={}'.format( task_name, ae_consts.get_status(status=task_res['status']), task_res['err'], work['label'])) # if/else debug else: log.info('connecting to broker={} backend={}'.format( broker_url, backend_url)) # Get the Celery app app = get_celery_app.get_celery_app( name=__name__, auth_url=broker_url, backend_url=backend_url, path_to_config_module=celery_config_module, ssl_options=ssl_options, transport_options=transport_options, include_tasks=include_tasks) log.info('calling task={} - work={}'.format( task_name, ae_consts.ppj(work))) job_id = app.send_task(task_name, (work, )) log.info('calling task={} - success job_id={}'.format( task_name, job_id))
def run_aws_backup(): """run_aws_backup Run buy and sell analysis on a stock to send alerts to subscribed users """ log.debug('start - aws-backup') parser = argparse.ArgumentParser(description=('stock analysis tool')) parser.add_argument('-t', help=('ticker'), required=True, dest='ticker') parser.add_argument('-e', help=('file path to extract an ' 'algorithm-ready datasets from redis'), required=False, dest='algo_extract_loc') parser.add_argument('-l', help=('show dataset in this file'), required=False, dest='show_from_file') parser.add_argument('-H', help=('show trading history dataset in this file'), required=False, dest='show_history_from_file') parser.add_argument( '-E', help=('show trading performance report dataset in this file'), required=False, dest='show_report_from_file') parser.add_argument( '-L', help=('restore an algorithm-ready dataset file back into redis'), required=False, dest='restore_algo_file') parser.add_argument('-f', help=('run in mode: prepare dataset from ' 'redis key or s3 key'), required=False, dest='prepare_mode', action='store_true') parser.add_argument( '-J', help=('plot action - after preparing you can use: ' '-J show to open the image (good for debugging)'), required=False, dest='plot_action') parser.add_argument( '-b', help=('run a backtest using the dataset in ' 'a file path/s3 key/redis key formats: ' 'file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' 's3://algoready/SPY-latest.json or ' 'redis:SPY-latest'), required=False, dest='backtest_loc') parser.add_argument('-B', help=('optional - broker url for Celery'), required=False, dest='broker_url') parser.add_argument('-C', help=('optional - broker url for Celery'), required=False, dest='backend_url') parser.add_argument( '-w', help=('optional - flag for publishing an algorithm job ' 'using Celery to the ae workers'), required=False, dest='run_on_engine', action='store_true') parser.add_argument('-k', help=('optional - s3 access key'), required=False, dest='s3_access_key') parser.add_argument('-K', help=('optional - s3 secret key'), required=False, dest='s3_secret_key') parser.add_argument('-a', help=('optional - s3 address format: <host:port>'), required=False, dest='s3_address') parser.add_argument('-Z', help=('optional - s3 secure: default False'), required=False, dest='s3_secure') parser.add_argument('-s', help=('optional - start date: YYYY-MM-DD'), required=False, dest='start_date') parser.add_argument('-n', help=('optional - end date: YYYY-MM-DD'), required=False, dest='end_date') parser.add_argument('-u', help=('optional - s3 bucket name'), required=False, dest='s3_bucket_name') parser.add_argument('-G', help=('optional - s3 region name'), required=False, dest='s3_region_name') parser.add_argument( '-g', help=('Path to a custom algorithm module file ' 'on disk. This module must have a single ' 'class that inherits from: ' 'https://github.com/AlgoTraders/stock-analysis-engine/' 'blob/master/' 'analysis_engine/algo.py Additionally you ' 'can find the Example-Minute-Algorithm here: ' 'https://github.com/AlgoTraders/stock-analysis-engine/' 'blob/master/analysis_engine/mocks/' 'example_algo_minute.py'), required=False, dest='run_algo_in_file') parser.add_argument('-p', help=('optional - s3 bucket/file for trading history'), required=False, dest='algo_history_loc') parser.add_argument( '-o', help=('optional - s3 bucket/file for trading performance report'), required=False, dest='algo_report_loc') parser.add_argument('-r', help=('optional - redis_address format: <host:port>'), required=False, dest='redis_address') parser.add_argument('-R', help=('optional - redis and s3 key name'), required=False, dest='keyname') parser.add_argument( '-m', help=('optional - redis database number (0 by default)'), required=False, dest='redis_db') parser.add_argument('-x', help=('optional - redis expiration in seconds'), required=False, dest='redis_expire') parser.add_argument('-z', help=('optional - strike price'), required=False, dest='strike') parser.add_argument( '-c', help=('optional - algorithm config_file path for setting ' 'up internal algorithm trading strategies and ' 'indicators'), required=False, dest='config_file') parser.add_argument( '-P', help=('optional - get pricing data if "1" or "0" disabled'), required=False, dest='get_pricing') parser.add_argument( '-N', help=('optional - get news data if "1" or "0" disabled'), required=False, dest='get_news') parser.add_argument( '-O', help=('optional - get options data if "1" or "0" disabled'), required=False, dest='get_options') parser.add_argument( '-i', help=('optional - ignore column names (comma separated)'), required=False, dest='ignore_columns') parser.add_argument('-d', help=('debug'), required=False, dest='debug', action='store_true') args = parser.parse_args() mode = 'prepare' plot_action = ae_consts.PLOT_ACTION_SHOW ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID ssl_options = ae_consts.SSL_OPTIONS transport_options = ae_consts.TRANSPORT_OPTIONS broker_url = ae_consts.WORKER_BROKER_URL backend_url = ae_consts.WORKER_BACKEND_URL path_to_config_module = ae_consts.WORKER_CELERY_CONFIG_MODULE include_tasks = ae_consts.INCLUDE_TASKS s3_access_key = ae_consts.S3_ACCESS_KEY s3_secret_key = ae_consts.S3_SECRET_KEY s3_region_name = ae_consts.S3_REGION_NAME s3_address = ae_consts.S3_ADDRESS s3_secure = ae_consts.S3_SECURE s3_bucket_name = ae_consts.S3_BUCKET s3_key = ae_consts.S3_KEY redis_address = ae_consts.REDIS_ADDRESS redis_key = ae_consts.REDIS_KEY redis_password = ae_consts.REDIS_PASSWORD redis_db = ae_consts.REDIS_DB redis_expire = ae_consts.REDIS_EXPIRE dataset_type = ae_consts.SA_DATASET_TYPE_ALGO_READY serialize_datasets = ae_consts.DEFAULT_SERIALIZED_DATASETS output_redis_key = None output_s3_bucket = None output_s3_key = None ignore_columns = None compress = False encoding = 'utf-8' slack_enabled = False slack_code_block = False slack_full_width = False verbose = False debug = False redis_serializer = 'json' redis_encoding = 'utf-8' output_redis_key = None output_s3_bucket = None output_s3_key = None s3_enabled = True redis_enabled = True ignore_columns = None debug = False run_on_engine = False show_from_file = None show_history_from_file = None show_report_from_file = None restore_algo_file = None backtest_loc = None use_custom_algo = False algo_history_loc = 's3://algohistory' algo_report_loc = 's3://algoreport' algo_extract_loc = 's3://algoready' use_balance = 5000.0 use_commission = 6.0 auto_fill = True use_start_date = None use_end_date = None use_config_file = None use_name = 'myalgo' if args.ticker: ticker = args.ticker.upper() if args.broker_url: broker_url = args.broker_url if args.backend_url: backend_url = args.backend_url if args.s3_access_key: s3_access_key = args.s3_access_key if args.s3_secret_key: s3_secret_key = args.s3_secret_key if args.s3_region_name: s3_region_name = args.s3_region_name if args.s3_address: s3_address = args.s3_address s3_enabled = True if args.s3_secure: s3_secure = args.s3_secure if args.s3_bucket_name: s3_bucket_name = args.s3_bucket_name if args.keyname: s3_key = args.keyname redis_key = args.keyname if args.redis_address: redis_address = args.redis_address if args.redis_db: redis_db = args.redis_db if args.redis_expire: redis_expire = args.redis_expire if args.prepare_mode: mode = ae_consts.SA_MODE_PREPARE if args.ignore_columns: ignore_columns_org = args.ignore_columns ignore_columns = ignore_columns_org.split(",") if args.plot_action: if str(args.plot_action).lower() == 'show': plot_action = ae_consts.PLOT_ACTION_SHOW elif str(args.plot_action).lower() == 's3': plot_action = ae_consts.PLOT_ACTION_SAVE_TO_S3 elif str(args.plot_action).lower() == 'save': plot_action = ae_consts.PLOT_ACTION_SAVE_AS_FILE else: plot_action = ae_consts.PLOT_ACTION_SHOW log.warning('unsupported plot_action: {}'.format(args.plot_action)) if args.debug: debug = True if args.algo_extract_loc: mode = ae_consts.SA_MODE_EXTRACT if args.show_from_file: show_from_file = args.show_from_file mode = ae_consts.SA_MODE_SHOW_DATASET if args.show_history_from_file: show_history_from_file = args.show_history_from_file mode = ae_consts.SA_MODE_SHOW_HISTORY_DATASET if args.show_report_from_file: show_report_from_file = args.show_report_from_file mode = ae_consts.SA_MODE_SHOW_REPORT_DATASET if args.restore_algo_file: restore_algo_file = args.restore_algo_file mode = ae_consts.SA_MODE_RESTORE_REDIS_DATASET if args.run_algo_in_file: mode = ae_consts.SA_MODE_RUN_ALGO if args.backtest_loc: mode = ae_consts.SA_MODE_RUN_ALGO if args.start_date: try: use_start_date = '{} 00:00:00'.format(str(args.start_date)) datetime.datetime.strptime(args.start_date, ae_consts.COMMON_DATE_FORMAT) except Exception as e: msg = ('please use a start date formatted as: {}' '\n' 'error was: {}'.format(ae_consts.COMMON_DATE_FORMAT, e)) log.error(msg) sys.exit(1) # end of testing for a valid date # end of args.start_date if args.end_date: try: use_end_date = '{} 00:00:00'.format(str(args.end_date)) datetime.datetime.strptime(args.end_date, ae_consts.COMMON_DATE_FORMAT) except Exception as e: msg = ('please use an end date formatted as: {}' '\n' 'error was: {}'.format(ae_consts.COMMON_DATE_FORMAT, e)) log.error(msg) sys.exit(1) # end of testing for a valid date # end of args.end_date if args.config_file: use_config_file = args.config_file if not os.path.exists(use_config_file): log.error('Failed: unable to find config file: -c {}'.format( use_config_file)) sys.exit(1) config_dict = None load_from_s3_bucket = None load_from_s3_key = None load_from_redis_key = None load_from_file = None load_compress = False load_publish = True load_config = None report_redis_key = None report_s3_bucket = None report_s3_key = None report_file = None report_compress = False report_publish = False report_config = None history_redis_key = None history_s3_bucket = None history_s3_key = None history_file = None history_compress = False history_publish = False history_config = None extract_redis_key = None extract_s3_bucket = None extract_s3_key = None extract_file = None extract_save_dir = None extract_compress = False extract_publish = True extract_config = None publish_to_slack = False publish_to_s3 = True publish_to_redis = False use_timeseries = 'day' use_trade_strategy = 'count' valid = False required_task = False work = None task_name = None work = {} path_to_tasks = 'analysis_engine.work_tasks' if mode == ae_consts.SA_MODE_PREPARE: task_name = ('{}.' 'prepare_pricing_dataset.prepare_pricing_dataset' ).format(path_to_tasks) work = api_requests.build_prepare_dataset_request() if output_s3_key: work['prepared_s3_key'] = output_s3_key if output_s3_bucket: work['prepared_s3_bucket'] = output_s3_bucket if output_redis_key: work['prepared_redis_key'] = output_redis_key work['ignore_columns'] = ignore_columns valid = True required_task = True elif mode == ae_consts.SA_MODE_EXTRACT: if args.algo_extract_loc: algo_extract_loc = args.algo_extract_loc if ('file:/' not in algo_extract_loc and 's3://' not in algo_extract_loc and 'redis://' not in algo_extract_loc): log.error( 'invalid -e <extract_to_file_or_s3_key_or_redis_key> ' 'specified. please use either: ' '-e file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-e s3://algoready/SPY-latest.json or ' '-e redis://SPY-latest') sys.exit(1) if 's3://' in algo_extract_loc: extract_s3_bucket = algo_extract_loc.split('/')[-2] extract_s3_key = algo_extract_loc.split('/')[-1] elif 'redis://' in algo_extract_loc: extract_redis_key = algo_extract_loc.split('/')[-1] elif 'file:/' in algo_extract_loc: extract_file = algo_extract_loc.split(':')[-1] # end of parsing supported transport for loading use_custom_algo = True elif mode == ae_consts.SA_MODE_SHOW_DATASET: examine_dataset_in_file(ticker=ticker, path_to_file=show_from_file) log.info('done showing {} dataset from file={}'.format( ticker, show_from_file)) sys.exit(0) elif mode == ae_consts.SA_MODE_SHOW_HISTORY_DATASET: examine_dataset_in_file( ticker=ticker, dataset_type=ae_consts.SA_DATASET_TYPE_TRADING_HISTORY, path_to_file=show_history_from_file) log.info('done showing trading history {} dataset from ' 'file={}'.format(ticker, show_from_file)) sys.exit(0) elif mode == ae_consts.SA_MODE_SHOW_REPORT_DATASET: examine_dataset_in_file( ticker=ticker, dataset_type=ae_consts.SA_DATASET_TYPE_TRADING_REPORT, path_to_file=show_report_from_file) log.info('done showing trading performance report {} dataset from ' 'file={}'.format(ticker, show_from_file)) sys.exit(0) elif mode == ae_consts.SA_MODE_RESTORE_REDIS_DATASET: restore_missing_dataset_values_from_algo_ready_file( ticker=ticker, path_to_file=restore_algo_file, redis_address=redis_address, redis_password=redis_password, redis_db=redis_db, output_redis_db=redis_db, dataset_type=ae_consts.SA_DATASET_TYPE_ALGO_READY, serialize_datasets=ae_consts.DEFAULT_SERIALIZED_DATASETS) log.info( 'done restoring {} dataset from file={} into redis_db={}'.format( ticker, restore_algo_file, redis_db)) sys.exit(0) elif mode == ae_consts.SA_MODE_RUN_ALGO: if args.run_algo_in_file: if not os.path.exists(args.run_algo_in_file): log.error('missing algorithm module file: {}'.format( args.run_algo_in_file)) sys.exit(1) if args.backtest_loc: backtest_loc = args.backtest_loc if ('file:/' not in backtest_loc and 's3://' not in backtest_loc and 'redis://' not in backtest_loc): log.error( 'invalid -b <backtest dataset file> specified. ' '{} ' 'please use either: ' '-b file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-b s3://algoready/SPY-latest.json or ' '-b redis://SPY-latest'.format(backtest_loc)) sys.exit(1) if 's3://' in backtest_loc: load_from_s3_bucket = backtest_loc.split('/')[-2] load_from_s3_key = backtest_loc.split('/')[-1] elif 'redis://' in backtest_loc: load_from_redis_key = backtest_loc.split('/')[-1] elif 'file:/' in backtest_loc: load_from_file = backtest_loc.split(':')[-1] load_publish = True # end of parsing supported transport - loading an algo-ready if args.algo_history_loc: algo_history_loc = args.algo_history_loc if ('file:/' not in algo_history_loc and 's3://' not in algo_history_loc and 'redis://' not in algo_history_loc): log.error( 'invalid -p <backtest dataset file> specified. ' '{} ' 'please use either: ' '-p file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-p s3://algoready/SPY-latest.json or ' '-p redis://SPY-latest'.format(algo_history_loc)) sys.exit(1) if 's3://' in algo_history_loc: history_s3_bucket = algo_history_loc.split('/')[-2] history_s3_key = algo_history_loc.split('/')[-1] elif 'redis://' in algo_history_loc: history_redis_key = algo_history_loc.split('/')[-1] elif 'file:/' in algo_history_loc: history_file = algo_history_loc.split(':')[-1] history_publish = True # end of parsing supported transport - trading history if args.algo_report_loc: algo_report_loc = args.algo_report_loc if ('file:/' not in algo_report_loc and 's3://' not in algo_report_loc and 'redis://' not in algo_report_loc): log.error( 'invalid -o <backtest dataset file> specified. ' '{} ' 'please use either: ' '-o file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-o s3://algoready/SPY-latest.json or ' '-o redis://SPY-latest'.format(algo_report_loc)) sys.exit(1) if 's3://' in algo_report_loc: report_s3_bucket = algo_report_loc.split('/')[-2] report_s3_key = algo_report_loc.split('/')[-1] elif 'redis://' in algo_report_loc: report_redis_key = algo_report_loc.split('/')[-1] elif 'file:/' in algo_report_loc: report_file = algo_report_loc.split(':')[-1] report_publish = True # end of parsing supported transport - trading performance report if args.algo_extract_loc: algo_extract_loc = args.algo_extract_loc if ('file:/' not in algo_extract_loc and 's3://' not in algo_extract_loc and 'redis://' not in algo_extract_loc): log.error( 'invalid -e <backtest dataset file> specified. ' '{} ' 'please use either: ' '-e file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-e s3://algoready/SPY-latest.json or ' '-e redis://SPY-latest'.format(algo_extract_loc)) sys.exit(1) if 's3://' in algo_extract_loc: extract_s3_bucket = algo_extract_loc.split('/')[-2] extract_s3_key = algo_extract_loc.split('/')[-1] elif 'redis://' in algo_extract_loc: extract_redis_key = algo_extract_loc.split('/')[-1] elif 'file:/' in algo_extract_loc: extract_file = algo_extract_loc.split(':')[-1] extract_publish = True # end of parsing supported transport - extract algorithm-ready use_custom_algo = True # end of set up for backtest if use_custom_algo: if args.run_on_engine: run_on_engine = True log.info('starting algo on the engine') else: log.info('starting algo') algo_res = run_custom_algo.run_custom_algo( mod_path=args.run_algo_in_file, ticker=ticker, balance=use_balance, commission=use_commission, start_date=use_start_date, end_date=use_end_date, config_file=use_config_file, name=use_name, auto_fill=auto_fill, config_dict=config_dict, load_from_s3_bucket=load_from_s3_bucket, load_from_s3_key=load_from_s3_key, load_from_redis_key=load_from_redis_key, load_from_file=load_from_file, load_compress=load_compress, load_publish=load_publish, load_config=load_config, report_redis_key=report_redis_key, report_s3_bucket=report_s3_bucket, report_s3_key=report_s3_key, report_file=report_file, report_compress=report_compress, report_publish=report_publish, report_config=report_config, history_redis_key=history_redis_key, history_s3_bucket=history_s3_bucket, history_s3_key=history_s3_key, history_file=history_file, history_compress=history_compress, history_publish=history_publish, history_config=history_config, extract_redis_key=extract_redis_key, extract_s3_bucket=extract_s3_bucket, extract_s3_key=extract_s3_key, extract_file=extract_file, extract_save_dir=extract_save_dir, extract_compress=extract_compress, extract_publish=extract_publish, extract_config=extract_config, publish_to_slack=publish_to_slack, publish_to_s3=publish_to_s3, publish_to_redis=publish_to_redis, dataset_type=dataset_type, serialize_datasets=serialize_datasets, compress=compress, encoding=encoding, redis_enabled=redis_enabled, redis_key=redis_key, redis_address=redis_address, redis_db=redis_db, redis_password=redis_password, redis_expire=redis_expire, redis_serializer=redis_serializer, redis_encoding=redis_encoding, s3_enabled=s3_enabled, s3_key=s3_key, s3_address=s3_address, s3_bucket=s3_bucket_name, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, slack_enabled=slack_enabled, slack_code_block=slack_code_block, slack_full_width=slack_full_width, dataset_publish_extract=extract_publish, dataset_publish_history=history_publish, dataset_publish_report=report_publish, run_on_engine=run_on_engine, auth_url=broker_url, backend_url=backend_url, include_tasks=include_tasks, ssl_options=ssl_options, transport_options=transport_options, path_to_config_module=path_to_config_module, timeseries=use_timeseries, trade_strategy=use_trade_strategy, verbose=verbose) show_label = 'algo.name={}'.format(use_name) show_extract = '{}'.format(algo_extract_loc) show_history = '{}'.format(algo_history_loc) show_report = '{}'.format(algo_report_loc) base_label = ('load={} ' 'extract={} ' 'history={} ' 'report={}'.format(args.run_algo_in_file, show_extract, show_history, show_report)) show_label = ('{} running in engine task_id={} {}'.format( ticker, algo_res['rec'].get('task_id', 'missing-task-id'), base_label)) if not run_on_engine: algo_trade_history_recs = algo_res['rec'].get('history', []) show_label = ('{} algo.name={} {} trade_history_len={}'.format( ticker, use_name, base_label, len(algo_trade_history_recs))) if args.debug: log.info('algo_res={}'.format(algo_res)) if algo_res['status'] == ae_consts.SUCCESS: log.info('{} - done running {}'.format( ae_consts.get_status(status=algo_res['status']), show_label)) else: log.error('{} - done running {}'.format( ae_consts.get_status(status=algo_res['status']), show_label)) else: if algo_res['status'] == ae_consts.SUCCESS: log.info('{} - done running {}'.format( ae_consts.get_status(status=algo_res['status']), show_label)) else: log.error('run_custom_algo returned error: {}'.format( algo_res['err'])) sys.exit(1) # end of running the custom algo handler if mode == ae_consts.SA_MODE_EXTRACT: log.info('done extracting dataset - {}'.format(ticker)) elif mode == ae_consts.SA_MODE_RUN_ALGO: log.info('done running algo - {}'.format(ticker)) sys.exit(0) # end of handling mode-specific arg assignments # sanity checking the work and task are valid if not valid: log.error('usage error: missing a supported mode: ' '-f (for prepare a dataset) ') sys.exit(1) if required_task and not task_name: log.error('usage error: missing a supported task_name') sys.exit(1) # end of sanity checks work['ticker'] = ticker work['ticker_id'] = ticker_id work['s3_bucket'] = s3_bucket_name work['s3_key'] = s3_key work['redis_key'] = redis_key work['s3_access_key'] = s3_access_key work['s3_secret_key'] = s3_secret_key work['s3_region_name'] = s3_region_name work['s3_address'] = s3_address work['s3_secure'] = s3_secure work['redis_address'] = redis_address work['redis_password'] = redis_password work['redis_db'] = redis_db work['redis_expire'] = redis_expire work['s3_enabled'] = s3_enabled work['redis_enabled'] = redis_enabled work['debug'] = debug work['label'] = 'ticker={}'.format(ticker) task_res = None if ae_consts.is_celery_disabled(): work['celery_disabled'] = True log.debug('starting without celery work={}'.format( ae_consts.ppj(work))) if mode == ae_consts.SA_MODE_PREPARE: task_res = prep_dataset.prepare_pricing_dataset(work) if debug: log.info('done - result={} ' 'task={} status={} ' 'err={} label={}'.format( ae_consts.ppj(task_res), task_name, ae_consts.get_status(status=task_res['status']), task_res['err'], work['label'])) else: log.info('done - result ' 'task={} status={} ' 'err={} label={}'.format( task_name, ae_consts.get_status(status=task_res['status']), task_res['err'], work['label'])) if task_res['status'] == ae_consts.SUCCESS: image_res = None label = work['label'] ticker = work['ticker'] if plot_action == ae_consts.PLOT_ACTION_SHOW: log.info('showing plot') """ minute_key = '{}_minute'.format( redis_key) minute_df_res = build_df.build_df_from_redis( label=label', address=redis_address, db=redis_db, key=minute_key) minute_df = None if ( minute_df_res['status'] == SUCCESS and minute_df_res['rec']['valid_df']): minute_df = minute_df_res['rec']['data'] print(minute_df.columns.values) column_list = [ 'close', 'date' ] """ today_str = datetime.datetime.now().strftime('%Y-%m-%d') extract_req = work extract_req['redis_key'] = '{}_minute'.format( work['redis_key']) extract_status, minute_df = \ extract_utils.extract_minute_dataset( work_dict=work) if extract_status == ae_consts.SUCCESS: log.info('{} - ticker={} creating chart date={}'.format( label, ticker, today_str)) """ Plot Pricing with the Volume Overlay: """ image_res = ae_charts.plot_overlay_pricing_and_volume( log_label=label, ticker=ticker, date_format=ae_consts.IEX_MINUTE_DATE_FORMAT, df=minute_df, show_plot=True) """ Plot the High-Low-Open-Close Pricing: """ """ image_res = ae_charts.plot_hloc_pricing( log_label=label, ticker=ticker, title='{} - Minute Pricing - {}'.format( ticker, today_str), df=minute_df, show_plot=True) """ """ Plot by custom columns in the DataFrame """ """ column_list = minute_df.columns.values column_list = [ 'date', 'close', 'high', 'low', 'open' ] image_res = ae_charts.plot_df( log_label=label, title='Pricing Title', column_list=column_list, df=minute_df, xcol='date', xlabel='Date', ylabel='Pricing', show_plot=True) """ elif plot_action == ae_consts.PLOT_ACTION_SAVE_TO_S3: log.info('coming soon - support to save to s3') elif plot_action == ae_consts.PLOT_ACTION_SAVE_AS_FILE: log.info('coming soon - support to save as file') if image_res: log.info('{} show plot - status={} err={}'.format( label, ae_consts.get_status(image_res['status']), image_res['err'])) else: log.info('connecting to broker={} backend={}'.format( broker_url, backend_url)) # Get the Celery app app = get_celery_app.get_celery_app( name=__name__, auth_url=broker_url, backend_url=backend_url, path_to_config_module=path_to_config_module, ssl_options=ssl_options, transport_options=transport_options, include_tasks=include_tasks) log.info('calling task={} - work={}'.format(task_name, ae_consts.ppj(work))) job_id = app.send_task(task_name, (work, )) log.info('calling task={} - success job_id={}'.format( task_name, job_id))
def prepare_pricing_dataset( self, work_dict): """prepare_pricing_dataset Prepare dataset for analysis. Supports loading dataset from s3 if not found in redis. Outputs prepared artifact as a csv to s3 and redis. :param work_dict: dictionary for key/values """ label = 'prepare' log.info( 'task - {} - start ' 'work_dict={}'.format( label, work_dict)) initial_data = None ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID rec = { 'ticker': None, 'ticker_id': None, 's3_enabled': True, 'redis_enabled': True, 's3_bucket': None, 's3_key': None, 'redis_key': None, 'prepared_s3_key': None, 'prepared_s3_bucket': None, 'prepared_redis_key': None, 'prepared_data': None, 'prepared_size': None, 'initial_data': None, 'initial_size': None, 'ignore_columns': None, 'updated': None } res = build_result.build_result( status=ae_consts.NOT_RUN, err=None, rec=rec) try: ticker = work_dict.get( 'ticker', ae_consts.TICKER) ticker_id = int(work_dict.get( 'ticker_id', ae_consts.TICKER_ID)) if not ticker: res = build_result.build_result( status=ae_consts.ERR, err='missing ticker', rec=rec) return res label = work_dict.get( 'label', label) s3_key = work_dict.get( 's3_key', None) s3_bucket_name = work_dict.get( 's3_bucket', 'pricing') s3_access_key = work_dict.get( 's3_access_key', ae_consts.S3_ACCESS_KEY) s3_secret_key = work_dict.get( 's3_secret_key', ae_consts.S3_SECRET_KEY) s3_region_name = work_dict.get( 's3_region_name', ae_consts.S3_REGION_NAME) s3_address = work_dict.get( 's3_address', ae_consts.S3_ADDRESS) s3_secure = work_dict.get( 's3_secure', ae_consts.S3_SECURE) == '1' redis_address = work_dict.get( 'redis_address', ae_consts.REDIS_ADDRESS) redis_key = work_dict.get( 'redis_key', ae_consts.REDIS_KEY) redis_password = work_dict.get( 'redis_password', ae_consts.REDIS_PASSWORD) redis_db = work_dict.get( 'redis_db', None) if not redis_db: redis_db = ae_consts.REDIS_DB redis_expire = None if 'redis_expire' in work_dict: redis_expire = work_dict.get( 'redis_expire', ae_consts.REDIS_EXPIRE) updated = work_dict.get( 'updated', datetime.datetime.utcnow().strftime( '%Y_%m_%d_%H_%M_%S')) prepared_s3_key = work_dict.get( 'prepared_s3_key', '{}_{}.csv'.format( ticker, updated)) prepared_s3_bucket = work_dict.get( 'prepared_s3_bucket', 'prepared') prepared_redis_key = work_dict.get( 'prepared_redis_key', 'prepared') ignore_columns = work_dict.get( 'ignore_columns', None) log.info( '{} redis enabled address={}@{} ' 'key={} prepare_s3={}:{} prepare_redis={} ' 'ignore_columns={}'.format( label, redis_address, redis_db, redis_key, prepared_s3_bucket, prepared_s3_key, prepared_redis_key, ignore_columns)) redis_host = redis_address.split(':')[0] redis_port = redis_address.split(':')[1] enable_s3 = True enable_redis_publish = True rec['ticker'] = ticker rec['ticker_id'] = ticker_id rec['s3_bucket'] = s3_bucket_name rec['s3_key'] = s3_key rec['redis_key'] = redis_key rec['prepared_s3_key'] = prepared_s3_key rec['prepared_s3_bucket'] = prepared_s3_bucket rec['prepared_redis_key'] = prepared_redis_key rec['updated'] = updated rec['s3_enabled'] = enable_s3 rec['redis_enabled'] = enable_redis_publish try: log.info( '{} connecting redis={}:{} ' 'db={} key={} ' 'updated={} expire={}'.format( label, redis_host, redis_port, redis_db, redis_key, updated, redis_expire)) rc = redis.Redis( host=redis_host, port=redis_port, password=redis_password, db=redis_db) except Exception as e: err = ( '{} failed - redis connection to address={}@{} ' 'key={} ex={}'.format( label, redis_address, redis_key, redis_db, e)) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res # end of try/ex for connecting to redis initial_data_res = redis_get.get_data_from_redis_key( label=label, client=rc, key=redis_key) log.info( '{} get redis key={} status={} err={}'.format( label, redis_key, ae_consts.get_status(initial_data_res['status']), initial_data_res['err'])) initial_data = initial_data_res['rec'].get( 'data', None) if enable_s3 and not initial_data: log.info( '{} failed to find redis_key={} trying s3 ' 'from s3_key={} s3_bucket={} s3_address={}'.format( label, redis_key, s3_key, s3_bucket_name, s3_address)) get_from_s3_req = \ api_requests.build_publish_from_s3_to_redis_request() get_from_s3_req['s3_enabled'] = enable_s3 get_from_s3_req['s3_access_key'] = s3_access_key get_from_s3_req['s3_secret_key'] = s3_secret_key get_from_s3_req['s3_region_name'] = s3_region_name get_from_s3_req['s3_address'] = s3_address get_from_s3_req['s3_secure'] = s3_secure get_from_s3_req['s3_key'] = s3_key get_from_s3_req['s3_bucket'] = s3_bucket_name get_from_s3_req['redis_key'] = redis_key get_from_s3_req['label'] = ( '{}-run_publish_from_s3_to_redis'.format( label)) log.info( '{} load from s3={} to ' 'redis={}'.format( label, s3_key, redis_key)) try: # run in synchronous mode: get_from_s3_req['celery_disabled'] = True task_res = s3_to_redis.run_publish_from_s3_to_redis( get_from_s3_req) if task_res.get( 'status', ae_consts.ERR) == ae_consts.SUCCESS: log.info( '{} loaded s3={}:{} ' 'to redis={} retrying'.format( label, s3_bucket_name, s3_key, redis_key)) initial_data_res = redis_get.get_data_from_redis_key( label=label, client=rc, key=redis_key) log.info( '{} get redis try=2 key={} status={} err={}'.format( label, redis_key, ae_consts.get_status(initial_data_res['status']), initial_data_res['err'])) initial_data = initial_data_res['rec'].get( 'data', None) else: err = ( '{} ERR failed loading from bucket={} ' 's3_key={} to redis_key={} with res={}'.format( label, s3_bucket_name, s3_key, redis_key, task_res)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res except Exception as e: err = ( '{} extract from s3 and publish to redis failed loading ' 'data from bucket={} in ' 's3_key={} with publish to redis_key={} ' 'with ex={}'.format( label, s3_bucket_name, s3_key, redis_key, e)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res # end of try/ex for publishing from s3->redis # end of if enable_s3 if not initial_data: err = ( '{} did not find any data to prepare in redis_key={} or ' 's3_key={} in bucket={}'.format( label, redis_key, s3_key, s3_bucket_name)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res initial_data_num_chars = len(str(initial_data)) initial_size_value = None initial_size_str = None if initial_data_num_chars < ae_consts.PREPARE_DATA_MIN_SIZE: err = ( '{} not enough data={} in redis_key={} or ' 's3_key={} in bucket={}'.format( label, initial_data_num_chars, redis_key, s3_key, s3_bucket_name)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res else: initial_size_value = initial_data_num_chars / 1024000 initial_size_str = ae_consts.to_f(initial_size_value) if ae_consts.ev('DEBUG_PREPARE', '0') == '1': log.info( '{} initial - redis_key={} data={}'.format( label, redis_key, str(initial_data))) else: log.info( '{} initial - redis_key={} data size={} MB'.format( label, redis_key, initial_size_str)) # end of trying to get initial_data rec['initial_data'] = initial_data rec['initial_size'] = initial_data_num_chars prepare_data = None try: if ae_consts.ev('DEBUG_PREPARE', '0') == '1': log.info( '{} data={} - flatten - {} MB from ' 'redis_key={}'.format( label, ae_consts.ppj(initial_data), initial_size_str, redis_key)) else: log.info( '{} flatten - {} MB from ' 'redis_key={}'.format( label, initial_size_str, redis_key)) prepare_data = dict_to_csv.flatten_dict( data=initial_data) except Exception as e: prepare_data = None err = ( '{} flatten - convert to csv failed with ex={} ' 'redis_key={}'.format( label, e, redis_key)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res # end of try/ex if not prepare_data: err = ( '{} flatten - did not return any data from redis_key={} ' 'or s3_key={} in bucket={}'.format( label, redis_key, s3_key, s3_bucket_name)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res # end of prepare_data prepare_data_num_chars = len(str(prepare_data)) prepare_size_value = None if prepare_data_num_chars < ae_consts.PREPARE_DATA_MIN_SIZE: err = ( '{} prepare - there is not enough data={} in redis_key={}' ''.format( label, prepare_data_num_chars, redis_key)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res else: prepare_size_value = prepare_data_num_chars / 1024000 prepare_size_str = ae_consts.to_f(prepare_size_value) if ae_consts.ev('DEBUG_PREPARE', '0') == '1': log.info( '{} data={} - prepare - redis_key={}'.format( label, redis_key, ae_consts.ppj(prepare_data))) else: log.info( '{} prepare - redis_key={} data size={} MB'.format( label, redis_key, prepare_size_str)) # end of trying to the size of the prepared data rec['prepared_data'] = prepare_data rec['prepared_size'] = prepare_data_num_chars res = build_result.build_result( status=ae_consts.SUCCESS, err=None, rec=rec) rc = None except Exception as e: res = build_result.build_result( status=ae_consts.ERR, err=( 'failed - prepare_pricing_dataset ' 'dict={} with ex={}').format( work_dict, e), rec=rec) log.error( '{} - {}'.format( label, res['err'])) # end of try/ex log.info( 'task - prepare_pricing_dataset done - ' '{} - status={}'.format( label, ae_consts.get_status(res['status']))) return get_task_results.get_task_results( work_dict=work_dict, result=res)
def get_report(self, verbose=False): """get_report Get the indicator's current output node that is used for the trading performance report generated at the end of the algorithm .. note:: the report dict should mostly be numeric types to enable AI predictions after removing non-numeric columns :param verbose: optional - boolean for toggling to show the report """ cur_report_dict = {} # allow derived indicators to build their own report prefix report_prefix_key_name = self.get_report_prefix() for key in self.report_dict: is_valid = True if key in self.report_ignore_keys: is_valid = False if is_valid: report_key = self.build_report_key( key, prefix_key=report_prefix_key_name, key_type='report', cur_report_dict=cur_report_dict) cur_report_dict[report_key] = self.report_dict[key] # for all keys to output into the report buy_value = None sell_value = None for key in self.configurables: is_valid = True if key in self.report_ignore_keys: is_valid = False elif key not in self.__dict__: is_valid = False if is_valid: report_key = self.build_report_key( key, prefix_key=report_prefix_key_name, key_type='conf', cur_report_dict=cur_report_dict) use_value = None if key == 'is_buy': buy_value = self.__dict__[key] if buy_value: use_value = \ ae_consts.INDICATOR_ACTIONS[buy_value] else: use_value = \ ae_consts.INT_INDICATOR_NOT_PROCESSED elif key == 'is_sell': sell_value = self.__dict__[key] if sell_value: use_value = \ ae_consts.INDICATOR_ACTIONS[sell_value] else: use_value = \ ae_consts.INT_INDICATOR_NOT_PROCESSED else: use_value = self.__dict__[key] # end of deciding value cur_report_dict[report_key] = use_value # if valid # end of all configurables for this indicator if verbose or self.verbose: self.lg('indicator={} ' 'report={} ' 'buy={} sell={}'.format(self.name, ae_consts.ppj(cur_report_dict), buy_value, sell_value)) return cur_report_dict
def get_configurables(self, **kwargs): """get_configurables helper for setting up algorithm configs for this indicator and programmatically set the values based off the domain rules .. code-block:: python from analysis_engine.indicators.macd \ import IndicatorMACD ind = IndicatorMACD(config_dict={ 'verbose': True }).get_configurables() :param kwargs: keyword args dictionary """ self.ind_confs = [] # common: self.build_base_configurables(ind_type='momentum', category='technical', uses_data=self.config.get( 'uses_data', 'minute'), version=1) # custom: self.ind_confs.append( self.build_configurable_node(name='num_points', conf_type='int', max_value=200, min_value=2, default_value=100, inc_interval=1)) self.ind_confs.append( self.build_configurable_node(name='buy_below_percent', conf_type='percent', min_value=3.0, max_value=100.0, default_value=5.0, inc_interval=1)) self.ind_confs.append( self.build_configurable_node(name='buy_above_percent', conf_type='percent', min_value=3.0, max_value=100.0, default_value=5.0, inc_interval=1)) self.ind_confs.append( self.build_configurable_node(name='sell_below_percent', conf_type='percent', min_value=3.0, max_value=100.0, default_value=5.0, inc_interval=1)) self.ind_confs.append( self.build_configurable_node(name='sell_above_percent', conf_type='percent', min_value=3.0, max_value=100.0, default_value=5.0, inc_interval=1)) self.ind_confs.append( self.build_configurable_node(name='fast_period', conf_type='float', inc_interval=1)) self.ind_confs.append( self.build_configurable_node(name='slow_period', conf_type='float', inc_interval=1)) self.ind_confs.append( self.build_configurable_node(name='signal_period', conf_type='float', inc_interval=1)) # output / reporting: self.ind_confs.append( self.build_configurable_node(name='macd_value', conf_type='float', is_output_only=True)) self.ind_confs.append( self.build_configurable_node(name='macd_signal', conf_type='float', is_output_only=True)) self.ind_confs.append( self.build_configurable_node(name='macd_hist', conf_type='float', is_output_only=True)) self.ind_confs.append( self.build_configurable_node(name='close', conf_type='float', is_output_only=True)) self.ind_confs.append( self.build_configurable_node(name='percent_value', conf_type='percent', is_output_only=True)) default_values_dict = {} for node in self.ind_confs: name = node['name'] default_value = node.get('default', None) default_values_dict[name] = default_value use_file = None try: if __file__: use_file = __file__ except Exception: use_file = None self.starter_dict = { 'name': self.__class__.__name__.lower().replace('indicator', ''), 'module_path': use_file, 'category': default_values_dict.get('category', 'momentum'), 'type': default_values_dict.get('type', 'technical'), 'uses_data': default_values_dict.get('uses_data', 'minute'), 'verbose': default_values_dict.get('verbose', False) } self.starter_dict.update(default_values_dict) self.lg('configurables={} for class={} in file={} ' 'starter:\n {}'.format(ae_consts.ppj(self.ind_confs), self.__class__.__name__, use_file, ae_consts.ppj(self.starter_dict))) return self.ind_confs
def publish_ticker_aggregate_from_s3(self, work_dict): """publish_ticker_aggregate_from_s3 Publish Aggregated Ticker Data from S3 to Redis :param work_dict: dictionary for key/values """ label = 'pub-tic-agg-s3-to-redis' log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict)) ticker = TICKER ticker_id = TICKER_ID rec = { 'ticker': None, 'ticker_id': None, 's3_read_enabled': True, 's3_upload_enabled': True, 'redis_enabled': True, 's3_bucket': None, 's3_compiled_bucket': None, 's3_key': None, 'redis_key': None, 'updated': None } res = build_result.build_result(status=NOT_RUN, err=None, rec=rec) try: ticker = work_dict.get('ticker', TICKER) ticker_id = int(work_dict.get('ticker_id', TICKER_ID)) if not ticker: res = build_result.build_result(status=ERR, err='missing ticker', rec=rec) return res label = work_dict.get('label', label) s3_key = work_dict.get('s3_key', None) s3_bucket_name = work_dict.get('s3_bucket', 'pricing') s3_compiled_bucket_name = work_dict.get('s3_compiled_bucket', 'compileddatasets') redis_key = work_dict.get('redis_key', None) updated = work_dict.get('updated', None) enable_s3_upload = work_dict.get('s3_upload_enabled', ENABLED_S3_UPLOAD) enable_redis_publish = work_dict.get('redis_enabled', ENABLED_REDIS_PUBLISH) serializer = work_dict.get('serializer', 'json') encoding = work_dict.get('encoding', 'utf-8') enable_s3_read = True rec['ticker'] = ticker rec['ticker_id'] = ticker_id rec['s3_bucket'] = s3_bucket_name rec['s3_compiled_bucket'] = s3_compiled_bucket_name rec['s3_key'] = s3_key rec['redis_key'] = redis_key rec['updated'] = updated rec['s3_read_enabled'] = enable_s3_read rec['s3_upload_enabled'] = enable_s3_upload rec['redis_enabled'] = enable_redis_publish if enable_s3_read: log.info('{} parsing s3 values'.format(label)) access_key = work_dict.get('s3_access_key', S3_ACCESS_KEY) secret_key = work_dict.get('s3_secret_key', S3_SECRET_KEY) region_name = work_dict.get('s3_region_name', S3_REGION_NAME) service_address = work_dict.get('s3_address', S3_ADDRESS) secure = work_dict.get('s3_secure', S3_SECURE) == '1' endpoint_url = 'http://{}'.format(service_address) if secure: endpoint_url = 'https://{}'.format(service_address) log.info('{} building s3 endpoint_url={} ' 'region={}'.format(label, endpoint_url, region_name)) s3 = boto3.resource( 's3', endpoint_url=endpoint_url, aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region_name, config=boto3.session.Config(signature_version='s3v4')) try: log.info('{} checking bucket={} exists'.format( label, s3_bucket_name)) if s3.Bucket(s3_bucket_name) not in s3.buckets.all(): log.info('{} creating bucket={}'.format( label, s3_bucket_name)) s3.create_bucket(Bucket=s3_bucket_name) except Exception as e: log.info('{} failed creating bucket={} ' 'with ex={}'.format(label, s3_bucket_name, e)) # end of try/ex for creating bucket try: log.info('{} checking bucket={} keys'.format( label, s3_bucket_name)) date_keys = [] keys = [] # {TICKER}_YYYY-DD-MM regex reg = r'^.*_\d{4}-(0?[1-9]|1[012])-(0?[1-9]|[12][0-9]|3[01])$' for bucket in s3.buckets.all(): for key in bucket.objects.all(): if (ticker.lower() in key.key.lower() and bool(re.compile(reg).search(key.key))): keys.append(key.key) date_keys.append( key.key.split('{}_'.format(ticker))[1]) except Exception as e: log.info('{} failed to get bucket={} ' 'keys with ex={}'.format(label, s3_bucket_name, e)) # end of try/ex for getting bucket keys if keys: data = [] for idx, key in enumerate(keys): try: log.info('{} reading to s3={}/{} ' 'updated={}'.format(label, s3_bucket_name, key, updated)) loop_data = s3_read_contents_from_key.\ s3_read_contents_from_key( s3=s3, s3_bucket_name=s3_bucket_name, s3_key=key, encoding=encoding, convert_as_json=True) initial_size_value = \ len(str(loop_data)) / 1024000 initial_size_str = to_f(initial_size_value) if ev('DEBUG_S3', '0') == '1': log.info('{} read s3={}/{} data={}'.format( label, s3_bucket_name, key, ppj(loop_data))) else: log.info('{} read s3={}/{} data size={} MB'.format( label, s3_bucket_name, key, initial_size_str)) data.append({'{}'.format(date_keys[idx]): loop_data}) except Exception as e: err = ('{} failed reading bucket={} ' 'key={} ex={}').format(label, s3_bucket_name, key, e) log.error(err) res = build_result.build_result(status=NOT_RUN, err=err, rec=rec) # end of try/ex for creating bucket else: log.info('{} No keys found in ' 'S3 bucket={} for ticker={}'.format( label, s3_bucket_name, ticker)) else: log.info('{} SKIP S3 read bucket={} ' 'ticker={}'.format(label, s3_bucket_name, ticker)) # end of if enable_s3_read if data and enable_s3_upload: try: log.info('{} checking bucket={} exists'.format( label, s3_compiled_bucket_name)) if s3.Bucket(s3_compiled_bucket_name) not in s3.buckets.all(): log.info('{} creating bucket={}'.format( label, s3_compiled_bucket_name)) s3.create_bucket(Bucket=s3_compiled_bucket_name) except Exception as e: log.info('{} failed creating bucket={} ' 'with ex={}'.format(label, s3_compiled_bucket_name, e)) # end of try/ex for creating bucket try: cmpr_data = zlib.compress(json.dumps(data).encode(encoding), 9) if ev('DEBUG_S3', '0') == '1': log.info( '{} uploading to s3={}/{} data={} updated={}'.format( label, s3_compiled_bucket_name, s3_key, ppj(loop_data), updated)) else: sizes = { 'MB': 1024000, 'GB': 1024000000, 'TB': 1024000000000, 'PB': 1024000000000000 } initial_size_value = len(str(data)) org_data_size = 'MB' for key in sizes.keys(): size = float(initial_size_value) / float(sizes[key]) if size > 1024: continue org_data_size = key initial_size_value = size break initial_size_str = to_f(initial_size_value) cmpr_data_size_value = len(cmpr_data) cmpr_data_size = 'MB' for key in sizes.keys(): size = float(cmpr_data_size_value) / float(sizes[key]) if size > 1024: continue cmpr_data_size = key cmpr_data_size_value = size break cmpr_size_str = to_f(cmpr_data_size_value) log.info( '{} uploading to s3={}/{} data original_size={} {} ' 'compressed_size={} {} updated={}'.format( label, s3_compiled_bucket_name, s3_key, initial_size_str, org_data_size, cmpr_size_str, cmpr_data_size, updated)) s3.Bucket(s3_compiled_bucket_name).put_object(Key=s3_key, Body=cmpr_data) except Exception as e: log.error('{} failed uploading bucket={} ' 'key={} ex={}'.format(label, s3_compiled_bucket_name, s3_key, e)) # end of try/ex for creating bucket else: log.info('{} SKIP S3 upload bucket={} ' 'key={}'.format(label, s3_bucket_name, s3_key)) # end of if enable_s3_upload if data and enable_redis_publish: redis_address = work_dict.get('redis_address', REDIS_ADDRESS) redis_key = work_dict.get('redis_key', REDIS_KEY) redis_password = work_dict.get('redis_password', REDIS_PASSWORD) redis_db = work_dict.get('redis_db', None) if not redis_db: redis_db = REDIS_DB redis_expire = None if 'redis_expire' in work_dict: redis_expire = work_dict.get('redis_expire', REDIS_EXPIRE) log.info('redis enabled address={}@{} ' 'key={}'.format(redis_address, redis_db, redis_key)) redis_host = redis_address.split(':')[0] redis_port = redis_address.split(':')[1] try: if ev('DEBUG_REDIS', '0') == '1': log.info('{} publishing redis={}:{} ' 'db={} key={} ' 'updated={} expire={} ' 'data={}'.format(label, redis_host, redis_port, redis_db, redis_key, updated, redis_expire, ppj(data))) else: log.info('{} publishing redis={}:{} ' 'db={} key={} ' 'updated={} expire={}'.format( label, redis_host, redis_port, redis_db, redis_key, updated, redis_expire)) # end of if/else rc = redis.Redis(host=redis_host, port=redis_port, password=redis_password, db=redis_db) redis_set_res = redis_set.set_data_in_redis_key( label=label, client=rc, key=redis_key, data=data, serializer=serializer, encoding=encoding, expire=redis_expire, px=None, nx=False, xx=False) log.info('{} redis_set status={} err={}'.format( label, get_status(redis_set_res['status']), redis_set_res['err'])) except Exception as e: log.error('{} failed - redis publish to ' 'key={} ex={}'.format(label, redis_key, e)) # end of try/ex for creating bucket else: log.info('{} SKIP REDIS publish ' 'key={}'.format(label, redis_key)) # end of if enable_redis_publish res = build_result.build_result(status=SUCCESS, err=None, rec=rec) except Exception as e: res = build_result.build_result(status=ERR, err=('failed - publish_from_s3 ' 'dict={} with ex={}').format( work_dict, e), rec=rec) log.error('{} - {}'.format(label, res['err'])) # end of try/ex log.info('task - publish_from_s3 done - ' '{} - status={}'.format(label, get_status(res['status']))) return analysis_engine.get_task_results.get_task_results( work_dict=work_dict, result=res)
def build_buy_order(ticker, num_owned, close, balance, commission, date, details, use_key, minute=None, shares=None, version=1, auto_fill=True, is_live_trading=False, backtest_shares_default=10, reason=None): """build_buy_order Create an algorithm buy order as a dictionary .. note:: setting the ``minute`` is required to build a minute-by-minute ``Trading History`` :param ticker: ticker :param num_owned: integer current owned number of shares for this asset :param close: float closing price of the asset :param balance: float amount of available capital :param commission: float for commission costs :param date: string trade date for that row usually ``COMMON_DATE_FORMAT`` (``YYYY-MM-DD``) :param minute: optional - string with the minute that the order was placed. format is ``COMMON_TICK_DATE_FORMAT`` (``YYYY-MM-DD HH:MM:SS``) :param details: dictionary for full row of values to review all buys after the algorithm finishes. (usually ``row.to_json()``) :param use_key: string for redis and s3 publishing of the algorithm result dictionary as a json-serialized dictionary :param shares: optional - integer number of shares to buy if None buy max number of shares at the ``close`` with the available ``balance`` amount. :param version: optional - version tracking integer :param auto_fill: optional - bool for not assuming the trade filled (default ``True``) :param is_live_trading: optional - bool for filling trades for live trading or for backtest tuning filled (default ``False`` which is backtest mode) :param backtest_shares_default: optional - integer for simulating shares during a backtest even if there are not enough funds (default ``10``) :param reason: optional - string for recording why the algo decided to buy for review after the algorithm finishes """ status = ae_consts.TRADE_OPEN s3_bucket_name = ae_consts.ALGO_BUYS_S3_BUCKET_NAME s3_key = use_key redis_key = use_key s3_enabled = True redis_enabled = True cost_of_trade = None new_shares = num_owned new_balance = balance created_date = None tradable_funds = balance - (2.0 * commission) if not is_live_trading: if not shares: shares = backtest_shares_default tradable_funds = ((shares * close) + (2.0 * commission)) if close > 0.1 and tradable_funds > 10.0: can_buy_num_shares = shares if not can_buy_num_shares: can_buy_num_shares = int(tradable_funds / close) cost_of_trade = ae_consts.to_f(val=(can_buy_num_shares * close) + commission) if can_buy_num_shares > 0: if cost_of_trade > balance: status = ae_consts.TRADE_NOT_ENOUGH_FUNDS else: created_date = ae_utils.utc_now_str() if auto_fill: new_shares = int(num_owned + can_buy_num_shares) new_balance = ae_consts.to_f(balance - cost_of_trade) status = ae_consts.TRADE_FILLED else: new_shares = shares new_balance = balance else: status = ae_consts.TRADE_NOT_ENOUGH_FUNDS else: status = ae_consts.TRADE_NOT_ENOUGH_FUNDS order_dict = { 'ticker': ticker, 'status': status, 'balance': new_balance, 'shares': new_shares, 'buy_price': cost_of_trade, 'prev_balance': balance, 'prev_shares': num_owned, 'close': close, 'details': details, 'reason': reason, 'date': date, 'minute': minute, 'created': created_date, 's3_bucket': s3_bucket_name, 's3_key': s3_key, 'redis_key': redis_key, 's3_enabled': s3_enabled, 'redis_enabled': redis_enabled, 'version': version } use_date = minute if not use_date: use_date = date log.debug('{} {} buy {} order={}'.format( ticker, use_date, ae_consts.get_status(status=order_dict['status']), ae_consts.ppj(order_dict))) return order_dict
def examine_dataset_in_file( path_to_file, compress=False, encoding='utf-8', ticker=None, dataset_type=ae_consts.SA_DATASET_TYPE_ALGO_READY, serialize_datasets=ae_consts.DEFAULT_SERIALIZED_DATASETS, ): """examine_dataset_in_file Show the internal dataset dictionary structure in dataset file :param path_to_file: path to file :param compress: optional - boolean flag for decompressing the contents of the ``path_to_file`` if necessary (default is ``False`` and algorithms use ``zlib`` for compression) :param encoding: optional - string for data encoding :param ticker: optional - string ticker symbol to verify is in the dataset :param dataset_type: optional - dataset type (default is ``SA_DATASET_TYPE_ALGO_READY``) :param serialize_datasets: optional - list of dataset names to deserialize in the dataset """ if dataset_type == ae_consts.SA_DATASET_TYPE_ALGO_READY: log.info('show start - load dataset from file={}'.format(path_to_file)) show_dataset.show_dataset(path_to_file=path_to_file, compress=compress, encoding=encoding, dataset_type=dataset_type, serialize_datasets=serialize_datasets) log.info('show done - dataset in file={}'.format(path_to_file)) elif dataset_type == ae_consts.SA_DATASET_TYPE_TRADING_HISTORY: log.info('load trading history dataset ' 'from file={}'.format(path_to_file)) trading_history_dict = load_history.load_history_dataset_from_file( path_to_file=path_to_file, compress=compress, encoding=encoding) history_df = trading_history_dict[ticker] first_date = history_df['date'].iloc[0] end_date = history_df['date'].iloc[-1] title = ('Trading History {} for Algo {}\n' 'Backtest dates from {} to {}'.format( ticker, trading_history_dict['algo_name'], first_date, end_date)) xcol = 'date' xlabel = 'Dates vs {} values'.format(trading_history_dict['algo_name']) ylabel = 'Algo Values from columns:\n{}'.format( trading_history_dict['algo_name'], list(history_df.columns.values)) df_filter = (history_df['close'] > 0.01) # set default hloc columns: red = 'close' blue = 'low' green = 'high' orange = 'open' log.info('available columns to plot in dataset: {}'.format( ae_consts.ppj(list(history_df.columns.values)))) plot_trading_history.plot_trading_history(title=title, df=history_df, red=red, blue=blue, green=green, orange=orange, date_col=xcol, xlabel=xlabel, ylabel=ylabel, df_filter=df_filter, show_plot=True, dropna_for_all=False) elif dataset_type == ae_consts.SA_DATASET_TYPE_TRADING_REPORT: log.info('load trading performance report dataset ' 'from file={}'.format(path_to_file)) trading_report_dict = load_report.load_report_dataset_from_file( path_to_file=path_to_file, compress=compress, encoding=encoding) print(trading_report_dict) else: log.error('show unsupported dataset type={} for file={}'.format( dataset_type, path_to_file)) return