def get_result(self): """get_result""" log.info('building results') finished_date = ae_utils.utc_now_str() self.result = { 'name': self.name, 'created': self.created_date, 'updated': finished_date, 'open_positions': self.positions, 'buys': self.get_buys(), 'sells': self.get_sells(), 'num_processed': len(self.order_history), 'history': self.order_history, 'balance': self.balance, 'commission': self.commission } return self.result
def build_buy_order(ticker, num_owned, close, balance, commission, date, details, use_key, minute=None, shares=None, version=1, auto_fill=True, is_live_trading=False, backtest_shares_default=10, reason=None): """build_buy_order Create an algorithm buy order as a dictionary .. note:: setting the ``minute`` is required to build a minute-by-minute ``Trading History`` :param ticker: ticker :param num_owned: integer current owned number of shares for this asset :param close: float closing price of the asset :param balance: float amount of available capital :param commission: float for commission costs :param date: string trade date for that row usually ``COMMON_DATE_FORMAT`` (``YYYY-MM-DD``) :param minute: optional - string with the minute that the order was placed. format is ``COMMON_TICK_DATE_FORMAT`` (``YYYY-MM-DD HH:MM:SS``) :param details: dictionary for full row of values to review all buys after the algorithm finishes. (usually ``row.to_json()``) :param use_key: string for redis and s3 publishing of the algorithm result dictionary as a json-serialized dictionary :param shares: optional - integer number of shares to buy if None buy max number of shares at the ``close`` with the available ``balance`` amount. :param version: optional - version tracking integer :param auto_fill: optional - bool for not assuming the trade filled (default ``True``) :param is_live_trading: optional - bool for filling trades for live trading or for backtest tuning filled (default ``False`` which is backtest mode) :param backtest_shares_default: optional - integer for simulating shares during a backtest even if there are not enough funds (default ``10``) :param reason: optional - string for recording why the algo decided to buy for review after the algorithm finishes """ status = ae_consts.TRADE_OPEN s3_bucket_name = ae_consts.ALGO_BUYS_S3_BUCKET_NAME s3_key = use_key redis_key = use_key s3_enabled = True redis_enabled = True cost_of_trade = None new_shares = num_owned new_balance = balance created_date = None tradable_funds = balance - (2.0 * commission) if not is_live_trading: if not shares: shares = backtest_shares_default tradable_funds = ((shares * close) + (2.0 * commission)) if close > 0.1 and tradable_funds > 10.0: can_buy_num_shares = shares if not can_buy_num_shares: can_buy_num_shares = int(tradable_funds / close) cost_of_trade = ae_consts.to_f(val=(can_buy_num_shares * close) + commission) if can_buy_num_shares > 0: if cost_of_trade > balance: status = ae_consts.TRADE_NOT_ENOUGH_FUNDS else: created_date = ae_utils.utc_now_str() if auto_fill: new_shares = int(num_owned + can_buy_num_shares) new_balance = ae_consts.to_f(balance - cost_of_trade) status = ae_consts.TRADE_FILLED else: new_shares = shares new_balance = balance else: status = ae_consts.TRADE_NOT_ENOUGH_FUNDS else: status = ae_consts.TRADE_NOT_ENOUGH_FUNDS order_dict = { 'ticker': ticker, 'status': status, 'balance': new_balance, 'shares': new_shares, 'buy_price': cost_of_trade, 'prev_balance': balance, 'prev_shares': num_owned, 'close': close, 'details': details, 'reason': reason, 'date': date, 'minute': minute, 'created': created_date, 's3_bucket': s3_bucket_name, 's3_key': s3_key, 'redis_key': redis_key, 's3_enabled': s3_enabled, 'redis_enabled': redis_enabled, 'version': version } use_date = minute if not use_date: use_date = date log.debug('{} {} buy {} order={}'.format( ticker, use_date, ae_consts.get_status(status=order_dict['status']), ae_consts.ppj(order_dict))) return order_dict
def get_ds_dict(ticker, base_key=None, ds_id=None, label=None, service_dict=None): """get_ds_dict Get a dictionary with all cache keys for a ticker and return the dictionary. Use this method to decouple your apps from the underlying cache key implementations (if you do not need them). :param ticker: ticker :param base_key: optional - base key that is prepended in all cache keys :param ds_id: optional - dataset id (useful for external database id) :param label: optional - tracking label in the logs :param service_dict: optional - parent call functions and Celery tasks can use this dictionary to seed the common service routes and endpoints. Refer to ``analysis_engine.consts.SERVICE_VALS`` for automatically-copied over keys by this helper. """ if not ticker: raise Exception('please pass in a ticker') use_base_key = base_key if not use_base_key: use_base_key = '{}_{}'.format( ticker, ae_utils.get_last_close_str(fmt=ae_consts.COMMON_DATE_FORMAT)) date_str = ae_utils.utc_date_str(fmt=ae_consts.COMMON_DATE_FORMAT) now_str = ae_utils.utc_now_str(fmt=ae_consts.COMMON_TICK_DATE_FORMAT) daily_redis_key = '{}_{}'.format(use_base_key, ae_consts.DAILY_S3_BUCKET_NAME) minute_redis_key = '{}_{}'.format(use_base_key, ae_consts.MINUTE_S3_BUCKET_NAME) quote_redis_key = '{}_{}'.format(use_base_key, ae_consts.QUOTE_S3_BUCKET_NAME) stats_redis_key = '{}_{}'.format(use_base_key, ae_consts.STATS_S3_BUCKET_NAME) peers_redis_key = '{}_{}'.format(use_base_key, ae_consts.PEERS_S3_BUCKET_NAME) news_iex_redis_key = '{}_{}1'.format(use_base_key, ae_consts.NEWS_S3_BUCKET_NAME) financials_redis_key = '{}_{}'.format(use_base_key, ae_consts.FINANCIALS_S3_BUCKET_NAME) earnings_redis_key = '{}_{}'.format(use_base_key, ae_consts.EARNINGS_S3_BUCKET_NAME) dividends_redis_key = '{}_{}'.format(use_base_key, ae_consts.DIVIDENDS_S3_BUCKET_NAME) company_redis_key = '{}_{}'.format(use_base_key, ae_consts.COMPANY_S3_BUCKET_NAME) options_yahoo_redis_key = '{}_{}'.format(use_base_key, ae_consts.OPTIONS_S3_BUCKET_NAME) call_options_yahoo_redis_key = '{}_calls'.format(use_base_key) put_options_yahoo_redis_key = '{}_puts'.format(use_base_key) pricing_yahoo_redis_key = '{}_{}'.format(use_base_key, ae_consts.PRICING_S3_BUCKET_NAME) news_yahoo_redis_key = '{}_{}'.format(use_base_key, ae_consts.NEWS_S3_BUCKET_NAME) call_options_td_redis_key = '{}_tdcalls'.format(use_base_key) put_options_td_redis_key = '{}_tdputs'.format(use_base_key) ds_cache_dict = { 'daily': daily_redis_key, 'minute': minute_redis_key, 'quote': quote_redis_key, 'stats': stats_redis_key, 'peers': peers_redis_key, 'news1': news_iex_redis_key, 'financials': financials_redis_key, 'earnings': earnings_redis_key, 'dividends': dividends_redis_key, 'company': company_redis_key, 'options': options_yahoo_redis_key, 'calls': call_options_yahoo_redis_key, 'puts': put_options_yahoo_redis_key, 'pricing': pricing_yahoo_redis_key, 'news': news_yahoo_redis_key, 'tdcalls': call_options_td_redis_key, 'tdputs': put_options_td_redis_key, 'ticker': ticker, 'ds_id': ds_id, 'label': label, 'created': now_str, 'date': date_str, 'manifest_key': use_base_key, 'version': ae_consts.CACHE_DICT_VERSION } # set keys/values for redis/minio from the # service_dict - helper method for # launching job chains if service_dict: for k in ae_consts.SERVICE_VALS: ds_cache_dict[k] = service_dict[k] return ds_cache_dict
def build_sell_order(ticker, num_owned, close, balance, commission, date, details, use_key, minute=None, shares=None, version=1, auto_fill=True, is_live_trading=False, backtest_shares_default=10, reason=None): """build_sell_order Create an algorithm sell order as a dictionary :param ticker: ticker :param num_owned: integer current owned number of shares for this asset :param close: float closing price of the asset :param balance: float amount of available capital :param commission: float for commission costs :param date: string trade date for that row usually ``COMMON_DATE_FORMAT`` (``YYYY-MM-DD``) :param minute: optional - string with the minute that the order was placed. format is ``COMMON_TICK_DATE_FORMAT`` (``YYYY-MM-DD HH:MM:SS``) :param details: dictionary for full row of values to review all sells after the algorithm finishes. (usually ``row.to_json()``) :param use_key: string for redis and s3 publishing of the algorithm result dictionary as a json-serialized dictionary :param shares: optional - integer number of shares to sell if None sell all ``num_owned`` shares at the ``close``. :param version: optional - version tracking integer :param auto_fill: optional - bool for not assuming the trade filled (default ``True``) :param is_live_trading: optional - bool for filling trades for live trading or for backtest tuning filled (default ``False`` which is backtest mode) :param backtest_shares_default: optional - integer for simulating shares during a backtest even if there are not enough funds (default ``10``) :param reason: optional - string for recording why the algo decided to sell for review after the algorithm finishes """ status = ae_consts.TRADE_OPEN s3_bucket_name = ae_consts.ALGO_SELLS_S3_BUCKET_NAME s3_key = use_key redis_key = use_key s3_enabled = True redis_enabled = True cost_of_trade = None sell_price = 0.0 new_shares = num_owned new_balance = balance created_date = None tradable_funds = balance - commission if num_owned == 0: status = ae_consts.TRADE_NO_SHARES_TO_SELL elif close > 0.1 and tradable_funds > 10.0: cost_of_trade = commission if shares: if shares > num_owned: shares = num_owned else: shares = num_owned sell_price = ae_consts.to_f(val=(shares * close) + commission) if cost_of_trade > balance: status = ae_consts.TRADE_NOT_ENOUGH_FUNDS else: created_date = ae_utils.utc_now_str() if auto_fill: new_shares = num_owned - shares new_balance = ae_consts.to_f(balance + sell_price) status = ae_consts.TRADE_FILLED else: new_shares = shares new_balance = balance else: status = ae_consts.TRADE_NOT_ENOUGH_FUNDS order_dict = { 'ticker': ticker, 'status': status, 'balance': new_balance, 'shares': new_shares, 'sell_price': sell_price, 'prev_balance': balance, 'prev_shares': num_owned, 'close': close, 'details': details, 'reason': reason, 'date': date, 'minute': minute, 'created': created_date, 's3_bucket': s3_bucket_name, 's3_key': s3_key, 'redis_key': redis_key, 's3_enabled': s3_enabled, 'redis_enabled': redis_enabled, 'version': version } use_date = minute if not use_date: use_date = date log.debug(f'{ticker} {use_date} sell ' f'{ae_consts.get_status(status=order_dict["status"])} ' f'order={ae_consts.ppj(order_dict)}') return order_dict
def build_indicator_node(node, label=None): """build_indicator_node Parse a dictionary in the algorithm config ``indicators`` list and return a dicationry Supported values found in: `analysis_engine/consts.py <https://github.com/AlgoTraders/sto ck-analysis-engine/blob/master/analysis_engine/consts.py>`__ :param node: single dictionary from the config's ``indicators`` list :param label: optional - string log tracking this class in the logs (usually just the algo name is good enough to help debug issues when running distributed) :return: dictionary """ if not label: label = 'build_indicator_node' name = node.get('name', None) if not name: raise Exception( '{} - missing "name" in indicator dictionary={}'.format( label, node)) # end of name check ind_id = str(uuid.uuid4()).replace('-', '') uses_dataset_str = node.get('uses_data', 'daily') uses_dataset = ae_consts.get_indicator_uses_data_as_int( val=uses_dataset_str) if uses_dataset == ae_consts.INDICATOR_USES_DATA_UNSUPPORTED: uses_dataset = ae_consts.INDICATOR_USES_DATA_ANY log.debug('{} - unsupported indicator ' 'uses_dataset={} defaulting to "daily"'.format( label, uses_dataset_str)) # end of supported indicator dataset types ind_category_str = node.get('category', 'momentum') ind_category = ae_consts.get_indicator_category_as_int( val=ind_category_str) if ind_category == ae_consts.INDICATOR_CATEGORY_UNKNOWN: ind_category = ae_consts.INDICATOR_CATEGORY_MOMENTUM log.debug('{} - unsupported indicator ' 'category={} defaulting to "momentum"'.format( label, ind_category)) # end of supported indicator category ind_type_str = node.get('type', 'technical') ind_type = ae_consts.get_indicator_type_as_int(val=ind_type_str) if ind_type == ae_consts.INDICATOR_TYPE_UNKNOWN: ind_type = ae_consts.INDICATOR_TYPE_TECHNICAL log.debug('{} - unsupported indicator ' 'type={} defaulting to "technical"'.format(label, ind_type)) # end of supported indicator type # allow easier key discovery use_unique_id = node.get('unique_id', False) ind_name = '{}_{}_{}_{}'.format(name, ind_category, ind_type, uses_dataset) if use_unique_id: ind_name = '{}_{}_{}_{}_{}'.format(name, ind_category, ind_type, uses_dataset, ind_id) use_module_name = None use_path_to_module = None # none will use the BaseIndicator which does nothing use_path_to_module = node.get('module_path', ae_consts.INDICATOR_BASE_MODULE_PATH) if not use_path_to_module: raise Exception('Failed building Indicator node with missing ' 'module_path node={}'.format(node)) use_module_name = node.get('module_name', node.get('name', ind_id)) default_report_ignore_keys = \ ae_consts.INDICATOR_IGNORED_CONIGURABLE_KEYS report_dict = { 'id': ind_id, 'name': ind_name, 'created': ae_utils.utc_now_str(), 'version': 1, 'module_name': use_module_name, 'path_to_module': use_path_to_module, 'report_ignore_keys': default_report_ignore_keys, 'metrics': { 'type': ind_type, 'category': ind_category, 'uses_data': uses_dataset } } labeled_node = copy.deepcopy(node) # allow a node's sub report dir to be patched with this # tracking + reporting data # the algorithms will flatten: # indicators[ind_name]['report']['metrics'] # for trading performance report generation if 'report' in labeled_node: labeled_node['report'].update(report_dict) else: labeled_node['report'] = report_dict return labeled_node
def publish_trading_history(self, records_for_history, pt_s3_access_key=None, pt_s3_secret_key=None, pt_s3_address=None, pt_s3_region=None, pt_s3_bucket=None, pt_s3_key=None, pt_s3_secure=ae_consts.NOT_SET, **kwargs): """publish_trading_history Helper for publishing a trading history to another S3 service like AWS :param records_for_history: list of dictionaries for the history file :param pt_s3_access_key: access key :param pt_s3_secret_key: secret :param pt_s3_address: address :param pt_s3_region: region :param pt_s3_bucket: bucket :param pt_s3_key: key :param pt_s3_secure: secure flag :param kwargs: support for keyword arg dict """ use_s3_access_key = self.pt_s3_access_key use_s3_secret_key = self.pt_s3_secret_key use_s3_address = self.pt_s3_address use_s3_region = self.pt_s3_region use_s3_bucket = self.pt_s3_bucket use_s3_key = self.pt_s3_key use_s3_secure = self.pt_s3_secure use_s3_enabled = kwargs.get('s3_enabled', True) use_redis_enabled = kwargs.get('redis_enabled', False) use_redis_address = kwargs.get('redis_address', None) use_redis_db = kwargs.get('redis_db', None) use_redis_key = kwargs.get('redis_key', None) use_redis_password = kwargs.get('redis_password', None) use_redis_expire = kwargs.get('redis_expire', None) use_redis_serializer = kwargs.get('redis_serializer', 'json') use_redis_encoding = kwargs.get('redis_encoding', 'utf-8') verbose = kwargs.get('verbose', False) if pt_s3_access_key: use_s3_access_key = pt_s3_access_key if pt_s3_secret_key: use_s3_secret_key = pt_s3_secret_key if pt_s3_address: use_s3_address = pt_s3_address if pt_s3_region: use_s3_region = pt_s3_region if pt_s3_bucket: use_s3_bucket = pt_s3_bucket if pt_s3_key: use_s3_key = pt_s3_key if pt_s3_secure != ae_consts.NOT_SET: use_s3_secure = pt_s3_secure rec = { 'tickers': self.ticker, 'version': int(ae_consts.ALGO_HISTORY_VERSION), 'last_trade_date': ae_utils.get_last_close_str(), 'algo_config_dict': self.config_dict, 'algo_name': self.use_name, 'created': ae_utils.utc_now_str(), self.ticker: records_for_history } num_bytes = len(str(rec)) num_mb = ae_consts.get_mb(num_bytes) msg = ( f'publish - {self.ticker} - {rec["last_trade_date"]} ' # f'{use_s3_access_key} with: {use_s3_secret_key} ' f's3_loc={use_s3_address}/{use_s3_bucket}/{use_s3_key} ' f'mb={num_mb}MB') log.info(msg) publish.publish(data=rec, label='pub', df_compress=True, compress=False, convert_to_dict=False, output_file=None, redis_enabled=use_redis_enabled, redis_key=use_redis_key, redis_address=use_redis_address, redis_db=use_redis_db, redis_password=use_redis_password, redis_expire=use_redis_expire, redis_serializer=use_redis_serializer, redis_encoding=use_redis_encoding, s3_enabled=use_s3_enabled, s3_key=use_s3_key, s3_address=use_s3_address, s3_bucket=use_s3_bucket, s3_access_key=use_s3_access_key, s3_secret_key=use_s3_secret_key, s3_region_name=use_s3_region, s3_secure=use_s3_secure, slack_enabled=False, verbose=verbose)
def get_new_pricing_data(self, work_dict): """get_new_pricing_data Get Ticker information on: - prices - turn off with ``work_dict.get_pricing = False`` - news - turn off with ``work_dict.get_news = False`` - options - turn off with ``work_dict.get_options = False`` :param work_dict: dictionary for key/values """ label = 'get_new_pricing_data' log.debug(f'task - {label} - start ' f'work_dict={work_dict}') num_success = 0 ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID rec = { 'pricing': None, 'options': None, 'calls': None, 'puts': None, 'news': None, 'daily': None, 'minute': None, 'quote': None, 'stats': None, 'peers': None, 'iex_news': None, 'financials': None, 'earnings': None, 'dividends': None, 'company': None, 'exp_date': None, 'publish_pricing_update': None, 'num_success': num_success, 'date': ae_utils.utc_now_str(), 'updated': None, 'version': ae_consts.DATASET_COLLECTION_VERSION } res = {'status': ae_consts.NOT_RUN, 'err': None, 'rec': rec} try: ticker = work_dict.get('ticker', ticker) ticker_id = work_dict.get('ticker_id', ae_consts.TICKER_ID) s3_bucket = work_dict.get('s3_bucket', ae_consts.S3_BUCKET) s3_key = work_dict.get('s3_key', ae_consts.S3_KEY) redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY) exp_date = work_dict.get('exp_date', None) cur_date = ae_utils.last_close() cur_strike = work_dict.get('strike', None) contract_type = str(work_dict.get('contract', 'C')).upper() label = work_dict.get('label', label) iex_datasets = work_dict.get('iex_datasets', iex_consts.DEFAULT_FETCH_DATASETS) td_datasets = work_dict.get('td_datasets', td_consts.DEFAULT_FETCH_DATASETS_TD) fetch_mode = work_dict.get('fetch_mode', ae_consts.FETCH_MODE_ALL) iex_token = work_dict.get('iex_token', iex_consts.IEX_TOKEN) td_token = work_dict.get('td_token', td_consts.TD_TOKEN) str_fetch_mode = str(fetch_mode).lower() # control flags to deal with feed issues: get_iex_data = True get_td_data = True if (fetch_mode == ae_consts.FETCH_MODE_ALL or str_fetch_mode == 'initial'): get_iex_data = True get_td_data = True iex_datasets = ae_consts.IEX_INITIAL_DATASETS elif (fetch_mode == ae_consts.FETCH_MODE_ALL or str_fetch_mode == 'all'): get_iex_data = True get_td_data = True iex_datasets = ae_consts.IEX_DATASETS_DEFAULT elif (fetch_mode == ae_consts.FETCH_MODE_YHO or str_fetch_mode == 'yahoo'): get_iex_data = False get_td_data = False elif (fetch_mode == ae_consts.FETCH_MODE_IEX or str_fetch_mode == 'iex-all'): get_iex_data = True get_td_data = False iex_datasets = ae_consts.IEX_DATASETS_DEFAULT elif (fetch_mode == ae_consts.FETCH_MODE_IEX or str_fetch_mode == 'iex'): get_iex_data = True get_td_data = False iex_datasets = ae_consts.IEX_INTRADAY_DATASETS elif (fetch_mode == ae_consts.FETCH_MODE_INTRADAY or str_fetch_mode == 'intra'): get_iex_data = True get_td_data = True iex_datasets = ae_consts.IEX_INTRADAY_DATASETS elif (fetch_mode == ae_consts.FETCH_MODE_DAILY or str_fetch_mode == 'daily'): get_iex_data = True get_td_data = False iex_datasets = ae_consts.IEX_DAILY_DATASETS elif (fetch_mode == ae_consts.FETCH_MODE_WEEKLY or str_fetch_mode == 'weekly'): get_iex_data = True get_td_data = False iex_datasets = ae_consts.IEX_WEEKLY_DATASETS elif (fetch_mode == ae_consts.FETCH_MODE_TD or str_fetch_mode == 'td'): get_iex_data = False get_td_data = True else: get_iex_data = False get_td_data = False fetch_arr = str_fetch_mode.split(',') found_fetch = False iex_datasets = [] for fetch_name in fetch_arr: if fetch_name not in iex_datasets: if fetch_name == 'iex_min': iex_datasets.append('minute') elif fetch_name == 'iex_day': iex_datasets.append('daily') elif fetch_name == 'iex_quote': iex_datasets.append('quote') elif fetch_name == 'iex_stats': iex_datasets.append('stats') elif fetch_name == 'iex_peers': iex_datasets.append('peers') elif fetch_name == 'iex_news': iex_datasets.append('news') elif fetch_name == 'iex_fin': iex_datasets.append('financials') elif fetch_name == 'iex_earn': iex_datasets.append('earnings') elif fetch_name == 'iex_div': iex_datasets.append('dividends') elif fetch_name == 'iex_comp': iex_datasets.append('company') elif fetch_name == 'td': get_td_data = True else: log.warn('unsupported IEX dataset ' f'{fetch_name}') found_fetch = (len(iex_datasets) != 0) if not found_fetch: log.error(f'{label} - unsupported ' f'fetch_mode={fetch_mode} value') else: get_iex_data = True log.debug(f'{label} - ' f'fetching={len(iex_datasets)} ' f'{iex_datasets} ' f'fetch_mode={fetch_mode}') # end of screening custom fetch_mode settings num_tokens = 0 if get_iex_data: if not iex_token: log.warn(f'{label} - ' 'please set a valid IEX Cloud Account token (' 'https://iexcloud.io/cloud-login/#/register' ') to fetch data from IEX Cloud. It must be ' 'set as an environment variable like: ' 'export IEX_TOKEN=<token>') get_iex_data = False else: num_tokens += 1 # sanity check - disable IEX fetch if the token is not set if get_td_data: missing_td_token = [ 'MISSING_TD_TOKEN', 'SETYOURTDTOKEN', 'SETYOURTRADIERTOKENHERE' ] if td_token in missing_td_token: log.warn(f'{label} - ' 'please set a valid Tradier Account token (' 'https://developer.tradier.com/user/sign_up' ') to fetch pricing data from Tradier. It must be ' 'set as an environment variable like: ' 'export TD_TOKEN=<token>') get_td_data = False else: num_tokens += 1 # sanity check - disable Tradier fetch if the token is not set """ as of Thursday, Jan. 3, 2019: https://developer.yahoo.com/yql/ Important EOL Notice: As of Thursday, Jan. 3, 2019 the YQL service at query.yahooapis.com will be retired """ get_yahoo_data = False if (not get_iex_data and not get_td_data and not get_yahoo_data): err = None if num_tokens == 0: res['status'] = ae_consts.MISSING_TOKEN err = (f'Please set a valid IEX_TOKEN or TD_TOKEN ' f'environment variable') else: err = (f'Please set at least one supported datafeed from ' f'either: ' f'IEX Cloud (fetch -t TICKER -g iex) or ' f'Tradier (fetch -t TICKER -g td) ' f'for ' f'ticker={ticker} ' f'cur_date={cur_date} ' f'IEX enabled={get_iex_data} ' f'TD enabled={get_td_data} ' f'YHO enabled={get_yahoo_data}') res['status'] = ae_consts.ERR res['err'] = err return get_task_results.get_task_results(work_dict=work_dict, result=res) # end of checking that there is at least 1 feed on if not exp_date: exp_date = opt_dates.option_expiration(date=exp_date) else: exp_date = datetime.datetime.strptime(exp_date, '%Y-%m-%d') rec['updated'] = cur_date.strftime('%Y-%m-%d %H:%M:%S') log.debug(f'{label} getting pricing for ticker={ticker} ' f'cur_date={cur_date} exp_date={exp_date} ' f'IEX={get_iex_data} ' f'TD={get_td_data} ' f'YHO={get_yahoo_data}') yahoo_rec = { 'ticker': ticker, 'pricing': None, 'options': None, 'calls': None, 'puts': None, 'news': None, 'exp_date': None, 'publish_pricing_update': None, 'date': None, 'updated': None } # disabled on 2019-01-03 if get_yahoo_data: log.debug(f'{label} YHO ticker={ticker}') yahoo_res = yahoo_data.get_data_from_yahoo(work_dict=work_dict) status_str = ae_consts.get_status(status=yahoo_res['status']) if yahoo_res['status'] == ae_consts.SUCCESS: yahoo_rec = yahoo_res['rec'] msg = (f'{label} YHO ticker={ticker} ' f'status={status_str} err={yahoo_res["err"]}') if ae_consts.ev('SHOW_SUCCESS', '0') == '1': log.info(msg) else: log.debug(msg) rec['pricing'] = yahoo_rec.get('pricing', '{}') rec['news'] = yahoo_rec.get('news', '{}') rec['options'] = yahoo_rec.get('options', '{}') rec['calls'] = rec['options'].get('calls', ae_consts.EMPTY_DF_STR) rec['puts'] = rec['options'].get('puts', ae_consts.EMPTY_DF_STR) num_success += 1 else: log.error(f'{label} failed YHO ticker={ticker} ' f'status={status_str} err={yahoo_res["err"]}') # end of get from yahoo if get_iex_data: num_iex_ds = len(iex_datasets) log.debug(f'{label} IEX datasets={num_iex_ds}') for idx, ft_type in enumerate(iex_datasets): dataset_field = iex_consts.get_ft_str(ft_type=ft_type) log.debug(f'{label} IEX={idx}/{num_iex_ds} ' f'field={dataset_field} ticker={ticker}') iex_label = f'{label}-{dataset_field}' iex_req = copy.deepcopy(work_dict) iex_req['label'] = iex_label iex_req['ft_type'] = ft_type iex_req['field'] = dataset_field iex_req['ticker'] = ticker iex_res = iex_data.get_data_from_iex(work_dict=iex_req) status_str = (ae_consts.get_status(status=iex_res['status'])) if iex_res['status'] == ae_consts.SUCCESS: iex_rec = iex_res['rec'] msg = (f'{label} IEX ticker={ticker} ' f'field={dataset_field} ' f'status={status_str} ' f'err={iex_res["err"]}') if ae_consts.ev('SHOW_SUCCESS', '0') == '1': log.info(msg) else: log.debug(msg) if dataset_field == 'news': rec['iex_news'] = iex_rec['data'] else: rec[dataset_field] = iex_rec['data'] num_success += 1 else: log.debug(f'{label} failed IEX ticker={ticker} ' f'field={dataset_field} ' f'status={status_str} err={iex_res["err"]}') # end of if/else succcess # end idx, ft_type in enumerate(iex_datasets): # end of if get_iex_data if get_td_data: num_td_ds = len(td_datasets) log.debug(f'{label} TD datasets={num_td_ds}') for idx, ft_type in enumerate(td_datasets): dataset_field = td_consts.get_ft_str_td(ft_type=ft_type) log.debug(f'{label} TD={idx}/{num_td_ds} ' f'field={dataset_field} ticker={ticker}') td_label = (f'{label}-{dataset_field}') td_req = copy.deepcopy(work_dict) td_req['label'] = td_label td_req['ft_type'] = ft_type td_req['field'] = dataset_field td_req['ticker'] = ticker td_res = td_data.get_data_from_td(work_dict=td_req) status_str = (ae_consts.get_status(status=td_res['status'])) if td_res['status'] == ae_consts.SUCCESS: td_rec = td_res['rec'] msg = (f'{label} TD ticker={ticker} ' f'field={dataset_field} ' f'status={status_str} ' f'err={td_res["err"]}') if ae_consts.ev('SHOW_SUCCESS', '0') == '1': log.info(msg) else: log.debug(msg) if dataset_field == 'tdcalls': rec['tdcalls'] = td_rec['data'] if dataset_field == 'tdputs': rec['tdputs'] = td_rec['data'] else: rec[dataset_field] = td_rec['data'] num_success += 1 else: log.critical(f'{label} failed TD ticker={ticker} ' f'field={dataset_field} ' f'status={status_str} err={td_res["err"]}') # end of if/else succcess # end idx, ft_type in enumerate(td_datasets): # end of if get_td_data rec['num_success'] = num_success update_req = {'data': rec} update_req['ticker'] = ticker update_req['ticker_id'] = ticker_id update_req['strike'] = cur_strike update_req['contract'] = contract_type update_req['s3_enabled'] = work_dict.get('s3_enabled', ae_consts.ENABLED_S3_UPLOAD) update_req['redis_enabled'] = work_dict.get( 'redis_enabled', ae_consts.ENABLED_REDIS_PUBLISH) update_req['s3_bucket'] = s3_bucket update_req['s3_key'] = s3_key update_req['s3_access_key'] = work_dict.get('s3_access_key', ae_consts.S3_ACCESS_KEY) update_req['s3_secret_key'] = work_dict.get('s3_secret_key', ae_consts.S3_SECRET_KEY) update_req['s3_region_name'] = work_dict.get('s3_region_name', ae_consts.S3_REGION_NAME) update_req['s3_address'] = work_dict.get('s3_address', ae_consts.S3_ADDRESS) update_req['s3_secure'] = work_dict.get('s3_secure', ae_consts.S3_SECURE) update_req['redis_key'] = redis_key update_req['redis_address'] = work_dict.get('redis_address', ae_consts.REDIS_ADDRESS) update_req['redis_password'] = work_dict.get('redis_password', ae_consts.REDIS_PASSWORD) update_req['redis_db'] = int( work_dict.get('redis_db', ae_consts.REDIS_DB)) update_req['redis_expire'] = work_dict.get('redis_expire', ae_consts.REDIS_EXPIRE) update_req['updated'] = rec['updated'] update_req['label'] = label update_req['celery_disabled'] = True update_status = ae_consts.NOT_SET try: update_res = publisher.run_publish_pricing_update( work_dict=update_req) update_status = update_res.get('status', ae_consts.NOT_SET) status_str = ae_consts.get_status(status=update_status) if ae_consts.ev('DEBUG_RESULTS', '0') == '1': log.debug(f'{label} update_res ' f'status={status_str} ' f'data={ae_consts.ppj(update_res)}') else: log.debug(f'{label} run_publish_pricing_update ' f'status={status_str}') # end of if/else rec['publish_pricing_update'] = update_res res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) except Exception as f: err = (f'{label} publisher.run_publish_pricing_update failed ' f'with ex={f}') log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=rec) # end of trying to publish results to connected services except Exception as e: res = build_result.build_result(status=ae_consts.ERR, err=('failed - get_new_pricing_data ' f'dict={work_dict} with ex={e}'), rec=rec) log.error(f'{label} - {res["err"]}') # end of try/ex if ae_consts.ev('DATASET_COLLECTION_SLACK_ALERTS', '0') == '1': env_name = 'DEV' if ae_consts.ev('PROD_SLACK_ALERTS', '1') == '1': env_name = 'PROD' done_msg = (f'Dataset collected ticker=*{ticker}* on ' f'env=*{env_name}* ' f'redis_key={redis_key} s3_key={s3_key} ' f'IEX={get_iex_data} ' f'TD={get_td_data} ' f'YHO={get_yahoo_data}') log.debug(f'{label} sending slack msg={done_msg}') if res['status'] == ae_consts.SUCCESS: slack_utils.post_success(msg=done_msg, block=False, jupyter=True) else: slack_utils.post_failure(msg=done_msg, block=False, jupyter=True) # end of if/else success # end of publishing to slack log.debug('task - get_new_pricing_data done - ' f'{label} - status={ae_consts.get_status(res["status"])}') return get_task_results.get_task_results(work_dict=work_dict, result=res)
def get_new_pricing_data(self, work_dict): """get_new_pricing_data Get Ticker information on: - prices - turn off with ``work_dict.get_pricing = False`` - news - turn off with ``work_dict.get_news = False`` - options - turn off with ``work_dict.get_options = False`` :param work_dict: dictionary for key/values """ label = 'get_new_pricing_data' log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict)) ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID rec = { 'pricing': None, 'options': None, 'calls': None, 'puts': None, 'news': None, 'daily': None, 'minute': None, 'quote': None, 'stats': None, 'peers': None, 'iex_news': None, 'financials': None, 'earnings': None, 'dividends': None, 'company': None, 'exp_date': None, 'publish_pricing_update': None, 'date': ae_utils.utc_now_str(), 'updated': None, 'version': ae_consts.DATASET_COLLECTION_VERSION } res = {'status': ae_consts.NOT_RUN, 'err': None, 'rec': rec} try: ticker = work_dict.get('ticker', ticker) ticker_id = work_dict.get('ticker_id', ae_consts.TICKER_ID) s3_bucket = work_dict.get('s3_bucket', ae_consts.S3_BUCKET) s3_key = work_dict.get('s3_key', ae_consts.S3_KEY) redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY) exp_date = work_dict.get('exp_date', None) cur_date = datetime.datetime.utcnow() cur_strike = work_dict.get('strike', None) contract_type = str(work_dict.get('contract', 'C')).upper() label = work_dict.get('label', label) iex_datasets = work_dict.get('iex_datasets', iex_consts.DEFAULT_FETCH_DATASETS) td_datasets = work_dict.get('td_datasets', td_consts.DEFAULT_FETCH_DATASETS_TD) fetch_mode = work_dict.get('fetch_mode', ae_consts.FETCH_MODE_ALL) # control flags to deal with feed issues: get_iex_data = True get_td_data = True if (fetch_mode == ae_consts.FETCH_MODE_ALL or str(fetch_mode).lower() == 'all'): get_iex_data = True get_td_data = True elif (fetch_mode == ae_consts.FETCH_MODE_YHO or str(fetch_mode).lower() == 'yahoo'): get_iex_data = False get_td_data = False elif (fetch_mode == ae_consts.FETCH_MODE_IEX or str(fetch_mode).lower() == 'iex'): get_iex_data = True get_td_data = False elif (fetch_mode == ae_consts.FETCH_MODE_TD or str(fetch_mode).lower() == 'td'): get_iex_data = False get_td_data = True else: log.debug('{} - unsupported fetch_mode={} value'.format( label, fetch_mode)) """ as of Thursday, Jan. 3, 2019: https://developer.yahoo.com/yql/ Important EOL Notice: As of Thursday, Jan. 3, 2019 the YQL service at query.yahooapis.com will be retired """ get_yahoo_data = False if not exp_date: exp_date = opt_dates.option_expiration(date=exp_date) else: exp_date = datetime.datetime.strptime(exp_date, '%Y-%m-%d') rec['updated'] = cur_date.strftime('%Y-%m-%d %H:%M:%S') log.info('{} getting pricing for ticker={} ' 'cur_date={} exp_date={} ' 'yahoo={} iex={}'.format(label, ticker, cur_date, exp_date, get_yahoo_data, get_iex_data)) yahoo_rec = { 'ticker': ticker, 'pricing': None, 'options': None, 'calls': None, 'puts': None, 'news': None, 'exp_date': None, 'publish_pricing_update': None, 'date': None, 'updated': None } # disabled on 2019-01-03 if get_yahoo_data: log.info('{} yahoo ticker={}'.format(label, ticker)) yahoo_res = yahoo_data.get_data_from_yahoo(work_dict=work_dict) if yahoo_res['status'] == ae_consts.SUCCESS: yahoo_rec = yahoo_res['rec'] log.info('{} yahoo ticker={} ' 'status={} err={}'.format( label, ticker, ae_consts.get_status(status=yahoo_res['status']), yahoo_res['err'])) rec['pricing'] = yahoo_rec.get('pricing', '{}') rec['news'] = yahoo_rec.get('news', '{}') rec['options'] = yahoo_rec.get('options', '{}') rec['calls'] = rec['options'].get('calls', ae_consts.EMPTY_DF_STR) rec['puts'] = rec['options'].get('puts', ae_consts.EMPTY_DF_STR) else: log.error('{} failed YAHOO ticker={} ' 'status={} err={}'.format( label, ticker, ae_consts.get_status(status=yahoo_res['status']), yahoo_res['err'])) # end of get from yahoo if get_iex_data: num_iex_ds = len(iex_datasets) log.debug('{} iex datasets={}'.format(label, num_iex_ds)) for idx, ft_type in enumerate(iex_datasets): dataset_field = iex_consts.get_ft_str(ft_type=ft_type) log.info('{} iex={}/{} field={} ticker={}'.format( label, idx, num_iex_ds, dataset_field, ticker)) iex_label = '{}-{}'.format(label, dataset_field) iex_req = copy.deepcopy(work_dict) iex_req['label'] = iex_label iex_req['ft_type'] = ft_type iex_req['field'] = dataset_field iex_req['ticker'] = ticker iex_res = iex_data.get_data_from_iex(work_dict=iex_req) if iex_res['status'] == ae_consts.SUCCESS: iex_rec = iex_res['rec'] log.info( '{} iex ticker={} field={} ' 'status={} err={}'.format( label, ticker, dataset_field, ae_consts.get_status(status=iex_res['status']), iex_res['err'])) if dataset_field == 'news': rec['iex_news'] = iex_rec['data'] else: rec[dataset_field] = iex_rec['data'] else: log.debug( '{} failed IEX ticker={} field={} ' 'status={} err={}'.format( label, ticker, dataset_field, ae_consts.get_status(status=iex_res['status']), iex_res['err'])) # end of if/else succcess # end idx, ft_type in enumerate(iex_datasets): # end of if get_iex_data if get_td_data: num_td_ds = len(td_datasets) log.debug('{} td datasets={}'.format(label, num_td_ds)) for idx, ft_type in enumerate(td_datasets): dataset_field = td_consts.get_ft_str_td(ft_type=ft_type) log.info('{} td={}/{} field={} ticker={}'.format( label, idx, num_td_ds, dataset_field, ticker)) td_label = '{}-{}'.format(label, dataset_field) td_req = copy.deepcopy(work_dict) td_req['label'] = td_label td_req['ft_type'] = ft_type td_req['field'] = dataset_field td_req['ticker'] = ticker td_res = td_data.get_data_from_td(work_dict=td_req) if td_res['status'] == ae_consts.SUCCESS: td_rec = td_res['rec'] log.info('{} td ticker={} field={} ' 'status={} err={}'.format( label, ticker, dataset_field, ae_consts.get_status(status=td_res['status']), td_res['err'])) if dataset_field == 'tdcalls': rec['tdcalls'] = td_rec['data'] if dataset_field == 'tdputs': rec['tdputs'] = td_rec['data'] else: rec[dataset_field] = td_rec['data'] else: log.critical( '{} failed TD ticker={} field={} ' 'status={} err={}'.format( label, ticker, dataset_field, ae_consts.get_status(status=td_res['status']), td_res['err'])) # end of if/else succcess # end idx, ft_type in enumerate(td_datasets): # end of if get_td_data update_req = {'data': rec} update_req['ticker'] = ticker update_req['ticker_id'] = ticker_id update_req['strike'] = cur_strike update_req['contract'] = contract_type update_req['s3_enabled'] = work_dict.get('s3_enabled', ae_consts.ENABLED_S3_UPLOAD) update_req['redis_enabled'] = work_dict.get( 'redis_enabled', ae_consts.ENABLED_REDIS_PUBLISH) update_req['s3_bucket'] = s3_bucket update_req['s3_key'] = s3_key update_req['s3_access_key'] = work_dict.get('s3_access_key', ae_consts.S3_ACCESS_KEY) update_req['s3_secret_key'] = work_dict.get('s3_secret_key', ae_consts.S3_SECRET_KEY) update_req['s3_region_name'] = work_dict.get('s3_region_name', ae_consts.S3_REGION_NAME) update_req['s3_address'] = work_dict.get('s3_address', ae_consts.S3_ADDRESS) update_req['s3_secure'] = work_dict.get('s3_secure', ae_consts.S3_SECURE) update_req['redis_key'] = redis_key update_req['redis_address'] = work_dict.get('redis_address', ae_consts.REDIS_ADDRESS) update_req['redis_password'] = work_dict.get('redis_password', ae_consts.REDIS_PASSWORD) update_req['redis_db'] = int( work_dict.get('redis_db', ae_consts.REDIS_DB)) update_req['redis_expire'] = work_dict.get('redis_expire', ae_consts.REDIS_EXPIRE) update_req['updated'] = rec['updated'] update_req['label'] = label update_req['celery_disabled'] = True update_status = ae_consts.NOT_SET try: update_res = publisher.run_publish_pricing_update( work_dict=update_req) update_status = update_res.get('status', ae_consts.NOT_SET) if ae_consts.ev('DEBUG_RESULTS', '0') == '1': log.info('{} update_res status={} data={}'.format( label, ae_consts.get_status(status=update_status), ae_consts.ppj(update_res))) else: log.info('{} run_publish_pricing_update status={}'.format( label, ae_consts.get_status(status=update_status))) # end of if/else rec['publish_pricing_update'] = update_res res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) except Exception as f: err = ('{} publisher.run_publish_pricing_update failed ' 'with ex={}'.format(label, f)) log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=rec) # end of trying to publish results to connected services except Exception as e: res = build_result.build_result(status=ae_consts.ERR, err=('failed - get_new_pricing_data ' 'dict={} with ex={}').format( work_dict, e), rec=rec) log.error('{} - {}'.format(label, res['err'])) # end of try/ex if ae_consts.ev('DATASET_COLLECTION_SLACK_ALERTS', '0') == '1': env_name = 'DEV' if ae_consts.ev('PROD_SLACK_ALERTS', '1') == '1': env_name = 'PROD' done_msg = ('Dataset collected ticker=*{}* on env=*{}* ' 'redis_key={} s3_key={} iex={} yahoo={}'.format( ticker, env_name, redis_key, s3_key, get_iex_data, get_yahoo_data)) log.debug('{} sending slack msg={}'.format(label, done_msg)) if res['status'] == ae_consts.SUCCESS: slack_utils.post_success(msg=done_msg, block=False, jupyter=True) else: slack_utils.post_failure(msg=done_msg, block=False, jupyter=True) # end of if/else success # end of publishing to slack log.info('task - get_new_pricing_data done - ' '{} - status={}'.format(label, ae_consts.get_status(res['status']))) return get_task_results.get_task_results(work_dict=work_dict, result=res)