def extract_quote_dataset(work_dict, scrub_mode='sort-by-date'): """extract_quote_dataset Fetch the IEX quote data for a ticker and return it as a pandas Dataframe :param work_dict: dictionary of args :param scrub_mode: type of scrubbing handler to run """ label = work_dict.get('label', 'extract') df_type = iex_consts.DATAFEED_QUOTE df_str = iex_consts.get_datafeed_str(df_type=df_type) req = copy.deepcopy(work_dict) if 'redis_key' not in work_dict: # see if it's get dataset dictionary if 'quote' in work_dict: req['redis_key'] = req['quote'] req['s3_key'] = req['quote'] # end of support for the get dataset dictionary log.debug(f'{label} - {df_str} - start') return extract_utils.perform_extract(df_type=df_type, df_str=df_str, work_dict=req, scrub_mode=scrub_mode)
def extract_minute_dataset(work_dict, scrub_mode='sort-by-date'): """extract_minute_dataset Fetch the IEX minute intraday data for a ticker and return it as a pandas Dataframe :param work_dict: dictionary of args :param scrub_mode: type of scrubbing handler to run """ label = work_dict.get('label', 'extract') df_type = DATAFEED_MINUTE df_str = get_datafeed_str(df_type=df_type) req = copy.deepcopy(work_dict) if 'redis_key' not in work_dict: # see if it's get dataset dictionary if 'minute' in work_dict: req['redis_key'] = req['minute'] req['s3_key'] = req['minute'] # end of support for the get dataset dictionary log.debug('{} - {} - start'.format(label, df_str)) return extract_utils.perform_extract(df_type=df_type, df_str=df_str, work_dict=req, scrub_mode=scrub_mode)
def extract_company_dataset( work_dict, scrub_mode='NO_SORT'): """extract_company_dataset Fetch the IEX company data for a ticker and return it as a pandas Dataframe :param work_dict: dictionary of args :param scrub_mode: type of scrubbing handler to run """ label = work_dict.get('label', 'extract') df_type = iex_consts.DATAFEED_COMPANY df_str = iex_consts.get_datafeed_str(df_type=df_type) req = copy.deepcopy(work_dict) if 'redis_key' not in work_dict: # see if it's get dataset dictionary if 'company' in work_dict: req['redis_key'] = req['company'] req['s3_key'] = req['company'] # end of support for the get dataset dictionary log.debug( '{} - {} - start'.format( label, df_str)) return extract_utils.perform_extract( df_type=df_type, df_str=df_str, work_dict=req, scrub_mode=scrub_mode)
def extract_dataset(key, ticker=None, date=None, work_dict=None, scrub_mode='NO_SORT', verbose=False): """extract_dataset Extract the IEX key data for a ticker from Redis and return it as a tuple (status, ``pandas.Dataframe``) :param key: IEX dataset key :param ticker: string ticker to extract :param date: optional - string date to extract formatted ``YYYY-MM-DD`` :param work_dict: dictionary of args :param scrub_mode: type of scrubbing handler to run :param verbose: optional - boolean for turning on logging """ if not key or key not in keys: log.error(f'unsupported extract key={key} in keys={keys}') return None label = key df_type = keys[key] df_str = iex_consts.get_datafeed_str(df_type=df_type) latest_close_date = ae_utils.get_last_close_str() use_date = date if work_dict: if not ticker: ticker = work_dict.get('ticker', None) if not work_dict: work_dict = api_requests.get_ds_dict(ticker=ticker) req = copy.deepcopy(work_dict) if not use_date: use_date = latest_close_date redis_key = f'{ticker}_{use_date}_{key}' req['redis_key'] = redis_key req['s3_key'] = redis_key if verbose: log.info(f'{label} - {df_str} - ' f'date={date} ' f'redis_key={req["redis_key"]} ' f's3_key={req["s3_key"]} ' f'{ae_consts.ppj(req)}') return extract_utils.perform_extract(df_type=df_type, df_str=df_str, work_dict=req, scrub_mode=scrub_mode, verbose=verbose)
def debug_msg(label, datafeed_type, msg_format, date_str, df): """debug_msg Debug helper for debugging scrubbing handlers :param label: log label :param datafeed_type: fetch type :param msg_format: message to include :param date_str: date string :param df: ``pandas DataFrame`` or ``None`` """ msg = msg_format.format('_', date_str) dft_msg = '' if (datafeed_type == yahoo_consts.DATAFEED_PRICING_YAHOO or datafeed_type == yahoo_consts.DATAFEED_OPTIONS_YAHOO or datafeed_type == yahoo_consts.DATAFEED_NEWS_YAHOO): dft_msg = yahoo_consts.get_datafeed_str_yahoo(df_type=datafeed_type) elif (datafeed_type == td_consts.DATAFEED_TD_CALLS or datafeed_type == td_consts.DATAFEED_TD_PUTS): dft_msg = td_consts.get_datafeed_str_td(df_type=datafeed_type) else: dft_msg = iex_consts.get_datafeed_str(df_type=datafeed_type) if ae_consts.ev('DEBUG_FETCH', '0') == '1': if 'START' in msg: log.info('{} - {} -------------------------' '------------------------------------'.format( label, dft_msg)) msg = msg_format.format(df, date_str), if hasattr(df, 'empty'): log.info('{} - {} - {} found df={} ' 'columns={}'.format(label, dft_msg, msg, df, df.columns.values)) else: log.info('{} - {} - {} not df={}'.format(label, dft_msg, msg, df)) if 'END' in msg: log.info('{} - {} -------------------------' '------------------------------------'.format( label, dft_msg)) else: log.info('{} - {} - {}'.format(label, dft_msg, msg))