def extract_quote_dataset(work_dict, scrub_mode='sort-by-date'):
"""extract_quote_dataset
Fetch the IEX quote data for a ticker and
return it as a pandas Dataframe
:param work_dict: dictionary of args
:param scrub_mode: type of scrubbing handler to run
"""
label = work_dict.get('label', 'extract')
df_type = iex_consts.DATAFEED_QUOTE
df_str = iex_consts.get_datafeed_str(df_type=df_type)
req = copy.deepcopy(work_dict)
if 'redis_key' not in work_dict:
# see if it's get dataset dictionary
if 'quote' in work_dict:
req['redis_key'] = req['quote']
req['s3_key'] = req['quote']
# end of support for the get dataset dictionary
log.debug(f'{label} - {df_str} - start')
return extract_utils.perform_extract(df_type=df_type,
df_str=df_str,
work_dict=req,
scrub_mode=scrub_mode)
def extract_minute_dataset(work_dict, scrub_mode='sort-by-date'):
"""extract_minute_dataset
Fetch the IEX minute intraday data for a ticker and
return it as a pandas Dataframe
:param work_dict: dictionary of args
:param scrub_mode: type of scrubbing handler to run
"""
label = work_dict.get('label', 'extract')
df_type = DATAFEED_MINUTE
df_str = get_datafeed_str(df_type=df_type)
req = copy.deepcopy(work_dict)
if 'redis_key' not in work_dict:
# see if it's get dataset dictionary
if 'minute' in work_dict:
req['redis_key'] = req['minute']
req['s3_key'] = req['minute']
# end of support for the get dataset dictionary
log.debug('{} - {} - start'.format(label, df_str))
return extract_utils.perform_extract(df_type=df_type,
df_str=df_str,
work_dict=req,
scrub_mode=scrub_mode)
def extract_company_dataset(
work_dict,
scrub_mode='NO_SORT'):
"""extract_company_dataset
Fetch the IEX company data for a ticker and
return it as a pandas Dataframe
:param work_dict: dictionary of args
:param scrub_mode: type of scrubbing handler to run
"""
label = work_dict.get('label', 'extract')
df_type = iex_consts.DATAFEED_COMPANY
df_str = iex_consts.get_datafeed_str(df_type=df_type)
req = copy.deepcopy(work_dict)
if 'redis_key' not in work_dict:
# see if it's get dataset dictionary
if 'company' in work_dict:
req['redis_key'] = req['company']
req['s3_key'] = req['company']
# end of support for the get dataset dictionary
log.debug(
'{} - {} - start'.format(
label,
df_str))
return extract_utils.perform_extract(
df_type=df_type,
df_str=df_str,
work_dict=req,
scrub_mode=scrub_mode)
def extract_dataset(key,
ticker=None,
date=None,
work_dict=None,
scrub_mode='NO_SORT',
verbose=False):
"""extract_dataset
Extract the IEX key data for a ticker from Redis and
return it as a tuple (status, ``pandas.Dataframe``)
:param key: IEX dataset key
:param ticker: string ticker to extract
:param date: optional - string date to extract
formatted ``YYYY-MM-DD``
:param work_dict: dictionary of args
:param scrub_mode: type of scrubbing handler to run
:param verbose: optional - boolean for turning on logging
"""
if not key or key not in keys:
log.error(f'unsupported extract key={key} in keys={keys}')
return None
label = key
df_type = keys[key]
df_str = iex_consts.get_datafeed_str(df_type=df_type)
latest_close_date = ae_utils.get_last_close_str()
use_date = date
if work_dict:
if not ticker:
ticker = work_dict.get('ticker', None)
if not work_dict:
work_dict = api_requests.get_ds_dict(ticker=ticker)
req = copy.deepcopy(work_dict)
if not use_date:
use_date = latest_close_date
redis_key = f'{ticker}_{use_date}_{key}'
req['redis_key'] = redis_key
req['s3_key'] = redis_key
if verbose:
log.info(f'{label} - {df_str} - '
f'date={date} '
f'redis_key={req["redis_key"]} '
f's3_key={req["s3_key"]} '
f'{ae_consts.ppj(req)}')
return extract_utils.perform_extract(df_type=df_type,
df_str=df_str,
work_dict=req,
scrub_mode=scrub_mode,
verbose=verbose)
def debug_msg(label, datafeed_type, msg_format, date_str, df):
"""debug_msg
Debug helper for debugging scrubbing handlers
:param label: log label
:param datafeed_type: fetch type
:param msg_format: message to include
:param date_str: date string
:param df: ``pandas DataFrame`` or ``None``
"""
msg = msg_format.format('_', date_str)
dft_msg = ''
if (datafeed_type == yahoo_consts.DATAFEED_PRICING_YAHOO
or datafeed_type == yahoo_consts.DATAFEED_OPTIONS_YAHOO
or datafeed_type == yahoo_consts.DATAFEED_NEWS_YAHOO):
dft_msg = yahoo_consts.get_datafeed_str_yahoo(df_type=datafeed_type)
elif (datafeed_type == td_consts.DATAFEED_TD_CALLS
or datafeed_type == td_consts.DATAFEED_TD_PUTS):
dft_msg = td_consts.get_datafeed_str_td(df_type=datafeed_type)
else:
dft_msg = iex_consts.get_datafeed_str(df_type=datafeed_type)
if ae_consts.ev('DEBUG_FETCH', '0') == '1':
if 'START' in msg:
log.info('{} - {} -------------------------'
'------------------------------------'.format(
label, dft_msg))
msg = msg_format.format(df, date_str),
if hasattr(df, 'empty'):
log.info('{} - {} - {} found df={} '
'columns={}'.format(label, dft_msg, msg, df,
df.columns.values))
else:
log.info('{} - {} - {} not df={}'.format(label, dft_msg, msg, df))
if 'END' in msg:
log.info('{} - {} -------------------------'
'------------------------------------'.format(
label, dft_msg))
else:
log.info('{} - {} - {}'.format(label, dft_msg, msg))
