def extract_quote_dataset(work_dict, scrub_mode='sort-by-date'):
"""extract_quote_dataset
Fetch the IEX quote data for a ticker and
return it as a pandas Dataframe
:param work_dict: dictionary of args
:param scrub_mode: type of scrubbing handler to run
"""
label = work_dict.get('label', 'extract')
df_type = iex_consts.DATAFEED_QUOTE
df_str = iex_consts.get_datafeed_str(df_type=df_type)
req = copy.deepcopy(work_dict)
if 'redis_key' not in work_dict:
# see if it's get dataset dictionary
if 'quote' in work_dict:
req['redis_key'] = req['quote']
req['s3_key'] = req['quote']
# end of support for the get dataset dictionary
log.debug(f'{label} - {df_str} - start')
return extract_utils.perform_extract(df_type=df_type,
df_str=df_str,
work_dict=req,
scrub_mode=scrub_mode)
def extract_minute_dataset(work_dict, scrub_mode='sort-by-date'):
"""extract_minute_dataset
Fetch the IEX minute intraday data for a ticker and
return it as a pandas Dataframe
:param work_dict: dictionary of args
:param scrub_mode: type of scrubbing handler to run
"""
label = work_dict.get('label', 'extract')
df_type = DATAFEED_MINUTE
df_str = get_datafeed_str(df_type=df_type)
req = copy.deepcopy(work_dict)
if 'redis_key' not in work_dict:
# see if it's get dataset dictionary
if 'minute' in work_dict:
req['redis_key'] = req['minute']
req['s3_key'] = req['minute']
# end of support for the get dataset dictionary
log.debug('{} - {} - start'.format(label, df_str))
return extract_utils.perform_extract(df_type=df_type,
df_str=df_str,
work_dict=req,
scrub_mode=scrub_mode)
def extract_company_dataset(
work_dict,
scrub_mode='NO_SORT'):
"""extract_company_dataset
Fetch the IEX company data for a ticker and
return it as a pandas Dataframe
:param work_dict: dictionary of args
:param scrub_mode: type of scrubbing handler to run
"""
label = work_dict.get('label', 'extract')
df_type = iex_consts.DATAFEED_COMPANY
df_str = iex_consts.get_datafeed_str(df_type=df_type)
req = copy.deepcopy(work_dict)
if 'redis_key' not in work_dict:
# see if it's get dataset dictionary
if 'company' in work_dict:
req['redis_key'] = req['company']
req['s3_key'] = req['company']
# end of support for the get dataset dictionary
log.debug(
'{} - {} - start'.format(
label,
df_str))
return extract_utils.perform_extract(
df_type=df_type,
df_str=df_str,
work_dict=req,
scrub_mode=scrub_mode)
def extract_dataset(key,
ticker=None,
date=None,
work_dict=None,
scrub_mode='NO_SORT',
verbose=False):
"""extract_dataset
Extract the IEX key data for a ticker from Redis and
return it as a tuple (status, ``pandas.Dataframe``)
:param key: IEX dataset key
:param ticker: string ticker to extract
:param date: optional - string date to extract
formatted ``YYYY-MM-DD``
:param work_dict: dictionary of args
:param scrub_mode: type of scrubbing handler to run
:param verbose: optional - boolean for turning on logging
"""
if not key or key not in keys:
log.error(f'unsupported extract key={key} in keys={keys}')
return None
label = key
df_type = keys[key]
df_str = iex_consts.get_datafeed_str(df_type=df_type)
latest_close_date = ae_utils.get_last_close_str()
use_date = date
if work_dict:
if not ticker:
ticker = work_dict.get('ticker', None)
if not work_dict:
work_dict = api_requests.get_ds_dict(ticker=ticker)
req = copy.deepcopy(work_dict)
if not use_date:
use_date = latest_close_date
redis_key = f'{ticker}_{use_date}_{key}'
req['redis_key'] = redis_key
req['s3_key'] = redis_key
if verbose:
log.info(f'{label} - {df_str} - '
f'date={date} '
f'redis_key={req["redis_key"]} '
f's3_key={req["s3_key"]} '
f'{ae_consts.ppj(req)}')
return extract_utils.perform_extract(df_type=df_type,
df_str=df_str,
work_dict=req,
scrub_mode=scrub_mode,
verbose=verbose)
