def extract_quote_dataset(work_dict, scrub_mode='sort-by-date'): """extract_quote_dataset Fetch the IEX quote data for a ticker and return it as a pandas Dataframe :param work_dict: dictionary of args :param scrub_mode: type of scrubbing handler to run """ label = work_dict.get('label', 'extract') df_type = iex_consts.DATAFEED_QUOTE df_str = iex_consts.get_datafeed_str(df_type=df_type) req = copy.deepcopy(work_dict) if 'redis_key' not in work_dict: # see if it's get dataset dictionary if 'quote' in work_dict: req['redis_key'] = req['quote'] req['s3_key'] = req['quote'] # end of support for the get dataset dictionary log.debug(f'{label} - {df_str} - start') return extract_utils.perform_extract(df_type=df_type, df_str=df_str, work_dict=req, scrub_mode=scrub_mode)
def extract_minute_dataset(work_dict, scrub_mode='sort-by-date'): """extract_minute_dataset Fetch the IEX minute intraday data for a ticker and return it as a pandas Dataframe :param work_dict: dictionary of args :param scrub_mode: type of scrubbing handler to run """ label = work_dict.get('label', 'extract') df_type = DATAFEED_MINUTE df_str = get_datafeed_str(df_type=df_type) req = copy.deepcopy(work_dict) if 'redis_key' not in work_dict: # see if it's get dataset dictionary if 'minute' in work_dict: req['redis_key'] = req['minute'] req['s3_key'] = req['minute'] # end of support for the get dataset dictionary log.debug('{} - {} - start'.format(label, df_str)) return extract_utils.perform_extract(df_type=df_type, df_str=df_str, work_dict=req, scrub_mode=scrub_mode)
def extract_company_dataset( work_dict, scrub_mode='NO_SORT'): """extract_company_dataset Fetch the IEX company data for a ticker and return it as a pandas Dataframe :param work_dict: dictionary of args :param scrub_mode: type of scrubbing handler to run """ label = work_dict.get('label', 'extract') df_type = iex_consts.DATAFEED_COMPANY df_str = iex_consts.get_datafeed_str(df_type=df_type) req = copy.deepcopy(work_dict) if 'redis_key' not in work_dict: # see if it's get dataset dictionary if 'company' in work_dict: req['redis_key'] = req['company'] req['s3_key'] = req['company'] # end of support for the get dataset dictionary log.debug( '{} - {} - start'.format( label, df_str)) return extract_utils.perform_extract( df_type=df_type, df_str=df_str, work_dict=req, scrub_mode=scrub_mode)
def extract_dataset(key, ticker=None, date=None, work_dict=None, scrub_mode='NO_SORT', verbose=False): """extract_dataset Extract the IEX key data for a ticker from Redis and return it as a tuple (status, ``pandas.Dataframe``) :param key: IEX dataset key :param ticker: string ticker to extract :param date: optional - string date to extract formatted ``YYYY-MM-DD`` :param work_dict: dictionary of args :param scrub_mode: type of scrubbing handler to run :param verbose: optional - boolean for turning on logging """ if not key or key not in keys: log.error(f'unsupported extract key={key} in keys={keys}') return None label = key df_type = keys[key] df_str = iex_consts.get_datafeed_str(df_type=df_type) latest_close_date = ae_utils.get_last_close_str() use_date = date if work_dict: if not ticker: ticker = work_dict.get('ticker', None) if not work_dict: work_dict = api_requests.get_ds_dict(ticker=ticker) req = copy.deepcopy(work_dict) if not use_date: use_date = latest_close_date redis_key = f'{ticker}_{use_date}_{key}' req['redis_key'] = redis_key req['s3_key'] = redis_key if verbose: log.info(f'{label} - {df_str} - ' f'date={date} ' f'redis_key={req["redis_key"]} ' f's3_key={req["s3_key"]} ' f'{ae_consts.ppj(req)}') return extract_utils.perform_extract(df_type=df_type, df_str=df_str, work_dict=req, scrub_mode=scrub_mode, verbose=verbose)