def fetch_minute(work_dict, scrub_mode='sort-by-date'): """fetch_minute Fetch the IEX minute intraday data for a ticker and return it as a ``pandas.DataFrame``. https://iexcloud.io/docs/api/#historical-prices :param work_dict: dictionary of args :param scrub_mode: type of scrubbing handler to run """ ticker = work_dict.get('ticker', None) label = work_dict.get('label', None) use_date = work_dict.get('use_date', None) from_historical_date = None last_close_to_use = None dates = [] if 'from_historical_date' in work_dict: from_historical_date = work_dict['from_historical_date'] last_close_to_use = work_dict['last_close_to_use'] dates = ae_utils.get_days_between_dates( from_historical_date=work_dict['from_historical_date'], last_close_to_use=last_close_to_use) use_url = (f'/stock/{ticker}/chart/1d') if use_date: use_url = (f'/stock/{ticker}/chart/date/{use_date}') log.debug(f'{label} - minute - url={use_url} ' f'args={work_dict} ticker={ticker} ' f'fhdate={from_historical_date} ' f'last_close={last_close_to_use} ' f'dates={dates}') resp_json = iex_helpers.get_from_iex(url=use_url, token=iex_consts.IEX_TOKEN) df = pd.DataFrame(resp_json) iex_helpers.convert_datetime_columns(df=df) if not use_date: use_date = df['date'].iloc[-1].strftime('%Y-%m-%d') new_minutes = dataset_utils.build_dates_from_df_col(src_col='minute', use_date_str=use_date, df=df) df['minute'] = pd.to_datetime(new_minutes, format=iex_consts.IEX_TICK_FORMAT) df.set_index(['date', 'minute']) return df
def fetch_dividends(ticker=None, timeframe='3m', work_dict=None, scrub_mode='sort-by-date'): """fetch_dividends Fetch the IEX dividends data for a ticker and return it as a ``pandas.DataFrame``. https://iexcloud.io/docs/api/#dividends :param work_dict: dictionary of args :param ticker: optional - ticker string used for mostly testing :param timeframe: optional - string for setting dividend lookback period used for mostly testing :param scrub_mode: type of scrubbing handler to run """ label = None if work_dict: if not ticker: ticker = work_dict.get('ticker', None) label = work_dict.get('label', None) if not timeframe: timeframe = work_dict.get('timeframe', '3m') use_url = (f'/stock/{ticker}/dividends/{timeframe}') log.debug(f'{label} - divs - url={use_url} ' f'args={work_dict} ticker={ticker}') resp_json = iex_helpers.get_from_iex(url=use_url, token=iex_consts.IEX_TOKEN) df = pd.DataFrame(resp_json) iex_helpers.convert_datetime_columns(df=df) cols_to_drop = [] remove_these = None if len(cols_to_drop) > 0: for c in df: if c in cols_to_drop: if not remove_these: remove_these = [] remove_these.append(c) if remove_these: df = df.drop(columns=remove_these) return df
def fetch_daily(ticker=None, work_dict=None, scrub_mode='sort-by-date'): """fetch_daily Fetch the IEX daily data for a ticker and return it as a ``pandas.DataFrame``. https://iexcloud.io/docs/api/#historical-prices :param work_dict: dictionary of args :param ticker: optional - ticker string used for mostly testing :param scrub_mode: type of scrubbing handler to run """ label = None if work_dict: if not ticker: ticker = work_dict.get('ticker', None) label = work_dict.get('label', None) use_url = (f'/stock/{ticker}/chart/1m') log.debug(f'{label} - daily - url={use_url} ' f'args={work_dict} ' f'ticker={ticker}') resp_json = iex_helpers.get_from_iex(url=use_url, token=iex_consts.IEX_TOKEN) df = pd.DataFrame(resp_json) iex_helpers.convert_datetime_columns(df=df) cols_to_drop = ['uHigh', 'uLow', 'uOpen', 'uClose', 'uVolume'] remove_these = None for c in df: if c in cols_to_drop: if not remove_these: remove_these = [] remove_these.append(c) if remove_these: df = df.drop(columns=remove_these) df.set_index(['date']).sort_values(by=['date'], ascending=True) return df
def fetch_company(ticker=None, work_dict=None, scrub_mode='NO_SORT'): """fetch_company Fetch the IEX company data for a ticker and return it as a ``pandas.DataFrame``. https://iexcloud.io/docs/api/#company :param work_dict: dictionary of args :param ticker: optional - ticker string used for mostly testing :param scrub_mode: type of scrubbing handler to run """ label = None if work_dict: if not ticker: ticker = work_dict.get('ticker', None) label = work_dict.get('label', None) use_url = (f'/stock/{ticker}/company') log.debug(f'{label} - comp - url={use_url} ' f'args={work_dict} ticker={ticker}') resp_json = iex_helpers.get_from_iex(url=use_url, token=iex_consts.IEX_TOKEN) df = pd.DataFrame([resp_json]) iex_helpers.convert_datetime_columns(df=df) cols_to_drop = [] remove_these = None if len(cols_to_drop) > 0: for c in df: if c in cols_to_drop: if not remove_these: remove_these = [] remove_these.append(c) if remove_these: df = df.drop(columns=remove_these) return df
def fetch_earnings(work_dict, scrub_mode='sort-by-date'): """fetch_earnings Fetch the IEX earnings data for a ticker and return it as a ``pandas.DataFrame``. https://iexcloud.io/docs/api/#earnings :param work_dict: dictionary of args :param scrub_mode: type of scrubbing handler to run """ ticker = work_dict.get('ticker', None) label = work_dict.get('label', None) use_url = (f'/stock/{ticker}/earnings') log.debug(f'{label} - earns - url={use_url} ' f'args={work_dict} ticker={ticker}') resp_json = iex_helpers.get_from_iex(url=use_url, token=iex_consts.IEX_TOKEN) df = pd.DataFrame(resp_json.get('earnings', [])) iex_helpers.convert_datetime_columns(df=df) cols_to_drop = [] remove_these = None if len(cols_to_drop) > 0: for c in df: if c in cols_to_drop: if not remove_these: remove_these = [] remove_these.append(c) if remove_these: df = df.drop(columns=remove_these) return df
def fetch_company( ticker=None, work_dict=None, scrub_mode='NO_SORT', verbose=False): """fetch_company Fetch the IEX company data for a ticker and return it as a ``pandas.DataFrame``. https://iexcloud.io/docs/api/#company .. code-block:: python import analysis_engine.iex.fetch_api as iex_fetch comp_df = iex_fetch.fetch_company(ticker='SPY') print(comp_df) :param ticker: string ticker to fetch :param work_dict: dictionary of args used by the automation :param scrub_mode: optional - string type of scrubbing handler to run :param verbose: optional - bool to log for debugging """ label = None if work_dict: if not ticker: ticker = work_dict.get('ticker', None) label = work_dict.get('label', None) use_url = ( f'/stock/{ticker}/company') if verbose: log.info( f'{label} - comp - url={use_url} ' f'req={work_dict} ticker={ticker}') resp_json = iex_helpers.get_from_iex( url=use_url, token=iex_consts.IEX_TOKEN, verbose=verbose) df = pd.DataFrame([resp_json]) if verbose: log.info( f'{label} - comp - url={use_url} ' f'ticker={ticker} response ' f'df={df}') if len(df.index) == 0: return df iex_helpers.convert_datetime_columns( df=df) cols_to_drop = [] remove_these = None if len(cols_to_drop) > 0: for c in df: if c in cols_to_drop: if not remove_these: remove_these = [] remove_these.append(c) if remove_these: df = df.drop(columns=remove_these) return df
def fetch_dividends( ticker=None, timeframe='3m', work_dict=None, scrub_mode='sort-by-date', verbose=False): """fetch_dividends Fetch the IEX dividends data for a ticker and return it as a ``pandas.DataFrame``. https://iexcloud.io/docs/api/#dividends .. code-block:: python import analysis_engine.iex.fetch_api as iex_fetch div_df = iex_fetch.fetch_dividends(ticker='SPY') print(div_df) :param ticker: string ticker to fetch :param timeframe: optional - string for setting dividend lookback period used for (default is ``3m`` for three months) :param work_dict: dictionary of args used by the automation :param scrub_mode: optional - string type of scrubbing handler to run :param verbose: optional - bool to log for debugging """ label = None if work_dict: if not ticker: ticker = work_dict.get('ticker', None) label = work_dict.get('label', None) if not timeframe: timeframe = work_dict.get('timeframe', '3m') use_url = ( f'/stock/{ticker}/dividends/{timeframe}') if verbose: log.info( f'{label} - divs - url={use_url} ' f'req={work_dict} ticker={ticker}') resp_json = iex_helpers.get_from_iex( url=use_url, token=iex_consts.IEX_TOKEN, verbose=verbose) df = pd.DataFrame(resp_json) if verbose: log.info( f'{label} - divs - url={use_url} ' f'ticker={ticker} response ' f'df={df.tail(5)}') if len(df.index) == 0: return df iex_helpers.convert_datetime_columns( df=df) cols_to_drop = [] remove_these = None if len(cols_to_drop) > 0: for c in df: if c in cols_to_drop: if not remove_these: remove_these = [] remove_these.append(c) if remove_these: df = df.drop(columns=remove_these) return df
def fetch_minute( ticker=None, backfill_date=None, work_dict=None, scrub_mode='sort-by-date', verbose=False): """fetch_minute Fetch the IEX minute intraday data for a ticker and return it as a ``pandas.DataFrame``. https://iexcloud.io/docs/api/#historical-prices .. code-block:: python import analysis_engine.iex.fetch_api as iex_fetch minute_df = iex_fetch.fetch_minute(ticker='SPY') print(minute_df) :param ticker: string ticker to fetch :param backfill_date: optional - date string formatted ``YYYY-MM-DD`` for filling in missing minute data :param work_dict: dictionary of args used by the automation :param scrub_mode: optional - string type of scrubbing handler to run :param verbose: optional - bool to log for debugging """ label = None use_date = backfill_date from_historical_date = None last_close_to_use = None dates = [] if work_dict: label = work_dict.get('label', None) use_date = work_dict.get('use_date', None) if not ticker: ticker = work_dict.get('ticker', None) if not backfill_date: use_date = work_dict.get('backfill_date', None) if 'from_historical_date' in work_dict: from_historical_date = work_dict['from_historical_date'] if 'last_close_to_use' in work_dict: last_close_to_use = work_dict['last_close_to_use'] if from_historical_date and last_close_to_use: dates = ae_utils.get_days_between_dates( from_historical_date=work_dict['from_historical_date'], last_close_to_use=last_close_to_use) use_url = ( f'/stock/{ticker}/chart/1d') if use_date: # no - chars in the date use_url = ( f'/stock/{ticker}/chart/date/{use_date.replace("-", "")}') if verbose: log.info( f'{label} - minute - url={use_url} ' f'req={work_dict} ticker={ticker} ' f'fhdate={from_historical_date} ' f'last_close={last_close_to_use} ' f'dates={dates}') resp_json = iex_helpers.get_from_iex( url=use_url, token=iex_consts.IEX_TOKEN, verbose=verbose) df = pd.DataFrame(resp_json) if verbose: log.info( f'{label} - minute - url={use_url} ' f'ticker={ticker} response ' f'df={df.tail(5)}') if 'date' not in df: log.error( f'unable to download IEX Cloud minute ' f'data for {ticker} on backfill_date={use_date} ' f'df: {df} from url: {use_url} with response: {resp_json}') return df if len(df.index) == 0: return df iex_helpers.convert_datetime_columns( df=df) if not use_date: use_date = df['date'].iloc[-1].strftime('%Y-%m-%d') new_minutes = dataset_utils.build_dates_from_df_col( src_col='minute', use_date_str=use_date, df=df) df['date'] = pd.to_datetime( new_minutes, format=ae_consts.COMMON_TICK_DATE_FORMAT) # make sure dates are set as strings in the cache df['date'] = df['date'].dt.strftime( ae_consts.COMMON_TICK_DATE_FORMAT) df.set_index( [ 'date' ]) return df
def fetch_daily( ticker=None, work_dict=None, scrub_mode='sort-by-date', verbose=False): """fetch_daily Fetch the IEX daily data for a ticker and return it as a ``pandas.DataFrame``. https://iexcloud.io/docs/api/#historical-prices .. code-block:: python import analysis_engine.iex.fetch_api as iex_fetch daily_df = iex_fetch.fetch_daily(ticker='SPY') print(daily_df) :param ticker: string ticker to fetch :param work_dict: dictionary of args used by the automation :param scrub_mode: optional - string type of scrubbing handler to run :param verbose: optional - bool to log for debugging """ label = None if work_dict: if not ticker: ticker = work_dict.get('ticker', None) label = work_dict.get('label', None) use_url = ( f'/stock/{ticker}/chart/1m') if verbose: log.info( f'{label} - daily - url={use_url} ' f'req={work_dict} ' f'ticker={ticker}') resp_json = iex_helpers.get_from_iex( url=use_url, token=iex_consts.IEX_TOKEN, verbose=verbose) df = pd.DataFrame(resp_json) if verbose: log.info( f'{label} - daily - url={use_url} ' f'ticker={ticker} response ' f'df={df.tail(5)}') if len(df.index) == 0: return df iex_helpers.convert_datetime_columns( df=df) cols_to_drop = [ ] remove_these = None for c in df: if c in cols_to_drop: if not remove_these: remove_these = [] remove_these.append(c) if remove_these: df = df.drop(columns=remove_these) df.set_index( [ 'date' ]).sort_values(by=['date'], ascending=True) return df
def fetch_minute(ticker=None, backfill_date=None, work_dict=None, scrub_mode='sort-by-date'): """fetch_minute Fetch the IEX minute intraday data for a ticker and return it as a ``pandas.DataFrame``. https://iexcloud.io/docs/api/#historical-prices :param work_dict: dictionary of args :param ticker: optional - ticker string used for mostly testing :param backfill_date: optional - date string formatted ``YYYY-MM-DD`` for filling in missing minute data :param scrub_mode: type of scrubbing handler to run """ label = None use_date = backfill_date from_historical_date = None last_close_to_use = None dates = [] if work_dict: label = work_dict.get('label', None) use_date = work_dict.get('use_date', None) if not ticker: ticker = work_dict.get('ticker', None) if not backfill_date: use_date = work_dict.get('backfill_date', None) if 'from_historical_date' in work_dict: from_historical_date = work_dict['from_historical_date'] if 'last_close_to_use' in work_dict: last_close_to_use = work_dict['last_close_to_use'] if from_historical_date and last_close_to_use: dates = ae_utils.get_days_between_dates( from_historical_date=work_dict['from_historical_date'], last_close_to_use=last_close_to_use) use_url = (f'/stock/{ticker}/chart/1d') if use_date: # no - chars in the date use_url = (f'/stock/{ticker}/chart/date/{use_date.replace("-", "")}') log.debug(f'{label} - minute - url={use_url} ' f'args={work_dict} ticker={ticker} ' f'fhdate={from_historical_date} ' f'last_close={last_close_to_use} ' f'dates={dates}') resp_json = iex_helpers.get_from_iex(url=use_url, token=iex_consts.IEX_TOKEN) df = pd.DataFrame(resp_json) if 'date' not in df: log.error(f'unable to download IEX Cloud minute ' f'data for {ticker} on backfill_date={use_date}') return df iex_helpers.convert_datetime_columns(df=df) if not use_date: use_date = df['date'].iloc[-1].strftime('%Y-%m-%d') new_minutes = dataset_utils.build_dates_from_df_col(src_col='minute', use_date_str=use_date, df=df) df['date'] = pd.to_datetime(new_minutes, format=ae_consts.COMMON_TICK_DATE_FORMAT) # make sure dates are set as strings in the cache df['date'] = df['date'].dt.strftime(ae_consts.COMMON_TICK_DATE_FORMAT) df.set_index(['date']) return df