def get_pricing_on_date(ticker, date_str=None, label=None):
"""get_pricing_on_date
Get the latest pricing data from the
cached IEX data in redis. Use this to
keep costs down!
.. code-block:: python
import analysis_engine.iex.get_pricing_on_date as iex_cache
print(iex_cache.get_pricing_on_date('SPY'))
print(iex_cache.get_pricing_on_date(
ticker='SPY',
date_str='2019-02-07'))
:param ticker: ticker string
:param date_str: optional - string date
to pull data from redis. if ``None`` use
today's date. format is
``ae_consts.COMMON_TICK_DATE_FORMAT``
:param label: log label from tracking
"""
ret_dict = {
'status': ae_consts.NOT_SET,
'pricing_type': None,
'high': None,
'low': None,
'open': None,
'close': None,
'volume': None,
'date': None,
'minute': None,
'average': None,
'changeOverTime': None,
'label': None,
'marketAverage': None,
'marketChangeOverTime': None,
'marketClose': None,
'marketHigh': None,
'marketLow': None,
'marketNotional': None,
'marketNumberOfTrades': None,
'marketOpen': None,
'marketVolume': None,
'notional': None,
'numberOfTrades': None
}
use_date_str = None
if date_str:
use_date_str = (f'{ticker}_{date_str}')
all_extract_reqs = api_requests.get_ds_dict(ticker=ticker,
base_key=use_date_str,
label=label)
minute_key = all_extract_reqs['minute']
daily_key = all_extract_reqs['daily']
base_ex_req = {
'ticker': ticker,
's3_bucket': 'pricing',
's3_key': minute_key,
'redis_key': minute_key,
's3_enabled': True,
's3_access_key': ae_consts.S3_ACCESS_KEY,
's3_secret_key': ae_consts.S3_SECRET_KEY,
's3_region_name': ae_consts.S3_REGION_NAME,
's3_address': ae_consts.S3_ADDRESS,
's3_secure': ae_consts.S3_SECURE,
'redis_address': ae_consts.REDIS_ADDRESS,
'redis_password': ae_consts.REDIS_PASSWORD,
'redis_db': ae_consts.REDIS_DB,
'redis_expire': ae_consts.REDIS_EXPIRE,
'redis_enabled': True,
'fetch_mode': 'td',
'analysis_type': None,
'iex_datasets': [],
'debug': False,
'label': label,
'celery_disabled': True
}
log.debug(f'{ticker} - minute={minute_key} daily={daily_key}')
reqs = []
minute_ex_req = copy.deepcopy(base_ex_req)
minute_ex_req['ex_type'] = iex_consts.FETCH_MINUTE
minute_ex_req['iex_datasets'] = [iex_consts.FETCH_MINUTE]
reqs.append(minute_ex_req)
daily_ex_req = copy.deepcopy(base_ex_req)
daily_ex_req['ex_type'] = iex_consts.FETCH_DAILY
daily_ex_req['s3_key'] = daily_key
daily_ex_req['redis_key'] = daily_key
daily_ex_req['iex_datasets'] = [iex_consts.FETCH_DAILY]
reqs.append(daily_ex_req)
try:
for ex_req in reqs:
iex_status = ae_consts.FAILED
iex_df = None
if ex_req['ex_type'] == iex_consts.FETCH_MINUTE:
iex_status, iex_df = \
iex_extract_utils.extract_minute_dataset(
work_dict=ex_req)
else:
iex_status, iex_df = \
iex_extract_utils.extract_daily_dataset(
work_dict=ex_req)
# end of extracting
if ae_consts.is_df(df=iex_df):
if 'date' in iex_df:
iex_df.sort_values(by=['date'], ascending=True)
ret_dict = json.loads(iex_df.iloc[-1].to_json())
if 'date' in ret_dict:
try:
ret_dict['date'] = ae_utils.epoch_to_dt(
epoch=int(ret_dict['date'] / 1000),
use_utc=False,
convert_to_est=False).strftime(
ae_consts.COMMON_TICK_DATE_FORMAT)
except Exception as f:
log.critical(
f'failed converting {ret_dict} date to str '
f'with ex={f}')
if ex_req['ex_type'] == iex_consts.FETCH_MINUTE:
ret_dict['pricing_type'] = 'minute'
ret_dict['minute'] = ret_dict.get('date', None)
else:
ret_dict['pricing_type'] = 'daily'
ret_dict['status'] = iex_status
return ret_dict
# if a valid df then return it
except Exception as e:
log.critical(f'failed to get {ticker} iex minute data with ex={e}')
ret_dict['status'] = ae_consts.ERR
# end of try/ex to get latest pricing
return ret_dict
def fetch_calls(ticker=None,
work_dict=None,
scrub_mode='sort-by-date',
verbose=False):
"""fetch_calls
Fetch Tradier option calls for a ticker and
return a tuple: (status, ``pandas.DataFrame``)
.. code-block:: python
import analysis_engine.td.fetch_api as td_fetch
# Please set the TD_TOKEN environment variable to your token
calls_status, calls_df = td_fetch.fetch_calls(
ticker='SPY')
print(f'Fetched SPY Option Calls from Tradier status={calls_status}:')
print(calls_df)
:param ticker: string ticker to fetch
:param work_dict: dictionary of args
used by the automation
:param scrub_mode: optional - string type of
scrubbing handler to run
:param verbose: optional - bool for debugging
"""
label = 'fetch_calls'
datafeed_type = td_consts.DATAFEED_TD_CALLS
exp_date = None
latest_pricing = {}
latest_close = None
if work_dict:
ticker = work_dict.get('ticker', ticker)
label = work_dict.get('label', label)
exp_date = work_dict.get('exp_date', exp_date)
latest_pricing = work_dict.get('latest_pricing', latest_pricing)
latest_close = latest_pricing.get('close', latest_close)
log.debug(f'{label} - calls - close={latest_close} ' f'ticker={ticker}')
exp_date = opt_dates.option_expiration().strftime(
ae_consts.COMMON_DATE_FORMAT)
use_url = td_consts.TD_URLS['options'].format(ticker, exp_date)
headers = td_consts.get_auth_headers()
session = requests.Session()
session.headers = headers
res = url_helper.url_helper(sess=session).get(use_url)
if res.status_code != requests.codes.OK:
if res.status_code in [401, 403]:
log.critical('Please check the TD_TOKEN is correct '
f'received {res.status_code} during '
'fetch for: calls')
else:
log.info(f'failed to get call with response={res} '
f'code={res.status_code} '
f'text={res.text}')
return ae_consts.EMPTY, pd.DataFrame([{}])
records = json.loads(res.text)
org_records = records.get('options', {}).get('option', [])
if len(org_records) == 0:
log.info('failed to get call records ' 'text={}'.format(res.text))
return ae_consts.EMPTY, pd.DataFrame([{}])
options_list = []
# assumes UTC conversion will work with the system clock
created_minute = (
datetime.datetime.utcnow() -
datetime.timedelta(hours=5)).strftime('%Y-%m-%d %H:%M:00')
last_close_date = ae_utils.get_last_close_str(fmt='%Y-%m-%d %H:%M:00')
# hit bug where dates were None
if not last_close_date:
last_close_date = created_minute
for node in org_records:
node['date'] = last_close_date
node['created'] = created_minute
node['ticker'] = ticker
if (node['option_type'] == 'call'
and node['expiration_type'] == 'standard'
and float(node['bid']) > 0.01):
node['opt_type'] = int(ae_consts.OPTION_CALL)
node['exp_date'] = node['expiration_date']
new_node = {}
for col in td_consts.TD_OPTION_COLUMNS:
if col in node:
if col in td_consts.TD_EPOCH_COLUMNS:
# trade_date can be None
if node[col] == 0:
new_node[col] = None
else:
new_node[col] = ae_utils.epoch_to_dt(
epoch=node[col] / 1000,
use_utc=False,
convert_to_est=True).strftime(
ae_consts.COMMON_TICK_DATE_FORMAT)
"""
Debug epoch ms converter:
"""
"""
print('-----------')
print(col)
print(node[col])
print(new_node[col])
print('===========')
"""
# if/else valid date
else:
new_node[col] = node[col]
# if date column to convert
# if column is in the row
# convert all columns
options_list.append(new_node)
# end of records
full_df = pd.DataFrame(options_list).sort_values(by=['strike'],
ascending=True)
num_chains = len(full_df.index)
df = None
if latest_close:
df_filter = ((full_df['strike'] >=
(latest_close - ae_consts.OPTIONS_LOWER_STRIKE)) &
(full_df['strike'] <=
(latest_close + ae_consts.OPTIONS_UPPER_STRIKE)))
df = full_df[df_filter].copy().sort_values(
by=['date', 'strike']).reset_index()
else:
mid_chain_idx = int(num_chains / 2)
low_idx = int(mid_chain_idx - ae_consts.MAX_OPTIONS_LOWER_STRIKE)
high_idx = int(mid_chain_idx + ae_consts.MAX_OPTIONS_UPPER_STRIKE)
if low_idx < 0:
low_idx = 0
if high_idx > num_chains:
high_idx = num_chains
df = full_df[low_idx:high_idx].copy().sort_values(
by=['date', 'strike']).reset_index()
scrubbed_df = scrub_utils.ingress_scrub_dataset(
label=label,
scrub_mode=scrub_mode,
datafeed_type=datafeed_type,
msg_format='df={} date_str={}',
ds_id=ticker,
date_str=exp_date,
df=df)
return ae_consts.SUCCESS, scrubbed_df
def fetch_calls(work_dict, scrub_mode='sort-by-date'):
"""fetch_calls
Fetch the Tradier daily data for a ticker and
return it as a ``pandas.DataFrame``.
:param work_dict: dictionary of args
:param scrub_mode: type of scrubbing handler to run
"""
datafeed_type = td_consts.DATAFEED_TD_CALLS
ticker = work_dict.get('ticker', None)
label = work_dict.get('label', None)
exp_date = work_dict.get('exp_date', None)
log.debug(f'{label} - call - scrub_mode={scrub_mode} '
f'args={work_dict} ticker={ticker}')
exp_date = opt_dates.option_expiration().strftime(
ae_consts.COMMON_DATE_FORMAT)
use_url = td_consts.TD_URLS['options'].format(ticker, exp_date)
headers = td_consts.get_auth_headers()
session = requests.Session()
session.headers = headers
res = url_helper.url_helper(sess=session).get(use_url)
if res.status_code != requests.codes.OK:
if res.status_code in [401, 403]:
log.critical('Please check the TD_TOKEN is correct '
f'received {res.status_code} during '
'fetch for: calls')
else:
log.info(f'failed to get call with response={res} '
f'code={res.status_code} '
f'text={res.text}')
return ae_consts.EMPTY, pd.DataFrame([{}])
records = json.loads(res.text)
org_records = records.get('options', {}).get('option', [])
if len(org_records) == 0:
log.info('failed to get call records ' f'text={res.text}')
return ae_consts.EMPTY, pd.DataFrame([{}])
options_list = []
# assumes UTC conversion will work with the system clock
created_minute = (
datetime.datetime.utcnow() -
datetime.timedelta(hours=5)).strftime('%Y-%m-%d %H:%M:00')
last_close_date = ae_utils.get_last_close_str(fmt='%Y-%m-%d %H:%M:00')
# hit bug where dates were None
if not last_close_date:
last_close_date = created_minute
for node in org_records:
node['date'] = last_close_date
node['created'] = created_minute
node['ticker'] = ticker
if (node['option_type'] == 'call'
and node['expiration_type'] == 'standard'):
node['opt_type'] = int(ae_consts.OPTION_CALL)
node['exp_date'] = node['expiration_date']
new_node = {}
for col in td_consts.TD_OPTION_COLUMNS:
if col in node:
if col in td_consts.TD_EPOCH_COLUMNS:
# trade_date can be None
if node[col] == 0:
new_node[col] = None
else:
new_node[col] = ae_utils.epoch_to_dt(
epoch=node[col] / 1000,
use_utc=False,
convert_to_est=True).strftime(
ae_consts.COMMON_TICK_DATE_FORMAT)
"""
Debug epoch ms converter:
"""
"""
print('-----------')
print(col)
print(node[col])
print(new_node[col])
print('===========')
"""
# if/else valid date
else:
new_node[col] = node[col]
# if date column to convert
# if column is in the row
# convert all columns
options_list.append(new_node)
# end of records
full_df = pd.DataFrame(options_list).sort_values(by=['strike'],
ascending=True)
num_chains = len(full_df.index)
mid_chain_idx = int(num_chains / 2)
low_idx = int(mid_chain_idx - 20)
high_idx = int(mid_chain_idx + 30)
if low_idx < 0:
low_idx = 0
if high_idx > num_chains:
high_idx = num_chains
df = full_df[low_idx:high_idx].copy().sort_values(
by=['date', 'strike']).reset_index()
scrubbed_df = scrub_utils.ingress_scrub_dataset(
label=label,
scrub_mode=scrub_mode,
datafeed_type=datafeed_type,
msg_format='df={} date_str={}',
ds_id=ticker,
date_str=exp_date,
df=df)
return ae_consts.SUCCESS, scrubbed_df