def build_td_fetch_puts_request(label=None):
"""build_td_fetch_puts_request
Fetch tradier puts
:param label: log label to use
"""
ticker = ae_consts.TICKER
base_key = f'''{ticker}_tdputs_{datetime.datetime.utcnow().strftime(
'%Y_%m_%d_%H_%M_%S')}'''
s3_bucket_name = 'tdputs'
s3_key = base_key
redis_key = base_key
s3_enabled = True
redis_enabled = True
exp_date = opt_dates.option_expiration().strftime(
ae_consts.COMMON_DATE_FORMAT)
work = {
'ft_type': td_consts.FETCH_TD_PUTS,
'fd_type': td_consts.DATAFEED_TD_PUTS,
'ticker': ticker,
'exp_date': exp_date,
's3_bucket': s3_bucket_name,
's3_key': s3_key,
'redis_key': redis_key,
's3_enabled': s3_enabled,
'redis_enabled': redis_enabled
}
if label:
work['label'] = label
return work
def test_integration_account_credentials(self):
"""test_integration_account_credentials"""
if ae_consts.ev('INT_TESTS', '0') == '0':
return
headers = td_consts.get_auth_headers()
session = requests.Session()
session.headers = headers
self.exp_date = opt_dates.option_expiration().strftime(
ae_consts.COMMON_DATE_FORMAT)
use_url = td_consts.TD_URLS['options'].format(self.ticker,
self.exp_date)
response = url_helper.url_helper(sess=session).get(use_url)
self.assertEqual(response.status_code, 200)
self.assertTrue(len(json.loads(response.text)) > 0)
'analysis_engine.work_tasks.publish_from_s3_to_redis,'
'analysis_engine.work_tasks.publish_pricing_update,'
'analysis_engine.work_tasks.task_screener_analysis,'
'analysis_engine.work_tasks.publish_ticker_aggregate_from_s3'))
INCLUDE_TASKS = WORKER_TASKS.split(',')
CELERY_DISABLED = ev('CELERY_DISABLED', '0') == '1'
########################################
#
# Custom Variables
#
########################################
TICKER = ev('TICKER', 'SPY')
TICKER_ID = int(ev('TICKER_ID', '1'))
DEFAULT_TICKERS = ev('DEFAULT_TICKERS', 'SPY,AMZN,TSLA,NFLX').split(',')
NEXT_EXP = opt_dates.option_expiration()
NEXT_EXP_STR = NEXT_EXP.strftime('%Y-%m-%d')
DAILY_S3_BUCKET_NAME = ev('DAILY_S3_BUCKET_NAME', 'daily')
MINUTE_S3_BUCKET_NAME = ev('MINUTE_S3_BUCKET_NAME', 'minute')
QUOTE_S3_BUCKET_NAME = ev('QUOTE_S3_BUCKET_NAME', 'quote')
STATS_S3_BUCKET_NAME = ev('STATS_S3_BUCKET_NAME', 'stats')
PEERS_S3_BUCKET_NAME = ev('PEERS_S3_BUCKET_NAME', 'peers')
NEWS_S3_BUCKET_NAME = ev('NEWS_S3_BUCKET_NAME', 'news')
FINANCIALS_S3_BUCKET_NAME = ev('FINANCIALS_S3_BUCKET_NAME', 'financials')
EARNINGS_S3_BUCKET_NAME = ev('EARNINGS_S3_BUCKET_NAME', 'earnings')
DIVIDENDS_S3_BUCKET_NAME = ev('DIVIDENDS_S3_BUCKET_NAME', 'dividends')
COMPANY_S3_BUCKET_NAME = ev('COMPANY_S3_BUCKET_NAME', 'company')
FETCH_MODE = ev('FETCH_MODE', 'all')
PREPARE_S3_BUCKET_NAME = ev('PREPARE_S3_BUCKET_NAME', 'prepared')
ANALYZE_S3_BUCKET_NAME = ev('ANALYZE_S3_BUCKET_NAME', 'analyzed')
SCREENER_S3_BUCKET_NAME = ev('SCREENER_S3_BUCKET_NAME', 'screener-data')
def get_new_pricing_data(self, work_dict):
"""get_new_pricing_data
Get Ticker information on:
- prices - turn off with ``work_dict.get_pricing = False``
- news - turn off with ``work_dict.get_news = False``
- options - turn off with ``work_dict.get_options = False``
:param work_dict: dictionary for key/values
"""
label = 'get_new_pricing_data'
log.debug(f'task - {label} - start ' f'work_dict={work_dict}')
num_success = 0
ticker = ae_consts.TICKER
ticker_id = ae_consts.TICKER_ID
rec = {
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'daily': None,
'minute': None,
'quote': None,
'stats': None,
'peers': None,
'iex_news': None,
'financials': None,
'earnings': None,
'dividends': None,
'company': None,
'exp_date': None,
'publish_pricing_update': None,
'num_success': num_success,
'date': ae_utils.utc_now_str(),
'updated': None,
'version': ae_consts.DATASET_COLLECTION_VERSION
}
res = {'status': ae_consts.NOT_RUN, 'err': None, 'rec': rec}
try:
ticker = work_dict.get('ticker', ticker)
ticker_id = work_dict.get('ticker_id', ae_consts.TICKER_ID)
s3_bucket = work_dict.get('s3_bucket', ae_consts.S3_BUCKET)
s3_key = work_dict.get('s3_key', ae_consts.S3_KEY)
redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY)
exp_date = work_dict.get('exp_date', None)
cur_date = ae_utils.last_close()
cur_strike = work_dict.get('strike', None)
contract_type = str(work_dict.get('contract', 'C')).upper()
label = work_dict.get('label', label)
iex_datasets = work_dict.get('iex_datasets',
iex_consts.DEFAULT_FETCH_DATASETS)
td_datasets = work_dict.get('td_datasets',
td_consts.DEFAULT_FETCH_DATASETS_TD)
fetch_mode = work_dict.get('fetch_mode', ae_consts.FETCH_MODE_ALL)
iex_token = work_dict.get('iex_token', iex_consts.IEX_TOKEN)
td_token = work_dict.get('td_token', td_consts.TD_TOKEN)
str_fetch_mode = str(fetch_mode).lower()
# control flags to deal with feed issues:
get_iex_data = True
get_td_data = True
if (fetch_mode == ae_consts.FETCH_MODE_ALL
or str_fetch_mode == 'initial'):
get_iex_data = True
get_td_data = True
iex_datasets = ae_consts.IEX_INITIAL_DATASETS
elif (fetch_mode == ae_consts.FETCH_MODE_ALL
or str_fetch_mode == 'all'):
get_iex_data = True
get_td_data = True
iex_datasets = ae_consts.IEX_DATASETS_DEFAULT
elif (fetch_mode == ae_consts.FETCH_MODE_YHO
or str_fetch_mode == 'yahoo'):
get_iex_data = False
get_td_data = False
elif (fetch_mode == ae_consts.FETCH_MODE_IEX
or str_fetch_mode == 'iex-all'):
get_iex_data = True
get_td_data = False
iex_datasets = ae_consts.IEX_DATASETS_DEFAULT
elif (fetch_mode == ae_consts.FETCH_MODE_IEX
or str_fetch_mode == 'iex'):
get_iex_data = True
get_td_data = False
iex_datasets = ae_consts.IEX_INTRADAY_DATASETS
elif (fetch_mode == ae_consts.FETCH_MODE_INTRADAY
or str_fetch_mode == 'intra'):
get_iex_data = True
get_td_data = True
iex_datasets = ae_consts.IEX_INTRADAY_DATASETS
elif (fetch_mode == ae_consts.FETCH_MODE_DAILY
or str_fetch_mode == 'daily'):
get_iex_data = True
get_td_data = False
iex_datasets = ae_consts.IEX_DAILY_DATASETS
elif (fetch_mode == ae_consts.FETCH_MODE_WEEKLY
or str_fetch_mode == 'weekly'):
get_iex_data = True
get_td_data = False
iex_datasets = ae_consts.IEX_WEEKLY_DATASETS
elif (fetch_mode == ae_consts.FETCH_MODE_TD or str_fetch_mode == 'td'):
get_iex_data = False
get_td_data = True
else:
get_iex_data = False
get_td_data = False
fetch_arr = str_fetch_mode.split(',')
found_fetch = False
iex_datasets = []
for fetch_name in fetch_arr:
if fetch_name not in iex_datasets:
if fetch_name == 'iex_min':
iex_datasets.append('minute')
elif fetch_name == 'iex_day':
iex_datasets.append('daily')
elif fetch_name == 'iex_quote':
iex_datasets.append('quote')
elif fetch_name == 'iex_stats':
iex_datasets.append('stats')
elif fetch_name == 'iex_peers':
iex_datasets.append('peers')
elif fetch_name == 'iex_news':
iex_datasets.append('news')
elif fetch_name == 'iex_fin':
iex_datasets.append('financials')
elif fetch_name == 'iex_earn':
iex_datasets.append('earnings')
elif fetch_name == 'iex_div':
iex_datasets.append('dividends')
elif fetch_name == 'iex_comp':
iex_datasets.append('company')
elif fetch_name == 'td':
get_td_data = True
else:
log.warn('unsupported IEX dataset ' f'{fetch_name}')
found_fetch = (len(iex_datasets) != 0)
if not found_fetch:
log.error(f'{label} - unsupported '
f'fetch_mode={fetch_mode} value')
else:
get_iex_data = True
log.debug(f'{label} - '
f'fetching={len(iex_datasets)} '
f'{iex_datasets} '
f'fetch_mode={fetch_mode}')
# end of screening custom fetch_mode settings
num_tokens = 0
if get_iex_data:
if not iex_token:
log.warn(f'{label} - '
'please set a valid IEX Cloud Account token ('
'https://iexcloud.io/cloud-login/#/register'
') to fetch data from IEX Cloud. It must be '
'set as an environment variable like: '
'export IEX_TOKEN=<token>')
get_iex_data = False
else:
num_tokens += 1
# sanity check - disable IEX fetch if the token is not set
if get_td_data:
missing_td_token = [
'MISSING_TD_TOKEN', 'SETYOURTDTOKEN', 'SETYOURTRADIERTOKENHERE'
]
if td_token in missing_td_token:
log.warn(f'{label} - '
'please set a valid Tradier Account token ('
'https://developer.tradier.com/user/sign_up'
') to fetch pricing data from Tradier. It must be '
'set as an environment variable like: '
'export TD_TOKEN=<token>')
get_td_data = False
else:
num_tokens += 1
# sanity check - disable Tradier fetch if the token is not set
"""
as of Thursday, Jan. 3, 2019:
https://developer.yahoo.com/yql/
Important EOL Notice: As of Thursday, Jan. 3, 2019
the YQL service at query.yahooapis.com will be retired
"""
get_yahoo_data = False
if (not get_iex_data and not get_td_data and not get_yahoo_data):
err = None
if num_tokens == 0:
res['status'] = ae_consts.MISSING_TOKEN
err = (f'Please set a valid IEX_TOKEN or TD_TOKEN '
f'environment variable')
else:
err = (f'Please set at least one supported datafeed from '
f'either: '
f'IEX Cloud (fetch -t TICKER -g iex) or '
f'Tradier (fetch -t TICKER -g td) '
f'for '
f'ticker={ticker} '
f'cur_date={cur_date} '
f'IEX enabled={get_iex_data} '
f'TD enabled={get_td_data} '
f'YHO enabled={get_yahoo_data}')
res['status'] = ae_consts.ERR
res['err'] = err
return get_task_results.get_task_results(work_dict=work_dict,
result=res)
# end of checking that there is at least 1 feed on
if not exp_date:
exp_date = opt_dates.option_expiration(date=exp_date)
else:
exp_date = datetime.datetime.strptime(exp_date, '%Y-%m-%d')
rec['updated'] = cur_date.strftime('%Y-%m-%d %H:%M:%S')
log.debug(f'{label} getting pricing for ticker={ticker} '
f'cur_date={cur_date} exp_date={exp_date} '
f'IEX={get_iex_data} '
f'TD={get_td_data} '
f'YHO={get_yahoo_data}')
yahoo_rec = {
'ticker': ticker,
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'exp_date': None,
'publish_pricing_update': None,
'date': None,
'updated': None
}
# disabled on 2019-01-03
if get_yahoo_data:
log.debug(f'{label} YHO ticker={ticker}')
yahoo_res = yahoo_data.get_data_from_yahoo(work_dict=work_dict)
status_str = ae_consts.get_status(status=yahoo_res['status'])
if yahoo_res['status'] == ae_consts.SUCCESS:
yahoo_rec = yahoo_res['rec']
msg = (f'{label} YHO ticker={ticker} '
f'status={status_str} err={yahoo_res["err"]}')
if ae_consts.ev('SHOW_SUCCESS', '0') == '1':
log.info(msg)
else:
log.debug(msg)
rec['pricing'] = yahoo_rec.get('pricing', '{}')
rec['news'] = yahoo_rec.get('news', '{}')
rec['options'] = yahoo_rec.get('options', '{}')
rec['calls'] = rec['options'].get('calls',
ae_consts.EMPTY_DF_STR)
rec['puts'] = rec['options'].get('puts',
ae_consts.EMPTY_DF_STR)
num_success += 1
else:
log.error(f'{label} failed YHO ticker={ticker} '
f'status={status_str} err={yahoo_res["err"]}')
# end of get from yahoo
if get_iex_data:
num_iex_ds = len(iex_datasets)
log.debug(f'{label} IEX datasets={num_iex_ds}')
for idx, ft_type in enumerate(iex_datasets):
dataset_field = iex_consts.get_ft_str(ft_type=ft_type)
log.debug(f'{label} IEX={idx}/{num_iex_ds} '
f'field={dataset_field} ticker={ticker}')
iex_label = f'{label}-{dataset_field}'
iex_req = copy.deepcopy(work_dict)
iex_req['label'] = iex_label
iex_req['ft_type'] = ft_type
iex_req['field'] = dataset_field
iex_req['ticker'] = ticker
iex_res = iex_data.get_data_from_iex(work_dict=iex_req)
status_str = (ae_consts.get_status(status=iex_res['status']))
if iex_res['status'] == ae_consts.SUCCESS:
iex_rec = iex_res['rec']
msg = (f'{label} IEX ticker={ticker} '
f'field={dataset_field} '
f'status={status_str} '
f'err={iex_res["err"]}')
if ae_consts.ev('SHOW_SUCCESS', '0') == '1':
log.info(msg)
else:
log.debug(msg)
if dataset_field == 'news':
rec['iex_news'] = iex_rec['data']
else:
rec[dataset_field] = iex_rec['data']
num_success += 1
else:
log.debug(f'{label} failed IEX ticker={ticker} '
f'field={dataset_field} '
f'status={status_str} err={iex_res["err"]}')
# end of if/else succcess
# end idx, ft_type in enumerate(iex_datasets):
# end of if get_iex_data
if get_td_data:
num_td_ds = len(td_datasets)
log.debug(f'{label} TD datasets={num_td_ds}')
for idx, ft_type in enumerate(td_datasets):
dataset_field = td_consts.get_ft_str_td(ft_type=ft_type)
log.debug(f'{label} TD={idx}/{num_td_ds} '
f'field={dataset_field} ticker={ticker}')
td_label = (f'{label}-{dataset_field}')
td_req = copy.deepcopy(work_dict)
td_req['label'] = td_label
td_req['ft_type'] = ft_type
td_req['field'] = dataset_field
td_req['ticker'] = ticker
td_res = td_data.get_data_from_td(work_dict=td_req)
status_str = (ae_consts.get_status(status=td_res['status']))
if td_res['status'] == ae_consts.SUCCESS:
td_rec = td_res['rec']
msg = (f'{label} TD ticker={ticker} '
f'field={dataset_field} '
f'status={status_str} '
f'err={td_res["err"]}')
if ae_consts.ev('SHOW_SUCCESS', '0') == '1':
log.info(msg)
else:
log.debug(msg)
if dataset_field == 'tdcalls':
rec['tdcalls'] = td_rec['data']
if dataset_field == 'tdputs':
rec['tdputs'] = td_rec['data']
else:
rec[dataset_field] = td_rec['data']
num_success += 1
else:
log.critical(f'{label} failed TD ticker={ticker} '
f'field={dataset_field} '
f'status={status_str} err={td_res["err"]}')
# end of if/else succcess
# end idx, ft_type in enumerate(td_datasets):
# end of if get_td_data
rec['num_success'] = num_success
update_req = {'data': rec}
update_req['ticker'] = ticker
update_req['ticker_id'] = ticker_id
update_req['strike'] = cur_strike
update_req['contract'] = contract_type
update_req['s3_enabled'] = work_dict.get('s3_enabled',
ae_consts.ENABLED_S3_UPLOAD)
update_req['redis_enabled'] = work_dict.get(
'redis_enabled', ae_consts.ENABLED_REDIS_PUBLISH)
update_req['s3_bucket'] = s3_bucket
update_req['s3_key'] = s3_key
update_req['s3_access_key'] = work_dict.get('s3_access_key',
ae_consts.S3_ACCESS_KEY)
update_req['s3_secret_key'] = work_dict.get('s3_secret_key',
ae_consts.S3_SECRET_KEY)
update_req['s3_region_name'] = work_dict.get('s3_region_name',
ae_consts.S3_REGION_NAME)
update_req['s3_address'] = work_dict.get('s3_address',
ae_consts.S3_ADDRESS)
update_req['s3_secure'] = work_dict.get('s3_secure',
ae_consts.S3_SECURE)
update_req['redis_key'] = redis_key
update_req['redis_address'] = work_dict.get('redis_address',
ae_consts.REDIS_ADDRESS)
update_req['redis_password'] = work_dict.get('redis_password',
ae_consts.REDIS_PASSWORD)
update_req['redis_db'] = int(
work_dict.get('redis_db', ae_consts.REDIS_DB))
update_req['redis_expire'] = work_dict.get('redis_expire',
ae_consts.REDIS_EXPIRE)
update_req['updated'] = rec['updated']
update_req['label'] = label
update_req['celery_disabled'] = True
update_status = ae_consts.NOT_SET
try:
update_res = publisher.run_publish_pricing_update(
work_dict=update_req)
update_status = update_res.get('status', ae_consts.NOT_SET)
status_str = ae_consts.get_status(status=update_status)
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
log.debug(f'{label} update_res '
f'status={status_str} '
f'data={ae_consts.ppj(update_res)}')
else:
log.debug(f'{label} run_publish_pricing_update '
f'status={status_str}')
# end of if/else
rec['publish_pricing_update'] = update_res
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as f:
err = (f'{label} publisher.run_publish_pricing_update failed '
f'with ex={f}')
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=rec)
# end of trying to publish results to connected services
except Exception as e:
res = build_result.build_result(status=ae_consts.ERR,
err=('failed - get_new_pricing_data '
f'dict={work_dict} with ex={e}'),
rec=rec)
log.error(f'{label} - {res["err"]}')
# end of try/ex
if ae_consts.ev('DATASET_COLLECTION_SLACK_ALERTS', '0') == '1':
env_name = 'DEV'
if ae_consts.ev('PROD_SLACK_ALERTS', '1') == '1':
env_name = 'PROD'
done_msg = (f'Dataset collected ticker=*{ticker}* on '
f'env=*{env_name}* '
f'redis_key={redis_key} s3_key={s3_key} '
f'IEX={get_iex_data} '
f'TD={get_td_data} '
f'YHO={get_yahoo_data}')
log.debug(f'{label} sending slack msg={done_msg}')
if res['status'] == ae_consts.SUCCESS:
slack_utils.post_success(msg=done_msg, block=False, jupyter=True)
else:
slack_utils.post_failure(msg=done_msg, block=False, jupyter=True)
# end of if/else success
# end of publishing to slack
log.debug('task - get_new_pricing_data done - '
f'{label} - status={ae_consts.get_status(res["status"])}')
return get_task_results.get_task_results(work_dict=work_dict, result=res)
def get_data_from_yahoo(work_dict):
"""get_data_from_yahoo
Get data from yahoo
:param work_dict: request dictionary
"""
label = 'get_data_from_yahoo'
log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict))
num_news_rec = 0
num_option_calls = 0
num_option_puts = 0
cur_high = -1
cur_low = -1
cur_open = -1
cur_close = -1
cur_volume = -1
rec = {
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'exp_date': None,
'publish_pricing_update': None,
'date': None,
'updated': None
}
res = {'status': NOT_RUN, 'err': None, 'rec': rec}
try:
ticker = work_dict.get('ticker', TICKER)
exp_date = work_dict.get('exp_date', None)
cur_strike = work_dict.get('strike', None)
contract_type = str(work_dict.get('contract', 'C')).upper()
get_pricing = work_dict.get('get_pricing', True)
get_news = work_dict.get('get_news', True)
get_options = work_dict.get('get_options', True)
orient = work_dict.get('orient', 'records')
label = work_dict.get('label', label)
ticker_results = pinance.Pinance(ticker)
num_news_rec = 0
use_date = exp_date
if not exp_date:
exp_date = opt_dates.option_expiration(date=exp_date)
use_date = exp_date.strftime('%Y-%m-%d')
"""
Debug control flags
Quickly turn specific fetches off:
get_news = False
get_pricing = False
get_options = False
"""
if get_pricing:
log.info('{} getting ticker={} pricing'.format(label, ticker))
ticker_results.get_quotes()
if ticker_results.quotes_data:
pricing_dict = ticker_results.quotes_data
cur_high = pricing_dict.get('regularMarketDayHigh', None)
cur_low = pricing_dict.get('regularMarketDayLow', None)
cur_open = pricing_dict.get('regularMarketOpen', None)
cur_close = pricing_dict.get('regularMarketPreviousClose',
None)
cur_volume = pricing_dict.get('regularMarketVolume', None)
pricing_dict['high'] = cur_high
pricing_dict['low'] = cur_low
pricing_dict['open'] = cur_open
pricing_dict['close'] = cur_close
pricing_dict['volume'] = cur_volume
pricing_dict['date'] = get_last_close_str()
if 'regularMarketTime' in pricing_dict:
pricing_dict['market_time'] = \
datetime.datetime.fromtimestamp(
pricing_dict['regularMarketTime']).strftime(
COMMON_TICK_DATE_FORMAT)
if 'postMarketTime' in pricing_dict:
pricing_dict['post_market_time'] = \
datetime.datetime.fromtimestamp(
pricing_dict['postMarketTime']).strftime(
COMMON_TICK_DATE_FORMAT)
log.info('{} ticker={} converting pricing to '
'df orient={}'.format(label, ticker, orient))
try:
rec['pricing'] = pricing_dict
except Exception as f:
rec['pricing'] = '{}'
log.info('{} ticker={} failed converting pricing '
'data={} to df ex={}'.format(
label, ticker, ppj(pricing_dict), f))
# try/ex
log.info('{} ticker={} done converting pricing to '
'df orient={}'.format(label, ticker, orient))
else:
log.error('{} ticker={} missing quotes_data={}'.format(
label, ticker, ticker_results.quotes_data))
# end of if ticker_results.quotes_data
log.info('{} ticker={} close={} vol={}'.format(
label, ticker, cur_close, cur_volume))
else:
log.info('{} skip - getting ticker={} pricing'.format(
label, ticker, get_pricing))
# if get_pricing
if get_news:
log.info('{} getting ticker={} news'.format(label, ticker))
ticker_results.get_news()
if ticker_results.news_data:
news_list = None
try:
news_list = ticker_results.news_data
log.info('{} ticker={} converting news to '
'df orient={}'.format(label, ticker, orient))
num_news_rec = len(news_list)
rec['news'] = news_list
except Exception as f:
rec['news'] = '{}'
log.info('{} ticker={} failed converting news '
'data={} to df ex={}'.format(
label, ticker, news_list, f))
# try/ex
log.info('{} ticker={} done converting news to '
'df orient={}'.format(label, ticker, orient))
else:
log.info('{} ticker={} Yahoo NO news={}'.format(
label, ticker, ticker_results.news_data))
# end of if ticker_results.news_data
else:
log.info('{} skip - getting ticker={} news'.format(label, ticker))
# end if get_news
if get_options:
get_all_strikes = True
if get_all_strikes:
cur_strike = None
else:
if cur_close:
cur_strike = int(cur_close)
if not cur_strike:
cur_strike = 287
log.info('{} ticker={} num_news={} get options close={} '
'exp_date={} contract={} strike={}'.format(
label, ticker, num_news_rec, cur_close, use_date,
contract_type, cur_strike))
options_dict = \
yahoo_get_pricing.get_options(
ticker=ticker,
exp_date_str=use_date,
contract_type=contract_type,
strike=cur_strike)
rec['options'] = '{}'
try:
log.info('{} ticker={} converting options to '
'df orient={}'.format(label, ticker, orient))
num_option_calls = options_dict.get('num_calls', None)
num_option_puts = options_dict.get('num_puts', None)
rec['options'] = {
'exp_date': options_dict.get('exp_date', None),
'calls': options_dict.get('calls', None),
'puts': options_dict.get('puts', None),
'num_calls': num_option_calls,
'num_puts': num_option_puts
}
rec['calls'] = rec['options'].get('calls', EMPTY_DF_STR)
rec['puts'] = rec['options'].get('puts', EMPTY_DF_STR)
except Exception as f:
rec['options'] = '{}'
log.info('{} ticker={} failed converting options '
'data={} to df ex={}'.format(label, ticker,
options_dict, f))
# try/ex
log.info('{} ticker={} done converting options to '
'df orient={} num_calls={} num_puts={}'.format(
label, ticker, orient, num_option_calls,
num_option_puts))
else:
log.info('{} skip - getting ticker={} options'.format(
label, ticker))
# end of if get_options
log.info('{} yahoo pricing for ticker={} close={} '
'num_calls={} num_puts={} news={}'.format(
label, ticker, cur_close, num_option_calls,
num_option_puts, num_news_rec))
fields_to_upload = ['pricing', 'options', 'calls', 'puts', 'news']
for field_name in fields_to_upload:
upload_and_cache_req = copy.deepcopy(work_dict)
upload_and_cache_req['celery_disabled'] = True
upload_and_cache_req['data'] = rec[field_name]
if not upload_and_cache_req['data']:
upload_and_cache_req['data'] = '{}'
if 'redis_key' in work_dict:
upload_and_cache_req['redis_key'] = '{}_{}'.format(
work_dict.get('redis_key',
'{}_{}'.format(ticker, field_name)),
field_name)
if 's3_key' in work_dict:
upload_and_cache_req['s3_key'] = '{}_{}'.format(
work_dict.get('s3_key', '{}_{}'.format(ticker,
field_name)),
field_name)
try:
update_res = publisher.run_publish_pricing_update(
work_dict=upload_and_cache_req)
update_status = update_res.get('status', NOT_SET)
log.info('{} publish update status={} data={}'.format(
label, get_status(status=update_status), update_res))
except Exception as f:
err = ('{} - failed to upload YAHOO data={} to '
'to s3_key={} and redis_key={}'.format(
label, upload_and_cache_req,
upload_and_cache_req['s3_key'],
upload_and_cache_req['redis_key']))
log.error(err)
# end of try/ex to upload and cache
if not rec[field_name]:
log.debug('{} - ticker={} no data from YAHOO for '
'field_name={}'.format(label, ticker, field_name))
# end of for all fields
res = build_result.build_result(status=SUCCESS, err=None, rec=rec)
except Exception as e:
res = build_result.build_result(status=ERR,
err=('failed - get_data_from_yahoo '
'dict={} with ex={}').format(
work_dict, e),
rec=rec)
log.error('{} - {}'.format(label, res['err']))
# end of try/ex
log.info('task - get_data_from_yahoo done - '
'{} - status={}'.format(label, get_status(res['status'])))
return res
def get_new_pricing_data(self, work_dict):
"""get_new_pricing_data
Get Ticker information on:
- prices - turn off with ``work_dict.get_pricing = False``
- news - turn off with ``work_dict.get_news = False``
- options - turn off with ``work_dict.get_options = False``
:param work_dict: dictionary for key/values
"""
label = 'get_new_pricing_data'
log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict))
ticker = ae_consts.TICKER
ticker_id = ae_consts.TICKER_ID
rec = {
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'daily': None,
'minute': None,
'quote': None,
'stats': None,
'peers': None,
'iex_news': None,
'financials': None,
'earnings': None,
'dividends': None,
'company': None,
'exp_date': None,
'publish_pricing_update': None,
'date': ae_utils.utc_now_str(),
'updated': None,
'version': ae_consts.DATASET_COLLECTION_VERSION
}
res = {'status': ae_consts.NOT_RUN, 'err': None, 'rec': rec}
try:
ticker = work_dict.get('ticker', ticker)
ticker_id = work_dict.get('ticker_id', ae_consts.TICKER_ID)
s3_bucket = work_dict.get('s3_bucket', ae_consts.S3_BUCKET)
s3_key = work_dict.get('s3_key', ae_consts.S3_KEY)
redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY)
exp_date = work_dict.get('exp_date', None)
cur_date = datetime.datetime.utcnow()
cur_strike = work_dict.get('strike', None)
contract_type = str(work_dict.get('contract', 'C')).upper()
label = work_dict.get('label', label)
iex_datasets = work_dict.get('iex_datasets',
iex_consts.DEFAULT_FETCH_DATASETS)
td_datasets = work_dict.get('td_datasets',
td_consts.DEFAULT_FETCH_DATASETS_TD)
fetch_mode = work_dict.get('fetch_mode', ae_consts.FETCH_MODE_ALL)
# control flags to deal with feed issues:
get_iex_data = True
get_td_data = True
if (fetch_mode == ae_consts.FETCH_MODE_ALL
or str(fetch_mode).lower() == 'all'):
get_iex_data = True
get_td_data = True
elif (fetch_mode == ae_consts.FETCH_MODE_YHO
or str(fetch_mode).lower() == 'yahoo'):
get_iex_data = False
get_td_data = False
elif (fetch_mode == ae_consts.FETCH_MODE_IEX
or str(fetch_mode).lower() == 'iex'):
get_iex_data = True
get_td_data = False
elif (fetch_mode == ae_consts.FETCH_MODE_TD
or str(fetch_mode).lower() == 'td'):
get_iex_data = False
get_td_data = True
else:
log.debug('{} - unsupported fetch_mode={} value'.format(
label, fetch_mode))
"""
as of Thursday, Jan. 3, 2019:
https://developer.yahoo.com/yql/
Important EOL Notice: As of Thursday, Jan. 3, 2019
the YQL service at query.yahooapis.com will be retired
"""
get_yahoo_data = False
if not exp_date:
exp_date = opt_dates.option_expiration(date=exp_date)
else:
exp_date = datetime.datetime.strptime(exp_date, '%Y-%m-%d')
rec['updated'] = cur_date.strftime('%Y-%m-%d %H:%M:%S')
log.info('{} getting pricing for ticker={} '
'cur_date={} exp_date={} '
'yahoo={} iex={}'.format(label, ticker, cur_date, exp_date,
get_yahoo_data, get_iex_data))
yahoo_rec = {
'ticker': ticker,
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'exp_date': None,
'publish_pricing_update': None,
'date': None,
'updated': None
}
# disabled on 2019-01-03
if get_yahoo_data:
log.info('{} yahoo ticker={}'.format(label, ticker))
yahoo_res = yahoo_data.get_data_from_yahoo(work_dict=work_dict)
if yahoo_res['status'] == ae_consts.SUCCESS:
yahoo_rec = yahoo_res['rec']
log.info('{} yahoo ticker={} '
'status={} err={}'.format(
label, ticker,
ae_consts.get_status(status=yahoo_res['status']),
yahoo_res['err']))
rec['pricing'] = yahoo_rec.get('pricing', '{}')
rec['news'] = yahoo_rec.get('news', '{}')
rec['options'] = yahoo_rec.get('options', '{}')
rec['calls'] = rec['options'].get('calls',
ae_consts.EMPTY_DF_STR)
rec['puts'] = rec['options'].get('puts',
ae_consts.EMPTY_DF_STR)
else:
log.error('{} failed YAHOO ticker={} '
'status={} err={}'.format(
label, ticker,
ae_consts.get_status(status=yahoo_res['status']),
yahoo_res['err']))
# end of get from yahoo
if get_iex_data:
num_iex_ds = len(iex_datasets)
log.debug('{} iex datasets={}'.format(label, num_iex_ds))
for idx, ft_type in enumerate(iex_datasets):
dataset_field = iex_consts.get_ft_str(ft_type=ft_type)
log.info('{} iex={}/{} field={} ticker={}'.format(
label, idx, num_iex_ds, dataset_field, ticker))
iex_label = '{}-{}'.format(label, dataset_field)
iex_req = copy.deepcopy(work_dict)
iex_req['label'] = iex_label
iex_req['ft_type'] = ft_type
iex_req['field'] = dataset_field
iex_req['ticker'] = ticker
iex_res = iex_data.get_data_from_iex(work_dict=iex_req)
if iex_res['status'] == ae_consts.SUCCESS:
iex_rec = iex_res['rec']
log.info(
'{} iex ticker={} field={} '
'status={} err={}'.format(
label, ticker, dataset_field,
ae_consts.get_status(status=iex_res['status']),
iex_res['err']))
if dataset_field == 'news':
rec['iex_news'] = iex_rec['data']
else:
rec[dataset_field] = iex_rec['data']
else:
log.debug(
'{} failed IEX ticker={} field={} '
'status={} err={}'.format(
label, ticker, dataset_field,
ae_consts.get_status(status=iex_res['status']),
iex_res['err']))
# end of if/else succcess
# end idx, ft_type in enumerate(iex_datasets):
# end of if get_iex_data
if get_td_data:
num_td_ds = len(td_datasets)
log.debug('{} td datasets={}'.format(label, num_td_ds))
for idx, ft_type in enumerate(td_datasets):
dataset_field = td_consts.get_ft_str_td(ft_type=ft_type)
log.info('{} td={}/{} field={} ticker={}'.format(
label, idx, num_td_ds, dataset_field, ticker))
td_label = '{}-{}'.format(label, dataset_field)
td_req = copy.deepcopy(work_dict)
td_req['label'] = td_label
td_req['ft_type'] = ft_type
td_req['field'] = dataset_field
td_req['ticker'] = ticker
td_res = td_data.get_data_from_td(work_dict=td_req)
if td_res['status'] == ae_consts.SUCCESS:
td_rec = td_res['rec']
log.info('{} td ticker={} field={} '
'status={} err={}'.format(
label, ticker, dataset_field,
ae_consts.get_status(status=td_res['status']),
td_res['err']))
if dataset_field == 'tdcalls':
rec['tdcalls'] = td_rec['data']
if dataset_field == 'tdputs':
rec['tdputs'] = td_rec['data']
else:
rec[dataset_field] = td_rec['data']
else:
log.critical(
'{} failed TD ticker={} field={} '
'status={} err={}'.format(
label, ticker, dataset_field,
ae_consts.get_status(status=td_res['status']),
td_res['err']))
# end of if/else succcess
# end idx, ft_type in enumerate(td_datasets):
# end of if get_td_data
update_req = {'data': rec}
update_req['ticker'] = ticker
update_req['ticker_id'] = ticker_id
update_req['strike'] = cur_strike
update_req['contract'] = contract_type
update_req['s3_enabled'] = work_dict.get('s3_enabled',
ae_consts.ENABLED_S3_UPLOAD)
update_req['redis_enabled'] = work_dict.get(
'redis_enabled', ae_consts.ENABLED_REDIS_PUBLISH)
update_req['s3_bucket'] = s3_bucket
update_req['s3_key'] = s3_key
update_req['s3_access_key'] = work_dict.get('s3_access_key',
ae_consts.S3_ACCESS_KEY)
update_req['s3_secret_key'] = work_dict.get('s3_secret_key',
ae_consts.S3_SECRET_KEY)
update_req['s3_region_name'] = work_dict.get('s3_region_name',
ae_consts.S3_REGION_NAME)
update_req['s3_address'] = work_dict.get('s3_address',
ae_consts.S3_ADDRESS)
update_req['s3_secure'] = work_dict.get('s3_secure',
ae_consts.S3_SECURE)
update_req['redis_key'] = redis_key
update_req['redis_address'] = work_dict.get('redis_address',
ae_consts.REDIS_ADDRESS)
update_req['redis_password'] = work_dict.get('redis_password',
ae_consts.REDIS_PASSWORD)
update_req['redis_db'] = int(
work_dict.get('redis_db', ae_consts.REDIS_DB))
update_req['redis_expire'] = work_dict.get('redis_expire',
ae_consts.REDIS_EXPIRE)
update_req['updated'] = rec['updated']
update_req['label'] = label
update_req['celery_disabled'] = True
update_status = ae_consts.NOT_SET
try:
update_res = publisher.run_publish_pricing_update(
work_dict=update_req)
update_status = update_res.get('status', ae_consts.NOT_SET)
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
log.info('{} update_res status={} data={}'.format(
label, ae_consts.get_status(status=update_status),
ae_consts.ppj(update_res)))
else:
log.info('{} run_publish_pricing_update status={}'.format(
label, ae_consts.get_status(status=update_status)))
# end of if/else
rec['publish_pricing_update'] = update_res
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as f:
err = ('{} publisher.run_publish_pricing_update failed '
'with ex={}'.format(label, f))
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=rec)
# end of trying to publish results to connected services
except Exception as e:
res = build_result.build_result(status=ae_consts.ERR,
err=('failed - get_new_pricing_data '
'dict={} with ex={}').format(
work_dict, e),
rec=rec)
log.error('{} - {}'.format(label, res['err']))
# end of try/ex
if ae_consts.ev('DATASET_COLLECTION_SLACK_ALERTS', '0') == '1':
env_name = 'DEV'
if ae_consts.ev('PROD_SLACK_ALERTS', '1') == '1':
env_name = 'PROD'
done_msg = ('Dataset collected ticker=*{}* on env=*{}* '
'redis_key={} s3_key={} iex={} yahoo={}'.format(
ticker, env_name, redis_key, s3_key, get_iex_data,
get_yahoo_data))
log.debug('{} sending slack msg={}'.format(label, done_msg))
if res['status'] == ae_consts.SUCCESS:
slack_utils.post_success(msg=done_msg, block=False, jupyter=True)
else:
slack_utils.post_failure(msg=done_msg, block=False, jupyter=True)
# end of if/else success
# end of publishing to slack
log.info('task - get_new_pricing_data done - '
'{} - status={}'.format(label,
ae_consts.get_status(res['status'])))
return get_task_results.get_task_results(work_dict=work_dict, result=res)
def fetch_calls(ticker=None,
work_dict=None,
scrub_mode='sort-by-date',
verbose=False):
"""fetch_calls
Fetch Tradier option calls for a ticker and
return a tuple: (status, ``pandas.DataFrame``)
.. code-block:: python
import analysis_engine.td.fetch_api as td_fetch
# Please set the TD_TOKEN environment variable to your token
calls_status, calls_df = td_fetch.fetch_calls(
ticker='SPY')
print(f'Fetched SPY Option Calls from Tradier status={calls_status}:')
print(calls_df)
:param ticker: string ticker to fetch
:param work_dict: dictionary of args
used by the automation
:param scrub_mode: optional - string type of
scrubbing handler to run
:param verbose: optional - bool for debugging
"""
label = 'fetch_calls'
datafeed_type = td_consts.DATAFEED_TD_CALLS
exp_date = None
latest_pricing = {}
latest_close = None
if work_dict:
ticker = work_dict.get('ticker', ticker)
label = work_dict.get('label', label)
exp_date = work_dict.get('exp_date', exp_date)
latest_pricing = work_dict.get('latest_pricing', latest_pricing)
latest_close = latest_pricing.get('close', latest_close)
log.debug(f'{label} - calls - close={latest_close} ' f'ticker={ticker}')
exp_date = opt_dates.option_expiration().strftime(
ae_consts.COMMON_DATE_FORMAT)
use_url = td_consts.TD_URLS['options'].format(ticker, exp_date)
headers = td_consts.get_auth_headers()
session = requests.Session()
session.headers = headers
res = url_helper.url_helper(sess=session).get(use_url)
if res.status_code != requests.codes.OK:
if res.status_code in [401, 403]:
log.critical('Please check the TD_TOKEN is correct '
f'received {res.status_code} during '
'fetch for: calls')
else:
log.info(f'failed to get call with response={res} '
f'code={res.status_code} '
f'text={res.text}')
return ae_consts.EMPTY, pd.DataFrame([{}])
records = json.loads(res.text)
org_records = records.get('options', {}).get('option', [])
if len(org_records) == 0:
log.info('failed to get call records ' 'text={}'.format(res.text))
return ae_consts.EMPTY, pd.DataFrame([{}])
options_list = []
# assumes UTC conversion will work with the system clock
created_minute = (
datetime.datetime.utcnow() -
datetime.timedelta(hours=5)).strftime('%Y-%m-%d %H:%M:00')
last_close_date = ae_utils.get_last_close_str(fmt='%Y-%m-%d %H:%M:00')
# hit bug where dates were None
if not last_close_date:
last_close_date = created_minute
for node in org_records:
node['date'] = last_close_date
node['created'] = created_minute
node['ticker'] = ticker
if (node['option_type'] == 'call'
and node['expiration_type'] == 'standard'
and float(node['bid']) > 0.01):
node['opt_type'] = int(ae_consts.OPTION_CALL)
node['exp_date'] = node['expiration_date']
new_node = {}
for col in td_consts.TD_OPTION_COLUMNS:
if col in node:
if col in td_consts.TD_EPOCH_COLUMNS:
# trade_date can be None
if node[col] == 0:
new_node[col] = None
else:
new_node[col] = ae_utils.epoch_to_dt(
epoch=node[col] / 1000,
use_utc=False,
convert_to_est=True).strftime(
ae_consts.COMMON_TICK_DATE_FORMAT)
"""
Debug epoch ms converter:
"""
"""
print('-----------')
print(col)
print(node[col])
print(new_node[col])
print('===========')
"""
# if/else valid date
else:
new_node[col] = node[col]
# if date column to convert
# if column is in the row
# convert all columns
options_list.append(new_node)
# end of records
full_df = pd.DataFrame(options_list).sort_values(by=['strike'],
ascending=True)
num_chains = len(full_df.index)
df = None
if latest_close:
df_filter = ((full_df['strike'] >=
(latest_close - ae_consts.OPTIONS_LOWER_STRIKE)) &
(full_df['strike'] <=
(latest_close + ae_consts.OPTIONS_UPPER_STRIKE)))
df = full_df[df_filter].copy().sort_values(
by=['date', 'strike']).reset_index()
else:
mid_chain_idx = int(num_chains / 2)
low_idx = int(mid_chain_idx - ae_consts.MAX_OPTIONS_LOWER_STRIKE)
high_idx = int(mid_chain_idx + ae_consts.MAX_OPTIONS_UPPER_STRIKE)
if low_idx < 0:
low_idx = 0
if high_idx > num_chains:
high_idx = num_chains
df = full_df[low_idx:high_idx].copy().sort_values(
by=['date', 'strike']).reset_index()
scrubbed_df = scrub_utils.ingress_scrub_dataset(
label=label,
scrub_mode=scrub_mode,
datafeed_type=datafeed_type,
msg_format='df={} date_str={}',
ds_id=ticker,
date_str=exp_date,
df=df)
return ae_consts.SUCCESS, scrubbed_df
def fetch_data(
work_dict,
fetch_type=None):
"""fetch_data
Factory method for fetching data from
TD using an enum or string alias. Returns
a pandas ``DataFrame`` and only supports
one ticker at a time.
Supported enums from: ``analysis_engine.td.consts``
::
fetch_type = FETCH_TD_CALLS
fetch_type = FETCH_TD_PUTS
Supported ``work_dict['ft_type']`` string values:
::
work_dict['ft_type'] = 'tdcalls'
work_dict['ft_type'] = 'tdputs'
:param work_dict: dictionary of args for the Tradier api
:param fetch_type: optional - name or enum of the fetcher to create
can also be a lower case string
in work_dict['ft_type']
"""
use_fetch_name = None
ticker = work_dict.get(
'ticker',
None)
if not fetch_type:
fetch_type = work_dict.get(
'ft_type',
None)
if fetch_type:
use_fetch_name = str(fetch_type).lower()
if 'exp_date' not in work_dict:
work_dict['exp_date'] = opt_dates.option_expiration().strftime(
ae_consts.COMMON_DATE_FORMAT)
log.debug(f'name={use_fetch_name} type={fetch_type} args={work_dict}')
status_df = ae_consts.NOT_SET
df = pd.DataFrame([{}])
if (
use_fetch_name == 'tdcalls' or
fetch_type == td_consts.FETCH_TD_CALLS):
status_df, fetch_df = td_fetch.fetch_calls(
work_dict=work_dict)
if status_df == ae_consts.SUCCESS:
log.debug(
'call - merge df')
work_copy = copy.deepcopy(
work_dict)
work_copy['ft_type'] = td_consts.FETCH_TD_CALLS
work_copy['fd_type'] = 'tdcalls'
if 'tdcalls' in work_dict:
work_copy['redis_key'] = work_dict['tdcalls']
work_copy['s3_key'] = f'{work_dict["tdcalls"]}.json'
else:
work_copy['redis_key'] = f'{work_dict["redis_key"]}_tdcalls'
work_copy['s3_key'] = f'{work_dict["redis_key"]}_tdcalls'
ext_status, ext_df = \
td_extract.extract_option_calls_dataset(
work_dict=work_copy)
if ext_status == ae_consts.SUCCESS and len(ext_df.index) > 0:
log.debug(
f'call - merging fetch={len(fetch_df.index)} '
f'with ext={len(ext_df.index)}')
"""
for testing compression:
"""
"""
import sys
print(ext_df['date'])
print(ext_df['ask_date'])
print(ext_df['bid_date'])
print(ext_df['trade_date'])
sys.exit(1)
"""
extracted_records = json.loads(ext_df.to_json(
orient='records'))
fetched_records = json.loads(fetch_df.to_json(
orient='records'))
new_records = []
dates_by_strike_dict = {}
for ex_row in extracted_records:
date_strike_name = (
f'{ex_row["created"]}_{ex_row["strike"]}')
if date_strike_name not in dates_by_strike_dict:
new_node = {}
for c in td_consts.TD_OPTION_COLUMNS:
if c in ex_row:
new_node[c] = ex_row[c]
# end of for all columns to copy over
new_node.pop('index', None)
new_node.pop('level_0', None)
new_records.append(new_node)
dates_by_strike_dict[date_strike_name] = True
# build extracted records
for ft_row in fetched_records:
date_strike_name = (
f'{ft_row["created"]}_{ft_row["strike"]}')
try:
if date_strike_name not in dates_by_strike_dict:
new_node = {}
for c in td_consts.TD_OPTION_COLUMNS:
if c in ft_row:
new_node[c] = ft_row[c]
# end of for all columns to copy over
new_node.pop('index', None)
new_node.pop('level_0', None)
new_records.append(new_node)
dates_by_strike_dict[date_strike_name] = True
else:
log.error(
f'already have {ticker} call - '
f'date={ft_row["created"]} '
f'strike={ft_row["strike"]}')
except Exception as p:
log.critical(f'failed fetching call with ex={p}')
return ae_consts.ERR, None
# end of adding fetched records after the extracted
df = pd.DataFrame(new_records)
df.sort_values(
by=[
'date',
'strike'
],
ascending=True)
log.debug(f'call - merged={len(df.index)}')
else:
df = fetch_df.sort_values(
by=[
'date',
'strike'
],
ascending=True)
else:
log.warn(
f'{ticker} - no data found for calls')
# if able to merge fetch + last for today
elif (
use_fetch_name == 'tdputs' or
fetch_type == td_consts.FETCH_TD_PUTS):
status_df, fetch_df = td_fetch.fetch_puts(
work_dict=work_dict)
if status_df == ae_consts.SUCCESS:
log.debug(
'put - merge df')
work_copy = copy.deepcopy(
work_dict)
work_copy['ft_type'] = td_consts.FETCH_TD_PUTS
work_copy['fd_type'] = 'tdputs'
if 'tdputs' in work_dict:
work_copy['redis_key'] = work_dict['tdputs']
work_copy['s3_key'] = f'{work_dict["tdputs"]}.json'
else:
work_copy['redis_key'] = f'{work_dict["redis_key"]}_tdputs'
work_copy['s3_key'] = f'{work_dict["s3_key"]}_tdputs'
ext_status, ext_df = \
td_extract.extract_option_puts_dataset(
work_dict=work_copy)
if ext_status == ae_consts.SUCCESS and len(ext_df.index) > 0:
log.debug(
f'put - merging fetch={len(fetch_df.index)} with '
f'ext={len(ext_df.index)}')
"""
for testing compression:
"""
"""
import sys
print(ext_df['date'])
sys.exit(1)
"""
extracted_records = json.loads(ext_df.to_json(
orient='records'))
fetched_records = json.loads(fetch_df.to_json(
orient='records'))
new_records = []
dates_by_strike_dict = {}
for ex_row in extracted_records:
date_strike_name = (
f'{ex_row["created"]}_{ex_row["strike"]}')
if date_strike_name not in dates_by_strike_dict:
new_node = {}
for c in td_consts.TD_OPTION_COLUMNS:
if c in ex_row:
new_node[c] = ex_row[c]
# end of for all columns to copy over
new_node.pop('index', None)
new_node.pop('level_0', None)
new_records.append(new_node)
dates_by_strike_dict[date_strike_name] = True
# build extracted records
for ft_row in fetched_records:
date_strike_name = (
f'{ft_row["created"]}_{ft_row["strike"]}')
try:
if date_strike_name not in dates_by_strike_dict:
new_node = {}
for c in td_consts.TD_OPTION_COLUMNS:
if c in ft_row:
new_node[c] = ft_row[c]
# end of for all columns to copy over
new_node.pop('index', None)
new_node.pop('level_0', None)
new_records.append(new_node)
dates_by_strike_dict[date_strike_name] = True
else:
log.error(
f'already have {ticker} put - '
f'date={ft_row["created"]} '
f'strike={ft_row["strike"]}')
except Exception as p:
log.critical(f'failed fetching puts with ex={p}')
return ae_consts.ERR, None
# end of adding fetched records after the extracted
df = pd.DataFrame(new_records)
df.sort_values(
by=[
'date',
'strike'
],
ascending=True)
log.debug(f'put - merged={len(df.index)}')
else:
df = fetch_df.sort_values(
by=[
'date',
'strike'
],
ascending=True)
else:
log.warn(
f'{ticker} - no data found for puts')
# if able to merge fetch + last for today
else:
log.error(
f'label={work_dict.get("label", None)} - '
f'unsupported fetch_data('
f'work_dict={work_dict}, '
f'fetch_type={fetch_type}'
f')')
raise NotImplementedError
# end of supported fetchers
return status_df, df
def fetch_calls(work_dict, scrub_mode='sort-by-date'):
"""fetch_calls
Fetch the Tradier daily data for a ticker and
return it as a ``pandas.DataFrame``.
:param work_dict: dictionary of args
:param scrub_mode: type of scrubbing handler to run
"""
datafeed_type = td_consts.DATAFEED_TD_CALLS
ticker = work_dict.get('ticker', None)
label = work_dict.get('label', None)
exp_date = work_dict.get('exp_date', None)
log.debug(f'{label} - call - scrub_mode={scrub_mode} '
f'args={work_dict} ticker={ticker}')
exp_date = opt_dates.option_expiration().strftime(
ae_consts.COMMON_DATE_FORMAT)
use_url = td_consts.TD_URLS['options'].format(ticker, exp_date)
headers = td_consts.get_auth_headers()
session = requests.Session()
session.headers = headers
res = url_helper.url_helper(sess=session).get(use_url)
if res.status_code != requests.codes.OK:
if res.status_code in [401, 403]:
log.critical('Please check the TD_TOKEN is correct '
f'received {res.status_code} during '
'fetch for: calls')
else:
log.info(f'failed to get call with response={res} '
f'code={res.status_code} '
f'text={res.text}')
return ae_consts.EMPTY, pd.DataFrame([{}])
records = json.loads(res.text)
org_records = records.get('options', {}).get('option', [])
if len(org_records) == 0:
log.info('failed to get call records ' f'text={res.text}')
return ae_consts.EMPTY, pd.DataFrame([{}])
options_list = []
# assumes UTC conversion will work with the system clock
created_minute = (
datetime.datetime.utcnow() -
datetime.timedelta(hours=5)).strftime('%Y-%m-%d %H:%M:00')
last_close_date = ae_utils.get_last_close_str(fmt='%Y-%m-%d %H:%M:00')
# hit bug where dates were None
if not last_close_date:
last_close_date = created_minute
for node in org_records:
node['date'] = last_close_date
node['created'] = created_minute
node['ticker'] = ticker
if (node['option_type'] == 'call'
and node['expiration_type'] == 'standard'):
node['opt_type'] = int(ae_consts.OPTION_CALL)
node['exp_date'] = node['expiration_date']
new_node = {}
for col in td_consts.TD_OPTION_COLUMNS:
if col in node:
if col in td_consts.TD_EPOCH_COLUMNS:
# trade_date can be None
if node[col] == 0:
new_node[col] = None
else:
new_node[col] = ae_utils.epoch_to_dt(
epoch=node[col] / 1000,
use_utc=False,
convert_to_est=True).strftime(
ae_consts.COMMON_TICK_DATE_FORMAT)
"""
Debug epoch ms converter:
"""
"""
print('-----------')
print(col)
print(node[col])
print(new_node[col])
print('===========')
"""
# if/else valid date
else:
new_node[col] = node[col]
# if date column to convert
# if column is in the row
# convert all columns
options_list.append(new_node)
# end of records
full_df = pd.DataFrame(options_list).sort_values(by=['strike'],
ascending=True)
num_chains = len(full_df.index)
mid_chain_idx = int(num_chains / 2)
low_idx = int(mid_chain_idx - 20)
high_idx = int(mid_chain_idx + 30)
if low_idx < 0:
low_idx = 0
if high_idx > num_chains:
high_idx = num_chains
df = full_df[low_idx:high_idx].copy().sort_values(
by=['date', 'strike']).reset_index()
scrubbed_df = scrub_utils.ingress_scrub_dataset(
label=label,
scrub_mode=scrub_mode,
datafeed_type=datafeed_type,
msg_format='df={} date_str={}',
ds_id=ticker,
date_str=exp_date,
df=df)
return ae_consts.SUCCESS, scrubbed_df
def get_data_from_yahoo(work_dict):
"""get_data_from_yahoo
Get data from yahoo
:param work_dict: request dictionary
"""
label = 'get_data_from_yahoo'
log.info(f'task - {label} - start work_dict={work_dict}')
num_news_rec = 0
num_option_calls = 0
num_option_puts = 0
cur_high = -1
cur_low = -1
cur_open = -1
cur_close = -1
cur_volume = -1
rec = {
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'exp_date': None,
'publish_pricing_update': None,
'date': None,
'updated': None
}
res = {'status': NOT_RUN, 'err': None, 'rec': rec}
log.error('sorry - yahoo is disabled and '
'pinance is no longer supported '
'https://github.com/neberej/pinance')
return res
try:
ticker = work_dict.get('ticker', TICKER)
exp_date = work_dict.get('exp_date', None)
cur_strike = work_dict.get('strike', None)
contract_type = str(work_dict.get('contract', 'C')).upper()
get_pricing = work_dict.get('get_pricing', True)
get_news = work_dict.get('get_news', True)
get_options = work_dict.get('get_options', True)
orient = work_dict.get('orient', 'records')
label = work_dict.get('label', label)
ticker_results = None
num_news_rec = 0
use_date = exp_date
if not exp_date:
exp_date = opt_dates.option_expiration(date=exp_date)
use_date = exp_date.strftime('%Y-%m-%d')
"""
Debug control flags
Quickly turn specific fetches off:
get_news = False
get_pricing = False
get_options = False
"""
if get_pricing:
log.info(f'{label} getting ticker={ticker} pricing')
ticker_results.get_quotes()
if ticker_results.quotes_data:
pricing_dict = ticker_results.quotes_data
cur_high = pricing_dict.get('regularMarketDayHigh', None)
cur_low = pricing_dict.get('regularMarketDayLow', None)
cur_open = pricing_dict.get('regularMarketOpen', None)
cur_close = pricing_dict.get('regularMarketPreviousClose',
None)
cur_volume = pricing_dict.get('regularMarketVolume', None)
pricing_dict['high'] = cur_high
pricing_dict['low'] = cur_low
pricing_dict['open'] = cur_open
pricing_dict['close'] = cur_close
pricing_dict['volume'] = cur_volume
pricing_dict['date'] = get_last_close_str()
if 'regularMarketTime' in pricing_dict:
pricing_dict['market_time'] = \
datetime.datetime.fromtimestamp(
pricing_dict['regularMarketTime']).strftime(
COMMON_TICK_DATE_FORMAT)
if 'postMarketTime' in pricing_dict:
pricing_dict['post_market_time'] = \
datetime.datetime.fromtimestamp(
pricing_dict['postMarketTime']).strftime(
COMMON_TICK_DATE_FORMAT)
log.info(f'{label} ticker={ticker} converting pricing to '
f'df orient={orient}')
try:
rec['pricing'] = pricing_dict
except Exception as f:
rec['pricing'] = '{}'
log.info(
f'{label} ticker={ticker} failed converting pricing '
f'data={ppj(pricing_dict)} to df ex={f}')
# try/ex
log.info(f'{label} ticker={ticker} done converting pricing to '
f'df orient={orient}')
else:
log.error(f'{label} ticker={ticker} '
f'missing quotes_data={ticker_results.quotes_data}')
# end of if ticker_results.quotes_data
log.info(
f'{label} ticker={ticker} close={cur_close} vol={cur_volume}')
else:
log.info(f'{label} skip - getting ticker={ticker} pricing')
# if get_pricing
if get_news:
log.info(f'{label} getting ticker={ticker} news')
ticker_results.get_news()
if ticker_results.news_data:
news_list = None
try:
news_list = ticker_results.news_data
log.info(f'{label} ticker={ticker} converting news to '
f'df orient={orient}')
num_news_rec = len(news_list)
rec['news'] = news_list
except Exception as f:
rec['news'] = '{}'
log.info(f'{label} ticker={ticker} failed converting news '
f'data={news_list} to df ex={f}')
# try/ex
log.info(f'{label} ticker={ticker} done converting news to '
f'df orient={orient}')
else:
log.info(f'{label} ticker={ticker} Yahoo NO '
f'news={ticker_results.news_data}')
# end of if ticker_results.news_data
else:
log.info(f'{label} skip - getting ticker={ticker} news')
# end if get_news
if get_options:
get_all_strikes = True
if get_all_strikes:
cur_strike = None
else:
if cur_close:
cur_strike = int(cur_close)
if not cur_strike:
cur_strike = 287
log.info(
f'{label} ticker={ticker} num_news={num_news_rec} get options '
f'close={cur_close} exp_date={use_date} '
f'contract={contract_type} strike={cur_strike}')
options_dict = \
yahoo_get_pricing.get_options(
ticker=ticker,
exp_date_str=use_date,
contract_type=contract_type,
strike=cur_strike)
rec['options'] = '{}'
try:
log.info(f'{label} ticker={ticker} converting options to '
f'df orient={orient}')
num_option_calls = options_dict.get('num_calls', None)
num_option_puts = options_dict.get('num_puts', None)
rec['options'] = {
'exp_date': options_dict.get('exp_date', None),
'calls': options_dict.get('calls', None),
'puts': options_dict.get('puts', None),
'num_calls': num_option_calls,
'num_puts': num_option_puts
}
rec['calls'] = rec['options'].get('calls', EMPTY_DF_STR)
rec['puts'] = rec['options'].get('puts', EMPTY_DF_STR)
except Exception as f:
rec['options'] = '{}'
log.info(f'{label} ticker={ticker} failed converting options '
f'data={options_dict} to df ex={f}')
# try/ex
log.info(f'{label} ticker={ticker} done converting options to '
f'df orient={orient} num_calls={num_option_calls} '
f'num_puts={num_option_puts}')
else:
log.info(f'{label} skip - getting ticker={ticker} options')
# end of if get_options
log.info(
f'{label} yahoo pricing for ticker={ticker} close={cur_close} '
f'num_calls={num_option_calls} num_puts={num_option_puts} '
f'news={num_news_rec}')
fields_to_upload = ['pricing', 'options', 'calls', 'puts', 'news']
for field_name in fields_to_upload:
upload_and_cache_req = copy.deepcopy(work_dict)
upload_and_cache_req['celery_disabled'] = True
upload_and_cache_req['data'] = rec[field_name]
if not upload_and_cache_req['data']:
upload_and_cache_req['data'] = '{}'
if 'redis_key' in work_dict:
upload_and_cache_req['redis_key'] = f'''{work_dict.get(
'redis_key',
f'{ticker}_{field_name}')}_{field_name}'''
if 's3_key' in work_dict:
upload_and_cache_req['s3_key'] = f'''{work_dict.get(
's3_key',
f'{ticker}_{field_name}')}_{field_name}'''
try:
update_res = publisher.run_publish_pricing_update(
work_dict=upload_and_cache_req)
update_status = update_res.get('status', NOT_SET)
log.info(f'{label} publish update '
f'status={get_status(status=update_status)} '
f'data={update_res}')
except Exception:
err = (f'{label} - failed to upload YAHOO '
f'data={upload_and_cache_req} to '
f's3_key={upload_and_cache_req["s3_key"]} and '
f'redis_key={upload_and_cache_req["redis_key"]}')
log.error(err)
# end of try/ex to upload and cache
if not rec[field_name]:
log.debug(f'{label} - ticker={ticker} no data from YAHOO for '
f'field_name={field_name}')
# end of for all fields
res = build_result.build_result(status=SUCCESS, err=None, rec=rec)
except Exception as e:
res = build_result.build_result(status=ERR,
err=('failed - get_data_from_yahoo '
f'dict={work_dict} with ex={e}'),
rec=rec)
log.error(f'{label} - {res["err"]}')
# end of try/ex
log.info('task - get_data_from_yahoo done - '
f'{label} - status={get_status(res["status"])}')
return res
