def run_screener_analysis(work_dict):
"""run_screener_analysis
Celery wrapper for running without celery
:param work_dict: task data
"""
fn_name = 'run_screener_analysis'
label = f'''{fn_name} - {work_dict.get(
'label',
'')}'''
log.info(f'{label} - start')
response = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec={})
task_res = {}
# allow running without celery
if ae_consts.is_celery_disabled(work_dict=work_dict):
work_dict['celery_disabled'] = True
task_res = task_screener_analysis(work_dict)
if task_res:
response = task_res.get('result', task_res)
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
response_details = response
try:
response_details = ae_consts.ppj(response)
except Exception:
response_details = response
log.info(f'{label} task result={response_details}')
else:
log.error(f'{label} celery was disabled but the task={response} '
'did not return anything')
# end of if response
else:
task_res = task_screener_analysis.delay(work_dict=work_dict)
rec = {'task_id': task_res}
response = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
# if celery enabled
if response:
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
log.info(f'{label} - done '
f'status={ae_consts.get_status(response["status"])} '
f'err={response["err"]} rec={response["rec"]}')
else:
log.info(f'{label} - done '
f'status={ae_consts.get_status(response["status"])} '
f'err={response["err"]}')
else:
log.info(f'{label} - done no response')
# end of if/else response
return response
def run_screener_analysis(work_dict):
"""run_screener_analysis
Celery wrapper for running without celery
:param work_dict: task data
"""
fn_name = 'run_screener_analysis'
label = '{} - {}'.format(fn_name, work_dict.get('label', ''))
log.info('{} - start'.format(label))
response = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec={})
task_res = {}
# allow running without celery
if ae_consts.is_celery_disabled(work_dict=work_dict):
work_dict['celery_disabled'] = True
task_res = task_screener_analysis(work_dict)
if task_res:
response = task_res.get('result', task_res)
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
response_details = response
try:
response_details = ae_consts.ppj(response)
except Exception:
response_details = response
log.info('{} task result={}'.format(label, response_details))
else:
log.error('{} celery was disabled but the task={} '
'did not return anything'.format(label, response))
# end of if response
else:
task_res = task_screener_analysis.delay(work_dict=work_dict)
rec = {'task_id': task_res}
response = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
# if celery enabled
if response:
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
log.info('{} - done '
'status={} err={} rec={}'.format(
label, ae_consts.get_status(response['status']),
response['err'], response['rec']))
else:
log.info('{} - done '
'status={} err={}'.format(
label, ae_consts.get_status(response['status']),
response['err']))
else:
log.info('{} - done ' 'no response'.format(label))
# end of if/else response
return response
def run_get_new_pricing_data(work_dict):
"""run_get_new_pricing_data
Celery wrapper for running without celery
:param work_dict: task data
"""
label = work_dict.get('label', '')
log.debug(f'run_get_new_pricing_data - {label} - start')
response = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec={})
task_res = {}
# allow running without celery
if ae_consts.is_celery_disabled(work_dict=work_dict):
work_dict['celery_disabled'] = True
task_res = get_new_pricing_data(work_dict)
if task_res:
response = task_res.get('result', task_res)
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
response_details = response
try:
response_details = ae_consts.ppj(response)
except Exception:
response_details = response
log.debug(f'{label} task result={response_details}')
else:
log.error(f'{label} celery was disabled but the task={response} '
'did not return anything')
# end of if response
else:
task_res = get_new_pricing_data.delay(work_dict=work_dict)
rec = {'task_id': task_res}
response = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
# if celery enabled
if response:
status_str = ae_consts.get_status(response['status'])
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
log.debug(f'run_get_new_pricing_data - {label} - done '
f'status={status_str} '
f'err={response["err"]} '
f'rec={response["rec"]}')
else:
log.debug(f'run_get_new_pricing_data - {label} - done '
f'status={status_str} '
f'rec={response["rec"]}')
else:
log.debug(f'run_get_new_pricing_data - {label} - done ' 'no response')
# end of if/else response
return response
def run_publish_ticker_aggregate_from_s3(work_dict):
"""run_publish_ticker_aggregate_from_s3
Celery wrapper for running without celery
:param work_dict: task data
"""
label = work_dict.get('label', '')
log.info('run_publish_ticker_aggregate_from_s3 - {} - start'.format(label))
response = build_result.build_result(status=NOT_RUN, err=None, rec={})
task_res = {}
# allow running without celery
if is_celery_disabled(work_dict=work_dict):
work_dict['celery_disabled'] = True
task_res = publish_ticker_aggregate_from_s3(work_dict=work_dict)
if task_res:
response = task_res.get('result', task_res)
if ev('DEBUG_RESULTS', '0') == '1':
response_details = response
try:
response_details = ppj(response)
except Exception:
response_details = response
log.info('{} task result={}'.format(label, response_details))
else:
log.error('{} celery was disabled but the task={} '
'did not return anything'.format(label, response))
# end of if response
else:
task_res = publish_ticker_aggregate_from_s3.delay(work_dict=work_dict)
rec = {'task_id': task_res}
response = build_result.build_result(status=SUCCESS, err=None, rec=rec)
# if celery enabled
if response:
if ev('DEBUG_RESULTS', '0') == '1':
log.info('run_publish_ticker_aggregate_from_s3 - {} - done '
'status={} err={} rec={}'.format(
label, get_status(response['status']),
response['err'], response['rec']))
else:
log.info('run_publish_ticker_aggregate_from_s3 - {} - done '
'status={} err={}'.format(label,
get_status(response['status']),
response['err']))
else:
log.info('run_publish_ticker_aggregate_from_s3 - {} - done '
'no response'.format(label))
# end of if/else response
return response
def mock_extract_news_from_redis_success(label, host, port, db, password, key,
**kwargs):
"""mock_extract_news_from_redis_success
:param label: test label
:param address: test address
:param db: test db
:param key: test key
:param kwargs: additional keyword args as a dictionary
"""
sample_record = build_cache_ready_pricing_dataset(
label=('{}.{}.{}.{}.{}.'
'mock_extract_news_from_redis_success'.format(
label, host, port, db, key)))
rec = {'data': sample_record['news']}
res = build_result(status=SUCCESS, err=None, rec=rec)
return res
def mock_extract_news_from_redis_success(label, host, port, db, password, key,
**kwargs):
"""mock_extract_news_from_redis_success
:param label: test label
:param address: test address
:param db: test db
:param key: test key
:param kwargs: additional keyword args as a dictionary
"""
sample_record = api_requests.build_cache_ready_pricing_dataset(
label=(f'{label}.{host}.{port}.{db}.{key}.'
'mock_extract_news_from_redis_success'))
rec = {'data': sample_record['news']}
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
return res
def plot_hloc_pricing(
log_label,
ticker,
df,
title,
show_plot=True,
dropna_for_all=True):
"""plot_hloc_pricing
Plot the high, low, open and close columns together on a chart
:param log_label: log identifier
:param ticker: ticker
:param df: initialized ``pandas.DataFrame``
:param title: title for the chart
:param show_plot: bool to show the plot
:param dropna_for_all: optional - bool to toggle keep None's in
the plot ``df`` (default is drop them
for display purposes)
"""
rec = {
'ax': None,
'fig': None
}
result = build_result.build_result(
status=ae_consts.NOT_RUN,
err=None,
rec=rec)
try:
log.info(
f'{log_label} - '
'plot_hloc_pricing'
' - start')
set_common_seaborn_fonts()
fig, ax = plt.subplots(
figsize=(15.0, 10.0))
use_df = df
if dropna_for_all:
log.info(
f'{log_label} - '
'plot_hloc_pricing'
' - dropna_for_all')
use_df = df.dropna(axis=0, how='any')
# end of pre-plot dataframe scrubbing
plt.plot(
use_df['date'],
use_df['high'],
label='High',
color=ae_consts.PLOT_COLORS['high'],
alpha=0.4)
plt.plot(
use_df['date'],
use_df['low'],
label='Low',
color=ae_consts.PLOT_COLORS['low'],
alpha=0.4)
plt.plot(
use_df['date'],
use_df['close'],
label='Close',
color=ae_consts.PLOT_COLORS['close'],
alpha=0.4)
plt.plot(
use_df['date'],
use_df['open'],
label='Open',
color=ae_consts.PLOT_COLORS['open'],
alpha=0.4)
xlabel = 'Dates'
ylabel = 'Prices'
plt.grid(True)
plt.xlabel(xlabel)
plt.ylabel(ylabel)
# Build a date vs Close DataFrame
start_date = str(use_df.iloc[0]['date'].strftime('%Y-%m-%d'))
end_date = str(use_df.iloc[-1]['date'].strftime('%Y-%m-%d'))
if not title:
title = (
f'{ticker} Pricing from: {start_date} to {end_date}')
ax.set_title(title)
# Build out the xtick chart by the dates
ax.xaxis.grid(True, which='minor')
legend_list = [
'high',
'low',
'close',
'open'
]
fig.autofmt_xdate()
show_with_entities(
log_label=log_label,
xlabel=xlabel,
ylabel=ylabel,
title=title,
ax=ax,
fig=fig,
legend_list=legend_list,
show_plot=show_plot)
rec['ax'] = ax
rec['fig'] = fig
result = build_result.build_result(
status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
err = (
f'failed plot_hloc_pricing with ex={e}')
log.error(err)
result = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
# end of try/ex
send_final_log(
log_label=log_label,
fn_name='plot_hloc_pricing',
result=result)
return result
def run_distributed_algorithm(self, algo_req):
"""run_distributed_algorithm
Process a distributed Algorithm
:param algo_req: dictionary for key/values for
running an algorithm using Celery workers
"""
label = algo_req.get('name', 'ae-algo')
verbose = algo_req.get('verbose', False)
debug = algo_req.get('debug', False)
# please be careful logging prod passwords:
if verbose or debug:
log.info('task - {} - start ' 'algo_req={}'.format(label, algo_req))
else:
log.info('task - {} - start '.format(label))
# end of start log
rec = {}
res = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec=rec)
created_algo_object = None
custom_algo_module = None
new_algo_object = None
use_custom_algo = False
found_algo_module = True # assume the BaseAlgo
should_publish_extract_dataset = False
should_publish_history_dataset = False
should_publish_report_dataset = False
ticker = algo_req.get('ticker', 'SPY')
num_days_back = algo_req.get('num_days_back', 75)
name = algo_req.get('name', 'ae-algo')
algo_module_path = algo_req.get('mod_path', None)
module_name = algo_req.get('module_name', 'BaseAlgo')
custom_algo_module = algo_req.get('custom_algo_module', None)
new_algo_object = algo_req.get('new_algo_object', None)
use_custom_algo = algo_req.get('use_custom_algo', False)
should_publish_extract_dataset = algo_req.get(
'should_publish_extract_dataset', False)
should_publish_history_dataset = algo_req.get(
'should_publish_history_dataset', False)
should_publish_report_dataset = algo_req.get(
'should_publish_report_dataset', False)
start_date = algo_req.get('start_date', None)
end_date = algo_req.get('end_date', None)
raise_on_err = algo_req.get('raise_on_err', False)
report_config = algo_req.get('report_config', None)
history_config = algo_req.get('history_config', None)
extract_config = algo_req.get('extract_config', None)
err = None
if algo_module_path:
found_algo_module = False
module_name = algo_module_path.split('/')[-1]
loader = importlib.machinery.SourceFileLoader(module_name,
algo_module_path)
custom_algo_module = types.ModuleType(loader.name)
loader.exec_module(custom_algo_module)
use_custom_algo = True
for member in inspect.getmembers(custom_algo_module):
if module_name in str(member):
found_algo_module = True
break
# for all members in this custom module file
# if loading a custom algorithm module from a file on disk
if not found_algo_module:
err = ('{} - unable to find custom algorithm module={} '
'module_path={}'.format(label, custom_algo_module,
algo_module_path))
if algo_module_path:
err = (
'{} - analysis_engine.work_tasks.run_distributed_algorithm '
'was unable '
'to find custom algorithm module={} with provided path to \n '
'file: {} \n'
'\n'
'Please confirm '
'that the class inherits from the BaseAlgo class like:\n'
'\n'
'import analysis_engine.algo\n'
'class MyAlgo(analysis_engine.algo.BaseAlgo):\n '
'\n'
'If it is then please file an issue on github:\n '
'https://github.com/AlgoTraders/stock-analysis-engine/'
'issues/new \n\nFor now this error results in a shutdown'
'\n'.format(label, custom_algo_module, algo_module_path))
# if algo_module_path set
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# if not found_algo_module
use_start_date = start_date
use_end_date = end_date
if not use_end_date:
end_date = datetime.datetime.utcnow()
use_end_date = end_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT)
if not use_start_date:
start_date = end_date - datetime.timedelta(days=num_days_back)
use_start_date = start_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT)
dataset_publish_extract = algo_req.get('dataset_publish_extract', False)
dataset_publish_history = algo_req.get('dataset_publish_history', False)
dataset_publish_report = algo_req.get('dataset_publish_report', False)
try:
if use_custom_algo:
log.info('inspecting {} for class {}'.format(
custom_algo_module, module_name))
use_class_member_object = None
for member in inspect.getmembers(custom_algo_module):
if module_name in str(member):
log.info('start {} with {}'.format(name, member[1]))
use_class_member_object = member
break
# end of looking over the class definition but did not find it
if use_class_member_object:
new_algo_object = member[1](**algo_req)
else:
err = ('{} - did not find a derived '
'analysis_engine.algo.BaseAlgo '
'class in the module file={} '
'for ticker={} algo_name={}'.format(
label, algo_module_path, ticker, name))
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# end of finding a valid algorithm object
else:
new_algo_object = ae_algo.BaseAlgo(**algo_req)
# if using a custom module path or the BaseAlgo
if new_algo_object:
# heads up - logging this might have passwords in the algo_req
# log.debug(
# '{} algorithm request: {}'.format(
# name,
# algo_req))
log.info('{} - run ticker={} from {} to {}'.format(
name, ticker, use_start_date, use_end_date))
algo_res = run_algo.run_algo(algo=new_algo_object,
raise_on_err=raise_on_err,
**algo_req)
created_algo_object = new_algo_object
log.info('{} - run ticker={} from {} to {}'.format(
name, ticker, use_start_date, use_end_date))
if custom_algo_module:
log.info(
'{} - done run_algo custom_algo_module={} module_name={} '
'ticker={} from {} to {}'.format(name, custom_algo_module,
module_name, ticker,
use_start_date,
use_end_date))
else:
log.info('{} - done run_algo BaseAlgo ticker={} from {} '
'to {}'.format(name, ticker, use_start_date,
use_end_date))
else:
err = ('{} - missing a derived analysis_engine.algo.BaseAlgo '
'class in the module file={} for '
'ticker={} algo_name={}'.format(label, algo_module_path,
ticker, name))
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# end of finding a valid algorithm object
if not created_algo_object:
err = ('{} - failed creating algorithm object - '
'ticker={} status={} error={}'
'algo name={} custom_algo_module={} module_name={} '
'from {} to {}'.format(
label, ticker,
ae_consts.get_status(status=algo_res['status']),
algo_res['err'], name, custom_algo_module, module_name,
use_start_date, use_end_date))
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# end of stop early
if should_publish_extract_dataset or dataset_publish_extract:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (extract_config['redis_address'] and extract_config['redis_db']
and extract_config['redis_key']):
redis_log = 'redis://{}@{}/{}'.format(
extract_config['redis_address'],
extract_config['redis_db'], extract_config['redis_key'])
use_log += ' {}'.format(redis_log)
else:
extract_config['redis_enabled'] = False
if (extract_config['s3_address'] and extract_config['s3_bucket']
and extract_config['s3_key']):
s3_log = 's3://{}/{}/{}'.format(extract_config['s3_address'],
extract_config['s3_bucket'],
extract_config['s3_key'])
use_log += ' {}'.format(s3_log)
else:
extract_config['s3_enabled'] = False
if extract_config['output_file']:
file_log = 'file:{}'.format(extract_config['output_file'])
use_log += ' {}'.format(file_log)
log.info('{} - publish - start ticker={} algorithm-ready {}'
''.format(name, ticker, use_log))
publish_status = created_algo_object.publish_input_dataset(
**extract_config)
if publish_status != ae_consts.SUCCESS:
msg = ('failed to publish algorithm-ready datasets '
'with status {} attempted to {}'.format(
ae_consts.get_status(status=publish_status),
use_log))
log.error(msg)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# end of stop early
log.info('{} - publish - done ticker={} algorithm-ready {}'
''.format(name, ticker, use_log))
# if publish the algorithm-ready dataset
if should_publish_history_dataset or dataset_publish_history:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (history_config['redis_address'] and history_config['redis_db']
and history_config['redis_key']):
redis_log = 'redis://{}@{}/{}'.format(
history_config['redis_address'],
history_config['redis_db'], history_config['redis_key'])
use_log += ' {}'.format(redis_log)
if (history_config['s3_address'] and history_config['s3_bucket']
and history_config['s3_key']):
s3_log = 's3://{}/{}/{}'.format(history_config['s3_address'],
history_config['s3_bucket'],
history_config['s3_key'])
use_log += ' {}'.format(s3_log)
if history_config['output_file']:
file_log = 'file:{}'.format(history_config['output_file'])
use_log += ' {}'.format(file_log)
log.info('{} - publish - start ticker={} trading history {}'
''.format(name, ticker, use_log))
publish_status = \
created_algo_object.publish_trade_history_dataset(
**history_config)
if publish_status != ae_consts.SUCCESS:
msg = ('failed to publish trading history datasets '
'with status {} attempted to {}'.format(
ae_consts.get_status(status=publish_status),
use_log))
log.error(msg)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# end of stop early
log.info('{} - publish - done ticker={} trading history {}'
''.format(name, ticker, use_log))
# if publish an trading history dataset
if should_publish_report_dataset or dataset_publish_report:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (report_config['redis_address'] and report_config['redis_db']
and report_config['redis_key']):
redis_log = 'redis://{}@{}/{}'.format(
report_config['redis_address'], report_config['redis_db'],
report_config['redis_key'])
use_log += ' {}'.format(redis_log)
if (report_config['s3_address'] and report_config['s3_bucket']
and report_config['s3_key']):
s3_log = 's3://{}/{}/{}'.format(report_config['s3_address'],
report_config['s3_bucket'],
report_config['s3_key'])
use_log += ' {}'.format(s3_log)
if report_config['output_file']:
file_log = ' file:{}'.format(report_config['output_file'])
use_log += ' {}'.format(file_log)
log.info('{} - publishing ticker={} trading performance report {}'
''.format(name, ticker, use_log))
publish_status = created_algo_object.publish_report_dataset(
**report_config)
if publish_status != ae_consts.SUCCESS:
msg = ('failed to publish trading performance report datasets '
'with status {} attempted to {}'.format(
ae_consts.get_status(status=publish_status),
use_log))
log.error(msg)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# end of stop early
log.info(
'{} - publish - done ticker={} trading performance report {}'
''.format(name, ticker, use_log))
# if publish an trading performance report dataset
log.info(
'{} - done publishing datasets for ticker={} from {} to {}'.format(
name, ticker, use_start_date, use_end_date))
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
res = build_result.build_result(
status=ae_consts.ERR,
err=('failed - run_distributed_algorithm '
'dict={} with ex={}').format(algo_req, e),
rec=rec)
log.error('{} - {}'.format(label, res['err']))
# end of try/ex
log.info('task - run_distributed_algorithm done - '
'{} - status={}'.format(label,
ae_consts.get_status(res['status'])))
return get_task_results.get_task_results(work_dict=algo_req, result=res)
def task_screener_analysis(self, work_dict):
"""task_screener_analysis
:param work_dict: task dictionary
"""
label = work_dict.get('label', 'screener')
log.info('{} - start'.format(label))
rec = {}
res = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec=rec)
"""
Input - Set up dataset sources to collect
"""
ticker = work_dict.get('ticker', None)
org_tickers = work_dict.get('tickers', None)
if not ticker and not org_tickers:
res = build_result.build_result(status=ae_consts.ERR,
err='missing ticker or tickers',
rec=rec)
tickers = []
if not org_tickers:
if ticker:
tickers = [ticker]
else:
for t in org_tickers:
upper_cased_ticker = str(t).upper()
if upper_cased_ticker not in tickers:
tickers.append(upper_cased_ticker)
# build a unique ticker list
# end of ensuring tickers is a unique list of
# upper-cased ticker symbol strings
# fetch from: 'all', 'iex' or 'yahoo'
fetch_mode = work_dict.get('fetch_mode', os.getenv('FETCH_MODE', 'iex'))
iex_datasets = work_dict.get(
'iex_datasets',
os.getenv('IEX_DATASETS_DEFAULT', ae_consts.IEX_DATASETS_DEFAULT))
# if defined, these are task functions for
# calling customiized determine Celery tasks
determine_sells_callback = work_dict.get('determine_sells', None)
determine_buys_callback = work_dict.get('determine_buys', None)
try:
log.info('{} fetch={} tickers={} '
'iex_datasets={} '
'sell_task={} '
'buy_task={}'.format(label, fetch_mode, tickers, iex_datasets,
determine_sells_callback,
determine_buys_callback))
"""
Input - Set up required urls for building buckets
"""
fv_urls = work_dict.get('urls', )
if not fv_urls:
res = build_result.build_result(
status=ae_consts.ERR,
err='missing required urls list of screeners',
rec=rec)
# stop if something errored out with the
# celery helper for turning off celery to debug
# without an engine running
if res['err']:
log.error('{} - tickers={} fetch={} iex_datasets={} '
'hit validation err={}'.format(label, tickers,
fetch_mode, iex_datasets,
res['err']))
return get_task_results.get_task_results(work_dict=work_dict,
result=res)
# end of input validation checks
num_urls = len(fv_urls)
log.info('{} - running urls={}'.format(label, fv_urls))
fv_dfs = []
for uidx, url in enumerate(fv_urls):
log.info('{} - url={}/{} url={}'.format(label, uidx, num_urls,
url))
fv_res = finviz_utils.fetch_tickers_from_screener(url=url)
if fv_res['status'] == ae_consts.SUCCESS:
fv_dfs.append(fv_res['rec']['data'])
for ft_tick in fv_res['rec']['tickers']:
upper_ft_ticker = ft_tick.upper()
if upper_ft_ticker not in tickers:
tickers.append(upper_ft_ticker)
# end of for all found tickers
else:
log.error('{} - failed url={}/{} url={}'.format(
label, uidx, num_urls, url))
# if success vs log the error
# end of urls to get pandas.DataFrame and unique tickers
"""
Find tickers in screens
"""
num_tickers = len(tickers)
log.info('{} - fetching tickers={} from urls={}'.format(
label, num_tickers, num_urls))
"""
pull ticker data
"""
fetch_recs = fetch_utils.fetch(tickers=tickers,
fetch_mode=fetch_mode,
iex_datasets=iex_datasets)
if fetch_recs:
rec = fetch_recs
"""
Output - Where is data getting cached and archived?
(this helps to retroactively evaluate trading performance)
"""
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
else:
err = ('{} - tickers={} failed fetch={} '
'iex_datasets={}'.format(label, tickers, fetch_mode,
iex_datasets))
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=rec)
log.info('{} - done'.format(label))
except Exception as e:
err = ('{} - tickers={} fetch={} hit ex={} '.format(
label, tickers, fetch_mode, e))
log.error(err)
res = build_result.build_result(status=ae_consts.ERR, err=err, rec=rec)
# end of try/ex
return get_task_results.get_task_results(work_dict=work_dict, result=res)
def get_data_from_iex(work_dict):
"""get_data_from_iex
Get data from IEX - this requires an account
:param work_dict: request dictionary
"""
label = 'get_data_from_iex'
log.debug(f'task - {label} - start ' f'work_dict={work_dict}')
rec = {'data': None, 'updated': None}
res = {'status': ae_consts.NOT_RUN, 'err': None, 'rec': rec}
ticker = None
field = None
ft_type = None
try:
ticker = work_dict.get('ticker', ae_consts.TICKER)
field = work_dict.get('field', 'daily')
ft_type = work_dict.get('ft_type', None)
ft_str = str(ft_type).lower()
label = work_dict.get('label', label)
orient = work_dict.get('orient', 'records')
backfill_date = work_dict.get('backfill_date', None)
iex_req = None
if ft_type == iex_consts.FETCH_DAILY or ft_str == 'daily':
ft_type == iex_consts.FETCH_DAILY
iex_req = api_requests.build_iex_fetch_daily_request(label=label)
elif ft_type == iex_consts.FETCH_MINUTE or ft_str == 'minute':
ft_type == iex_consts.FETCH_MINUTE
iex_req = api_requests.build_iex_fetch_minute_request(label=label)
elif ft_type == iex_consts.FETCH_QUOTE or ft_str == 'quote':
ft_type == iex_consts.FETCH_QUOTE
iex_req = api_requests.build_iex_fetch_quote_request(label=label)
elif ft_type == iex_consts.FETCH_STATS or ft_str == 'stats':
ft_type == iex_consts.FETCH_STATS
iex_req = api_requests.build_iex_fetch_stats_request(label=label)
elif ft_type == iex_consts.FETCH_PEERS or ft_str == 'peers':
ft_type == iex_consts.FETCH_PEERS
iex_req = api_requests.build_iex_fetch_peers_request(label=label)
elif ft_type == iex_consts.FETCH_NEWS or ft_str == 'news':
ft_type == iex_consts.FETCH_NEWS
iex_req = api_requests.build_iex_fetch_news_request(label=label)
elif ft_type == iex_consts.FETCH_FINANCIALS or ft_str == 'financials':
ft_type == iex_consts.FETCH_FINANCIALS
iex_req = api_requests.build_iex_fetch_financials_request(
label=label)
elif ft_type == iex_consts.FETCH_EARNINGS or ft_str == 'earnings':
ft_type == iex_consts.FETCH_EARNINGS
iex_req = api_requests.build_iex_fetch_earnings_request(
label=label)
elif ft_type == iex_consts.FETCH_DIVIDENDS or ft_str == 'dividends':
ft_type == iex_consts.FETCH_DIVIDENDS
iex_req = api_requests.build_iex_fetch_dividends_request(
label=label)
elif ft_type == iex_consts.FETCH_COMPANY or ft_str == 'company':
ft_type == iex_consts.FETCH_COMPANY
iex_req = api_requests.build_iex_fetch_company_request(label=label)
else:
log.error(f'{label} - unsupported ft_type={ft_type} '
f'ft_str={ft_str} ticker={ticker}')
raise NotImplementedError
# if supported fetch request type
iex_req['ticker'] = ticker
clone_keys = [
'ticker', 's3_address', 's3_bucket', 's3_key', 'redis_address',
'redis_db', 'redis_password', 'redis_key'
]
for k in clone_keys:
if k in iex_req:
iex_req[k] = work_dict.get(k, f'{k}-missing-in-{label}')
# end of cloning keys
if not iex_req:
err = (f'{label} - ticker={ticker} '
f'did not build an IEX request '
f'for work={work_dict}')
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=rec)
return res
else:
log.debug(f'{label} - ticker={ticker} '
f'field={field} '
f'orient={orient} fetch')
# if invalid iex request
df = None
try:
if 'from' in work_dict:
iex_req['from'] = datetime.datetime.strptime(
'%Y-%m-%d %H:%M:%S', work_dict['from'])
if backfill_date:
iex_req['backfill_date'] = backfill_date
iex_req['redis_key'] = (f'{ticker}_{backfill_date}_{field}')
iex_req['s3_key'] = (f'{ticker}_{backfill_date}_{field}')
if os.getenv('SHOW_SUCCESS', '0') == '1':
log.info(f'fetching IEX {field} req={iex_req}')
else:
log.debug(f'fetching IEX {field} req={iex_req}')
df = iex_fetch_data.fetch_data(work_dict=iex_req,
fetch_type=ft_type)
rec['data'] = df.to_json(orient=orient, date_format='iso')
rec['updated'] = datetime.datetime.utcnow().strftime(
'%Y-%m-%d %H:%M:%S')
except Exception as f:
log.error(f'{label} - ticker={ticker} field={ft_type} '
f'failed fetch_data '
f'with ex={f}')
# end of try/ex
if ae_consts.ev('DEBUG_IEX_DATA', '0') == '1':
log.debug(f'{label} ticker={ticker} '
f'field={field} data={rec["data"]} to_json')
else:
log.debug(f'{label} ticker={ticker} field={field} to_json')
# end of if/else found data
upload_and_cache_req = copy.deepcopy(iex_req)
upload_and_cache_req['celery_disabled'] = True
upload_and_cache_req['data'] = rec['data']
if not upload_and_cache_req['data']:
upload_and_cache_req['data'] = '{}'
use_field = field
if use_field == 'news':
use_field = 'news1'
if 'redis_key' in work_dict:
rk = work_dict.get('redis_key', iex_req['redis_key'])
if backfill_date:
rk = f'{ticker}_{backfill_date}'
upload_and_cache_req['redis_key'] = (f'{rk}_{use_field}')
if 's3_key' in work_dict:
sk = work_dict.get('s3_key', iex_req['s3_key'])
if backfill_date:
sk = f'{ticker}_{backfill_date}'
upload_and_cache_req['s3_key'] = (f'{sk}_{use_field}')
try:
update_res = publisher.run_publish_pricing_update(
work_dict=upload_and_cache_req)
update_status = update_res.get('status', ae_consts.NOT_SET)
log.debug(f'{label} publish update '
f'status={ae_consts.get_status(status=update_status)} '
f'data={update_res}')
except Exception:
err = (f'{label} - failed to upload iex '
f'data={upload_and_cache_req} to '
f'to s3_key={upload_and_cache_req["s3_key"]} '
f'and redis_key={upload_and_cache_req["redis_key"]}')
log.error(err)
# end of try/ex to upload and cache
if not rec['data']:
log.debug(f'{label} - ticker={ticker} no IEX data '
f'field={field} to publish')
# end of if/else
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
res = build_result.build_result(status=ae_consts.ERR,
err=(f'failed - get_data_from_iex '
f'dict={work_dict} with ex={e}'),
rec=rec)
# end of try/ex
log.debug(f'task - get_data_from_iex done - '
f'{label} - '
f'status={ae_consts.get_status(res["status"])} '
f'err={res["err"]}')
return res
def set_data_in_redis_key(label=None,
data=None,
client=None,
host=None,
port=None,
password=None,
db=None,
key=None,
expire=None,
px=None,
nx=False,
xx=False,
serializer='json',
encoding='utf-8'):
"""set_data_in_redis_key
:param label: log tracking label
:param data: data to set in redis
:param client: initialized redis client
:param host: not used yet - redis host
:param port: not used yet - redis port
:param password: not used yet - redis password
:param db: not used yet - redis db
:param key: not used yet - redis key
:param expire: redis expire
:param px: redis px
:param nx: redis nx
:param xx: redis xx
:param serializer: not used yet - support for future
pickle objects in redis
:param encoding: format of the encoded key in redis
"""
data_str = None
encoded_data = None
rec = {}
res = build_result.build_result(status=NOT_RUN, err=None, rec=rec)
log_id = label if label else 'set-redis'
try:
log.debug('{} serializer={} encoding={} for key={}'.format(
log_id, serializer, encoding, key))
if serializer == 'json':
data_str = json.dumps(data)
encoded_data = data_str.encode(encoding)
else:
encoded_data = None
err = ('{} unsupported serializer={} '
'encoding={} key={}'.format(log_id, serializer, encoding,
key))
log.error(err)
res = build_result.build_result(status=ERR, err=err, rec=rec)
return res
# if supported serializer
if encoded_data:
if ev('DEBUG_REDIS', '0') == '1':
log.debug('{} set - key={} data={}'.format(
log_id, key, encoded_data))
use_client = client
if not use_client:
log.debug('{} set key={} new client={}:{}@{}'.format(
log_id, key, host, port, db))
use_client = redis.Redis(host=host,
port=port,
password=password,
db=db)
else:
log.debug('{} set key={} client'.format(log_id, key))
# create Redis client if not set
use_client.set(name=key,
value=encoded_data,
ex=expire,
px=px,
nx=nx,
xx=xx)
res = build_result.build_result(status=SUCCESS, err=None, rec=rec)
return res
else:
err = ('{} no data for key={}'.format(log_id, key))
log.error(err)
res = build_result.build_result(status=ERR, err=err, rec=rec)
return res
# end of if have data to set
except Exception as e:
err = ('{} failed - redis set from data={} encoded_data={} '
'key={} ex={}'.format(log_id, data, encoded_data, key, e))
log.error(err)
res = build_result.build_result(status=ERR, err=err, rec=rec)
# end of try/ex for setting redis data
return res
def run_distributed_algorithm(self, algo_req):
"""run_distributed_algorithm
Process an Algorithm using a Celery task that is
processed by a Celery worker
:param algo_req: dictionary for key/values for
running an algorithm using Celery workers
"""
label = algo_req.get('name', 'ae-algo')
verbose = algo_req.get('verbose_task', False)
debug = algo_req.get('debug', False)
# please be careful logging prod passwords:
if debug:
log.info(f'task - {label} - start algo_req={algo_req}')
elif verbose:
log.info(f'task - {label} - start ')
# end of start log
rec = {}
res = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec=rec)
created_algo_object = None
custom_algo_module = None
new_algo_object = None
use_custom_algo = False
found_algo_module = True # assume the BaseAlgo
should_publish_extract_dataset = False
should_publish_history_dataset = False
should_publish_report_dataset = False
ticker = algo_req.get('ticker', 'SPY')
num_days_back = algo_req.get('num_days_back', 75)
name = algo_req.get('name', 'ae-algo')
algo_module_path = algo_req.get('mod_path', None)
module_name = algo_req.get('module_name', 'BaseAlgo')
custom_algo_module = algo_req.get('custom_algo_module', None)
new_algo_object = algo_req.get('new_algo_object', None)
use_custom_algo = algo_req.get('use_custom_algo', False)
should_publish_extract_dataset = algo_req.get(
'should_publish_extract_dataset', False)
should_publish_history_dataset = algo_req.get(
'should_publish_history_dataset', False)
should_publish_report_dataset = algo_req.get(
'should_publish_report_dataset', False)
start_date = algo_req.get('start_date', None)
end_date = algo_req.get('end_date', None)
raise_on_err = algo_req.get('raise_on_err', True)
report_config = algo_req.get('report_config', None)
history_config = algo_req.get('history_config', None)
extract_config = algo_req.get('extract_config', None)
err = None
if algo_module_path:
found_algo_module = False
module_name = algo_module_path.split('/')[-1]
loader = importlib.machinery.SourceFileLoader(module_name,
algo_module_path)
custom_algo_module = types.ModuleType(loader.name)
loader.exec_module(custom_algo_module)
use_custom_algo = True
for member in inspect.getmembers(custom_algo_module):
if module_name in str(member):
found_algo_module = True
break
# for all members in this custom module file
# if loading a custom algorithm module from a file on disk
if not found_algo_module:
err = (f'{label} - unable to find custom algorithm '
f'module={custom_algo_module} module_path={algo_module_path}')
if algo_module_path:
err = (
f'{label} - analysis_engine.'
'work_tasks.run_distributed_algorithm was unable '
f'to find custom algorithm module={custom_algo_module} with '
f'provided path to \n file: {algo_module_path} \n'
'\n'
'Please confirm '
'that the class inherits from the BaseAlgo class like:\n'
'\n'
'import analysis_engine.algo\n'
'class MyAlgo(analysis_engine.algo.BaseAlgo):\n '
'\n'
'If it is then please file an issue on github:\n '
'https://github.com/AlgoTraders/stock-analysis-engine/'
'issues/new \n\nFor now this error results in a shutdown'
'\n')
# if algo_module_path set
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# if not found_algo_module
use_start_date = start_date
use_end_date = end_date
if not use_end_date:
end_date = datetime.datetime.utcnow()
use_end_date = end_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT)
if not use_start_date:
start_date = end_date - datetime.timedelta(days=num_days_back)
use_start_date = start_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT)
dataset_publish_extract = algo_req.get('dataset_publish_extract', False)
dataset_publish_history = algo_req.get('dataset_publish_history', False)
dataset_publish_report = algo_req.get('dataset_publish_report', False)
try:
if use_custom_algo:
if verbose:
log.info(
f'inspecting {custom_algo_module} for class {module_name}')
use_class_member_object = None
for member in inspect.getmembers(custom_algo_module):
if module_name in str(member):
if verbose:
log.info(f'start {name} with {member[1]}')
use_class_member_object = member
break
# end of looking over the class definition but did not find it
if use_class_member_object:
if algo_req.get('backtest', False):
new_algo_object = member[1](ticker=algo_req['ticker'],
config_dict=algo_req)
else:
new_algo_object = member[1](**algo_req)
else:
err = (f'{label} - did not find a derived '
'analysis_engine.algo.BaseAlgo '
f'class in the module file={algo_module_path} '
f'for ticker={ticker} algo_name={name}')
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# end of finding a valid algorithm object
else:
new_algo_object = ae_algo.BaseAlgo(**algo_req)
# if using a custom module path or the BaseAlgo
if new_algo_object:
# heads up - logging this might have passwords in the algo_req
# log.debug(f'{name} algorithm request: {algo_req}')
if verbose:
log.info(f'{name} - run START ticker={ticker} '
f'from {use_start_date} to {use_end_date}')
if algo_req.get('backtest', False):
algo_res = run_algo.run_algo(algo=new_algo_object,
config_dict=algo_req)
created_algo_object = new_algo_object
else:
algo_res = run_algo.run_algo(algo=new_algo_object, **algo_req)
created_algo_object = new_algo_object
if verbose:
log.info(f'{name} - run DONE ticker={ticker} '
f'from {use_start_date} to {use_end_date}')
if debug:
if custom_algo_module:
log.info(f'{name} - done run_algo '
f'custom_algo_module={custom_algo_module} '
f'module_name={module_name} ticker={ticker} '
f'from {use_start_date} to {use_end_date}')
else:
log.info(
f'{name} - done run_algo BaseAlgo ticker={ticker} '
f'from {use_start_date} to {use_end_date}')
else:
err = (
f'{label} - missing a derived analysis_engine.algo.BaseAlgo '
f'class in the module file={algo_module_path} for '
f'ticker={ticker} algo_name={name}')
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# end of finding a valid algorithm object
if not created_algo_object:
err = (f'{label} - failed creating algorithm object - '
f'ticker={ticker} '
f'status={ae_consts.get_status(status=algo_res["status"])} '
f'error={algo_res["err"]} algo name={name} '
f'custom_algo_module={custom_algo_module} '
f'module_name={module_name} '
f'from {use_start_date} to {use_end_date}')
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# end of stop early
if should_publish_extract_dataset or dataset_publish_extract:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (extract_config['redis_address'] and extract_config['redis_db']
and extract_config['redis_key']):
redis_log = (f'redis://{extract_config["redis_address"]}'
f'@{extract_config["redis_db"]}'
f'/{extract_config["redis_key"]}')
use_log += f' {redis_log}'
else:
extract_config['redis_enabled'] = False
if (extract_config['s3_address'] and extract_config['s3_bucket']
and extract_config['s3_key']):
s3_log = (f's3://{extract_config["s3_address"]}'
f'/{extract_config["s3_bucket"]}'
f'/{extract_config["s3_key"]}')
use_log += f' {s3_log}'
else:
extract_config['s3_enabled'] = False
if extract_config['output_file']:
file_log = f'file:{extract_config["output_file"]}'
use_log += f' {file_log}'
if verbose:
log.info(f'{name} - publish - start ticker={ticker} '
f'algorithm-ready {use_log}')
publish_status = created_algo_object.publish_input_dataset(
**extract_config)
if publish_status != ae_consts.SUCCESS:
msg = ('failed to publish algorithm-ready datasets with '
f'status {ae_consts.get_status(status=publish_status)} '
f'attempted to {use_log}')
log.error(msg)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# end of stop early
if verbose:
log.info(f'{name} - publish - done ticker={ticker} '
f'algorithm-ready {use_log}')
# if publish the algorithm-ready dataset
if should_publish_history_dataset or dataset_publish_history:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (history_config['redis_address'] and history_config['redis_db']
and history_config['redis_key']):
redis_log = (f'redis://{history_config["redis_address"]}'
f'@{history_config["redis_db"]}'
f'/{history_config["redis_key"]}')
use_log += f' {redis_log}'
if (history_config['s3_address'] and history_config['s3_bucket']
and history_config['s3_key']):
s3_log = (f's3://{history_config["s3_address"]}'
f'/{history_config["s3_bucket"]}'
f'/{history_config["s3_key"]}')
use_log += f' {s3_log}'
if history_config['output_file']:
file_log = f'file:{history_config["output_file"]}'
use_log += f' {file_log}'
if verbose:
log.info(f'{name} - publish - start ticker={ticker} trading '
f'history {use_log}')
publish_status = \
created_algo_object.publish_trade_history_dataset(
**history_config)
if publish_status != ae_consts.SUCCESS:
msg = ('failed to publish trading history datasets with '
f'status {ae_consts.get_status(status=publish_status)} '
f'attempted to {use_log}')
log.error(msg)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# end of stop early
if verbose:
log.info(f'{name} - publish - done ticker={ticker} trading '
f'history {use_log}')
# if publish an trading history dataset
if should_publish_report_dataset or dataset_publish_report:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (report_config['redis_address'] and report_config['redis_db']
and report_config['redis_key']):
redis_log = (f'redis://{report_config["redis_address"]}'
f'@{report_config["redis_db"]}'
f'/{report_config["redis_key"]}')
use_log += f' {redis_log}'
if (report_config['s3_address'] and report_config['s3_bucket']
and report_config['s3_key']):
s3_log = (f's3://{report_config["s3_address"]}'
f'/{report_config["s3_bucket"]}'
f'/{report_config["s3_key"]}')
use_log += f' {s3_log}'
if report_config['output_file']:
file_log = f' file:{report_config["output_file"]}'
use_log += f' {file_log}'
if verbose:
log.info(
f'{name} - publishing ticker={ticker} trading performance '
f'report {use_log}')
publish_status = created_algo_object.publish_report_dataset(
**report_config)
if publish_status != ae_consts.SUCCESS:
msg = ('failed to publish trading performance '
'report datasets with '
f'status {ae_consts.get_status(status=publish_status)} '
f'attempted to {use_log}')
log.error(msg)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# end of stop early
if verbose:
log.info(f'{name} - publish - done ticker={ticker} trading '
f'performance report {use_log}')
# if publish an trading performance report dataset
if verbose:
log.info(f'{name} - done publishing datasets for ticker={ticker} '
f'from {use_start_date} to {use_end_date}')
rec['history_config'] = history_config
rec['report_config'] = report_config
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
res = build_result.build_result(
status=ae_consts.ERR,
err=('failed - run_distributed_algorithm '
f'dict={algo_req} with ex={e}'),
rec=rec)
if raise_on_err:
raise e
else:
log.error(f'{label} - {res["err"]}')
# end of try/ex
if verbose:
log.info('task - run_distributed_algorithm done - '
f'{label} - status={ae_consts.get_status(res["status"])}')
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
def get_data_from_td(work_dict):
"""get_data_from_td
Get pricing data from Tradier
:param work_dict: request dictionary
"""
label = 'get_data_from_td'
log.debug(f'task - {label} - start work_dict={work_dict}')
rec = {'data': None, 'updated': None}
res = {'status': ae_consts.NOT_RUN, 'err': None, 'rec': rec}
ticker = None
field = None
ft_type = None
try:
ticker = work_dict.get('ticker', ae_consts.TICKER)
field = work_dict.get('field', 'daily')
ft_type = work_dict.get('ft_type', None)
ft_str = str(ft_type).lower()
label = work_dict.get('label', label)
orient = work_dict.get('orient', 'records')
td_req = None
if ft_type == td_consts.FETCH_TD_CALLS or ft_str == 'tdcalls':
ft_type == td_consts.FETCH_TD_CALLS
td_req = api_requests.build_td_fetch_calls_request(label=label)
elif ft_type == td_consts.FETCH_TD_PUTS or ft_str == 'tdputs':
ft_type == td_consts.FETCH_TD_PUTS
td_req = api_requests.build_td_fetch_puts_request(label=label)
else:
log.error(
f'{label} - unsupported ft_type={ft_type} ft_str={ft_str} '
f'ticker={ticker}')
raise NotImplementedError
# if supported fetch request type
clone_keys = [
'ticker', 's3_address', 's3_bucket', 's3_key', 'redis_address',
'redis_db', 'redis_password', 'redis_key'
]
for k in clone_keys:
td_req[k] = work_dict.get(k, f'{k}-missing-in-{label}')
# end of cloning keys
if not td_req:
err = (f'{label} - ticker={td_req["ticker"]} did not build a TD '
f'request for work={work_dict}')
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=rec)
return res
else:
log.debug(f'{label} - ticker={td_req["ticker"]} field={field} '
f'orient={orient} fetch')
# if invalid td request
df = None
try:
if 'from' in work_dict:
td_req['from'] = datetime.datetime.strptime(
'%Y-%m-%d %H:%M:%S', work_dict['from'])
status_df, df = td_fetch_data.fetch_data(work_dict=td_req,
fetch_type=ft_type)
if status_df == ae_consts.SUCCESS:
rec['data'] = df.to_json(orient=orient)
rec['updated'] = datetime.datetime.utcnow().strftime(
'%Y-%m-%d %H:%M:%S')
elif status_df == ae_consts.EMPTY:
res = build_result.build_result(
status=ae_consts.ERR,
err=(f'did not fetch any data'),
rec=rec)
return res
else:
err = (f'{label} - ticker={td_req["ticker"]} '
f'td_fetch_data.fetch_data field={ft_type} '
'failed fetch_data')
log.critical(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=rec)
return res
except Exception as f:
err = (f'{label} - ticker={td_req["ticker"]} field={ft_type} '
f'failed fetch_data with ex={f}')
log.critical(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=rec)
return res
# end of try/ex
if ae_consts.ev('DEBUG_TD_DATA', '0') == '1':
log.debug(f'{label} ticker={td_req["ticker"]} field={field} '
f'data={rec["data"]} to_json')
else:
log.debug(
f'{label} ticker={td_req["ticker"]} field={field} to_json')
# end of if/else found data
upload_and_cache_req = copy.deepcopy(td_req)
upload_and_cache_req['celery_disabled'] = True
upload_and_cache_req['data'] = rec['data']
if not upload_and_cache_req['data']:
upload_and_cache_req['data'] = '{}'
use_field = field
if use_field == 'news':
use_field = 'news1'
if 'redis_key' in work_dict:
upload_and_cache_req['redis_key'] = (f'''{work_dict.get(
'redis_key',
td_req['redis_key'])}_'''
f'{use_field}')
if 's3_key' in work_dict:
upload_and_cache_req['s3_key'] = (f'''{work_dict.get(
's3_key',
td_req['s3_key'])}_'''
f'{use_field}')
try:
update_res = publisher.run_publish_pricing_update(
work_dict=upload_and_cache_req)
update_status = update_res.get('status', ae_consts.NOT_SET)
log.debug(f'{label} publish update '
f'status={ae_consts.get_status(status=update_status)} '
f'data={update_res}')
except Exception:
err = (
f'{label} - failed to upload td data={upload_and_cache_req} '
f'to s3_key={upload_and_cache_req["s3_key"]} and '
f'redis_key={upload_and_cache_req["redis_key"]}')
log.error(err)
# end of try/ex to upload and cache
if not rec['data']:
log.debug(f'{label} - ticker={td_req["ticker"]} no Tradier data '
f'field={field} to publish')
# end of if/else
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
res = build_result.build_result(status=ae_consts.ERR,
err=('failed - get_data_from_td '
f'dict={work_dict} with ex={e}'),
rec=rec)
# end of try/ex
log.debug('task - get_data_from_td done - '
f'{label} - status={ae_consts.get_status(res["status"])} '
f'err={res["err"]}')
return res
def get_new_pricing_data(self, work_dict):
"""get_new_pricing_data
Get Ticker information on:
- prices - turn off with ``work_dict.get_pricing = False``
- news - turn off with ``work_dict.get_news = False``
- options - turn off with ``work_dict.get_options = False``
:param work_dict: dictionary for key/values
"""
label = 'get_new_pricing_data'
log.debug(f'task - {label} - start ' f'work_dict={work_dict}')
num_success = 0
ticker = ae_consts.TICKER
ticker_id = ae_consts.TICKER_ID
rec = {
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'daily': None,
'minute': None,
'quote': None,
'stats': None,
'peers': None,
'iex_news': None,
'financials': None,
'earnings': None,
'dividends': None,
'company': None,
'exp_date': None,
'publish_pricing_update': None,
'num_success': num_success,
'date': ae_utils.utc_now_str(),
'updated': None,
'version': ae_consts.DATASET_COLLECTION_VERSION
}
res = {'status': ae_consts.NOT_RUN, 'err': None, 'rec': rec}
try:
ticker = work_dict.get('ticker', ticker)
ticker_id = work_dict.get('ticker_id', ae_consts.TICKER_ID)
s3_bucket = work_dict.get('s3_bucket', ae_consts.S3_BUCKET)
s3_key = work_dict.get('s3_key', ae_consts.S3_KEY)
redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY)
exp_date = work_dict.get('exp_date', None)
cur_date = ae_utils.last_close()
cur_strike = work_dict.get('strike', None)
contract_type = str(work_dict.get('contract', 'C')).upper()
label = work_dict.get('label', label)
iex_datasets = work_dict.get('iex_datasets',
iex_consts.DEFAULT_FETCH_DATASETS)
td_datasets = work_dict.get('td_datasets',
td_consts.DEFAULT_FETCH_DATASETS_TD)
fetch_mode = work_dict.get('fetch_mode', ae_consts.FETCH_MODE_ALL)
iex_token = work_dict.get('iex_token', iex_consts.IEX_TOKEN)
td_token = work_dict.get('td_token', td_consts.TD_TOKEN)
str_fetch_mode = str(fetch_mode).lower()
# control flags to deal with feed issues:
get_iex_data = True
get_td_data = True
if (fetch_mode == ae_consts.FETCH_MODE_ALL
or str_fetch_mode == 'initial'):
get_iex_data = True
get_td_data = True
iex_datasets = ae_consts.IEX_INITIAL_DATASETS
elif (fetch_mode == ae_consts.FETCH_MODE_ALL
or str_fetch_mode == 'all'):
get_iex_data = True
get_td_data = True
iex_datasets = ae_consts.IEX_DATASETS_DEFAULT
elif (fetch_mode == ae_consts.FETCH_MODE_YHO
or str_fetch_mode == 'yahoo'):
get_iex_data = False
get_td_data = False
elif (fetch_mode == ae_consts.FETCH_MODE_IEX
or str_fetch_mode == 'iex-all'):
get_iex_data = True
get_td_data = False
iex_datasets = ae_consts.IEX_DATASETS_DEFAULT
elif (fetch_mode == ae_consts.FETCH_MODE_IEX
or str_fetch_mode == 'iex'):
get_iex_data = True
get_td_data = False
iex_datasets = ae_consts.IEX_INTRADAY_DATASETS
elif (fetch_mode == ae_consts.FETCH_MODE_INTRADAY
or str_fetch_mode == 'intra'):
get_iex_data = True
get_td_data = True
iex_datasets = ae_consts.IEX_INTRADAY_DATASETS
elif (fetch_mode == ae_consts.FETCH_MODE_DAILY
or str_fetch_mode == 'daily'):
get_iex_data = True
get_td_data = False
iex_datasets = ae_consts.IEX_DAILY_DATASETS
elif (fetch_mode == ae_consts.FETCH_MODE_WEEKLY
or str_fetch_mode == 'weekly'):
get_iex_data = True
get_td_data = False
iex_datasets = ae_consts.IEX_WEEKLY_DATASETS
elif (fetch_mode == ae_consts.FETCH_MODE_TD or str_fetch_mode == 'td'):
get_iex_data = False
get_td_data = True
else:
get_iex_data = False
get_td_data = False
fetch_arr = str_fetch_mode.split(',')
found_fetch = False
iex_datasets = []
for fetch_name in fetch_arr:
if fetch_name not in iex_datasets:
if fetch_name == 'iex_min':
iex_datasets.append('minute')
elif fetch_name == 'iex_day':
iex_datasets.append('daily')
elif fetch_name == 'iex_quote':
iex_datasets.append('quote')
elif fetch_name == 'iex_stats':
iex_datasets.append('stats')
elif fetch_name == 'iex_peers':
iex_datasets.append('peers')
elif fetch_name == 'iex_news':
iex_datasets.append('news')
elif fetch_name == 'iex_fin':
iex_datasets.append('financials')
elif fetch_name == 'iex_earn':
iex_datasets.append('earnings')
elif fetch_name == 'iex_div':
iex_datasets.append('dividends')
elif fetch_name == 'iex_comp':
iex_datasets.append('company')
elif fetch_name == 'td':
get_td_data = True
else:
log.warn('unsupported IEX dataset ' f'{fetch_name}')
found_fetch = (len(iex_datasets) != 0)
if not found_fetch:
log.error(f'{label} - unsupported '
f'fetch_mode={fetch_mode} value')
else:
get_iex_data = True
log.debug(f'{label} - '
f'fetching={len(iex_datasets)} '
f'{iex_datasets} '
f'fetch_mode={fetch_mode}')
# end of screening custom fetch_mode settings
num_tokens = 0
if get_iex_data:
if not iex_token:
log.warn(f'{label} - '
'please set a valid IEX Cloud Account token ('
'https://iexcloud.io/cloud-login/#/register'
') to fetch data from IEX Cloud. It must be '
'set as an environment variable like: '
'export IEX_TOKEN=<token>')
get_iex_data = False
else:
num_tokens += 1
# sanity check - disable IEX fetch if the token is not set
if get_td_data:
missing_td_token = [
'MISSING_TD_TOKEN', 'SETYOURTDTOKEN', 'SETYOURTRADIERTOKENHERE'
]
if td_token in missing_td_token:
log.warn(f'{label} - '
'please set a valid Tradier Account token ('
'https://developer.tradier.com/user/sign_up'
') to fetch pricing data from Tradier. It must be '
'set as an environment variable like: '
'export TD_TOKEN=<token>')
get_td_data = False
else:
num_tokens += 1
# sanity check - disable Tradier fetch if the token is not set
"""
as of Thursday, Jan. 3, 2019:
https://developer.yahoo.com/yql/
Important EOL Notice: As of Thursday, Jan. 3, 2019
the YQL service at query.yahooapis.com will be retired
"""
get_yahoo_data = False
if (not get_iex_data and not get_td_data and not get_yahoo_data):
err = None
if num_tokens == 0:
res['status'] = ae_consts.MISSING_TOKEN
err = (f'Please set a valid IEX_TOKEN or TD_TOKEN '
f'environment variable')
else:
err = (f'Please set at least one supported datafeed from '
f'either: '
f'IEX Cloud (fetch -t TICKER -g iex) or '
f'Tradier (fetch -t TICKER -g td) '
f'for '
f'ticker={ticker} '
f'cur_date={cur_date} '
f'IEX enabled={get_iex_data} '
f'TD enabled={get_td_data} '
f'YHO enabled={get_yahoo_data}')
res['status'] = ae_consts.ERR
res['err'] = err
return get_task_results.get_task_results(work_dict=work_dict,
result=res)
# end of checking that there is at least 1 feed on
if not exp_date:
exp_date = opt_dates.option_expiration(date=exp_date)
else:
exp_date = datetime.datetime.strptime(exp_date, '%Y-%m-%d')
rec['updated'] = cur_date.strftime('%Y-%m-%d %H:%M:%S')
log.debug(f'{label} getting pricing for ticker={ticker} '
f'cur_date={cur_date} exp_date={exp_date} '
f'IEX={get_iex_data} '
f'TD={get_td_data} '
f'YHO={get_yahoo_data}')
yahoo_rec = {
'ticker': ticker,
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'exp_date': None,
'publish_pricing_update': None,
'date': None,
'updated': None
}
# disabled on 2019-01-03
if get_yahoo_data:
log.debug(f'{label} YHO ticker={ticker}')
yahoo_res = yahoo_data.get_data_from_yahoo(work_dict=work_dict)
status_str = ae_consts.get_status(status=yahoo_res['status'])
if yahoo_res['status'] == ae_consts.SUCCESS:
yahoo_rec = yahoo_res['rec']
msg = (f'{label} YHO ticker={ticker} '
f'status={status_str} err={yahoo_res["err"]}')
if ae_consts.ev('SHOW_SUCCESS', '0') == '1':
log.info(msg)
else:
log.debug(msg)
rec['pricing'] = yahoo_rec.get('pricing', '{}')
rec['news'] = yahoo_rec.get('news', '{}')
rec['options'] = yahoo_rec.get('options', '{}')
rec['calls'] = rec['options'].get('calls',
ae_consts.EMPTY_DF_STR)
rec['puts'] = rec['options'].get('puts',
ae_consts.EMPTY_DF_STR)
num_success += 1
else:
log.error(f'{label} failed YHO ticker={ticker} '
f'status={status_str} err={yahoo_res["err"]}')
# end of get from yahoo
if get_iex_data:
num_iex_ds = len(iex_datasets)
log.debug(f'{label} IEX datasets={num_iex_ds}')
for idx, ft_type in enumerate(iex_datasets):
dataset_field = iex_consts.get_ft_str(ft_type=ft_type)
log.debug(f'{label} IEX={idx}/{num_iex_ds} '
f'field={dataset_field} ticker={ticker}')
iex_label = f'{label}-{dataset_field}'
iex_req = copy.deepcopy(work_dict)
iex_req['label'] = iex_label
iex_req['ft_type'] = ft_type
iex_req['field'] = dataset_field
iex_req['ticker'] = ticker
iex_res = iex_data.get_data_from_iex(work_dict=iex_req)
status_str = (ae_consts.get_status(status=iex_res['status']))
if iex_res['status'] == ae_consts.SUCCESS:
iex_rec = iex_res['rec']
msg = (f'{label} IEX ticker={ticker} '
f'field={dataset_field} '
f'status={status_str} '
f'err={iex_res["err"]}')
if ae_consts.ev('SHOW_SUCCESS', '0') == '1':
log.info(msg)
else:
log.debug(msg)
if dataset_field == 'news':
rec['iex_news'] = iex_rec['data']
else:
rec[dataset_field] = iex_rec['data']
num_success += 1
else:
log.debug(f'{label} failed IEX ticker={ticker} '
f'field={dataset_field} '
f'status={status_str} err={iex_res["err"]}')
# end of if/else succcess
# end idx, ft_type in enumerate(iex_datasets):
# end of if get_iex_data
if get_td_data:
num_td_ds = len(td_datasets)
log.debug(f'{label} TD datasets={num_td_ds}')
for idx, ft_type in enumerate(td_datasets):
dataset_field = td_consts.get_ft_str_td(ft_type=ft_type)
log.debug(f'{label} TD={idx}/{num_td_ds} '
f'field={dataset_field} ticker={ticker}')
td_label = (f'{label}-{dataset_field}')
td_req = copy.deepcopy(work_dict)
td_req['label'] = td_label
td_req['ft_type'] = ft_type
td_req['field'] = dataset_field
td_req['ticker'] = ticker
td_res = td_data.get_data_from_td(work_dict=td_req)
status_str = (ae_consts.get_status(status=td_res['status']))
if td_res['status'] == ae_consts.SUCCESS:
td_rec = td_res['rec']
msg = (f'{label} TD ticker={ticker} '
f'field={dataset_field} '
f'status={status_str} '
f'err={td_res["err"]}')
if ae_consts.ev('SHOW_SUCCESS', '0') == '1':
log.info(msg)
else:
log.debug(msg)
if dataset_field == 'tdcalls':
rec['tdcalls'] = td_rec['data']
if dataset_field == 'tdputs':
rec['tdputs'] = td_rec['data']
else:
rec[dataset_field] = td_rec['data']
num_success += 1
else:
log.critical(f'{label} failed TD ticker={ticker} '
f'field={dataset_field} '
f'status={status_str} err={td_res["err"]}')
# end of if/else succcess
# end idx, ft_type in enumerate(td_datasets):
# end of if get_td_data
rec['num_success'] = num_success
update_req = {'data': rec}
update_req['ticker'] = ticker
update_req['ticker_id'] = ticker_id
update_req['strike'] = cur_strike
update_req['contract'] = contract_type
update_req['s3_enabled'] = work_dict.get('s3_enabled',
ae_consts.ENABLED_S3_UPLOAD)
update_req['redis_enabled'] = work_dict.get(
'redis_enabled', ae_consts.ENABLED_REDIS_PUBLISH)
update_req['s3_bucket'] = s3_bucket
update_req['s3_key'] = s3_key
update_req['s3_access_key'] = work_dict.get('s3_access_key',
ae_consts.S3_ACCESS_KEY)
update_req['s3_secret_key'] = work_dict.get('s3_secret_key',
ae_consts.S3_SECRET_KEY)
update_req['s3_region_name'] = work_dict.get('s3_region_name',
ae_consts.S3_REGION_NAME)
update_req['s3_address'] = work_dict.get('s3_address',
ae_consts.S3_ADDRESS)
update_req['s3_secure'] = work_dict.get('s3_secure',
ae_consts.S3_SECURE)
update_req['redis_key'] = redis_key
update_req['redis_address'] = work_dict.get('redis_address',
ae_consts.REDIS_ADDRESS)
update_req['redis_password'] = work_dict.get('redis_password',
ae_consts.REDIS_PASSWORD)
update_req['redis_db'] = int(
work_dict.get('redis_db', ae_consts.REDIS_DB))
update_req['redis_expire'] = work_dict.get('redis_expire',
ae_consts.REDIS_EXPIRE)
update_req['updated'] = rec['updated']
update_req['label'] = label
update_req['celery_disabled'] = True
update_status = ae_consts.NOT_SET
try:
update_res = publisher.run_publish_pricing_update(
work_dict=update_req)
update_status = update_res.get('status', ae_consts.NOT_SET)
status_str = ae_consts.get_status(status=update_status)
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
log.debug(f'{label} update_res '
f'status={status_str} '
f'data={ae_consts.ppj(update_res)}')
else:
log.debug(f'{label} run_publish_pricing_update '
f'status={status_str}')
# end of if/else
rec['publish_pricing_update'] = update_res
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as f:
err = (f'{label} publisher.run_publish_pricing_update failed '
f'with ex={f}')
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=rec)
# end of trying to publish results to connected services
except Exception as e:
res = build_result.build_result(status=ae_consts.ERR,
err=('failed - get_new_pricing_data '
f'dict={work_dict} with ex={e}'),
rec=rec)
log.error(f'{label} - {res["err"]}')
# end of try/ex
if ae_consts.ev('DATASET_COLLECTION_SLACK_ALERTS', '0') == '1':
env_name = 'DEV'
if ae_consts.ev('PROD_SLACK_ALERTS', '1') == '1':
env_name = 'PROD'
done_msg = (f'Dataset collected ticker=*{ticker}* on '
f'env=*{env_name}* '
f'redis_key={redis_key} s3_key={s3_key} '
f'IEX={get_iex_data} '
f'TD={get_td_data} '
f'YHO={get_yahoo_data}')
log.debug(f'{label} sending slack msg={done_msg}')
if res['status'] == ae_consts.SUCCESS:
slack_utils.post_success(msg=done_msg, block=False, jupyter=True)
else:
slack_utils.post_failure(msg=done_msg, block=False, jupyter=True)
# end of if/else success
# end of publishing to slack
log.debug('task - get_new_pricing_data done - '
f'{label} - status={ae_consts.get_status(res["status"])}')
return get_task_results.get_task_results(work_dict=work_dict, result=res)
def plot_dnn_fit_history(
title,
df,
red,
red_color=None,
red_label=None,
blue=None,
blue_color=None,
blue_label=None,
green=None,
green_color=None,
green_label=None,
orange=None,
orange_color=None,
orange_label=None,
xlabel='Training Epochs',
ylabel='Error Values',
linestyle='-',
width=9.0,
height=9.0,
date_format='%d\n%b',
df_filter=None,
start_date=None,
footnote_text=None,
footnote_xpos=0.70,
footnote_ypos=0.01,
footnote_color='#888888',
footnote_fontsize=8,
scale_y=False,
show_plot=True,
dropna_for_all=False,
verbose=False,
send_plots_to_slack=False):
"""plot_dnn_fit_history
Plot a DNN's fit history using `Keras fit history object <https://ker
as.io/visualization/#training-history-visualization>`__
:param title: title of the plot
:param df: dataset which is ``pandas.DataFrame``
:param red: string - column name to plot in
``red_color`` (or default ``ae_consts.PLOT_COLORS[red]``)
where the column is in the ``df`` and
accessible with:``df[red]``
:param red_color: hex color code to plot the data in the
``df[red]`` (default is ``ae_consts.PLOT_COLORS['red']``)
:param red_label: optional - string for the label used
to identify the ``red`` line in the legend
:param blue: string - column name to plot in
``blue_color`` (or default ``ae_consts.PLOT_COLORS['blue']``)
where the column is in the ``df`` and
accessible with:``df[blue]``
:param blue_color: hex color code to plot the data in the
``df[blue]`` (default is ``ae_consts.PLOT_COLORS['blue']``)
:param blue_label: optional - string for the label used
to identify the ``blue`` line in the legend
:param green: string - column name to plot in
``green_color`` (or default ``ae_consts.PLOT_COLORS['darkgreen']``)
where the column is in the ``df`` and
accessible with:``df[green]``
:param green_color: hex color code to plot the data in the
``df[green]`` (default is ``ae_consts.PLOT_COLORS['darkgreen']``)
:param green_label: optional - string for the label used
to identify the ``green`` line in the legend
:param orange: string - column name to plot in
``orange_color`` (or default ``ae_consts.PLOT_COLORS['orange']``)
where the column is in the ``df`` and
accessible with:``df[orange]``
:param orange_color: hex color code to plot the data in the
``df[orange]`` (default is ``ae_consts.PLOT_COLORS['orange']``)
:param orange_label: optional - string for the label used
to identify the ``orange`` line in the legend
:param xlabel: x-axis label
:param ylabel: y-axis label
:param linestyle: style of the plot line
:param width: float - width of the image
:param height: float - height of the image
:param date_format: string - format for dates
:param df_filter: optional - initialized ``pandas.DataFrame`` query
for reducing the ``df`` records before plotting. As an eaxmple
``df_filter=(df['close'] > 0.01)`` would find only records in
the ``df`` with a ``close`` value greater than ``0.01``
:param start_date: optional - string ``datetime``
for plotting only from a date formatted as
``YYYY-MM-DD HH\\:MM\\:SS``
:param footnote_text: optional - string footnote text
(default is ``algotraders <DATE>``)
:param footnote_xpos: optional - float for footnote position
on the x-axies
(default is ``0.75``)
:param footnote_ypos: optional - float for footnote position
on the y-axies
(default is ``0.01``)
:param footnote_color: optional - string hex color code for
the footnote text
(default is ``#888888``)
:param footnote_fontsize: optional - float footnote font size
(default is ``8``)
:param scale_y: optional - bool to scale the y-axis with
.. code-block:: python
use_ax.set_ylim(
[0, use_ax.get_ylim()[1] * 3])
:param show_plot: bool to show the plot
:param dropna_for_all: optional - bool to toggle keep None's in
the plot ``df`` (default is drop them for display purposes)
:param verbose: optional - bool to show logs for debugging
a dataset
:param send_plots_to_slack: optional - bool to send the dnn plot to slack
"""
rec = {
'ax1': None,
'fig': None
}
result = build_result.build_result(
status=ae_consts.NOT_RUN,
err=None,
rec=rec)
if verbose:
log.info(
f'plot_dnn_fit_history - start')
use_red = red_color
use_blue = blue_color
use_green = green_color
use_orange = orange_color
if not use_red:
use_red = ae_consts.PLOT_COLORS['red']
if not use_blue:
use_blue = ae_consts.PLOT_COLORS['blue']
if not use_green:
use_green = ae_consts.PLOT_COLORS['darkgreen']
if not use_orange:
use_orange = ae_consts.PLOT_COLORS['orange']
use_footnote = footnote_text
if not use_footnote:
use_footnote = f'''algotraders - {datetime.datetime.now().strftime(
ae_consts.COMMON_TICK_DATE_FORMAT)}'''
column_list = []
all_plots = []
if red:
column_list.append(red)
all_plots.append({
'column': red,
'color': use_red})
if blue:
column_list.append(blue)
all_plots.append({
'column': blue,
'color': use_blue})
if green:
column_list.append(green)
all_plots.append({
'column': green,
'color': use_green})
if orange:
column_list.append(orange)
all_plots.append({
'column': orange,
'color': use_orange})
use_df = df
if hasattr(df_filter, 'to_json'):
# Was seeing this warning below in Jupyter:
# UserWarning: Boolean Series key
# will be reindexed to match DataFrame index
# use_df = use_df[df_filter][column_list]
# now using:
use_df = use_df.loc[df_filter, column_list]
if verbose:
log.info(
f'plot_dnn_fit_history '
f'filter df.index={len(use_df.index)} '
f'column_list={column_list}')
ae_charts.set_common_seaborn_fonts()
hex_color = ae_consts.PLOT_COLORS['blue']
fig, ax = plt.subplots(
sharex=True,
sharey=True,
figsize=(
width,
height))
all_axes = [
ax
]
num_plots = len(all_plots)
for idx, node in enumerate(all_plots):
column_name = node['column']
hex_color = node['color']
use_ax = ax
if verbose:
log.info(
f'plot_dnn_fit_history - '
f'{idx + 1}/{num_plots} - '
f'{column_name} '
f'in '
f'{hex_color} - '
f'ax={use_ax}')
use_ax.plot(
use_df[column_name],
label=column_name,
linestyle=linestyle,
color=hex_color)
# end if this is not the fist axis
# end of for all plots
lines = []
for idx, cur_ax in enumerate(all_axes):
ax_lines = cur_ax.get_lines()
for line in ax_lines:
label_name = str(line.get_label())
use_label = label_name
if idx == 0:
if red_label:
use_label = red_label
elif idx == 1:
if blue_label:
use_label = blue_label
elif idx == 2:
use_label = label_name[-20:]
if green_label:
use_label = green_label
elif idx == 3:
use_label = label_name[-20:]
if orange_label:
use_label = orange_label
else:
if len(label_name) > 10:
use_label = label_name[-20:]
# end of fixing the labels in the legend
line.set_label(use_label)
if line.get_label() not in lines:
lines.append(line)
rec[f'ax{idx + 1}'] = cur_ax
# end of compiling a new-shortened legend while removing dupes
for idx, cur_ax in enumerate(all_axes):
if cur_ax:
if cur_ax.get_legend():
cur_ax.get_legend().remove()
# end of removing all previous legends
if verbose:
log.info(
f'legend lines={[l.get_label() for l in lines]}')
# log what's going to be in the legend
ax.legend(
lines,
[l.get_label() for l in lines],
loc='best',
shadow=True)
fig.autofmt_xdate()
plt.xlabel(xlabel)
plt.ylabel(ylabel)
ax.set_title(title)
ae_charts.add_footnote(
fig=fig,
xpos=footnote_xpos,
ypos=footnote_ypos,
text=use_footnote,
color=footnote_color,
fontsize=footnote_fontsize)
plt.tight_layout()
if send_plots_to_slack:
post_plot(plt, title=title)
if show_plot:
plt.show()
else:
plt.plot()
rec['fig'] = fig
result = build_result.build_result(
status=ae_consts.SUCCESS,
err=None,
rec=rec)
return result
def plot_overlay_pricing_and_volume(
log_label,
ticker,
df,
xlabel=None,
ylabel=None,
high_color=ae_consts.PLOT_COLORS['high'],
close_color=ae_consts.PLOT_COLORS['blue'],
volume_color=ae_consts.PLOT_COLORS['green'],
date_format=ae_consts.IEX_MINUTE_DATE_FORMAT,
show_plot=True,
dropna_for_all=True):
"""plot_overlay_pricing_and_volume
Plot pricing (high, low, open, close) and volume as
an overlay off the x-axis
Here is a sample chart from the
`Stock Analysis Jupyter Intro Notebook <https://github.com/Al
goTraders/stock-analysis-engine/blob/master/co
mpose/docker/notebooks/Stock-Analysis-Intro.ipynb>`__
.. image:: https://i.imgur.com/pH368gy.png
:param log_label: log identifier
:param ticker: ticker name
:param df: timeseries ``pandas.DateFrame``
:param xlabel: x-axis label
:param ylabel: y-axis label
:param high_color: optional - high plot color
:param close_color: optional - close plot color
:param volume_color: optional - volume color
:param data_format: optional - date format string this must
be a valid value for the ``df['date']`` column
that would work with:
``datetime.datetime.stftime(date_format)``
:param show_plot: optional - bool to show the plot
:param dropna_for_all: optional - bool to toggle keep None's in
the plot ``df`` (default is drop them
for display purposes)
"""
rec = {
'fig': None,
'ax': None,
'ax2': None
}
result = build_result.build_result(
status=ae_consts.NOT_RUN,
err=None,
rec=rec)
try:
log.info(
f'{log_label} - '
'plot_overlay_pricing_and_volume'
' - start')
set_common_seaborn_fonts()
use_df = df
if dropna_for_all:
log.info(
f'{log_label} - '
'plot_overlay_pricing_and_volume'
' - dropna_for_all')
use_df = df.dropna(axis=0, how='any')
# end of pre-plot dataframe scrubbing
fig, ax = plt.subplots(
sharex=True,
sharey=True,
figsize=(15.0, 10.0))
ax.plot(
use_df['date'],
use_df['close'],
color=close_color)
ax.plot(
use_df['date'],
use_df['high'],
color=high_color)
# use a second axis to display the
# volume since it is in a different range
# this will fill under the
# volume's y values as well
ax2 = ax.twinx()
ax2.plot(
use_df['date'],
use_df['volume'],
linestyle='-',
color=volume_color,
alpha=0.6)
ax2.fill_between(
use_df['date'].values,
0,
use_df['volume'].values,
color=volume_color,
alpha=0.5)
# setup the second plot for volume
ax2.set_ylim([0, ax2.get_ylim()[1] * 3])
plt.grid(True)
use_xlabel = xlabel
use_ylabel = ylabel
if not use_xlabel:
xlabel = 'Minute Dates'
if not use_ylabel:
ylabel = f'{ticker} High and Close Prices'
plt.xlabel(use_xlabel)
plt.ylabel(use_ylabel)
# Build a date vs Close DataFrame
start_date = ''
end_date = ''
try:
start_date = str(use_df.iloc[0]['date'].strftime(date_format))
end_date = str(use_df.iloc[-1]['date'].strftime(date_format))
except Exception:
date_format = '%Y-%m-%d'
start_date = str(use_df.iloc[0]['date'].strftime(date_format))
end_date = str(use_df.iloc[-1]['date'].strftime(date_format))
use_title = (
f'{ticker} Pricing from: {start_date} to {end_date}')
ax.set_title(use_title)
# Merge in the second axis (volume) Legend
handles, labels = plt.gca().get_legend_handles_labels()
newLabels, newHandles = [], []
for handle, label in zip(handles, labels):
if label not in newLabels:
newLabels.append(label)
newHandles.append(handle)
lines = ax.get_lines() + ax2.get_lines() + newHandles
ax.legend(
lines,
[l.get_label() for l in lines],
loc='best',
shadow=True)
# Build out the xtick chart by the dates
ax.xaxis.grid(True, which='minor')
ax.fmt_xdata = mdates.DateFormatter(date_format)
ax.xaxis.set_major_formatter(ax.fmt_xdata)
ax.xaxis.set_minor_formatter(ax.fmt_xdata)
# turn off the grids on volume
ax2.fmt_xdata = mdates.DateFormatter(date_format)
ax2.xaxis.grid(False)
ax2.yaxis.grid(False)
ax2.yaxis.set_ticklabels([])
fig.autofmt_xdate()
show_with_entities(
log_label=log_label,
xlabel=xlabel,
ylabel=ylabel,
title=use_title,
ax=ax,
fig=fig,
show_plot=show_plot)
rec['fig'] = fig
rec['ax'] = ax
rec['ax2'] = ax2
result = build_result.build_result(
status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
err = (
'failed plot_overlay_pricing_and_volume '
f'and volume with ex={e}')
result = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
# end of try/ex
send_final_log(
log_label=log_label,
fn_name='plot_overlay_pricing_and_volume',
result=result)
return result
def get_data_from_iex(work_dict):
"""get_data_from_iex
Get pricing from iex
:param work_dict: request dictionary
"""
label = 'get_data_from_iex'
log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict))
rec = {'data': None, 'updated': None}
res = {'status': ae_consts.NOT_RUN, 'err': None, 'rec': rec}
ticker = None
field = None
ft_type = None
try:
ticker = work_dict.get('ticker', ae_consts.TICKER)
field = work_dict.get('field', 'daily')
ft_type = work_dict.get('ft_type', None)
ft_str = str(ft_type).lower()
label = work_dict.get('label', label)
orient = work_dict.get('orient', 'records')
iex_req = None
if ft_type == iex_consts.FETCH_DAILY or ft_str == 'daily':
ft_type == iex_consts.FETCH_DAILY
iex_req = api_requests.build_iex_fetch_daily_request(label=label)
elif ft_type == iex_consts.FETCH_MINUTE or ft_str == 'minute':
ft_type == iex_consts.FETCH_MINUTE
iex_req = api_requests.build_iex_fetch_minute_request(label=label)
elif ft_type == iex_consts.FETCH_QUOTE or ft_str == 'quote':
ft_type == iex_consts.FETCH_QUOTE
iex_req = api_requests.build_iex_fetch_quote_request(label=label)
elif ft_type == iex_consts.FETCH_STATS or ft_str == 'stats':
ft_type == iex_consts.FETCH_STATS
iex_req = api_requests.build_iex_fetch_stats_request(label=label)
elif ft_type == iex_consts.FETCH_PEERS or ft_str == 'peers':
ft_type == iex_consts.FETCH_PEERS
iex_req = api_requests.build_iex_fetch_peers_request(label=label)
elif ft_type == iex_consts.FETCH_NEWS or ft_str == 'news':
ft_type == iex_consts.FETCH_NEWS
iex_req = api_requests.build_iex_fetch_news_request(label=label)
elif ft_type == iex_consts.FETCH_FINANCIALS or ft_str == 'financials':
ft_type == iex_consts.FETCH_FINANCIALS
iex_req = api_requests.build_iex_fetch_financials_request(
label=label)
elif ft_type == iex_consts.FETCH_EARNINGS or ft_str == 'earnings':
ft_type == iex_consts.FETCH_EARNINGS
iex_req = api_requests.build_iex_fetch_earnings_request(
label=label)
elif ft_type == iex_consts.FETCH_DIVIDENDS or ft_str == 'dividends':
ft_type == iex_consts.FETCH_DIVIDENDS
iex_req = api_requests.build_iex_fetch_dividends_request(
label=label)
elif ft_type == iex_consts.FETCH_COMPANY or ft_str == 'company':
ft_type == iex_consts.FETCH_COMPANY
iex_req = api_requests.build_iex_fetch_company_request(label=label)
else:
log.error('{} - unsupported ft_type={} ft_str={} ticker={}'.format(
label, ft_type, ft_str, ticker))
raise NotImplemented
# if supported fetch request type
clone_keys = [
'ticker', 's3_address', 's3_bucket', 's3_key', 'redis_address',
'redis_db', 'redis_password', 'redis_key'
]
for k in clone_keys:
iex_req[k] = work_dict.get(k, '{}-missing-in-{}'.format(k, label))
# end of cloning keys
if not iex_req:
err = ('{} - ticker={} did not build an IEX request '
'for work={}'.format(label, iex_req['ticker'], work_dict))
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=rec)
return res
else:
log.info('{} - ticker={} field={} '
'orient={} fetch'.format(label, iex_req['ticker'], field,
orient))
# if invalid iex request
df = None
try:
if 'from' in work_dict:
iex_req['from'] = datetime.datetime.strptime(
'%Y-%m-%d %H:%M:%S', work_dict['from'])
df = iex_fetch_data.fetch_data(work_dict=iex_req,
fetch_type=ft_type)
rec['data'] = df.to_json(orient=orient, date_format='iso')
rec['updated'] = datetime.datetime.utcnow().strftime(
'%Y-%m-%d %H:%M:%S')
except Exception as f:
log.error('{} - ticker={} field={} failed fetch_data '
'with ex={}'.format(label, iex_req['ticker'], ft_type,
f))
# end of try/ex
if ae_consts.ev('DEBUG_IEX_DATA', '0') == '1':
log.info('{} ticker={} field={} data={} to_json'.format(
label, iex_req['ticker'], field, rec['data']))
else:
log.info('{} ticker={} field={} to_json'.format(
label, iex_req['ticker'], field))
# end of if/else found data
upload_and_cache_req = copy.deepcopy(iex_req)
upload_and_cache_req['celery_disabled'] = True
upload_and_cache_req['data'] = rec['data']
if not upload_and_cache_req['data']:
upload_and_cache_req['data'] = '{}'
use_field = field
if use_field == 'news':
use_field = 'news1'
if 'redis_key' in work_dict:
upload_and_cache_req['redis_key'] = '{}_{}'.format(
work_dict.get('redis_key', iex_req['redis_key']), use_field)
if 's3_key' in work_dict:
upload_and_cache_req['s3_key'] = '{}_{}'.format(
work_dict.get('s3_key', iex_req['s3_key']), use_field)
try:
update_res = publisher.run_publish_pricing_update(
work_dict=upload_and_cache_req)
update_status = update_res.get('status', ae_consts.NOT_SET)
log.info('{} publish update status={} data={}'.format(
label, ae_consts.get_status(status=update_status), update_res))
except Exception as f:
err = ('{} - failed to upload iex data={} to '
'to s3_key={} and redis_key={}'.format(
label, upload_and_cache_req,
upload_and_cache_req['s3_key'],
upload_and_cache_req['redis_key']))
log.error(err)
# end of try/ex to upload and cache
if not rec['data']:
log.info('{} - ticker={} no IEX data field={} to publish'.format(
label, iex_req['ticker'], field))
# end of if/else
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
res = build_result.build_result(status=ae_consts.ERR,
err=('failed - get_data_from_iex '
'dict={} with ex={}').format(
work_dict, e),
rec=rec)
# end of try/ex
log.info('task - get_data_from_iex done - '
'{} - status={} err={}'.format(
label, ae_consts.get_status(res['status']), res['err']))
return res
def get_new_pricing_data(self, work_dict):
"""get_new_pricing_data
Get Ticker information on:
- prices - turn off with ``work_dict.get_pricing = False``
- news - turn off with ``work_dict.get_news = False``
- options - turn off with ``work_dict.get_options = False``
:param work_dict: dictionary for key/values
"""
label = 'get_new_pricing_data'
log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict))
ticker = ae_consts.TICKER
ticker_id = ae_consts.TICKER_ID
rec = {
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'daily': None,
'minute': None,
'quote': None,
'stats': None,
'peers': None,
'iex_news': None,
'financials': None,
'earnings': None,
'dividends': None,
'company': None,
'exp_date': None,
'publish_pricing_update': None,
'date': ae_utils.utc_now_str(),
'updated': None,
'version': ae_consts.DATASET_COLLECTION_VERSION
}
res = {'status': ae_consts.NOT_RUN, 'err': None, 'rec': rec}
try:
ticker = work_dict.get('ticker', ticker)
ticker_id = work_dict.get('ticker_id', ae_consts.TICKER_ID)
s3_bucket = work_dict.get('s3_bucket', ae_consts.S3_BUCKET)
s3_key = work_dict.get('s3_key', ae_consts.S3_KEY)
redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY)
exp_date = work_dict.get('exp_date', None)
cur_date = datetime.datetime.utcnow()
cur_strike = work_dict.get('strike', None)
contract_type = str(work_dict.get('contract', 'C')).upper()
label = work_dict.get('label', label)
iex_datasets = work_dict.get('iex_datasets',
iex_consts.DEFAULT_FETCH_DATASETS)
td_datasets = work_dict.get('td_datasets',
td_consts.DEFAULT_FETCH_DATASETS_TD)
fetch_mode = work_dict.get('fetch_mode', ae_consts.FETCH_MODE_ALL)
# control flags to deal with feed issues:
get_iex_data = True
get_td_data = True
if (fetch_mode == ae_consts.FETCH_MODE_ALL
or str(fetch_mode).lower() == 'all'):
get_iex_data = True
get_td_data = True
elif (fetch_mode == ae_consts.FETCH_MODE_YHO
or str(fetch_mode).lower() == 'yahoo'):
get_iex_data = False
get_td_data = False
elif (fetch_mode == ae_consts.FETCH_MODE_IEX
or str(fetch_mode).lower() == 'iex'):
get_iex_data = True
get_td_data = False
elif (fetch_mode == ae_consts.FETCH_MODE_TD
or str(fetch_mode).lower() == 'td'):
get_iex_data = False
get_td_data = True
else:
log.debug('{} - unsupported fetch_mode={} value'.format(
label, fetch_mode))
"""
as of Thursday, Jan. 3, 2019:
https://developer.yahoo.com/yql/
Important EOL Notice: As of Thursday, Jan. 3, 2019
the YQL service at query.yahooapis.com will be retired
"""
get_yahoo_data = False
if not exp_date:
exp_date = opt_dates.option_expiration(date=exp_date)
else:
exp_date = datetime.datetime.strptime(exp_date, '%Y-%m-%d')
rec['updated'] = cur_date.strftime('%Y-%m-%d %H:%M:%S')
log.info('{} getting pricing for ticker={} '
'cur_date={} exp_date={} '
'yahoo={} iex={}'.format(label, ticker, cur_date, exp_date,
get_yahoo_data, get_iex_data))
yahoo_rec = {
'ticker': ticker,
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'exp_date': None,
'publish_pricing_update': None,
'date': None,
'updated': None
}
# disabled on 2019-01-03
if get_yahoo_data:
log.info('{} yahoo ticker={}'.format(label, ticker))
yahoo_res = yahoo_data.get_data_from_yahoo(work_dict=work_dict)
if yahoo_res['status'] == ae_consts.SUCCESS:
yahoo_rec = yahoo_res['rec']
log.info('{} yahoo ticker={} '
'status={} err={}'.format(
label, ticker,
ae_consts.get_status(status=yahoo_res['status']),
yahoo_res['err']))
rec['pricing'] = yahoo_rec.get('pricing', '{}')
rec['news'] = yahoo_rec.get('news', '{}')
rec['options'] = yahoo_rec.get('options', '{}')
rec['calls'] = rec['options'].get('calls',
ae_consts.EMPTY_DF_STR)
rec['puts'] = rec['options'].get('puts',
ae_consts.EMPTY_DF_STR)
else:
log.error('{} failed YAHOO ticker={} '
'status={} err={}'.format(
label, ticker,
ae_consts.get_status(status=yahoo_res['status']),
yahoo_res['err']))
# end of get from yahoo
if get_iex_data:
num_iex_ds = len(iex_datasets)
log.debug('{} iex datasets={}'.format(label, num_iex_ds))
for idx, ft_type in enumerate(iex_datasets):
dataset_field = iex_consts.get_ft_str(ft_type=ft_type)
log.info('{} iex={}/{} field={} ticker={}'.format(
label, idx, num_iex_ds, dataset_field, ticker))
iex_label = '{}-{}'.format(label, dataset_field)
iex_req = copy.deepcopy(work_dict)
iex_req['label'] = iex_label
iex_req['ft_type'] = ft_type
iex_req['field'] = dataset_field
iex_req['ticker'] = ticker
iex_res = iex_data.get_data_from_iex(work_dict=iex_req)
if iex_res['status'] == ae_consts.SUCCESS:
iex_rec = iex_res['rec']
log.info(
'{} iex ticker={} field={} '
'status={} err={}'.format(
label, ticker, dataset_field,
ae_consts.get_status(status=iex_res['status']),
iex_res['err']))
if dataset_field == 'news':
rec['iex_news'] = iex_rec['data']
else:
rec[dataset_field] = iex_rec['data']
else:
log.debug(
'{} failed IEX ticker={} field={} '
'status={} err={}'.format(
label, ticker, dataset_field,
ae_consts.get_status(status=iex_res['status']),
iex_res['err']))
# end of if/else succcess
# end idx, ft_type in enumerate(iex_datasets):
# end of if get_iex_data
if get_td_data:
num_td_ds = len(td_datasets)
log.debug('{} td datasets={}'.format(label, num_td_ds))
for idx, ft_type in enumerate(td_datasets):
dataset_field = td_consts.get_ft_str_td(ft_type=ft_type)
log.info('{} td={}/{} field={} ticker={}'.format(
label, idx, num_td_ds, dataset_field, ticker))
td_label = '{}-{}'.format(label, dataset_field)
td_req = copy.deepcopy(work_dict)
td_req['label'] = td_label
td_req['ft_type'] = ft_type
td_req['field'] = dataset_field
td_req['ticker'] = ticker
td_res = td_data.get_data_from_td(work_dict=td_req)
if td_res['status'] == ae_consts.SUCCESS:
td_rec = td_res['rec']
log.info('{} td ticker={} field={} '
'status={} err={}'.format(
label, ticker, dataset_field,
ae_consts.get_status(status=td_res['status']),
td_res['err']))
if dataset_field == 'tdcalls':
rec['tdcalls'] = td_rec['data']
if dataset_field == 'tdputs':
rec['tdputs'] = td_rec['data']
else:
rec[dataset_field] = td_rec['data']
else:
log.critical(
'{} failed TD ticker={} field={} '
'status={} err={}'.format(
label, ticker, dataset_field,
ae_consts.get_status(status=td_res['status']),
td_res['err']))
# end of if/else succcess
# end idx, ft_type in enumerate(td_datasets):
# end of if get_td_data
update_req = {'data': rec}
update_req['ticker'] = ticker
update_req['ticker_id'] = ticker_id
update_req['strike'] = cur_strike
update_req['contract'] = contract_type
update_req['s3_enabled'] = work_dict.get('s3_enabled',
ae_consts.ENABLED_S3_UPLOAD)
update_req['redis_enabled'] = work_dict.get(
'redis_enabled', ae_consts.ENABLED_REDIS_PUBLISH)
update_req['s3_bucket'] = s3_bucket
update_req['s3_key'] = s3_key
update_req['s3_access_key'] = work_dict.get('s3_access_key',
ae_consts.S3_ACCESS_KEY)
update_req['s3_secret_key'] = work_dict.get('s3_secret_key',
ae_consts.S3_SECRET_KEY)
update_req['s3_region_name'] = work_dict.get('s3_region_name',
ae_consts.S3_REGION_NAME)
update_req['s3_address'] = work_dict.get('s3_address',
ae_consts.S3_ADDRESS)
update_req['s3_secure'] = work_dict.get('s3_secure',
ae_consts.S3_SECURE)
update_req['redis_key'] = redis_key
update_req['redis_address'] = work_dict.get('redis_address',
ae_consts.REDIS_ADDRESS)
update_req['redis_password'] = work_dict.get('redis_password',
ae_consts.REDIS_PASSWORD)
update_req['redis_db'] = int(
work_dict.get('redis_db', ae_consts.REDIS_DB))
update_req['redis_expire'] = work_dict.get('redis_expire',
ae_consts.REDIS_EXPIRE)
update_req['updated'] = rec['updated']
update_req['label'] = label
update_req['celery_disabled'] = True
update_status = ae_consts.NOT_SET
try:
update_res = publisher.run_publish_pricing_update(
work_dict=update_req)
update_status = update_res.get('status', ae_consts.NOT_SET)
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
log.info('{} update_res status={} data={}'.format(
label, ae_consts.get_status(status=update_status),
ae_consts.ppj(update_res)))
else:
log.info('{} run_publish_pricing_update status={}'.format(
label, ae_consts.get_status(status=update_status)))
# end of if/else
rec['publish_pricing_update'] = update_res
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as f:
err = ('{} publisher.run_publish_pricing_update failed '
'with ex={}'.format(label, f))
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=rec)
# end of trying to publish results to connected services
except Exception as e:
res = build_result.build_result(status=ae_consts.ERR,
err=('failed - get_new_pricing_data '
'dict={} with ex={}').format(
work_dict, e),
rec=rec)
log.error('{} - {}'.format(label, res['err']))
# end of try/ex
if ae_consts.ev('DATASET_COLLECTION_SLACK_ALERTS', '0') == '1':
env_name = 'DEV'
if ae_consts.ev('PROD_SLACK_ALERTS', '1') == '1':
env_name = 'PROD'
done_msg = ('Dataset collected ticker=*{}* on env=*{}* '
'redis_key={} s3_key={} iex={} yahoo={}'.format(
ticker, env_name, redis_key, s3_key, get_iex_data,
get_yahoo_data))
log.debug('{} sending slack msg={}'.format(label, done_msg))
if res['status'] == ae_consts.SUCCESS:
slack_utils.post_success(msg=done_msg, block=False, jupyter=True)
else:
slack_utils.post_failure(msg=done_msg, block=False, jupyter=True)
# end of if/else success
# end of publishing to slack
log.info('task - get_new_pricing_data done - '
'{} - status={}'.format(label,
ae_consts.get_status(res['status'])))
return get_task_results.get_task_results(work_dict=work_dict, result=res)
def fetch_tickers_from_screener(
url,
columns=DEFAULT_FINVIZ_COLUMNS,
as_json=False,
soup_selector='td.screener-body-table-nw',
label='fz-screen-converter'):
"""fetch_tickers_from_screener
Convert all the tickers on a FinViz screener
url to a ``pandas.DataFrame``. Returns a dictionary
with a ticker list and DataFrame or a json-serialized
DataFrame in a string (by default ``as_json=False`` will
return a ``pandas.DataFrame`` if the
``returned-dictionary['status'] == SUCCESS``
Works with urls created on:
https://finviz.com/screener.ashx
.. code-block:: python
import analysis_engine.finviz.fetch_api as fv
url = (
'https://finviz.com/screener.ashx?'
'v=111&'
'f=cap_midunder,exch_nyse,fa_div_o5,idx_sp500'
'&ft=4')
res = fv.fetch_tickers_from_screener(url=url)
print(res)
:param url: FinViz screener url
:param columns: ordered header column as a list of strings
and corresponds to the header row from the
FinViz screener table
:param soup_selector: ``bs4.BeautifulSoup.selector`` string
for pulling selected html data
(by default ``td.screener-body-table-nw``)
:param as_json: FinViz screener url
:param label: log tracking label string
"""
rec = {
'data': None,
'created': get_last_close_str(),
'tickers': []
}
res = req_utils.build_result(
status=NOT_RUN,
err=None,
rec=rec)
try:
log.info(
'{} fetching url={}'.format(
label,
url))
response = requests.get(url)
if response.status_code != requests.codes.ok:
err = (
'{} finviz returned non-ok HTTP (200) '
'status_code={} with text={} for url={}'.format(
label,
response.status_code,
response.text,
url))
log.error(err)
return req_utils.build_result(
status=ERR,
err=err,
rec=rec)
# end of checking for a good HTTP response status code
soup = bs4.BeautifulSoup(
response.text,
features='html.parser')
selected = soup.select(soup_selector)
log.debug(
'{} found={} url={}'.format(
label,
len(selected),
url))
ticker_list = []
rows = []
use_columns = columns
num_columns = len(use_columns)
new_row = {}
col_idx = 0
for idx, node in enumerate(selected):
if col_idx >= num_columns:
col_idx = 0
column_name = use_columns[col_idx]
test_text = str(node.text).lower().strip()
col_idx += 1
if column_name != 'ignore' and (
test_text != 'save as portfolio'
and test_text != 'export'):
cur_text = str(node.text).strip()
if column_name == 'ticker':
ticker_list.append(cur_text)
new_row[column_name] = cur_text.upper()
else:
new_row[column_name] = cur_text
# end of filtering bad sections around table
if len(new_row) >= num_columns:
log.debug(
'{} adding ticker={}'.format(
label,
new_row['ticker']))
rows.append(new_row)
new_row = {}
col_idx = 0
# end of if valid row
# end if column is valid
# end of walking through all matched html data on the screener
log.debug(
'{} done convert url={} to tickers={} '
'rows={}'.format(
label,
url,
ticker_list,
len(rows)))
df = pd.DataFrame(
rows)
log.info(
'{} fetch done - df={} from url={} with tickers={} '
'rows={}'.format(
label,
len(df.index),
url,
ticker_list,
len(rows)))
rec['tickers'] = ticker_list
rec['data'] = df
res = req_utils.build_result(
status=SUCCESS,
err=None,
rec=rec)
except Exception as e:
rec['tickers'] = []
rec['data'] = None
err = (
'{} failed converting screen url={} to list '
'with ex={}'.format(
label,
url,
e))
log.error(err)
res = req_utils.build_result(
status=EX,
err=err,
rec=rec)
# end of try/ex
return res
def publish_from_s3_to_redis(self, work_dict):
"""publish_from_s3_to_redis
Publish Ticker Data from S3 to Redis
:param work_dict: dictionary for key/values
"""
label = 'pub-s3-to-redis'
log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict))
ticker = ae_consts.TICKER
ticker_id = ae_consts.TICKER_ID
rec = {
'ticker': None,
'ticker_id': None,
's3_enabled': True,
'redis_enabled': True,
's3_bucket': None,
's3_key': None,
'redis_key': None,
'updated': None
}
res = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec=rec)
try:
ticker = work_dict.get('ticker', ae_consts.TICKER)
ticker_id = int(work_dict.get('ticker_id', ae_consts.TICKER_ID))
if not ticker:
res = build_result.build_result(status=ae_consts.ERR,
err='missing ticker',
rec=rec)
return res
s3_key = work_dict.get('s3_key', None)
s3_bucket_name = work_dict.get('s3_bucket', 'pricing')
redis_key = work_dict.get('redis_key', None)
updated = work_dict.get('updated', None)
serializer = work_dict.get('serializer', 'json')
encoding = work_dict.get('encoding', 'utf-8')
label = work_dict.get('label', label)
enable_s3_read = True
enable_redis_publish = True
rec['ticker'] = ticker
rec['ticker_id'] = ticker_id
rec['s3_bucket'] = s3_bucket_name
rec['s3_key'] = s3_key
rec['redis_key'] = redis_key
rec['updated'] = updated
rec['s3_enabled'] = enable_s3_read
rec['redis_enabled'] = enable_redis_publish
data = None
if enable_s3_read:
log.info('{} parsing s3 values'.format(label))
access_key = work_dict.get('s3_access_key',
ae_consts.S3_ACCESS_KEY)
secret_key = work_dict.get('s3_secret_key',
ae_consts.S3_SECRET_KEY)
region_name = work_dict.get('s3_region_name',
ae_consts.S3_REGION_NAME)
service_address = work_dict.get('s3_address', ae_consts.S3_ADDRESS)
secure = work_dict.get('s3_secure', ae_consts.S3_SECURE) == '1'
endpoint_url = 'http://{}'.format(service_address)
if secure:
endpoint_url = 'https://{}'.format(service_address)
log.info('{} building s3 endpoint_url={} '
'region={}'.format(label, endpoint_url, region_name))
s3 = boto3.resource(
's3',
endpoint_url=endpoint_url,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region_name,
config=boto3.session.Config(signature_version='s3v4'))
try:
log.info('{} checking bucket={} exists'.format(
label, s3_bucket_name))
if s3.Bucket(s3_bucket_name) not in s3.buckets.all():
log.info('{} creating bucket={}'.format(
label, s3_bucket_name))
s3.create_bucket(Bucket=s3_bucket_name)
except Exception as e:
log.info('{} failed creating bucket={} '
'with ex={}'.format(label, s3_bucket_name, e))
# end of try/ex for creating bucket
try:
log.info('{} reading to s3={}/{} '
'updated={}'.format(label, s3_bucket_name, s3_key,
updated))
data = s3_read_contents_from_key.s3_read_contents_from_key(
s3=s3,
s3_bucket_name=s3_bucket_name,
s3_key=s3_key,
encoding=encoding,
convert_as_json=True)
initial_size_value = \
len(str(data)) / 1024000
initial_size_str = ae_consts.to_f(initial_size_value)
if ae_consts.ev('DEBUG_S3', '0') == '1':
log.info('{} read s3={}/{} data={}'.format(
label, s3_bucket_name, s3_key, ae_consts.ppj(data)))
else:
log.info('{} read s3={}/{} data size={} MB'.format(
label, s3_bucket_name, s3_key, initial_size_str))
except Exception as e:
err = ('{} failed reading bucket={} '
'key={} ex={}').format(label, s3_bucket_name, s3_key, e)
log.error(err)
res = build_result.build_result(status=ae_consts.NOT_RUN,
err=err,
rec=rec)
# end of try/ex for creating bucket
else:
log.info('{} SKIP S3 read bucket={} '
'key={}'.format(label, s3_bucket_name, s3_key))
# end of if enable_s3_read
if enable_redis_publish:
redis_address = work_dict.get('redis_address',
ae_consts.REDIS_ADDRESS)
redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY)
redis_password = work_dict.get('redis_password',
ae_consts.REDIS_PASSWORD)
redis_db = work_dict.get('redis_db', None)
if not redis_db:
redis_db = ae_consts.REDIS_DB
redis_expire = None
if 'redis_expire' in work_dict:
redis_expire = work_dict.get('redis_expire',
ae_consts.REDIS_EXPIRE)
log.info('redis enabled address={}@{} '
'key={}'.format(redis_address, redis_db, redis_key))
redis_host = redis_address.split(':')[0]
redis_port = redis_address.split(':')[1]
try:
if ae_consts.ev('DEBUG_REDIS', '0') == '1':
log.info('{} publishing redis={}:{} '
'db={} key={} '
'updated={} expire={} '
'data={}'.format(label, redis_host, redis_port,
redis_db, redis_key,
updated, redis_expire,
ae_consts.ppj(data)))
else:
log.info('{} publishing redis={}:{} '
'db={} key={} '
'updated={} expire={}'.format(
label, redis_host, redis_port, redis_db,
redis_key, updated, redis_expire))
# end of if/else
rc = redis.Redis(host=redis_host,
port=redis_port,
password=redis_password,
db=redis_db)
redis_set_res = redis_set.set_data_in_redis_key(
label=label,
client=rc,
key=redis_key,
data=data,
serializer=serializer,
encoding=encoding,
expire=redis_expire,
px=None,
nx=False,
xx=False)
log.info('{} redis_set status={} err={}'.format(
label, ae_consts.get_status(redis_set_res['status']),
redis_set_res['err']))
except Exception as e:
log.error('{} failed - redis publish to '
'key={} ex={}'.format(label, redis_key, e))
# end of try/ex for creating bucket
else:
log.info('{} SKIP REDIS publish '
'key={}'.format(label, redis_key))
# end of if enable_redis_publish
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
res = build_result.build_result(
status=ae_consts.ERR,
err=('failed - publish_from_s3_to_redis '
'dict={} with ex={}').format(work_dict, e),
rec=rec)
log.error('{} - {}'.format(label, res['err']))
# end of try/ex
log.info('task - publish_from_s3_to_redis done - '
'{} - status={}'.format(label,
ae_consts.get_status(res['status'])))
return get_task_results.get_task_results(work_dict=work_dict, result=res)
def prepare_pricing_dataset(
self,
work_dict):
"""prepare_pricing_dataset
Prepare dataset for analysis. Supports loading dataset from
s3 if not found in redis. Outputs prepared artifact as a csv
to s3 and redis.
:param work_dict: dictionary for key/values
"""
label = 'prepare'
log.info(
'task - {} - start '
'work_dict={}'.format(
label,
work_dict))
initial_data = None
ticker = ae_consts.TICKER
ticker_id = ae_consts.TICKER_ID
rec = {
'ticker': None,
'ticker_id': None,
's3_enabled': True,
'redis_enabled': True,
's3_bucket': None,
's3_key': None,
'redis_key': None,
'prepared_s3_key': None,
'prepared_s3_bucket': None,
'prepared_redis_key': None,
'prepared_data': None,
'prepared_size': None,
'initial_data': None,
'initial_size': None,
'ignore_columns': None,
'updated': None
}
res = build_result.build_result(
status=ae_consts.NOT_RUN,
err=None,
rec=rec)
try:
ticker = work_dict.get(
'ticker',
ae_consts.TICKER)
ticker_id = int(work_dict.get(
'ticker_id',
ae_consts.TICKER_ID))
if not ticker:
res = build_result.build_result(
status=ae_consts.ERR,
err='missing ticker',
rec=rec)
return res
label = work_dict.get(
'label',
label)
s3_key = work_dict.get(
's3_key',
None)
s3_bucket_name = work_dict.get(
's3_bucket',
'pricing')
s3_access_key = work_dict.get(
's3_access_key',
ae_consts.S3_ACCESS_KEY)
s3_secret_key = work_dict.get(
's3_secret_key',
ae_consts.S3_SECRET_KEY)
s3_region_name = work_dict.get(
's3_region_name',
ae_consts.S3_REGION_NAME)
s3_address = work_dict.get(
's3_address',
ae_consts.S3_ADDRESS)
s3_secure = work_dict.get(
's3_secure',
ae_consts.S3_SECURE) == '1'
redis_address = work_dict.get(
'redis_address',
ae_consts.REDIS_ADDRESS)
redis_key = work_dict.get(
'redis_key',
ae_consts.REDIS_KEY)
redis_password = work_dict.get(
'redis_password',
ae_consts.REDIS_PASSWORD)
redis_db = work_dict.get(
'redis_db',
None)
if not redis_db:
redis_db = ae_consts.REDIS_DB
redis_expire = None
if 'redis_expire' in work_dict:
redis_expire = work_dict.get(
'redis_expire',
ae_consts.REDIS_EXPIRE)
updated = work_dict.get(
'updated',
datetime.datetime.utcnow().strftime(
'%Y_%m_%d_%H_%M_%S'))
prepared_s3_key = work_dict.get(
'prepared_s3_key',
'{}_{}.csv'.format(
ticker,
updated))
prepared_s3_bucket = work_dict.get(
'prepared_s3_bucket',
'prepared')
prepared_redis_key = work_dict.get(
'prepared_redis_key',
'prepared')
ignore_columns = work_dict.get(
'ignore_columns',
None)
log.info(
'{} redis enabled address={}@{} '
'key={} prepare_s3={}:{} prepare_redis={} '
'ignore_columns={}'.format(
label,
redis_address,
redis_db,
redis_key,
prepared_s3_bucket,
prepared_s3_key,
prepared_redis_key,
ignore_columns))
redis_host = redis_address.split(':')[0]
redis_port = redis_address.split(':')[1]
enable_s3 = True
enable_redis_publish = True
rec['ticker'] = ticker
rec['ticker_id'] = ticker_id
rec['s3_bucket'] = s3_bucket_name
rec['s3_key'] = s3_key
rec['redis_key'] = redis_key
rec['prepared_s3_key'] = prepared_s3_key
rec['prepared_s3_bucket'] = prepared_s3_bucket
rec['prepared_redis_key'] = prepared_redis_key
rec['updated'] = updated
rec['s3_enabled'] = enable_s3
rec['redis_enabled'] = enable_redis_publish
try:
log.info(
'{} connecting redis={}:{} '
'db={} key={} '
'updated={} expire={}'.format(
label,
redis_host,
redis_port,
redis_db,
redis_key,
updated,
redis_expire))
rc = redis.Redis(
host=redis_host,
port=redis_port,
password=redis_password,
db=redis_db)
except Exception as e:
err = (
'{} failed - redis connection to address={}@{} '
'key={} ex={}'.format(
label,
redis_address,
redis_key,
redis_db,
e))
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
# end of try/ex for connecting to redis
initial_data_res = redis_get.get_data_from_redis_key(
label=label,
client=rc,
key=redis_key)
log.info(
'{} get redis key={} status={} err={}'.format(
label,
redis_key,
ae_consts.get_status(initial_data_res['status']),
initial_data_res['err']))
initial_data = initial_data_res['rec'].get(
'data',
None)
if enable_s3 and not initial_data:
log.info(
'{} failed to find redis_key={} trying s3 '
'from s3_key={} s3_bucket={} s3_address={}'.format(
label,
redis_key,
s3_key,
s3_bucket_name,
s3_address))
get_from_s3_req = \
api_requests.build_publish_from_s3_to_redis_request()
get_from_s3_req['s3_enabled'] = enable_s3
get_from_s3_req['s3_access_key'] = s3_access_key
get_from_s3_req['s3_secret_key'] = s3_secret_key
get_from_s3_req['s3_region_name'] = s3_region_name
get_from_s3_req['s3_address'] = s3_address
get_from_s3_req['s3_secure'] = s3_secure
get_from_s3_req['s3_key'] = s3_key
get_from_s3_req['s3_bucket'] = s3_bucket_name
get_from_s3_req['redis_key'] = redis_key
get_from_s3_req['label'] = (
'{}-run_publish_from_s3_to_redis'.format(
label))
log.info(
'{} load from s3={} to '
'redis={}'.format(
label,
s3_key,
redis_key))
try:
# run in synchronous mode:
get_from_s3_req['celery_disabled'] = True
task_res = s3_to_redis.run_publish_from_s3_to_redis(
get_from_s3_req)
if task_res.get(
'status',
ae_consts.ERR) == ae_consts.SUCCESS:
log.info(
'{} loaded s3={}:{} '
'to redis={} retrying'.format(
label,
s3_bucket_name,
s3_key,
redis_key))
initial_data_res = redis_get.get_data_from_redis_key(
label=label,
client=rc,
key=redis_key)
log.info(
'{} get redis try=2 key={} status={} err={}'.format(
label,
redis_key,
ae_consts.get_status(initial_data_res['status']),
initial_data_res['err']))
initial_data = initial_data_res['rec'].get(
'data',
None)
else:
err = (
'{} ERR failed loading from bucket={} '
's3_key={} to redis_key={} with res={}'.format(
label,
s3_bucket_name,
s3_key,
redis_key,
task_res))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
except Exception as e:
err = (
'{} extract from s3 and publish to redis failed loading '
'data from bucket={} in '
's3_key={} with publish to redis_key={} '
'with ex={}'.format(
label,
s3_bucket_name,
s3_key,
redis_key,
e))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
# end of try/ex for publishing from s3->redis
# end of if enable_s3
if not initial_data:
err = (
'{} did not find any data to prepare in redis_key={} or '
's3_key={} in bucket={}'.format(
label,
redis_key,
s3_key,
s3_bucket_name))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
initial_data_num_chars = len(str(initial_data))
initial_size_value = None
initial_size_str = None
if initial_data_num_chars < ae_consts.PREPARE_DATA_MIN_SIZE:
err = (
'{} not enough data={} in redis_key={} or '
's3_key={} in bucket={}'.format(
label,
initial_data_num_chars,
redis_key,
s3_key,
s3_bucket_name))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
else:
initial_size_value = initial_data_num_chars / 1024000
initial_size_str = ae_consts.to_f(initial_size_value)
if ae_consts.ev('DEBUG_PREPARE', '0') == '1':
log.info(
'{} initial - redis_key={} data={}'.format(
label,
redis_key,
str(initial_data)))
else:
log.info(
'{} initial - redis_key={} data size={} MB'.format(
label,
redis_key,
initial_size_str))
# end of trying to get initial_data
rec['initial_data'] = initial_data
rec['initial_size'] = initial_data_num_chars
prepare_data = None
try:
if ae_consts.ev('DEBUG_PREPARE', '0') == '1':
log.info(
'{} data={} - flatten - {} MB from '
'redis_key={}'.format(
label,
ae_consts.ppj(initial_data),
initial_size_str,
redis_key))
else:
log.info(
'{} flatten - {} MB from '
'redis_key={}'.format(
label,
initial_size_str,
redis_key))
prepare_data = dict_to_csv.flatten_dict(
data=initial_data)
except Exception as e:
prepare_data = None
err = (
'{} flatten - convert to csv failed with ex={} '
'redis_key={}'.format(
label,
e,
redis_key))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
# end of try/ex
if not prepare_data:
err = (
'{} flatten - did not return any data from redis_key={} '
'or s3_key={} in bucket={}'.format(
label,
redis_key,
s3_key,
s3_bucket_name))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
# end of prepare_data
prepare_data_num_chars = len(str(prepare_data))
prepare_size_value = None
if prepare_data_num_chars < ae_consts.PREPARE_DATA_MIN_SIZE:
err = (
'{} prepare - there is not enough data={} in redis_key={}'
''.format(
label,
prepare_data_num_chars,
redis_key))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
else:
prepare_size_value = prepare_data_num_chars / 1024000
prepare_size_str = ae_consts.to_f(prepare_size_value)
if ae_consts.ev('DEBUG_PREPARE', '0') == '1':
log.info(
'{} data={} - prepare - redis_key={}'.format(
label,
redis_key,
ae_consts.ppj(prepare_data)))
else:
log.info(
'{} prepare - redis_key={} data size={} MB'.format(
label,
redis_key,
prepare_size_str))
# end of trying to the size of the prepared data
rec['prepared_data'] = prepare_data
rec['prepared_size'] = prepare_data_num_chars
res = build_result.build_result(
status=ae_consts.SUCCESS,
err=None,
rec=rec)
rc = None
except Exception as e:
res = build_result.build_result(
status=ae_consts.ERR,
err=(
'failed - prepare_pricing_dataset '
'dict={} with ex={}').format(
work_dict,
e),
rec=rec)
log.error(
'{} - {}'.format(
label,
res['err']))
# end of try/ex
log.info(
'task - prepare_pricing_dataset done - '
'{} - status={}'.format(
label,
ae_consts.get_status(res['status'])))
return get_task_results.get_task_results(
work_dict=work_dict,
result=res)
def plot_df(
log_label,
title,
column_list,
df,
xcol='date',
xlabel='Date',
ylabel='Pricing',
linestyle='-',
color='blue',
show_plot=True,
dropna_for_all=True):
"""plot_df
:param log_label: log identifier
:param title: title of the plot
:param column_list: list of columns in the df to show
:param df: initialized ``pandas.DataFrame``
:param xcol: x-axis column in the initialized ``pandas.DataFrame``
:param xlabel: x-axis label
:param ylabel: y-axis label
:param linestyle: style of the plot line
:param color: color to use
:param show_plot: bool to show the plot
:param dropna_for_all: optional - bool to toggle keep None's in
the plot ``df`` (default is drop them
for display purposes)
"""
rec = {
'ax': None,
'fig': None
}
result = build_result.build_result(
status=ae_consts.NOT_RUN,
err=None,
rec=rec)
try:
log.info(
f'{log_label} - '
'plot_df'
' - start')
use_df = df
if dropna_for_all:
log.info(
f'{log_label} - '
'plot_df'
' - dropna_for_all')
use_df = df.dropna(axis=0, how='any')
# end of pre-plot dataframe scrubbing
set_common_seaborn_fonts()
hex_color = ae_consts.PLOT_COLORS['blue']
fig, ax = plt.subplots(figsize=(15.0, 10.0))
if linestyle == '-':
use_df[column_list].plot(
x=xcol,
linestyle=linestyle,
ax=ax,
color=hex_color,
rot=0)
else:
use_df[column_list].plot(
kind='bar',
x=xcol,
ax=ax,
color=hex_color,
rot=0)
if 'date' in str(xcol).lower():
# plot the column
min_date = use_df.iloc[0][xcol]
max_date = use_df.iloc[-1][xcol]
# give a bit of space at each end of the plot - aesthetics
span = max_date - min_date
extra = int(span.days * 0.03) * datetime.timedelta(days=1)
ax.set_xlim([min_date - extra, max_date + extra])
# format the x tick marks
ax.xaxis.set_minor_formatter(mdates.DateFormatter('%b\n%Y'))
ax.xaxis.set_major_locator(plt.NullLocator())
ax.xaxis.set_minor_locator(
mdates.MonthLocator(
bymonthday=1,
interval=1))
# grid, legend and yLabel
ax.xaxis.grid(True, which='minor')
# end of date formatting
show_with_entities(
log_label=log_label,
xlabel=xlabel,
ylabel=ylabel,
title=title,
ax=ax,
fig=fig,
show_plot=show_plot)
rec['ax'] = ax
rec['fig'] = fig
result = build_result.build_result(
status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
err = (
f'failed plot_df title={title} with ex={e}')
result = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
# end of try/ex
send_final_log(
log_label=log_label,
fn_name='plot_df',
result=result)
return result
def run_algo(
ticker=None,
tickers=None,
algo=None, # optional derived ``analysis_engine.algo.Algo`` instance
balance=None, # float starting base capital
commission=None, # float for single trade commission for buy or sell
start_date=None, # string YYYY-MM-DD HH:MM:SS
end_date=None, # string YYYY-MM-DD HH:MM:SS
datasets=None, # string list of identifiers
num_owned_dict=None, # not supported
cache_freq='daily', # 'minute' not supported
auto_fill=True,
load_config=None,
report_config=None,
history_config=None,
extract_config=None,
use_key=None,
extract_mode='all',
iex_datasets=None,
redis_enabled=True,
redis_address=None,
redis_db=None,
redis_password=None,
redis_expire=None,
redis_key=None,
s3_enabled=True,
s3_address=None,
s3_bucket=None,
s3_access_key=None,
s3_secret_key=None,
s3_region_name=None,
s3_secure=False,
s3_key=None,
celery_disabled=True,
broker_url=None,
result_backend=None,
label=None,
name=None,
timeseries=None,
trade_strategy=None,
verbose=False,
publish_to_slack=True,
publish_to_s3=True,
publish_to_redis=True,
extract_datasets=None,
config_file=None,
config_dict=None,
version=1,
raise_on_err=True,
**kwargs):
"""run_algo
Run an algorithm with steps:
1) Extract redis keys between dates
2) Compile a data pipeline dictionary (call it ``data``)
3) Call algorithm's ``myalgo.handle_data(data=data)``
.. note:: If no ``algo`` is set, the
``analysis_engine.algo.BaseAlgo`` algorithm
is used.
.. note:: Please ensure Redis and Minio are running
before trying to extract tickers
**Stock tickers to extract**
:param ticker: single stock ticker/symbol/ETF to extract
:param tickers: optional - list of tickers to extract
:param use_key: optional - extract historical key from Redis
**Algo Configuration**
:param algo: derived instance of ``analysis_engine.algo.Algo`` object
:param balance: optional - float balance parameter
can also be set on the ``algo`` object if not
set on the args
:param commission: float for single trade commission for
buy or sell. can also be set on the ``algo`` objet
:param start_date: string ``YYYY-MM-DD_HH:MM:SS`` cache value
:param end_date: string ``YYYY-MM-DD_HH:MM:SS`` cache value
:param dataset_types: list of strings that are ``iex`` or ``yahoo``
datasets that are cached.
:param cache_freq: optional - depending on if you are running data feeds
on a ``daily`` cron (default) vs every ``minute`` (or faster)
:param num_owned_dict: not supported yet
:param auto_fill: optional - boolean for auto filling
buy/sell orders for backtesting (default is
``True``)
:param trading_calendar: ``trading_calendar.TradingCalendar``
object, by default ``analysis_engine.calendars.
always_open.AlwaysOpen`` trading calendar
# TradingCalendar by ``TFSExchangeCalendar``
:param config_file: path to a json file
containing custom algorithm object
member values (like indicator configuration and
predict future date units ahead for a backtest)
:param config_dict: optional - dictionary that
can be passed to derived class implementations
of: ``def load_from_config(config_dict=config_dict)``
**Timeseries**
:param timeseries: optional - string to
set ``day`` or ``minute`` backtesting
or live trading
(default is ``minute``)
**Trading Strategy**
:param trade_strategy: optional - string to
set the type of ``Trading Strategy``
for backtesting or live trading
(default is ``count``)
**Algorithm Dataset Loading, Extracting, Reporting
and Trading History arguments**
:param load_config: optional - dictionary
for setting member variables to load an
agorithm-ready dataset from
a file, s3 or redis
:param report_config: optional - dictionary
for setting member variables to publish
an algo ``trading performance report`` to s3,
redis, a file or slack
:param history_config: optional - dictionary
for setting member variables to publish
an algo ``trade history`` to s3, redis, a file
or slack
:param extract_config: optional - dictionary
for setting member variables to publish
an algo ``trading performance report`` to s3,
redis, a file or slack
**(Optional) Data sources, datafeeds and datasets to gather**
:param iex_datasets: list of strings for gathering specific `IEX
datasets <https://iexcloud.io/>`__
which are set as consts: ``analysis_engine.iex.consts.FETCH_*``.
**(Optional) Redis connectivity arguments**
:param redis_enabled: bool - toggle for auto-caching all
datasets in Redis
(default is ``True``)
:param redis_address: Redis connection string
format is ``host:port``
(default is ``localhost:6379``)
:param redis_db: Redis db to use
(default is ``0``)
:param redis_password: optional - Redis password
(default is ``None``)
:param redis_expire: optional - Redis expire value
(default is ``None``)
:param redis_key: optional - redis key not used
(default is ``None``)
**(Optional) Minio (S3) connectivity arguments**
:param s3_enabled: bool - toggle for auto-archiving on Minio (S3)
(default is ``True``)
:param s3_address: Minio S3 connection string
format ``host:port``
(default is ``localhost:9000``)
:param s3_bucket: S3 Bucket for storing the artifacts
(default is ``dev``) which should be viewable on a browser:
http://localhost:9000/minio/dev/
:param s3_access_key: S3 Access key
(default is ``trexaccesskey``)
:param s3_secret_key: S3 Secret key
(default is ``trex123321``)
:param s3_region_name: S3 region name
(default is ``us-east-1``)
:param s3_secure: Transmit using tls encryption
(default is ``False``)
:param s3_key: optional s3 key not used
(default is ``None``)
**(Optional) Celery worker broker connectivity arguments**
:param celery_disabled: bool - toggle synchronous mode or publish
to an engine connected to the `Celery broker and backend
<https://github.com/celery/celery#transports-and-backends>`__
(default is ``True`` - synchronous mode without an engine
or need for a broker or backend for Celery)
:param broker_url: Celery broker url
(default is ``redis://0.0.0.0:6379/13``)
:param result_backend: Celery backend url
(default is ``redis://0.0.0.0:6379/14``)
:param label: tracking log label
:param publish_to_slack: optional - boolean for
publishing to slack (coming soon)
:param publish_to_s3: optional - boolean for
publishing to s3 (coming soon)
:param publish_to_redis: optional - boolean for
publishing to redis (coming soon)
**(Optional) Debugging**
:param verbose: bool - show extract warnings
and other debug logging (default is False)
:param raise_on_err: optional - boolean for
unittests and developing algorithms with the
``analysis_engine.run_algo.run_algo`` helper.
When set to ``True`` exceptions will
are raised to the calling functions
:param kwargs: keyword arguments dictionary
"""
# dictionary structure with a list sorted on: ascending dates
# algo_data_req[ticker][list][dataset] = pd.DataFrame
algo_data_req = {}
extract_requests = []
return_algo = False # return created algo objects for use by caller
rec = {}
msg = None
use_tickers = tickers
use_balance = balance
use_commission = commission
if ticker:
use_tickers = [ticker]
else:
if not use_tickers:
use_tickers = []
# if these are not set as args, but the algo object
# has them, use them instead:
if algo:
if len(use_tickers) == 0:
use_tickers = algo.get_tickers()
if not use_balance:
use_balance = algo.get_balance()
if not use_commission:
use_commission = algo.get_commission()
default_iex_datasets = [
'daily', 'minute', 'quote', 'stats', 'peers', 'news', 'financials',
'earnings', 'dividends', 'company'
]
if not iex_datasets:
iex_datasets = default_iex_datasets
if redis_enabled:
if not redis_address:
redis_address = os.getenv('REDIS_ADDRESS', 'localhost:6379')
if not redis_password:
redis_password = os.getenv('REDIS_PASSWORD', None)
if not redis_db:
redis_db = int(os.getenv('REDIS_DB', '0'))
if not redis_expire:
redis_expire = os.getenv('REDIS_EXPIRE', None)
if s3_enabled:
if not s3_address:
s3_address = os.getenv('S3_ADDRESS', 'localhost:9000')
if not s3_access_key:
s3_access_key = os.getenv('AWS_ACCESS_KEY_ID', 'trexaccesskey')
if not s3_secret_key:
s3_secret_key = os.getenv('AWS_SECRET_ACCESS_KEY', 'trex123321')
if not s3_region_name:
s3_region_name = os.getenv('AWS_DEFAULT_REGION', 'us-east-1')
if not s3_secure:
s3_secure = os.getenv('S3_SECURE', '0') == '1'
if not s3_bucket:
s3_bucket = os.getenv('S3_BUCKET', 'dev')
if not broker_url:
broker_url = os.getenv('WORKER_BROKER_URL', 'redis://0.0.0.0:6379/11')
if not result_backend:
result_backend = os.getenv('WORKER_BACKEND_URL',
'redis://0.0.0.0:6379/12')
if not label:
label = 'run-algo'
num_tickers = len(use_tickers)
last_close_str = ae_utils.get_last_close_str()
if iex_datasets:
if verbose:
log.info(f'{label} - tickers={num_tickers} '
f'iex={json.dumps(iex_datasets)}')
else:
if verbose:
log.info(f'{label} - tickers={num_tickers}')
ticker_key = use_key
if not ticker_key:
ticker_key = f'{ticker}_{last_close_str}'
if not algo:
algo = base_algo.BaseAlgo(ticker=None,
tickers=use_tickers,
balance=use_balance,
commission=use_commission,
config_dict=config_dict,
name=label,
auto_fill=auto_fill,
timeseries=timeseries,
trade_strategy=trade_strategy,
publish_to_slack=publish_to_slack,
publish_to_s3=publish_to_s3,
publish_to_redis=publish_to_redis,
raise_on_err=raise_on_err)
return_algo = True
# the algo object is stored
# in the result at: res['rec']['algo']
if not algo:
msg = f'{label} - missing algo object'
log.error(msg)
return build_result.build_result(status=ae_consts.EMPTY,
err=msg,
rec=rec)
if raise_on_err:
log.debug(f'{label} - enabling algo exception raises')
algo.raise_on_err = True
indicator_datasets = algo.get_indicator_datasets()
if len(indicator_datasets) == 0:
indicator_datasets = ae_consts.BACKUP_DATASETS
log.info(f'using all datasets={indicator_datasets}')
verbose_extract = False
if config_dict:
verbose_extract = config_dict.get('verbose_extract', False)
common_vals = {}
common_vals['base_key'] = ticker_key
common_vals['celery_disabled'] = celery_disabled
common_vals['ticker'] = ticker
common_vals['label'] = label
common_vals['iex_datasets'] = iex_datasets
common_vals['s3_enabled'] = s3_enabled
common_vals['s3_bucket'] = s3_bucket
common_vals['s3_address'] = s3_address
common_vals['s3_secure'] = s3_secure
common_vals['s3_region_name'] = s3_region_name
common_vals['s3_access_key'] = s3_access_key
common_vals['s3_secret_key'] = s3_secret_key
common_vals['s3_key'] = ticker_key
common_vals['redis_enabled'] = redis_enabled
common_vals['redis_address'] = redis_address
common_vals['redis_password'] = redis_password
common_vals['redis_db'] = redis_db
common_vals['redis_key'] = ticker_key
common_vals['redis_expire'] = redis_expire
use_start_date_str = start_date
use_end_date_str = end_date
last_close_date = ae_utils.last_close()
end_date_val = None
cache_freq_fmt = ae_consts.COMMON_TICK_DATE_FORMAT
if not use_end_date_str:
use_end_date_str = last_close_date.strftime(cache_freq_fmt)
end_date_val = ae_utils.get_date_from_str(date_str=use_end_date_str,
fmt=cache_freq_fmt)
start_date_val = None
if not use_start_date_str:
start_date_val = end_date_val - datetime.timedelta(days=60)
use_start_date_str = start_date_val.strftime(cache_freq_fmt)
else:
start_date_val = datetime.datetime.strptime(
use_start_date_str, ae_consts.COMMON_TICK_DATE_FORMAT)
total_dates = (end_date_val - start_date_val).days
if end_date_val < start_date_val:
msg = (
f'{label} - invalid dates - start_date={start_date_val} is after '
f'end_date={end_date_val}')
raise Exception(msg)
if verbose:
log.info(f'{label} - days={total_dates} '
f'start={use_start_date_str} '
f'end={use_end_date_str} '
f'datasets={indicator_datasets}')
for ticker in use_tickers:
req = algo_utils.build_algo_request(ticker=ticker,
use_key=use_key,
start_date=use_start_date_str,
end_date=use_end_date_str,
datasets=datasets,
balance=use_balance,
cache_freq=cache_freq,
timeseries=timeseries,
trade_strategy=trade_strategy,
label=label)
ticker_key = f'{ticker}_{last_close_str}'
common_vals['ticker'] = ticker
common_vals['base_key'] = ticker_key
common_vals['redis_key'] = ticker_key
common_vals['s3_key'] = ticker_key
for date_key in req['extract_datasets']:
date_req = api_requests.get_ds_dict(ticker=ticker,
base_key=date_key,
ds_id=label,
service_dict=common_vals)
node_date_key = date_key.replace(f'{ticker}_', '')
extract_requests.append({
'id': date_key,
'ticker': ticker,
'date_key': date_key,
'date': node_date_key,
'req': date_req
})
# end of for all ticker in use_tickers
first_extract_date = None
last_extract_date = None
total_extract_requests = len(extract_requests)
cur_idx = 1
for idx, extract_node in enumerate(extract_requests):
extract_ticker = extract_node['ticker']
extract_date = extract_node['date']
ds_node_id = extract_node['id']
if not first_extract_date:
first_extract_date = extract_date
last_extract_date = extract_date
perc_progress = ae_consts.get_percent_done(
progress=cur_idx, total=total_extract_requests)
percent_label = (f'{label} '
f'ticker={extract_ticker} '
f'date={extract_date} '
f'{perc_progress} '
f'{idx}/{total_extract_requests} '
f'{indicator_datasets}')
if verbose:
log.info(f'extracting - {percent_label}')
ticker_bt_data = build_ds_node.build_dataset_node(
ticker=extract_ticker,
date=extract_date,
service_dict=common_vals,
datasets=indicator_datasets,
log_label=label,
verbose=verbose_extract)
if ticker not in algo_data_req:
algo_data_req[ticker] = []
algo_data_req[ticker].append({
'id': ds_node_id, # id is currently the cache key in redis
'date': extract_date, # used to confirm dates in asc order
'data': ticker_bt_data
})
if verbose:
log.info(f'extract - {percent_label} '
f'dataset={len(algo_data_req[ticker])}')
cur_idx += 1
# end of for service_dict in extract_requests
# this could be a separate celery task
status = ae_consts.NOT_RUN
if len(algo_data_req) == 0:
msg = (f'{label} - nothing to test - no data found for '
f'tickers={use_tickers} '
f'between {first_extract_date} and {last_extract_date}')
log.info(msg)
return build_result.build_result(status=ae_consts.EMPTY,
err=msg,
rec=rec)
# this could be a separate celery task
try:
if verbose:
log.info(f'handle_data START - {percent_label} from '
f'{first_extract_date} to {last_extract_date}')
algo.handle_data(data=algo_data_req)
if verbose:
log.info(f'handle_data END - {percent_label} from '
f'{first_extract_date} to {last_extract_date}')
except Exception as e:
a_name = algo.get_name()
a_debug_msg = algo.get_debug_msg()
if not a_debug_msg:
a_debug_msg = 'debug message not set'
a_config_dict = ae_consts.ppj(algo.config_dict)
msg = (f'{percent_label} - algo={a_name} '
f'encountered exception in handle_data tickers={use_tickers} '
f'from {first_extract_date} to {last_extract_date} ex={e} '
f'and failed during operation: {a_debug_msg}')
if raise_on_err:
if algo:
try:
ind_obj = \
algo.get_indicator_process_last_indicator()
if ind_obj:
ind_obj_path = ind_obj.get_path_to_module()
ind_obj_config = ae_consts.ppj(ind_obj.get_config())
found_error_hint = False
if hasattr(ind_obj.use_df, 'to_json'):
if len(ind_obj.use_df.index) == 0:
log.critical(
f'indicator failure report for '
f'last module: '
f'{ind_obj_path} '
f'indicator={ind_obj.get_name()} '
f'config={ind_obj_config} '
f'dataset={ind_obj.use_df.head(5)} '
f'name_of_dataset={ind_obj.uses_data}')
log.critical(
'--------------------------------------'
'--------------------------------------')
log.critical('Please check if this indicator: '
f'{ind_obj_path} '
'supports Empty Dataframes')
log.critical(
'--------------------------------------'
'--------------------------------------')
found_error_hint = True
# indicator error hints
if not found_error_hint:
log.critical(
f'indicator failure report for last module: '
f'{ind_obj_path} '
f'indicator={ind_obj.get_name()} '
f'config={ind_obj_config} '
f'dataset={ind_obj.use_df.head(5)} '
f'name_of_dataset={ind_obj.uses_data}')
except Exception as f:
log.critical(f'failed to pull indicator processor '
f'last indicator for debugging '
f'from ex={e} with parsing ex={f}')
# end of ignoring non-supported ways of creating
# indicator processors
log.error(msg)
log.error(f'algo failure report: '
f'algo={a_name} handle_data() '
f'config={a_config_dict} ')
log.critical(f'algo failed during operation: {a_debug_msg}')
raise e
else:
log.error(msg)
return build_result.build_result(status=ae_consts.ERR,
err=msg,
rec=rec)
# end of try/ex
# this could be a separate celery task
try:
if verbose:
log.info(f'get_result START - {percent_label} from '
f'{first_extract_date} to {last_extract_date}')
rec = algo.get_result()
status = ae_consts.SUCCESS
if verbose:
log.info(f'get_result END - {percent_label} from '
f'{first_extract_date} to {last_extract_date}')
except Exception as e:
msg = (
f'{percent_label} - algo={algo.get_name()} encountered exception '
f'in get_result tickers={use_tickers} from '
f'{first_extract_date} to {last_extract_date} ex={e}')
if raise_on_err:
if algo:
log.error(f'algo={algo.get_name()} failed in get_result with '
f'debug_msg={algo.get_debug_msg()}')
log.error(msg)
raise e
else:
log.error(msg)
return build_result.build_result(status=ae_consts.ERR,
err=msg,
rec=rec)
# end of try/ex
if return_algo:
rec['algo'] = algo
return build_result.build_result(status=status, err=msg, rec=rec)
def dist_plot(
log_label,
df,
width=10.0,
height=10.0,
title='Distribution Plot',
style='default',
xlabel='',
ylabel='',
show_plot=True,
dropna_for_all=True):
"""dist_plot
Show a distribution plot for the passed in dataframe: ``df``
:param log_label: log identifier
:param df: initialized ``pandas.DataFrame``
:param width: width of figure
:param height: height of figure
:param style: style to use
:param xlabel: x-axis label
:param ylabel: y-axis label
:param show_plot: bool to show plot or not
:param dropna_for_all: optional - bool to toggle keep None's in
the plot ``df`` (default is drop them
for display purposes)
"""
rec = {}
result = build_result.build_result(
status=ae_consts.NOT_RUN,
err=None,
rec=rec)
try:
log.info(
f'{log_label} - '
'dist_plot'
' - start')
set_common_seaborn_fonts()
fig, ax = plt.subplots(
figsize=(width, height))
if style == 'default':
sns.set_context('poster')
sns.axes_style('darkgrid')
use_df = df
if dropna_for_all:
log.info(
f'{log_label} - '
'dist_plot'
' - dropna_for_all')
use_df = df.dropna(axis=0, how='any')
# end of pre-plot dataframe scrubbing
sns.distplot(
use_df,
color=ae_consts.PLOT_COLORS['blue'],
ax=ax)
if xlabel != '':
ax.set_xlabel(xlabel)
if ylabel != '':
ax.set_ylabel(ylabel)
plt.tight_layout()
plt.subplots_adjust(top=0.9)
fig.suptitle(title)
show_with_entities(
log_label=log_label,
xlabel=xlabel,
ylabel=ylabel,
title=title,
ax=ax,
fig=fig,
show_plot=show_plot)
rec['ax'] = ax
rec['fig'] = fig
result = build_result.build_result(
status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
err = (
f'failed dist_plot title={title} with ex={e}')
log.error(err)
result = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
# end of try/ex
send_final_log(
log_label=log_label,
fn_name='dist_plot',
result=result)
return result
def publish_ticker_aggregate_from_s3(self, work_dict):
"""publish_ticker_aggregate_from_s3
Publish Aggregated Ticker Data from S3 to Redis
:param work_dict: dictionary for key/values
"""
label = 'pub-tic-agg-s3-to-redis'
log.info(f'task - {label} - start work_dict={work_dict}')
ticker = ae_consts.TICKER
ticker_id = ae_consts.TICKER_ID
rec = {
'ticker': None,
'ticker_id': None,
's3_read_enabled': True,
's3_upload_enabled': True,
'redis_enabled': True,
's3_bucket': None,
's3_compiled_bucket': None,
's3_key': None,
'redis_key': None,
'updated': None
}
res = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec=rec)
try:
ticker = work_dict.get('ticker', ae_consts.TICKER)
ticker_id = int(work_dict.get('ticker_id', ae_consts.TICKER_ID))
if not ticker:
res = build_result.build_result(status=ae_consts.ERR,
err='missing ticker',
rec=rec)
return res
label = work_dict.get('label', label)
s3_key = work_dict.get('s3_key', None)
s3_bucket_name = work_dict.get('s3_bucket', 'pricing')
s3_compiled_bucket_name = work_dict.get('s3_compiled_bucket',
'compileddatasets')
redis_key = work_dict.get('redis_key', None)
updated = work_dict.get('updated', None)
enable_s3_upload = work_dict.get('s3_upload_enabled',
ae_consts.ENABLED_S3_UPLOAD)
enable_redis_publish = work_dict.get('redis_enabled',
ae_consts.ENABLED_REDIS_PUBLISH)
serializer = work_dict.get('serializer', 'json')
encoding = work_dict.get('encoding', 'utf-8')
enable_s3_read = True
rec['ticker'] = ticker
rec['ticker_id'] = ticker_id
rec['s3_bucket'] = s3_bucket_name
rec['s3_compiled_bucket'] = s3_compiled_bucket_name
rec['s3_key'] = s3_key
rec['redis_key'] = redis_key
rec['updated'] = updated
rec['s3_read_enabled'] = enable_s3_read
rec['s3_upload_enabled'] = enable_s3_upload
rec['redis_enabled'] = enable_redis_publish
if enable_s3_read:
log.info(f'{label} parsing s3 values')
access_key = work_dict.get('s3_access_key',
ae_consts.S3_ACCESS_KEY)
secret_key = work_dict.get('s3_secret_key',
ae_consts.S3_SECRET_KEY)
region_name = work_dict.get('s3_region_name',
ae_consts.S3_REGION_NAME)
service_address = work_dict.get('s3_address', ae_consts.S3_ADDRESS)
secure = work_dict.get('s3_secure', ae_consts.S3_SECURE) == '1'
endpoint_url = f'http{"s" if secure else ""}://{service_address}'
log.info(f'{label} building s3 endpoint_url={endpoint_url} '
f'region={region_name}')
s3 = boto3.resource(
's3',
endpoint_url=endpoint_url,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region_name,
config=boto3.session.Config(signature_version='s3v4'))
try:
log.info(f'{label} checking bucket={s3_bucket_name} exists')
if s3.Bucket(s3_bucket_name) not in s3.buckets.all():
log.info(f'{label} creating bucket={s3_bucket_name}')
s3.create_bucket(Bucket=s3_bucket_name)
except Exception as e:
log.info(f'{label} failed creating bucket={s3_bucket_name} '
f'with ex={e}')
# end of try/ex for creating bucket
try:
log.info(f'{label} checking bucket={s3_bucket_name} keys')
date_keys = []
keys = []
# {TICKER}_YYYY-DD-MM regex
reg = r'^.*_\d{4}-(0?[1-9]|1[012])-(0?[1-9]|[12][0-9]|3[01])$'
for bucket in s3.buckets.all():
for key in bucket.objects.all():
if (ticker.lower() in key.key.lower()
and bool(re.compile(reg).search(key.key))):
keys.append(key.key)
date_keys.append(key.key.split(f'{ticker}_')[1])
except Exception as e:
log.info(f'{label} failed to get bucket={s3_bucket_name} '
f'keys with ex={e}')
# end of try/ex for getting bucket keys
if keys:
data = []
for idx, key in enumerate(keys):
try:
log.info(
f'{label} reading to s3={s3_bucket_name}/{key} '
f'updated={updated}')
loop_data = s3_read_contents_from_key.\
s3_read_contents_from_key(
s3=s3,
s3_bucket_name=s3_bucket_name,
s3_key=key,
encoding=encoding,
convert_as_json=True)
initial_size_value = \
len(str(loop_data)) / 1024000
initial_size_str = ae_consts.to_f(initial_size_value)
if ae_consts.ev('DEBUG_S3', '0') == '1':
log.info(f'{label} read s3={s3_bucket_name}/{key} '
f'data={ae_consts.ppj(loop_data)}')
else:
log.info(
f'{label} read s3={s3_bucket_name}/{key} data '
f'size={initial_size_str} MB')
data.append({f'{date_keys[idx]}': loop_data})
except Exception as e:
err = (
f'{label} failed reading bucket={s3_bucket_name} '
f'key={key} ex={e}')
log.error(err)
res = build_result.build_result(
status=ae_consts.NOT_RUN, err=err, rec=rec)
# end of try/ex for creating bucket
else:
log.info(f'{label} No keys found in S3 '
f'bucket={s3_bucket_name} for ticker={ticker}')
else:
log.info(f'{label} SKIP S3 read bucket={s3_bucket_name} '
f'ticker={ticker}')
# end of if enable_s3_read
if data and enable_s3_upload:
try:
log.info(f'{label} checking bucket={s3_compiled_bucket_name} '
'exists')
if s3.Bucket(s3_compiled_bucket_name) not in s3.buckets.all():
log.info(
f'{label} creating bucket={s3_compiled_bucket_name}')
s3.create_bucket(Bucket=s3_compiled_bucket_name)
except Exception as e:
log.info(f'{label} failed creating '
f'bucket={s3_compiled_bucket_name} with ex={e}')
# end of try/ex for creating bucket
try:
cmpr_data = zlib.compress(json.dumps(data).encode(encoding), 9)
if ae_consts.ev('DEBUG_S3', '0') == '1':
log.info(
f'{label} uploading to '
f's3={s3_compiled_bucket_name}/{s3_key} '
f'data={ae_consts.ppj(loop_data)} updated={updated}')
else:
sizes = {
'MB': 1024000,
'GB': 1024000000,
'TB': 1024000000000,
'PB': 1024000000000000
}
initial_size_value = len(str(data))
org_data_size = 'MB'
for key in sizes.keys():
size = float(initial_size_value) / float(sizes[key])
if size > 1024:
continue
org_data_size = key
initial_size_value = size
break
initial_size_str = ae_consts.to_f(initial_size_value)
cmpr_data_size_value = len(cmpr_data)
cmpr_data_size = 'MB'
for key in sizes.keys():
size = float(cmpr_data_size_value) / float(sizes[key])
if size > 1024:
continue
cmpr_data_size = key
cmpr_data_size_value = size
break
cmpr_size_str = ae_consts.to_f(cmpr_data_size_value)
log.info(
f'{label} uploading to '
f's3={s3_compiled_bucket_name}/{s3_key} data '
f'original_size={initial_size_str} {org_data_size} '
f'compressed_size={cmpr_size_str} {cmpr_data_size} '
f'updated={updated}')
s3.Bucket(s3_compiled_bucket_name).put_object(Key=s3_key,
Body=cmpr_data)
except Exception as e:
log.error(f'{label} failed '
f'uploading bucket={s3_compiled_bucket_name} '
f'key={s3_key} ex={e}')
# end of try/ex for creating bucket
else:
log.info(
f'{label} SKIP S3 upload bucket={s3_bucket_name} key={s3_key}')
# end of if enable_s3_upload
if data and enable_redis_publish:
redis_address = work_dict.get('redis_address',
ae_consts.REDIS_ADDRESS)
redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY)
redis_password = work_dict.get('redis_password',
ae_consts.REDIS_PASSWORD)
redis_db = work_dict.get('redis_db', None)
if not redis_db:
redis_db = ae_consts.REDIS_DB
redis_expire = None
if 'redis_expire' in work_dict:
redis_expire = work_dict.get('redis_expire',
ae_consts.REDIS_EXPIRE)
log.info(f'redis enabled address={redis_address}@{redis_db} '
f'key={redis_key}')
redis_host = redis_address.split(':')[0]
redis_port = redis_address.split(':')[1]
try:
if ae_consts.ev('DEBUG_REDIS', '0') == '1':
log.info(
f'{label} publishing redis={redis_host}:{redis_port} '
f'db={redis_db} key={redis_key} updated={updated} '
f'expire={redis_expire} data={ae_consts.ppj(data)}')
else:
log.info(
f'{label} publishing redis={redis_host}:{redis_port} '
f'db={redis_db} key={redis_key} '
f'updated={updated} expire={redis_expire}')
# end of if/else
rc = redis.Redis(host=redis_host,
port=redis_port,
password=redis_password,
db=redis_db)
redis_set_res = redis_set.set_data_in_redis_key(
label=label,
client=rc,
key=redis_key,
data=data,
serializer=serializer,
encoding=encoding,
expire=redis_expire,
px=None,
nx=False,
xx=False)
log.info(
f'{label} redis_set '
f'status={ae_consts.get_status(redis_set_res["status"])} '
f'err={redis_set_res["err"]}')
except Exception as e:
log.error(f'{label} failed - redis publish to '
f'key={redis_key} ex={e}')
# end of try/ex for creating bucket
else:
log.info(f'{label} SKIP REDIS publish key={redis_key}')
# end of if enable_redis_publish
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
res = build_result.build_result(
status=ae_consts.ERR,
err=(f'failed - publish_from_s3 dict={work_dict} with ex={e}'),
rec=rec)
log.error(f'{label} - {res["err"]}')
# end of try/ex
log.info('task - publish_from_s3 done - '
f'{label} - status={ae_consts.get_status(res["status"])}')
return get_task_results.get_task_results(work_dict=work_dict, result=res)
def get_data_from_redis_key(label=None,
client=None,
host=None,
port=None,
password=None,
db=None,
key=None,
expire=None,
serializer='json',
encoding='utf-8'):
"""get_data_from_redis_key
:param label: log tracking label
:param client: initialized redis client
:param host: not used yet - redis host
:param port: not used yet - redis port
:param password: not used yet - redis password
:param db: not used yet - redis db
:param key: not used yet - redis key
:param expire: not used yet - redis expire
:param serializer: not used yet - support for future
pickle objects in redis
:param encoding: format of the encoded key in redis
"""
decoded_data = None
data = None
rec = {'data': data}
res = build_result.build_result(status=NOT_RUN, err=None, rec=rec)
log_id = label if label else 'get-data'
try:
use_client = client
if not use_client:
log.debug('{} get key={} new client={}:{}@{}'.format(
log_id, key, host, port, db))
use_client = redis.Redis(host=host,
port=port,
password=password,
db=db)
else:
log.debug('{} get key={} client'.format(log_id, key))
# create Redis client if not set
# https://redis-py.readthedocs.io/en/latest/index.html#redis.StrictRedis.get # noqa
raw_data = use_client.get(name=key)
if raw_data:
log.debug('{} decoding key={} encoding={}'.format(
log_id, key, encoding))
decoded_data = raw_data.decode(encoding)
log.debug('{} deserial key={} serializer={}'.format(
log_id, key, serializer))
if serializer == 'json':
data = json.loads(decoded_data)
elif serializer == 'df':
data = decoded_data
else:
data = decoded_data
if data:
if ev('DEBUG_REDIS', '0') == '1':
log.info('{} - found key={} data={}'.format(
log_id, key, ppj(data)))
else:
log.debug('{} - found key={}'.format(log_id, key))
# log snippet - if data
rec['data'] = data
return build_result.build_result(status=SUCCESS, err=None, rec=rec)
else:
log.debug('{} no data key={}'.format(log_id, key))
return build_result.build_result(status=SUCCESS, err=None, rec=rec)
except Exception as e:
err = ('{} failed - redis get from decoded={} data={} '
'key={} ex={}'.format(log_id, decoded_data, data, key, e))
log.error(err)
res = build_result.build_result(status=ERR, err=err, rec=rec)
# end of try/ex for getting redis data
return res
def publish_pricing_update(
self,
work_dict):
"""publish_pricing_update
Publish Ticker Data to S3 and Redis
- prices - turn off with ``work_dict.get_pricing = False``
- news - turn off with ``work_dict.get_news = False``
- options - turn off with ``work_dict.get_options = False``
:param work_dict: dictionary for key/values
"""
label = 'publish_pricing'
log.info(
'task - {} - start'.format(
label))
ticker = TICKER
ticker_id = TICKER_ID
rec = {
'ticker': None,
'ticker_id': None,
's3_enabled': False,
'redis_enabled': False,
's3_bucket': None,
's3_key': None,
'redis_key': None,
'updated': None
}
res = build_result.build_result(
status=NOT_RUN,
err=None,
rec=rec)
try:
ticker = work_dict.get(
'ticker',
TICKER)
ticker_id = int(work_dict.get(
'ticker_id',
TICKER_ID))
if not ticker:
res = build_result.build_result(
status=ERR,
err='missing ticker',
rec=rec)
return res
label = work_dict.get(
'label',
label)
s3_key = work_dict.get(
's3_key',
None)
s3_bucket_name = work_dict.get(
's3_bucket',
'pricing')
redis_key = work_dict.get(
'redis_key',
None)
data = work_dict.get(
'data',
None)
updated = work_dict.get(
'updated',
None)
enable_s3_upload = work_dict.get(
's3_enabled',
ENABLED_S3_UPLOAD)
enable_redis_publish = work_dict.get(
'redis_enabled',
ENABLED_REDIS_PUBLISH)
serializer = work_dict.get(
'serializer',
'json')
encoding = work_dict.get(
'encoding',
'utf-8')
rec['ticker'] = ticker
rec['ticker_id'] = ticker_id
rec['s3_bucket'] = s3_bucket_name
rec['s3_key'] = s3_key
rec['redis_key'] = redis_key
rec['updated'] = updated
rec['s3_enabled'] = enable_s3_upload
rec['redis_enabled'] = enable_redis_publish
if enable_s3_upload:
access_key = work_dict.get(
's3_access_key',
S3_ACCESS_KEY)
secret_key = work_dict.get(
's3_secret_key',
S3_SECRET_KEY)
region_name = work_dict.get(
's3_region_name',
S3_REGION_NAME)
service_address = work_dict.get(
's3_address',
S3_ADDRESS)
secure = work_dict.get(
's3_secure',
S3_SECURE) == '1'
endpoint_url = 'http://{}'.format(
service_address)
if secure:
endpoint_url = 'https://{}'.format(
service_address)
log.info(
'{} building s3 endpoint_url={} '
'region={}'.format(
label,
endpoint_url,
region_name))
s3 = boto3.resource(
's3',
endpoint_url=endpoint_url,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region_name,
config=boto3.session.Config(
signature_version='s3v4')
)
try:
log.info(
'{} checking bucket={} exists'.format(
label,
s3_bucket_name))
if s3.Bucket(s3_bucket_name) not in s3.buckets.all():
log.info(
'{} creating bucket={}'.format(
label,
s3_bucket_name))
s3.create_bucket(
Bucket=s3_bucket_name)
except Exception as e:
log.info(
'{} failed creating bucket={} '
'with ex={}'.format(
label,
s3_bucket_name,
e))
# end of try/ex for creating bucket
try:
log.info(
'{} uploading to s3={}/{} '
'updated={}'.format(
label,
s3_bucket_name,
s3_key,
updated))
s3.Bucket(s3_bucket_name).put_object(
Key=s3_key,
Body=json.dumps(data).encode(encoding))
except Exception as e:
log.error(
'{} failed uploading bucket={} '
'key={} ex={}'.format(
label,
s3_bucket_name,
s3_key,
e))
# end of try/ex for creating bucket
else:
log.info(
'{} SKIP S3 upload bucket={} '
'key={}'.format(
label,
s3_bucket_name,
s3_key))
# end of if enable_s3_upload
if enable_redis_publish:
redis_address = work_dict.get(
'redis_address',
REDIS_ADDRESS)
redis_key = work_dict.get(
'redis_key',
REDIS_KEY)
redis_password = work_dict.get(
'redis_password',
REDIS_PASSWORD)
redis_db = work_dict.get(
'redis_db',
REDIS_DB)
redis_expire = None
if 'redis_expire' in work_dict:
redis_expire = work_dict.get(
'redis_expire',
REDIS_EXPIRE)
log.info(
'redis enabled address={}@{} '
'key={}'.format(
redis_address,
redis_db,
redis_key))
redis_host = None
redis_port = None
try:
redis_host = redis_address.split(':')[0]
redis_port = redis_address.split(':')[1]
except Exception as c:
err = (
'{} failed parsing redis_address={} '
'with ex={} '
'please set one with the format: '
'<hostname>:<port>'.format(
label,
redis_address,
c))
log.critical(err)
res = build_result.build_result(
status=ERR,
err=err,
rec=rec)
return res
# end of checking that redis_address is valid
try:
log.info(
'{} publishing redis={}:{} '
'db={} key={} '
'updated={} expire={}'.format(
label,
redis_host,
redis_port,
redis_db,
redis_key,
updated,
redis_expire))
rc = redis.Redis(
host=redis_host,
port=redis_port,
password=redis_password,
db=redis_db)
redis_set_res = redis_set.set_data_in_redis_key(
label=label,
client=rc,
key=redis_key,
data=data,
serializer=serializer,
encoding=encoding,
expire=redis_expire,
px=None,
nx=False,
xx=False)
log.info(
'{} redis_set status={} err={}'.format(
label,
get_status(redis_set_res['status']),
redis_set_res['err']))
except Exception as e:
log.error(
'{} failed - redis publish to '
'key={} ex={}'.format(
label,
redis_key,
e))
# end of try/ex for creating bucket
else:
log.info(
'{} SKIP REDIS publish '
'key={}'.format(
label,
redis_key))
# end of if enable_redis_publish
res = build_result.build_result(
status=SUCCESS,
err=None,
rec=rec)
except Exception as e:
res = build_result.build_result(
status=ERR,
err=(
'failed - publish_pricing_update '
'dict={} with ex={}').format(
work_dict,
e),
rec=rec)
log.error(
'{} - {}'.format(
label,
res['err']))
# end of try/ex
log.info(
'task - publish_pricing_update done - '
'{} - status={}'.format(
label,
get_status(res['status'])))
return analysis_engine.get_task_results.get_task_results(
work_dict=work_dict,
result=res)
def plot_trading_history(title,
df,
red,
red_color=None,
red_label=None,
blue=None,
blue_color=None,
blue_label=None,
green=None,
green_color=None,
green_label=None,
orange=None,
orange_color=None,
orange_label=None,
date_col='date',
xlabel='Date',
ylabel='Algo Values',
linestyle='-',
width=15.0,
height=15.0,
date_format='%d\n%b',
df_filter=None,
start_date=None,
footnote_text=None,
footnote_xpos=0.70,
footnote_ypos=0.01,
footnote_color='#888888',
footnote_fontsize=8,
scale_y=False,
show_plot=True,
dropna_for_all=False,
verbose=False):
"""plot_trading_history
Plot columns up to 4 lines from the ``Trading History`` dataset
:param title: title of the plot
:param df: dataset which is ``pandas.DataFrame``
:param red: string - column name to plot in
``red_color`` (or default ``ae_consts.PLOT_COLORS[red]``)
where the column is in the ``df`` and
accessible with:``df[red]``
:param red_color: hex color code to plot the data in the
``df[red]`` (default is ``ae_consts.PLOT_COLORS['red']``)
:param red_label: optional - string for the label used
to identify the ``red`` line in the legend
:param blue: string - column name to plot in
``blue_color`` (or default ``ae_consts.PLOT_COLORS['blue']``)
where the column is in the ``df`` and
accessible with:``df[blue]``
:param blue_color: hex color code to plot the data in the
``df[blue]`` (default is ``ae_consts.PLOT_COLORS['blue']``)
:param blue_label: optional - string for the label used
to identify the ``blue`` line in the legend
:param green: string - column name to plot in
``green_color`` (or default ``ae_consts.PLOT_COLORS['darkgreen']``)
where the column is in the ``df`` and
accessible with:``df[green]``
:param green_color: hex color code to plot the data in the
``df[green]`` (default is ``ae_consts.PLOT_COLORS['darkgreen']``)
:param green_label: optional - string for the label used
to identify the ``green`` line in the legend
:param orange: string - column name to plot in
``orange_color`` (or default ``ae_consts.PLOT_COLORS['orange']``)
where the column is in the ``df`` and
accessible with:``df[orange]``
:param orange_color: hex color code to plot the data in the
``df[orange]`` (default is ``ae_consts.PLOT_COLORS['orange']``)
:param orange_label: optional - string for the label used
to identify the ``orange`` line in the legend
:param date_col: string - date column name
(default is ``date``)
:param xlabel: x-axis label
:param ylabel: y-axis label
:param linestyle: style of the plot line
:param width: float - width of the image
:param height: float - height of the image
:param date_format: string - format for dates
:param df_filter: optional - initialized ``pandas.DataFrame`` query
for reducing the ``df`` records before plotting. As an eaxmple
``df_filter=(df['close'] > 0.01)`` would find only records in
the ``df`` with a ``close`` value greater than ``0.01``
:param start_date: optional - string ``datetime``
for plotting only from a date formatted as
``YYYY-MM-DD HH\:MM\:SS``
:param footnote_text: optional - string footnote text
(default is ``algotraders <DATE>``)
:param footnote_xpos: optional - float for footnote position
on the x-axies
(default is ``0.75``)
:param footnote_ypos: optional - float for footnote position
on the y-axies
(default is ``0.01``)
:param footnote_color: optional - string hex color code for
the footnote text
(default is ``#888888``)
:param footnote_fontsize: optional - float footnote font size
(default is ``8``)
:param scale_y: optional - bool to scale the y-axis with
.. code-block:: python
use_ax.set_ylim(
[0, use_ax.get_ylim()[1] * 3])
:param show_plot: bool to show the plot
:param dropna_for_all: optional - bool to toggle keep None's in
the plot ``df`` (default is drop them for display purposes)
:param verbose: optional - bool to show logs for debugging
a dataset
"""
rec = {'ax1': None, 'ax2': None, 'ax3': None, 'ax4': None, 'fig': None}
result = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec=rec)
if verbose:
log.info('plot_trading_history - start')
use_red = red_color
use_blue = blue_color
use_green = green_color
use_orange = orange_color
if not use_red:
use_red = ae_consts.PLOT_COLORS['red']
if not use_blue:
use_blue = ae_consts.PLOT_COLORS['blue']
if not use_green:
use_green = ae_consts.PLOT_COLORS['darkgreen']
if not use_orange:
use_orange = ae_consts.PLOT_COLORS['orange']
use_footnote = footnote_text
if not use_footnote:
use_footnote = ('algotraders - {}'.format(
datetime.datetime.now().strftime(
ae_consts.COMMON_TICK_DATE_FORMAT)))
column_list = [date_col]
all_plots = []
if red:
column_list.append(red)
all_plots.append({'column': red, 'color': use_red})
if blue:
column_list.append(blue)
all_plots.append({'column': blue, 'color': use_blue})
if green:
column_list.append(green)
all_plots.append({'column': green, 'color': use_green})
if orange:
column_list.append(orange)
all_plots.append({'column': orange, 'color': use_orange})
use_df = df
if start_date:
start_date_value = datetime.datetime.strptime(
start_date, ae_consts.COMMON_TICK_DATE_FORMAT)
use_df = df[(df[date_col] >= start_date_value)][column_list]
# end of filtering by start date
if verbose:
log.info(
'plot_history_df start_date={} df.index={} column_list={}'.format(
start_date, len(use_df.index), column_list))
if hasattr(df_filter, 'to_json'):
# Was seeing this warning below in Jupyter:
# UserWarning: Boolean Series key
# will be reindexed to match DataFrame index
# use_df = use_df[df_filter][column_list]
# now using:
use_df = use_df.loc[df_filter, column_list]
if verbose:
log.info('plot_history_df filter df.index={} column_list={}'.format(
start_date, len(use_df.index), column_list))
if dropna_for_all:
use_df = use_df.dropna(axis=0, how='any')
if verbose:
log.info('plot_history_df dropna_for_all')
# end of pre-plot dataframe scrubbing
ae_charts.set_common_seaborn_fonts()
hex_color = ae_consts.PLOT_COLORS['blue']
fig, ax = plt.subplots(sharex=True, sharey=True, figsize=(width, height))
# Convert matplotlib date numbers to strings for dates to
# avoid dealing with weekend date gaps in plots
date_strings, date_labels = \
ae_utils.get_trade_open_xticks_from_date_col(
use_df[date_col])
"""
hit the slice warning with this approach before
and one trying df[date_col] = df[date_col].dt.strftime
SettingWithCopyWarning
Try using .loc[row_indexer,col_indexer] = value instead
use_df[date_col].replace(
use_df[date_col].dt.strftime(
ae_consts.COMMON_TICK_DATE_FORMAT),
inplace=True)
trying this:
https://stackoverflow.com/questions/19738169/
convert-column-of-date-objects-in-pandas-dataframe-to-strings
"""
use_df[date_col] = use_df[date_col].apply(
lambda x: x.strftime(ae_consts.COMMON_TICK_DATE_FORMAT))
all_axes = []
num_plots = len(all_plots)
for idx, node in enumerate(all_plots):
column_name = node['column']
hex_color = node['color']
use_ax = ax
if idx > 0:
use_ax = ax.twinx()
if verbose:
log.info('plot_history_df - {}/{} - {} in {} - ax={}'.format(
(idx + 1), num_plots, column_name, hex_color, use_ax))
all_axes.append(use_ax)
use_ax.plot(use_df[date_col],
use_df[column_name],
linestyle=linestyle,
color=hex_color)
if idx > 0:
if scale_y:
use_ax.set_ylim([0, use_ax.get_ylim()[1] * 3])
use_ax.yaxis.set_ticklabels([])
use_ax.yaxis.set_ticks([])
use_ax.xaxis.grid(False)
use_ax.yaxis.grid(False)
# end if this is not the fist axis
use_ax.set_xticks(date_strings)
use_ax.set_xticklabels(date_labels, rotation=45, ha='right')
# end of for all plots
lines = []
for idx, cur_ax in enumerate(all_axes):
ax_lines = cur_ax.get_lines()
for line in ax_lines:
label_name = str(line.get_label())
use_label = label_name
if idx == 0:
if red_label:
use_label = red_label
elif idx == 1:
if blue_label:
use_label = blue_label
elif idx == 2:
use_label = label_name[-20:]
if green_label:
use_label = green_label
elif idx == 3:
use_label = label_name[-20:]
if orange_label:
use_label = orange_label
else:
if len(label_name) > 10:
use_label = label_name[-20:]
# end of fixing the labels in the legend
line.set_label(use_label)
if line.get_label() not in lines:
lines.append(line)
rec['ax{}'.format(idx + 1)] = cur_ax
# end of compiling a new-shortened legend while removing dupes
for idx, cur_ax in enumerate(all_axes):
if cur_ax:
if cur_ax.get_legend():
cur_ax.get_legend().remove()
# end of removing all previous legends
if verbose:
log.info('legend lines={}'.format([l.get_label() for l in lines]))
# log what's going to be in the legend
ax.legend(lines, [l.get_label() for l in lines], loc='best', shadow=True)
fig.autofmt_xdate()
plt.xlabel(xlabel)
plt.ylabel(ylabel)
ax.set_title(title)
ae_charts.add_footnote(fig=fig,
xpos=footnote_xpos,
ypos=footnote_ypos,
text=use_footnote,
color=footnote_color,
fontsize=footnote_fontsize)
plt.tight_layout()
plt.show()
if show_plot:
plt.show()
else:
plt.plot()
rec['fig'] = fig
result = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
return result
def build_df_from_redis(label=None,
client=None,
address=None,
host=None,
port=None,
password=None,
db=None,
key=None,
expire=None,
serializer='json',
encoding='utf-8',
orient='records'):
"""build_df_from_redis
:param label: log tracking label
:param client: initialized redis client
:param address: redis address: <host:port>
:param host: redis host
:param port: redis port
:param password: redis password
:param db: redis db
:param key: redis key
:param expire: not used yet - redis expire
:param serializer: support for future
pickle objects in redis
:param encoding: format of the encoded key in redis
:param orient: use the same orient value as
the ``to_json(orient='records')`` used
to deserialize the DataFrame correctly.
"""
data = None
valid_df = False
df = None
rec = {'valid_df': valid_df, 'data': data}
res = build_result.build_result(status=NOT_RUN, err=None, rec=rec)
log_id = label if label else 'build-df'
try:
log.debug('{} calling get redis key={}'.format(log_id, key))
use_host = host
use_port = port
if not use_host and not use_port:
if address:
use_host = address.split(':')[0]
use_port = int(address.split(':')[1])
use_client = client
if not use_client:
log.debug('{} connecting to redis={}:{}@{}'.format(
log_id, use_host, use_port, db))
use_client = redis.Redis(host=use_host,
port=use_port,
password=password,
db=db)
redis_res = redis_get.get_data_from_redis_key(label=log_id,
client=use_client,
host=use_host,
port=use_port,
password=password,
db=db,
key=key,
expire=expire,
serializer='json',
encoding=encoding)
valid_df = False
if redis_res['status'] == SUCCESS:
data = redis_res['rec'].get('data', None)
if data:
if ev('DEBUG_REDIS', '0') == '1':
log.info('{} - found key={} data={}'.format(
log_id, key, ppj(data)))
else:
log.debug('{} - loading df from key={}'.format(
log_id, key))
df = pd.read_json(data, orient='records')
valid_df = True
else:
log.debug('{} key={} no data'.format(log_id, key))
# if data
rec['data'] = df
rec['valid_df'] = valid_df
res = build_result.build_result(status=SUCCESS, err=None, rec=rec)
return res
else:
log.debug('{} no data key={}'.format(log_id, key))
res = build_result.build_result(status=SUCCESS, err=None, rec=rec)
return res
except Exception as e:
err = ('{} failed - build_df_from_redis data={} '
'key={} ex={}'.format(log_id, (data == '0'), key, e))
log.error(err)
res = build_result.build_result(status=ERR, err=err, rec=rec)
# end of try/ex for getting redis data
return res
def get_data_from_yahoo(work_dict):
"""get_data_from_yahoo
Get data from yahoo
:param work_dict: request dictionary
"""
label = 'get_data_from_yahoo'
log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict))
num_news_rec = 0
num_option_calls = 0
num_option_puts = 0
cur_high = -1
cur_low = -1
cur_open = -1
cur_close = -1
cur_volume = -1
rec = {
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'exp_date': None,
'publish_pricing_update': None,
'date': None,
'updated': None
}
res = {'status': NOT_RUN, 'err': None, 'rec': rec}
try:
ticker = work_dict.get('ticker', TICKER)
exp_date = work_dict.get('exp_date', None)
cur_strike = work_dict.get('strike', None)
contract_type = str(work_dict.get('contract', 'C')).upper()
get_pricing = work_dict.get('get_pricing', True)
get_news = work_dict.get('get_news', True)
get_options = work_dict.get('get_options', True)
orient = work_dict.get('orient', 'records')
label = work_dict.get('label', label)
ticker_results = pinance.Pinance(ticker)
num_news_rec = 0
use_date = exp_date
if not exp_date:
exp_date = opt_dates.option_expiration(date=exp_date)
use_date = exp_date.strftime('%Y-%m-%d')
"""
Debug control flags
Quickly turn specific fetches off:
get_news = False
get_pricing = False
get_options = False
"""
if get_pricing:
log.info('{} getting ticker={} pricing'.format(label, ticker))
ticker_results.get_quotes()
if ticker_results.quotes_data:
pricing_dict = ticker_results.quotes_data
cur_high = pricing_dict.get('regularMarketDayHigh', None)
cur_low = pricing_dict.get('regularMarketDayLow', None)
cur_open = pricing_dict.get('regularMarketOpen', None)
cur_close = pricing_dict.get('regularMarketPreviousClose',
None)
cur_volume = pricing_dict.get('regularMarketVolume', None)
pricing_dict['high'] = cur_high
pricing_dict['low'] = cur_low
pricing_dict['open'] = cur_open
pricing_dict['close'] = cur_close
pricing_dict['volume'] = cur_volume
pricing_dict['date'] = get_last_close_str()
if 'regularMarketTime' in pricing_dict:
pricing_dict['market_time'] = \
datetime.datetime.fromtimestamp(
pricing_dict['regularMarketTime']).strftime(
COMMON_TICK_DATE_FORMAT)
if 'postMarketTime' in pricing_dict:
pricing_dict['post_market_time'] = \
datetime.datetime.fromtimestamp(
pricing_dict['postMarketTime']).strftime(
COMMON_TICK_DATE_FORMAT)
log.info('{} ticker={} converting pricing to '
'df orient={}'.format(label, ticker, orient))
try:
rec['pricing'] = pricing_dict
except Exception as f:
rec['pricing'] = '{}'
log.info('{} ticker={} failed converting pricing '
'data={} to df ex={}'.format(
label, ticker, ppj(pricing_dict), f))
# try/ex
log.info('{} ticker={} done converting pricing to '
'df orient={}'.format(label, ticker, orient))
else:
log.error('{} ticker={} missing quotes_data={}'.format(
label, ticker, ticker_results.quotes_data))
# end of if ticker_results.quotes_data
log.info('{} ticker={} close={} vol={}'.format(
label, ticker, cur_close, cur_volume))
else:
log.info('{} skip - getting ticker={} pricing'.format(
label, ticker, get_pricing))
# if get_pricing
if get_news:
log.info('{} getting ticker={} news'.format(label, ticker))
ticker_results.get_news()
if ticker_results.news_data:
news_list = None
try:
news_list = ticker_results.news_data
log.info('{} ticker={} converting news to '
'df orient={}'.format(label, ticker, orient))
num_news_rec = len(news_list)
rec['news'] = news_list
except Exception as f:
rec['news'] = '{}'
log.info('{} ticker={} failed converting news '
'data={} to df ex={}'.format(
label, ticker, news_list, f))
# try/ex
log.info('{} ticker={} done converting news to '
'df orient={}'.format(label, ticker, orient))
else:
log.info('{} ticker={} Yahoo NO news={}'.format(
label, ticker, ticker_results.news_data))
# end of if ticker_results.news_data
else:
log.info('{} skip - getting ticker={} news'.format(label, ticker))
# end if get_news
if get_options:
get_all_strikes = True
if get_all_strikes:
cur_strike = None
else:
if cur_close:
cur_strike = int(cur_close)
if not cur_strike:
cur_strike = 287
log.info('{} ticker={} num_news={} get options close={} '
'exp_date={} contract={} strike={}'.format(
label, ticker, num_news_rec, cur_close, use_date,
contract_type, cur_strike))
options_dict = \
yahoo_get_pricing.get_options(
ticker=ticker,
exp_date_str=use_date,
contract_type=contract_type,
strike=cur_strike)
rec['options'] = '{}'
try:
log.info('{} ticker={} converting options to '
'df orient={}'.format(label, ticker, orient))
num_option_calls = options_dict.get('num_calls', None)
num_option_puts = options_dict.get('num_puts', None)
rec['options'] = {
'exp_date': options_dict.get('exp_date', None),
'calls': options_dict.get('calls', None),
'puts': options_dict.get('puts', None),
'num_calls': num_option_calls,
'num_puts': num_option_puts
}
rec['calls'] = rec['options'].get('calls', EMPTY_DF_STR)
rec['puts'] = rec['options'].get('puts', EMPTY_DF_STR)
except Exception as f:
rec['options'] = '{}'
log.info('{} ticker={} failed converting options '
'data={} to df ex={}'.format(label, ticker,
options_dict, f))
# try/ex
log.info('{} ticker={} done converting options to '
'df orient={} num_calls={} num_puts={}'.format(
label, ticker, orient, num_option_calls,
num_option_puts))
else:
log.info('{} skip - getting ticker={} options'.format(
label, ticker))
# end of if get_options
log.info('{} yahoo pricing for ticker={} close={} '
'num_calls={} num_puts={} news={}'.format(
label, ticker, cur_close, num_option_calls,
num_option_puts, num_news_rec))
fields_to_upload = ['pricing', 'options', 'calls', 'puts', 'news']
for field_name in fields_to_upload:
upload_and_cache_req = copy.deepcopy(work_dict)
upload_and_cache_req['celery_disabled'] = True
upload_and_cache_req['data'] = rec[field_name]
if not upload_and_cache_req['data']:
upload_and_cache_req['data'] = '{}'
if 'redis_key' in work_dict:
upload_and_cache_req['redis_key'] = '{}_{}'.format(
work_dict.get('redis_key',
'{}_{}'.format(ticker, field_name)),
field_name)
if 's3_key' in work_dict:
upload_and_cache_req['s3_key'] = '{}_{}'.format(
work_dict.get('s3_key', '{}_{}'.format(ticker,
field_name)),
field_name)
try:
update_res = publisher.run_publish_pricing_update(
work_dict=upload_and_cache_req)
update_status = update_res.get('status', NOT_SET)
log.info('{} publish update status={} data={}'.format(
label, get_status(status=update_status), update_res))
except Exception as f:
err = ('{} - failed to upload YAHOO data={} to '
'to s3_key={} and redis_key={}'.format(
label, upload_and_cache_req,
upload_and_cache_req['s3_key'],
upload_and_cache_req['redis_key']))
log.error(err)
# end of try/ex to upload and cache
if not rec[field_name]:
log.debug('{} - ticker={} no data from YAHOO for '
'field_name={}'.format(label, ticker, field_name))
# end of for all fields
res = build_result.build_result(status=SUCCESS, err=None, rec=rec)
except Exception as e:
res = build_result.build_result(status=ERR,
err=('failed - get_data_from_yahoo '
'dict={} with ex={}').format(
work_dict, e),
rec=rec)
log.error('{} - {}'.format(label, res['err']))
# end of try/ex
log.info('task - get_data_from_yahoo done - '
'{} - status={}'.format(label, get_status(res['status'])))
return res
