def test_integration_daily_indicator_with_algo_config(self):
"""test_integration_daily_indicator_with_algo_config"""
if ae_consts.ev('INT_TESTS', '0') == '0':
return
algo = base_algo.BaseAlgo(
ticker=self.ticker,
balance=self.balance,
start_date_str=self.start_date_str,
end_date_str=self.end_date_str,
config_dict=self.algo_config_dict)
self.assertEqual(
algo.name,
self.algo_config_dict['name'])
self.assertEqual(
algo.tickers,
[self.ticker])
algo.handle_data(
data=self.data)
res = algo.get_result()
print(ae_consts.ppj(res))
self.assertTrue(
res['history'][0]['total_sells'] >= 1)
self.assertTrue(
res['history'][0]['total_buys'] == 0)
def test_run_daily_indicator_with_algo_config_sell(self):
"""test_run_daily_indicator_with_algo_config_sell"""
algo = base_algo.BaseAlgo(
ticker=self.ticker,
balance=self.balance,
start_date_str=self.start_date_str,
end_date_str=self.end_date_str,
config_dict=self.algo_config_dict)
self.assertEqual(
algo.name,
self.algo_config_dict['name'])
self.assertEqual(
algo.tickers,
[self.ticker])
algo.handle_data(
data=self.data)
res = algo.get_result()
print(ae_consts.ppj(res))
self.assertEqual(
res['history'][0]['total_sells'],
1)
self.assertEqual(
res['history'][0]['total_buys'],
0)
def run_distributed_algorithm(self, algo_req):
"""run_distributed_algorithm
Process an Algorithm using a Celery task that is
processed by a Celery worker
:param algo_req: dictionary for key/values for
running an algorithm using Celery workers
"""
label = algo_req.get('name', 'ae-algo')
verbose = algo_req.get('verbose_task', False)
debug = algo_req.get('debug', False)
# please be careful logging prod passwords:
if debug:
log.info(f'task - {label} - start algo_req={algo_req}')
elif verbose:
log.info(f'task - {label} - start ')
# end of start log
rec = {}
res = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec=rec)
created_algo_object = None
custom_algo_module = None
new_algo_object = None
use_custom_algo = False
found_algo_module = True # assume the BaseAlgo
should_publish_extract_dataset = False
should_publish_history_dataset = False
should_publish_report_dataset = False
ticker = algo_req.get('ticker', 'SPY')
num_days_back = algo_req.get('num_days_back', 75)
name = algo_req.get('name', 'ae-algo')
algo_module_path = algo_req.get('mod_path', None)
module_name = algo_req.get('module_name', 'BaseAlgo')
custom_algo_module = algo_req.get('custom_algo_module', None)
new_algo_object = algo_req.get('new_algo_object', None)
use_custom_algo = algo_req.get('use_custom_algo', False)
should_publish_extract_dataset = algo_req.get(
'should_publish_extract_dataset', False)
should_publish_history_dataset = algo_req.get(
'should_publish_history_dataset', False)
should_publish_report_dataset = algo_req.get(
'should_publish_report_dataset', False)
start_date = algo_req.get('start_date', None)
end_date = algo_req.get('end_date', None)
raise_on_err = algo_req.get('raise_on_err', True)
report_config = algo_req.get('report_config', None)
history_config = algo_req.get('history_config', None)
extract_config = algo_req.get('extract_config', None)
err = None
if algo_module_path:
found_algo_module = False
module_name = algo_module_path.split('/')[-1]
loader = importlib.machinery.SourceFileLoader(module_name,
algo_module_path)
custom_algo_module = types.ModuleType(loader.name)
loader.exec_module(custom_algo_module)
use_custom_algo = True
for member in inspect.getmembers(custom_algo_module):
if module_name in str(member):
found_algo_module = True
break
# for all members in this custom module file
# if loading a custom algorithm module from a file on disk
if not found_algo_module:
err = (f'{label} - unable to find custom algorithm '
f'module={custom_algo_module} module_path={algo_module_path}')
if algo_module_path:
err = (
f'{label} - analysis_engine.'
'work_tasks.run_distributed_algorithm was unable '
f'to find custom algorithm module={custom_algo_module} with '
f'provided path to \n file: {algo_module_path} \n'
'\n'
'Please confirm '
'that the class inherits from the BaseAlgo class like:\n'
'\n'
'import analysis_engine.algo\n'
'class MyAlgo(analysis_engine.algo.BaseAlgo):\n '
'\n'
'If it is then please file an issue on github:\n '
'https://github.com/AlgoTraders/stock-analysis-engine/'
'issues/new \n\nFor now this error results in a shutdown'
'\n')
# if algo_module_path set
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# if not found_algo_module
use_start_date = start_date
use_end_date = end_date
if not use_end_date:
end_date = datetime.datetime.utcnow()
use_end_date = end_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT)
if not use_start_date:
start_date = end_date - datetime.timedelta(days=num_days_back)
use_start_date = start_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT)
dataset_publish_extract = algo_req.get('dataset_publish_extract', False)
dataset_publish_history = algo_req.get('dataset_publish_history', False)
dataset_publish_report = algo_req.get('dataset_publish_report', False)
try:
if use_custom_algo:
if verbose:
log.info(
f'inspecting {custom_algo_module} for class {module_name}')
use_class_member_object = None
for member in inspect.getmembers(custom_algo_module):
if module_name in str(member):
if verbose:
log.info(f'start {name} with {member[1]}')
use_class_member_object = member
break
# end of looking over the class definition but did not find it
if use_class_member_object:
if algo_req.get('backtest', False):
new_algo_object = member[1](ticker=algo_req['ticker'],
config_dict=algo_req)
else:
new_algo_object = member[1](**algo_req)
else:
err = (f'{label} - did not find a derived '
'analysis_engine.algo.BaseAlgo '
f'class in the module file={algo_module_path} '
f'for ticker={ticker} algo_name={name}')
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# end of finding a valid algorithm object
else:
new_algo_object = ae_algo.BaseAlgo(**algo_req)
# if using a custom module path or the BaseAlgo
if new_algo_object:
# heads up - logging this might have passwords in the algo_req
# log.debug(f'{name} algorithm request: {algo_req}')
if verbose:
log.info(f'{name} - run START ticker={ticker} '
f'from {use_start_date} to {use_end_date}')
if algo_req.get('backtest', False):
algo_res = run_algo.run_algo(algo=new_algo_object,
config_dict=algo_req)
created_algo_object = new_algo_object
else:
algo_res = run_algo.run_algo(algo=new_algo_object, **algo_req)
created_algo_object = new_algo_object
if verbose:
log.info(f'{name} - run DONE ticker={ticker} '
f'from {use_start_date} to {use_end_date}')
if debug:
if custom_algo_module:
log.info(f'{name} - done run_algo '
f'custom_algo_module={custom_algo_module} '
f'module_name={module_name} ticker={ticker} '
f'from {use_start_date} to {use_end_date}')
else:
log.info(
f'{name} - done run_algo BaseAlgo ticker={ticker} '
f'from {use_start_date} to {use_end_date}')
else:
err = (
f'{label} - missing a derived analysis_engine.algo.BaseAlgo '
f'class in the module file={algo_module_path} for '
f'ticker={ticker} algo_name={name}')
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# end of finding a valid algorithm object
if not created_algo_object:
err = (f'{label} - failed creating algorithm object - '
f'ticker={ticker} '
f'status={ae_consts.get_status(status=algo_res["status"])} '
f'error={algo_res["err"]} algo name={name} '
f'custom_algo_module={custom_algo_module} '
f'module_name={module_name} '
f'from {use_start_date} to {use_end_date}')
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# end of stop early
if should_publish_extract_dataset or dataset_publish_extract:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (extract_config['redis_address'] and extract_config['redis_db']
and extract_config['redis_key']):
redis_log = (f'redis://{extract_config["redis_address"]}'
f'@{extract_config["redis_db"]}'
f'/{extract_config["redis_key"]}')
use_log += f' {redis_log}'
else:
extract_config['redis_enabled'] = False
if (extract_config['s3_address'] and extract_config['s3_bucket']
and extract_config['s3_key']):
s3_log = (f's3://{extract_config["s3_address"]}'
f'/{extract_config["s3_bucket"]}'
f'/{extract_config["s3_key"]}')
use_log += f' {s3_log}'
else:
extract_config['s3_enabled'] = False
if extract_config['output_file']:
file_log = f'file:{extract_config["output_file"]}'
use_log += f' {file_log}'
if verbose:
log.info(f'{name} - publish - start ticker={ticker} '
f'algorithm-ready {use_log}')
publish_status = created_algo_object.publish_input_dataset(
**extract_config)
if publish_status != ae_consts.SUCCESS:
msg = ('failed to publish algorithm-ready datasets with '
f'status {ae_consts.get_status(status=publish_status)} '
f'attempted to {use_log}')
log.error(msg)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# end of stop early
if verbose:
log.info(f'{name} - publish - done ticker={ticker} '
f'algorithm-ready {use_log}')
# if publish the algorithm-ready dataset
if should_publish_history_dataset or dataset_publish_history:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (history_config['redis_address'] and history_config['redis_db']
and history_config['redis_key']):
redis_log = (f'redis://{history_config["redis_address"]}'
f'@{history_config["redis_db"]}'
f'/{history_config["redis_key"]}')
use_log += f' {redis_log}'
if (history_config['s3_address'] and history_config['s3_bucket']
and history_config['s3_key']):
s3_log = (f's3://{history_config["s3_address"]}'
f'/{history_config["s3_bucket"]}'
f'/{history_config["s3_key"]}')
use_log += f' {s3_log}'
if history_config['output_file']:
file_log = f'file:{history_config["output_file"]}'
use_log += f' {file_log}'
if verbose:
log.info(f'{name} - publish - start ticker={ticker} trading '
f'history {use_log}')
publish_status = \
created_algo_object.publish_trade_history_dataset(
**history_config)
if publish_status != ae_consts.SUCCESS:
msg = ('failed to publish trading history datasets with '
f'status {ae_consts.get_status(status=publish_status)} '
f'attempted to {use_log}')
log.error(msg)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# end of stop early
if verbose:
log.info(f'{name} - publish - done ticker={ticker} trading '
f'history {use_log}')
# if publish an trading history dataset
if should_publish_report_dataset or dataset_publish_report:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (report_config['redis_address'] and report_config['redis_db']
and report_config['redis_key']):
redis_log = (f'redis://{report_config["redis_address"]}'
f'@{report_config["redis_db"]}'
f'/{report_config["redis_key"]}')
use_log += f' {redis_log}'
if (report_config['s3_address'] and report_config['s3_bucket']
and report_config['s3_key']):
s3_log = (f's3://{report_config["s3_address"]}'
f'/{report_config["s3_bucket"]}'
f'/{report_config["s3_key"]}')
use_log += f' {s3_log}'
if report_config['output_file']:
file_log = f' file:{report_config["output_file"]}'
use_log += f' {file_log}'
if verbose:
log.info(
f'{name} - publishing ticker={ticker} trading performance '
f'report {use_log}')
publish_status = created_algo_object.publish_report_dataset(
**report_config)
if publish_status != ae_consts.SUCCESS:
msg = ('failed to publish trading performance '
'report datasets with '
f'status {ae_consts.get_status(status=publish_status)} '
f'attempted to {use_log}')
log.error(msg)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
# end of stop early
if verbose:
log.info(f'{name} - publish - done ticker={ticker} trading '
f'performance report {use_log}')
# if publish an trading performance report dataset
if verbose:
log.info(f'{name} - done publishing datasets for ticker={ticker} '
f'from {use_start_date} to {use_end_date}')
rec['history_config'] = history_config
rec['report_config'] = report_config
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
res = build_result.build_result(
status=ae_consts.ERR,
err=('failed - run_distributed_algorithm '
f'dict={algo_req} with ex={e}'),
rec=rec)
if raise_on_err:
raise e
else:
log.error(f'{label} - {res["err"]}')
# end of try/ex
if verbose:
log.info('task - run_distributed_algorithm done - '
f'{label} - status={ae_consts.get_status(res["status"])}')
task_result = {
'status': res['status'],
'err': res['err'],
'algo_req': algo_req,
'rec': rec
}
return task_result
def latest(self,
date_str=None,
start_row=-200,
extract_iex=True,
extract_yahoo=False,
extract_td=True,
verbose=False,
**kwargs):
"""latest
Run the algorithm with the latest pricing data. Also
supports running a backtest for a historical date in
the pricing history (format ``YYYY-MM-DD``)
:param date_str: optional - string start date ``YYYY-MM-DD``
default is the latest close date
:param start_row: negative number of rows back
from the end of the list in the data
default is ``-200`` where this means the algorithm
will process the latest 200 rows in the minute
dataset
:param extract_iex: bool flag for extracting from ``IEX``
:param extract_yahoo: bool flag for extracting from ``Yahoo``
which is disabled as of 1/2019
:param extract_td: bool flag for extracting from ``Tradier``
:param verbose: bool flag for logs
:param kwargs: keyword arg dict
"""
use_date_str = date_str
if not use_date_str:
use_date_str = ae_utils.get_last_close_str()
log.info('creating algo')
self.algo_obj = base_algo.BaseAlgo(
ticker=self.config_dict['ticker'],
balance=self.config_dict['balance'],
commission=self.config_dict['commission'],
name=self.use_name,
start_date=self.use_start_date,
end_date=self.use_end_date,
auto_fill=self.auto_fill,
config_dict=self.config_dict,
load_from_s3_bucket=self.load_from_s3_bucket,
load_from_s3_key=self.load_from_s3_key,
load_from_redis_key=self.load_from_redis_key,
load_from_file=self.load_from_file,
load_compress=self.load_compress,
load_publish=self.load_publish,
load_config=self.load_config,
report_redis_key=self.report_redis_key,
report_s3_bucket=self.report_s3_bucket,
report_s3_key=self.report_s3_key,
report_file=self.report_file,
report_compress=self.report_compress,
report_publish=self.report_publish,
report_config=self.report_config,
history_redis_key=self.history_redis_key,
history_s3_bucket=self.history_s3_bucket,
history_s3_key=self.history_s3_key,
history_file=self.history_file,
history_compress=self.history_compress,
history_publish=self.history_publish,
history_config=self.history_config,
extract_redis_key=self.extract_redis_key,
extract_s3_bucket=self.extract_s3_bucket,
extract_s3_key=self.extract_s3_key,
extract_file=self.extract_file,
extract_save_dir=self.extract_save_dir,
extract_compress=self.extract_compress,
extract_publish=self.extract_publish,
extract_config=self.extract_config,
publish_to_slack=self.publish_to_slack,
publish_to_s3=self.publish_to_s3,
publish_to_redis=self.publish_to_redis,
dataset_type=self.dataset_type,
serialize_datasets=self.serialize_datasets,
compress=self.compress,
encoding=self.encoding,
redis_enabled=self.redis_enabled,
redis_key=self.redis_key,
redis_address=self.redis_address,
redis_db=self.redis_db,
redis_password=self.redis_password,
redis_expire=self.redis_expire,
redis_serializer=self.redis_serializer,
redis_encoding=self.redis_encoding,
s3_enabled=self.s3_enabled,
s3_key=self.s3_key,
s3_address=self.s3_address,
s3_bucket=self.s3_bucket,
s3_access_key=self.s3_access_key,
s3_secret_key=self.s3_secret_key,
s3_region_name=self.s3_region_name,
s3_secure=self.s3_secure,
slack_enabled=self.slack_enabled,
slack_code_block=self.slack_code_block,
slack_full_width=self.slack_full_width,
dataset_publish_extract=self.extract_publish,
dataset_publish_history=self.history_publish,
dataset_publish_report=self.report_publish,
run_on_engine=self.run_on_engine,
auth_url=self.broker_url,
backend_url=self.backend_url,
include_tasks=self.include_tasks,
ssl_options=self.ssl_options,
transport_options=self.transport_options,
path_to_config_module=self.path_to_config_module,
timeseries=self.timeseries,
trade_strategy=self.trade_strategy,
verbose=False,
raise_on_err=self.raise_on_err)
log.info('run latest - start')
ticker = self.config_dict['ticker']
dataset_id = f'{ticker}_{use_date_str}'
self.common_fetch_vals['base_key'] = dataset_id
verbose_extract = self.config_dict.get('verbose_extract', False)
indicator_datasets = self.algo_obj.get_indicator_datasets()
ticker_data = build_dataset_node.build_dataset_node(
ticker=ticker,
date=use_date_str,
datasets=indicator_datasets,
service_dict=self.common_fetch_vals,
verbose=verbose_extract)
algo_data_req = {
ticker: [{
'id': dataset_id, # id is currently the cache key in redis
'date': use_date_str, # used to confirm dates in asc order
'data': ticker_data,
'start_row': start_row
}]
}
if verbose:
log.info(f'extract - {dataset_id} '
f'dataset={len(algo_data_req[ticker])}')
# this could be a separate celery task
try:
if verbose:
log.info(f'handle_data START - {dataset_id}')
self.algo_obj.handle_data(data=algo_data_req)
if verbose:
log.info(f'handle_data END - {dataset_id}')
except Exception as e:
a_name = self.algo_obj.get_name()
a_debug_msg = self.algo_obj.get_debug_msg()
if not a_debug_msg:
a_debug_msg = 'debug message not set'
# a_config_dict = ae_consts.ppj(self.algo_obj.config_dict)
msg = (f'{dataset_id} - algo={a_name} '
f'encountered exception in handle_data tickers={ticker} '
f'from ex={e} '
f'and failed during operation: {a_debug_msg}')
log.critical(f'{msg}')
# end try/ex
log.info('run latest - create history')
history_ds = self.algo_obj.create_history_dataset()
self.history_df = pd.DataFrame(history_ds[ticker])
self.determine_latest_times_in_history()
self.num_rows = len(self.history_df.index)
if verbose:
log.info(self.history_df[['minute', 'close']].tail(5))
log.info(f'run latest minute={self.end_date} - '
f'rows={self.num_rows} - done')
return self.get_history()
def run_algo(
ticker=None,
tickers=None,
algo=None, # optional derived ``analysis_engine.algo.Algo`` instance
balance=None, # float starting base capital
commission=None, # float for single trade commission for buy or sell
start_date=None, # string YYYY-MM-DD HH:MM:SS
end_date=None, # string YYYY-MM-DD HH:MM:SS
datasets=None, # string list of identifiers
num_owned_dict=None, # not supported
cache_freq='daily', # 'minute' not supported
auto_fill=True,
load_config=None,
report_config=None,
history_config=None,
extract_config=None,
use_key=None,
extract_mode='all',
iex_datasets=None,
redis_enabled=True,
redis_address=None,
redis_db=None,
redis_password=None,
redis_expire=None,
redis_key=None,
s3_enabled=True,
s3_address=None,
s3_bucket=None,
s3_access_key=None,
s3_secret_key=None,
s3_region_name=None,
s3_secure=False,
s3_key=None,
celery_disabled=True,
broker_url=None,
result_backend=None,
label=None,
name=None,
timeseries=None,
trade_strategy=None,
verbose=False,
publish_to_slack=True,
publish_to_s3=True,
publish_to_redis=True,
extract_datasets=None,
config_file=None,
config_dict=None,
version=1,
raise_on_err=True,
**kwargs):
"""run_algo
Run an algorithm with steps:
1) Extract redis keys between dates
2) Compile a data pipeline dictionary (call it ``data``)
3) Call algorithm's ``myalgo.handle_data(data=data)``
.. note:: If no ``algo`` is set, the
``analysis_engine.algo.BaseAlgo`` algorithm
is used.
.. note:: Please ensure Redis and Minio are running
before trying to extract tickers
**Stock tickers to extract**
:param ticker: single stock ticker/symbol/ETF to extract
:param tickers: optional - list of tickers to extract
:param use_key: optional - extract historical key from Redis
**Algo Configuration**
:param algo: derived instance of ``analysis_engine.algo.Algo`` object
:param balance: optional - float balance parameter
can also be set on the ``algo`` object if not
set on the args
:param commission: float for single trade commission for
buy or sell. can also be set on the ``algo`` objet
:param start_date: string ``YYYY-MM-DD_HH:MM:SS`` cache value
:param end_date: string ``YYYY-MM-DD_HH:MM:SS`` cache value
:param dataset_types: list of strings that are ``iex`` or ``yahoo``
datasets that are cached.
:param cache_freq: optional - depending on if you are running data feeds
on a ``daily`` cron (default) vs every ``minute`` (or faster)
:param num_owned_dict: not supported yet
:param auto_fill: optional - boolean for auto filling
buy/sell orders for backtesting (default is
``True``)
:param trading_calendar: ``trading_calendar.TradingCalendar``
object, by default ``analysis_engine.calendars.
always_open.AlwaysOpen`` trading calendar
# TradingCalendar by ``TFSExchangeCalendar``
:param config_file: path to a json file
containing custom algorithm object
member values (like indicator configuration and
predict future date units ahead for a backtest)
:param config_dict: optional - dictionary that
can be passed to derived class implementations
of: ``def load_from_config(config_dict=config_dict)``
**Timeseries**
:param timeseries: optional - string to
set ``day`` or ``minute`` backtesting
or live trading
(default is ``minute``)
**Trading Strategy**
:param trade_strategy: optional - string to
set the type of ``Trading Strategy``
for backtesting or live trading
(default is ``count``)
**Algorithm Dataset Loading, Extracting, Reporting
and Trading History arguments**
:param load_config: optional - dictionary
for setting member variables to load an
agorithm-ready dataset from
a file, s3 or redis
:param report_config: optional - dictionary
for setting member variables to publish
an algo ``trading performance report`` to s3,
redis, a file or slack
:param history_config: optional - dictionary
for setting member variables to publish
an algo ``trade history`` to s3, redis, a file
or slack
:param extract_config: optional - dictionary
for setting member variables to publish
an algo ``trading performance report`` to s3,
redis, a file or slack
**(Optional) Data sources, datafeeds and datasets to gather**
:param iex_datasets: list of strings for gathering specific `IEX
datasets <https://iexcloud.io/>`__
which are set as consts: ``analysis_engine.iex.consts.FETCH_*``.
**(Optional) Redis connectivity arguments**
:param redis_enabled: bool - toggle for auto-caching all
datasets in Redis
(default is ``True``)
:param redis_address: Redis connection string
format is ``host:port``
(default is ``localhost:6379``)
:param redis_db: Redis db to use
(default is ``0``)
:param redis_password: optional - Redis password
(default is ``None``)
:param redis_expire: optional - Redis expire value
(default is ``None``)
:param redis_key: optional - redis key not used
(default is ``None``)
**(Optional) Minio (S3) connectivity arguments**
:param s3_enabled: bool - toggle for auto-archiving on Minio (S3)
(default is ``True``)
:param s3_address: Minio S3 connection string
format ``host:port``
(default is ``localhost:9000``)
:param s3_bucket: S3 Bucket for storing the artifacts
(default is ``dev``) which should be viewable on a browser:
http://localhost:9000/minio/dev/
:param s3_access_key: S3 Access key
(default is ``trexaccesskey``)
:param s3_secret_key: S3 Secret key
(default is ``trex123321``)
:param s3_region_name: S3 region name
(default is ``us-east-1``)
:param s3_secure: Transmit using tls encryption
(default is ``False``)
:param s3_key: optional s3 key not used
(default is ``None``)
**(Optional) Celery worker broker connectivity arguments**
:param celery_disabled: bool - toggle synchronous mode or publish
to an engine connected to the `Celery broker and backend
<https://github.com/celery/celery#transports-and-backends>`__
(default is ``True`` - synchronous mode without an engine
or need for a broker or backend for Celery)
:param broker_url: Celery broker url
(default is ``redis://0.0.0.0:6379/13``)
:param result_backend: Celery backend url
(default is ``redis://0.0.0.0:6379/14``)
:param label: tracking log label
:param publish_to_slack: optional - boolean for
publishing to slack (coming soon)
:param publish_to_s3: optional - boolean for
publishing to s3 (coming soon)
:param publish_to_redis: optional - boolean for
publishing to redis (coming soon)
**(Optional) Debugging**
:param verbose: bool - show extract warnings
and other debug logging (default is False)
:param raise_on_err: optional - boolean for
unittests and developing algorithms with the
``analysis_engine.run_algo.run_algo`` helper.
When set to ``True`` exceptions will
are raised to the calling functions
:param kwargs: keyword arguments dictionary
"""
# dictionary structure with a list sorted on: ascending dates
# algo_data_req[ticker][list][dataset] = pd.DataFrame
algo_data_req = {}
extract_requests = []
return_algo = False # return created algo objects for use by caller
rec = {}
msg = None
use_tickers = tickers
use_balance = balance
use_commission = commission
if ticker:
use_tickers = [ticker]
else:
if not use_tickers:
use_tickers = []
# if these are not set as args, but the algo object
# has them, use them instead:
if algo:
if len(use_tickers) == 0:
use_tickers = algo.get_tickers()
if not use_balance:
use_balance = algo.get_balance()
if not use_commission:
use_commission = algo.get_commission()
default_iex_datasets = [
'daily', 'minute', 'quote', 'stats', 'peers', 'news', 'financials',
'earnings', 'dividends', 'company'
]
if not iex_datasets:
iex_datasets = default_iex_datasets
if redis_enabled:
if not redis_address:
redis_address = os.getenv('REDIS_ADDRESS', 'localhost:6379')
if not redis_password:
redis_password = os.getenv('REDIS_PASSWORD', None)
if not redis_db:
redis_db = int(os.getenv('REDIS_DB', '0'))
if not redis_expire:
redis_expire = os.getenv('REDIS_EXPIRE', None)
if s3_enabled:
if not s3_address:
s3_address = os.getenv('S3_ADDRESS', 'localhost:9000')
if not s3_access_key:
s3_access_key = os.getenv('AWS_ACCESS_KEY_ID', 'trexaccesskey')
if not s3_secret_key:
s3_secret_key = os.getenv('AWS_SECRET_ACCESS_KEY', 'trex123321')
if not s3_region_name:
s3_region_name = os.getenv('AWS_DEFAULT_REGION', 'us-east-1')
if not s3_secure:
s3_secure = os.getenv('S3_SECURE', '0') == '1'
if not s3_bucket:
s3_bucket = os.getenv('S3_BUCKET', 'dev')
if not broker_url:
broker_url = os.getenv('WORKER_BROKER_URL', 'redis://0.0.0.0:6379/11')
if not result_backend:
result_backend = os.getenv('WORKER_BACKEND_URL',
'redis://0.0.0.0:6379/12')
if not label:
label = 'run-algo'
num_tickers = len(use_tickers)
last_close_str = ae_utils.get_last_close_str()
if iex_datasets:
if verbose:
log.info(f'{label} - tickers={num_tickers} '
f'iex={json.dumps(iex_datasets)}')
else:
if verbose:
log.info(f'{label} - tickers={num_tickers}')
ticker_key = use_key
if not ticker_key:
ticker_key = f'{ticker}_{last_close_str}'
if not algo:
algo = base_algo.BaseAlgo(ticker=None,
tickers=use_tickers,
balance=use_balance,
commission=use_commission,
config_dict=config_dict,
name=label,
auto_fill=auto_fill,
timeseries=timeseries,
trade_strategy=trade_strategy,
publish_to_slack=publish_to_slack,
publish_to_s3=publish_to_s3,
publish_to_redis=publish_to_redis,
raise_on_err=raise_on_err)
return_algo = True
# the algo object is stored
# in the result at: res['rec']['algo']
if not algo:
msg = f'{label} - missing algo object'
log.error(msg)
return build_result.build_result(status=ae_consts.EMPTY,
err=msg,
rec=rec)
if raise_on_err:
log.debug(f'{label} - enabling algo exception raises')
algo.raise_on_err = True
indicator_datasets = algo.get_indicator_datasets()
if len(indicator_datasets) == 0:
indicator_datasets = ae_consts.BACKUP_DATASETS
log.info(f'using all datasets={indicator_datasets}')
verbose_extract = False
if config_dict:
verbose_extract = config_dict.get('verbose_extract', False)
common_vals = {}
common_vals['base_key'] = ticker_key
common_vals['celery_disabled'] = celery_disabled
common_vals['ticker'] = ticker
common_vals['label'] = label
common_vals['iex_datasets'] = iex_datasets
common_vals['s3_enabled'] = s3_enabled
common_vals['s3_bucket'] = s3_bucket
common_vals['s3_address'] = s3_address
common_vals['s3_secure'] = s3_secure
common_vals['s3_region_name'] = s3_region_name
common_vals['s3_access_key'] = s3_access_key
common_vals['s3_secret_key'] = s3_secret_key
common_vals['s3_key'] = ticker_key
common_vals['redis_enabled'] = redis_enabled
common_vals['redis_address'] = redis_address
common_vals['redis_password'] = redis_password
common_vals['redis_db'] = redis_db
common_vals['redis_key'] = ticker_key
common_vals['redis_expire'] = redis_expire
use_start_date_str = start_date
use_end_date_str = end_date
last_close_date = ae_utils.last_close()
end_date_val = None
cache_freq_fmt = ae_consts.COMMON_TICK_DATE_FORMAT
if not use_end_date_str:
use_end_date_str = last_close_date.strftime(cache_freq_fmt)
end_date_val = ae_utils.get_date_from_str(date_str=use_end_date_str,
fmt=cache_freq_fmt)
start_date_val = None
if not use_start_date_str:
start_date_val = end_date_val - datetime.timedelta(days=60)
use_start_date_str = start_date_val.strftime(cache_freq_fmt)
else:
start_date_val = datetime.datetime.strptime(
use_start_date_str, ae_consts.COMMON_TICK_DATE_FORMAT)
total_dates = (end_date_val - start_date_val).days
if end_date_val < start_date_val:
msg = (
f'{label} - invalid dates - start_date={start_date_val} is after '
f'end_date={end_date_val}')
raise Exception(msg)
if verbose:
log.info(f'{label} - days={total_dates} '
f'start={use_start_date_str} '
f'end={use_end_date_str} '
f'datasets={indicator_datasets}')
for ticker in use_tickers:
req = algo_utils.build_algo_request(ticker=ticker,
use_key=use_key,
start_date=use_start_date_str,
end_date=use_end_date_str,
datasets=datasets,
balance=use_balance,
cache_freq=cache_freq,
timeseries=timeseries,
trade_strategy=trade_strategy,
label=label)
ticker_key = f'{ticker}_{last_close_str}'
common_vals['ticker'] = ticker
common_vals['base_key'] = ticker_key
common_vals['redis_key'] = ticker_key
common_vals['s3_key'] = ticker_key
for date_key in req['extract_datasets']:
date_req = api_requests.get_ds_dict(ticker=ticker,
base_key=date_key,
ds_id=label,
service_dict=common_vals)
node_date_key = date_key.replace(f'{ticker}_', '')
extract_requests.append({
'id': date_key,
'ticker': ticker,
'date_key': date_key,
'date': node_date_key,
'req': date_req
})
# end of for all ticker in use_tickers
first_extract_date = None
last_extract_date = None
total_extract_requests = len(extract_requests)
cur_idx = 1
for idx, extract_node in enumerate(extract_requests):
extract_ticker = extract_node['ticker']
extract_date = extract_node['date']
ds_node_id = extract_node['id']
if not first_extract_date:
first_extract_date = extract_date
last_extract_date = extract_date
perc_progress = ae_consts.get_percent_done(
progress=cur_idx, total=total_extract_requests)
percent_label = (f'{label} '
f'ticker={extract_ticker} '
f'date={extract_date} '
f'{perc_progress} '
f'{idx}/{total_extract_requests} '
f'{indicator_datasets}')
if verbose:
log.info(f'extracting - {percent_label}')
ticker_bt_data = build_ds_node.build_dataset_node(
ticker=extract_ticker,
date=extract_date,
service_dict=common_vals,
datasets=indicator_datasets,
log_label=label,
verbose=verbose_extract)
if ticker not in algo_data_req:
algo_data_req[ticker] = []
algo_data_req[ticker].append({
'id': ds_node_id, # id is currently the cache key in redis
'date': extract_date, # used to confirm dates in asc order
'data': ticker_bt_data
})
if verbose:
log.info(f'extract - {percent_label} '
f'dataset={len(algo_data_req[ticker])}')
cur_idx += 1
# end of for service_dict in extract_requests
# this could be a separate celery task
status = ae_consts.NOT_RUN
if len(algo_data_req) == 0:
msg = (f'{label} - nothing to test - no data found for '
f'tickers={use_tickers} '
f'between {first_extract_date} and {last_extract_date}')
log.info(msg)
return build_result.build_result(status=ae_consts.EMPTY,
err=msg,
rec=rec)
# this could be a separate celery task
try:
if verbose:
log.info(f'handle_data START - {percent_label} from '
f'{first_extract_date} to {last_extract_date}')
algo.handle_data(data=algo_data_req)
if verbose:
log.info(f'handle_data END - {percent_label} from '
f'{first_extract_date} to {last_extract_date}')
except Exception as e:
a_name = algo.get_name()
a_debug_msg = algo.get_debug_msg()
if not a_debug_msg:
a_debug_msg = 'debug message not set'
a_config_dict = ae_consts.ppj(algo.config_dict)
msg = (f'{percent_label} - algo={a_name} '
f'encountered exception in handle_data tickers={use_tickers} '
f'from {first_extract_date} to {last_extract_date} ex={e} '
f'and failed during operation: {a_debug_msg}')
if raise_on_err:
if algo:
try:
ind_obj = \
algo.get_indicator_process_last_indicator()
if ind_obj:
ind_obj_path = ind_obj.get_path_to_module()
ind_obj_config = ae_consts.ppj(ind_obj.get_config())
found_error_hint = False
if hasattr(ind_obj.use_df, 'to_json'):
if len(ind_obj.use_df.index) == 0:
log.critical(
f'indicator failure report for '
f'last module: '
f'{ind_obj_path} '
f'indicator={ind_obj.get_name()} '
f'config={ind_obj_config} '
f'dataset={ind_obj.use_df.head(5)} '
f'name_of_dataset={ind_obj.uses_data}')
log.critical(
'--------------------------------------'
'--------------------------------------')
log.critical('Please check if this indicator: '
f'{ind_obj_path} '
'supports Empty Dataframes')
log.critical(
'--------------------------------------'
'--------------------------------------')
found_error_hint = True
# indicator error hints
if not found_error_hint:
log.critical(
f'indicator failure report for last module: '
f'{ind_obj_path} '
f'indicator={ind_obj.get_name()} '
f'config={ind_obj_config} '
f'dataset={ind_obj.use_df.head(5)} '
f'name_of_dataset={ind_obj.uses_data}')
except Exception as f:
log.critical(f'failed to pull indicator processor '
f'last indicator for debugging '
f'from ex={e} with parsing ex={f}')
# end of ignoring non-supported ways of creating
# indicator processors
log.error(msg)
log.error(f'algo failure report: '
f'algo={a_name} handle_data() '
f'config={a_config_dict} ')
log.critical(f'algo failed during operation: {a_debug_msg}')
raise e
else:
log.error(msg)
return build_result.build_result(status=ae_consts.ERR,
err=msg,
rec=rec)
# end of try/ex
# this could be a separate celery task
try:
if verbose:
log.info(f'get_result START - {percent_label} from '
f'{first_extract_date} to {last_extract_date}')
rec = algo.get_result()
status = ae_consts.SUCCESS
if verbose:
log.info(f'get_result END - {percent_label} from '
f'{first_extract_date} to {last_extract_date}')
except Exception as e:
msg = (
f'{percent_label} - algo={algo.get_name()} encountered exception '
f'in get_result tickers={use_tickers} from '
f'{first_extract_date} to {last_extract_date} ex={e}')
if raise_on_err:
if algo:
log.error(f'algo={algo.get_name()} failed in get_result with '
f'debug_msg={algo.get_debug_msg()}')
log.error(msg)
raise e
else:
log.error(msg)
return build_result.build_result(status=ae_consts.ERR,
err=msg,
rec=rec)
# end of try/ex
if return_algo:
rec['algo'] = algo
return build_result.build_result(status=status, err=msg, rec=rec)
def run_distributed_algorithm(self, algo_req):
"""run_distributed_algorithm
Process a distributed Algorithm
:param algo_req: dictionary for key/values for
running an algorithm using Celery workers
"""
label = algo_req.get('name', 'ae-algo')
verbose = algo_req.get('verbose', False)
debug = algo_req.get('debug', False)
# please be careful logging prod passwords:
if verbose or debug:
log.info('task - {} - start ' 'algo_req={}'.format(label, algo_req))
else:
log.info('task - {} - start '.format(label))
# end of start log
rec = {}
res = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec=rec)
created_algo_object = None
custom_algo_module = None
new_algo_object = None
use_custom_algo = False
found_algo_module = True # assume the BaseAlgo
should_publish_extract_dataset = False
should_publish_history_dataset = False
should_publish_report_dataset = False
ticker = algo_req.get('ticker', 'SPY')
num_days_back = algo_req.get('num_days_back', 75)
name = algo_req.get('name', 'ae-algo')
algo_module_path = algo_req.get('mod_path', None)
module_name = algo_req.get('module_name', 'BaseAlgo')
custom_algo_module = algo_req.get('custom_algo_module', None)
new_algo_object = algo_req.get('new_algo_object', None)
use_custom_algo = algo_req.get('use_custom_algo', False)
should_publish_extract_dataset = algo_req.get(
'should_publish_extract_dataset', False)
should_publish_history_dataset = algo_req.get(
'should_publish_history_dataset', False)
should_publish_report_dataset = algo_req.get(
'should_publish_report_dataset', False)
start_date = algo_req.get('start_date', None)
end_date = algo_req.get('end_date', None)
raise_on_err = algo_req.get('raise_on_err', False)
report_config = algo_req.get('report_config', None)
history_config = algo_req.get('history_config', None)
extract_config = algo_req.get('extract_config', None)
err = None
if algo_module_path:
found_algo_module = False
module_name = algo_module_path.split('/')[-1]
loader = importlib.machinery.SourceFileLoader(module_name,
algo_module_path)
custom_algo_module = types.ModuleType(loader.name)
loader.exec_module(custom_algo_module)
use_custom_algo = True
for member in inspect.getmembers(custom_algo_module):
if module_name in str(member):
found_algo_module = True
break
# for all members in this custom module file
# if loading a custom algorithm module from a file on disk
if not found_algo_module:
err = ('{} - unable to find custom algorithm module={} '
'module_path={}'.format(label, custom_algo_module,
algo_module_path))
if algo_module_path:
err = (
'{} - analysis_engine.work_tasks.run_distributed_algorithm '
'was unable '
'to find custom algorithm module={} with provided path to \n '
'file: {} \n'
'\n'
'Please confirm '
'that the class inherits from the BaseAlgo class like:\n'
'\n'
'import analysis_engine.algo\n'
'class MyAlgo(analysis_engine.algo.BaseAlgo):\n '
'\n'
'If it is then please file an issue on github:\n '
'https://github.com/AlgoTraders/stock-analysis-engine/'
'issues/new \n\nFor now this error results in a shutdown'
'\n'.format(label, custom_algo_module, algo_module_path))
# if algo_module_path set
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# if not found_algo_module
use_start_date = start_date
use_end_date = end_date
if not use_end_date:
end_date = datetime.datetime.utcnow()
use_end_date = end_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT)
if not use_start_date:
start_date = end_date - datetime.timedelta(days=num_days_back)
use_start_date = start_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT)
dataset_publish_extract = algo_req.get('dataset_publish_extract', False)
dataset_publish_history = algo_req.get('dataset_publish_history', False)
dataset_publish_report = algo_req.get('dataset_publish_report', False)
try:
if use_custom_algo:
log.info('inspecting {} for class {}'.format(
custom_algo_module, module_name))
use_class_member_object = None
for member in inspect.getmembers(custom_algo_module):
if module_name in str(member):
log.info('start {} with {}'.format(name, member[1]))
use_class_member_object = member
break
# end of looking over the class definition but did not find it
if use_class_member_object:
new_algo_object = member[1](**algo_req)
else:
err = ('{} - did not find a derived '
'analysis_engine.algo.BaseAlgo '
'class in the module file={} '
'for ticker={} algo_name={}'.format(
label, algo_module_path, ticker, name))
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# end of finding a valid algorithm object
else:
new_algo_object = ae_algo.BaseAlgo(**algo_req)
# if using a custom module path or the BaseAlgo
if new_algo_object:
# heads up - logging this might have passwords in the algo_req
# log.debug(
# '{} algorithm request: {}'.format(
# name,
# algo_req))
log.info('{} - run ticker={} from {} to {}'.format(
name, ticker, use_start_date, use_end_date))
algo_res = run_algo.run_algo(algo=new_algo_object,
raise_on_err=raise_on_err,
**algo_req)
created_algo_object = new_algo_object
log.info('{} - run ticker={} from {} to {}'.format(
name, ticker, use_start_date, use_end_date))
if custom_algo_module:
log.info(
'{} - done run_algo custom_algo_module={} module_name={} '
'ticker={} from {} to {}'.format(name, custom_algo_module,
module_name, ticker,
use_start_date,
use_end_date))
else:
log.info('{} - done run_algo BaseAlgo ticker={} from {} '
'to {}'.format(name, ticker, use_start_date,
use_end_date))
else:
err = ('{} - missing a derived analysis_engine.algo.BaseAlgo '
'class in the module file={} for '
'ticker={} algo_name={}'.format(label, algo_module_path,
ticker, name))
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# end of finding a valid algorithm object
if not created_algo_object:
err = ('{} - failed creating algorithm object - '
'ticker={} status={} error={}'
'algo name={} custom_algo_module={} module_name={} '
'from {} to {}'.format(
label, ticker,
ae_consts.get_status(status=algo_res['status']),
algo_res['err'], name, custom_algo_module, module_name,
use_start_date, use_end_date))
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# end of stop early
if should_publish_extract_dataset or dataset_publish_extract:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (extract_config['redis_address'] and extract_config['redis_db']
and extract_config['redis_key']):
redis_log = 'redis://{}@{}/{}'.format(
extract_config['redis_address'],
extract_config['redis_db'], extract_config['redis_key'])
use_log += ' {}'.format(redis_log)
else:
extract_config['redis_enabled'] = False
if (extract_config['s3_address'] and extract_config['s3_bucket']
and extract_config['s3_key']):
s3_log = 's3://{}/{}/{}'.format(extract_config['s3_address'],
extract_config['s3_bucket'],
extract_config['s3_key'])
use_log += ' {}'.format(s3_log)
else:
extract_config['s3_enabled'] = False
if extract_config['output_file']:
file_log = 'file:{}'.format(extract_config['output_file'])
use_log += ' {}'.format(file_log)
log.info('{} - publish - start ticker={} algorithm-ready {}'
''.format(name, ticker, use_log))
publish_status = created_algo_object.publish_input_dataset(
**extract_config)
if publish_status != ae_consts.SUCCESS:
msg = ('failed to publish algorithm-ready datasets '
'with status {} attempted to {}'.format(
ae_consts.get_status(status=publish_status),
use_log))
log.error(msg)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# end of stop early
log.info('{} - publish - done ticker={} algorithm-ready {}'
''.format(name, ticker, use_log))
# if publish the algorithm-ready dataset
if should_publish_history_dataset or dataset_publish_history:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (history_config['redis_address'] and history_config['redis_db']
and history_config['redis_key']):
redis_log = 'redis://{}@{}/{}'.format(
history_config['redis_address'],
history_config['redis_db'], history_config['redis_key'])
use_log += ' {}'.format(redis_log)
if (history_config['s3_address'] and history_config['s3_bucket']
and history_config['s3_key']):
s3_log = 's3://{}/{}/{}'.format(history_config['s3_address'],
history_config['s3_bucket'],
history_config['s3_key'])
use_log += ' {}'.format(s3_log)
if history_config['output_file']:
file_log = 'file:{}'.format(history_config['output_file'])
use_log += ' {}'.format(file_log)
log.info('{} - publish - start ticker={} trading history {}'
''.format(name, ticker, use_log))
publish_status = \
created_algo_object.publish_trade_history_dataset(
**history_config)
if publish_status != ae_consts.SUCCESS:
msg = ('failed to publish trading history datasets '
'with status {} attempted to {}'.format(
ae_consts.get_status(status=publish_status),
use_log))
log.error(msg)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# end of stop early
log.info('{} - publish - done ticker={} trading history {}'
''.format(name, ticker, use_log))
# if publish an trading history dataset
if should_publish_report_dataset or dataset_publish_report:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (report_config['redis_address'] and report_config['redis_db']
and report_config['redis_key']):
redis_log = 'redis://{}@{}/{}'.format(
report_config['redis_address'], report_config['redis_db'],
report_config['redis_key'])
use_log += ' {}'.format(redis_log)
if (report_config['s3_address'] and report_config['s3_bucket']
and report_config['s3_key']):
s3_log = 's3://{}/{}/{}'.format(report_config['s3_address'],
report_config['s3_bucket'],
report_config['s3_key'])
use_log += ' {}'.format(s3_log)
if report_config['output_file']:
file_log = ' file:{}'.format(report_config['output_file'])
use_log += ' {}'.format(file_log)
log.info('{} - publishing ticker={} trading performance report {}'
''.format(name, ticker, use_log))
publish_status = created_algo_object.publish_report_dataset(
**report_config)
if publish_status != ae_consts.SUCCESS:
msg = ('failed to publish trading performance report datasets '
'with status {} attempted to {}'.format(
ae_consts.get_status(status=publish_status),
use_log))
log.error(msg)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
return get_task_results.get_task_results(work_dict=algo_req,
result=res)
# end of stop early
log.info(
'{} - publish - done ticker={} trading performance report {}'
''.format(name, ticker, use_log))
# if publish an trading performance report dataset
log.info(
'{} - done publishing datasets for ticker={} from {} to {}'.format(
name, ticker, use_start_date, use_end_date))
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
res = build_result.build_result(
status=ae_consts.ERR,
err=('failed - run_distributed_algorithm '
'dict={} with ex={}').format(algo_req, e),
rec=rec)
log.error('{} - {}'.format(label, res['err']))
# end of try/ex
log.info('task - run_distributed_algorithm done - '
'{} - status={}'.format(label,
ae_consts.get_status(res['status'])))
return get_task_results.get_task_results(work_dict=algo_req, result=res)
def run_custom_algo(
mod_path,
ticker='SPY',
balance=50000,
commission=6.0,
start_date=None,
end_date=None,
name='myalgo',
auto_fill=True,
config_file=None,
config_dict=None,
load_from_s3_bucket=None,
load_from_s3_key=None,
load_from_redis_key=None,
load_from_file=None,
load_compress=False,
load_publish=True,
load_config=None,
report_redis_key=None,
report_s3_bucket=None,
report_s3_key=None,
report_file=None,
report_compress=False,
report_publish=True,
report_config=None,
history_redis_key=None,
history_s3_bucket=None,
history_s3_key=None,
history_file=None,
history_compress=False,
history_publish=True,
history_config=None,
extract_redis_key=None,
extract_s3_bucket=None,
extract_s3_key=None,
extract_file=None,
extract_save_dir=None,
extract_compress=False,
extract_publish=True,
extract_config=None,
publish_to_s3=True,
publish_to_redis=True,
publish_to_slack=True,
dataset_type=ae_consts.SA_DATASET_TYPE_ALGO_READY,
serialize_datasets=ae_consts.DEFAULT_SERIALIZED_DATASETS,
compress=False,
encoding='utf-8',
redis_enabled=True,
redis_key=None,
redis_address=None,
redis_db=None,
redis_password=None,
redis_expire=None,
redis_serializer='json',
redis_encoding='utf-8',
s3_enabled=True,
s3_key=None,
s3_address=None,
s3_bucket=None,
s3_access_key=None,
s3_secret_key=None,
s3_region_name=None,
s3_secure=False,
slack_enabled=False,
slack_code_block=False,
slack_full_width=False,
timeseries=None,
trade_strategy=None,
verbose=False,
debug=False,
dataset_publish_extract=False,
dataset_publish_history=False,
dataset_publish_report=False,
run_on_engine=False,
auth_url=ae_consts.WORKER_BROKER_URL,
backend_url=ae_consts.WORKER_BACKEND_URL,
include_tasks=ae_consts.INCLUDE_TASKS,
ssl_options=ae_consts.SSL_OPTIONS,
transport_options=ae_consts.TRANSPORT_OPTIONS,
path_to_config_module=ae_consts.WORKER_CELERY_CONFIG_MODULE,
raise_on_err=True):
"""run_custom_algo
Run a custom algorithm that derives the
``analysis_engine.algo.BaseAlgo`` class
.. note:: Make sure to only have **1**
class defined in an algo module. Imports from
other modules should work just fine.
**Algorithm arguments**
:param mod_path: file path to custom
algorithm class module
:param ticker: ticker symbol
:param balance: float - starting balance capital
for creating buys and sells
:param commission: float - cost pet buy or sell
:param name: string - name for tracking algorithm
in the logs
:param start_date: string - start date for backtest with
format ``YYYY-MM-DD HH:MM:SS``
:param end_date: end date for backtest with
format ``YYYY-MM-DD HH:MM:SS``
:param auto_fill: optional - boolean for auto filling
buy and sell orders for backtesting
(default is ``True``)
:param config_file: path to a json file
containing custom algorithm object
member values (like indicator configuration and
predict future date units ahead for a backtest)
:param config_dict: optional - dictionary that
can be passed to derived class implementations
of: ``def load_from_config(config_dict=config_dict)``
**Timeseries**
:param timeseries: optional - string to
set ``day`` or ``minute`` backtesting
or live trading
(default is ``minute``)
**Trading Strategy**
:param trade_strategy: optional - string to
set the type of ``Trading Strategy``
for backtesting or live trading
(default is ``count``)
**Running Distributed Algorithms on the Engine Workers**
:param run_on_engine: optional - boolean
flag for publishing custom algorithms
to Celery ae workers for distributing
algorithm workloads
(default is ``False`` which will run algos locally)
this is required for distributing algorithms
:param auth_url: Celery broker address
(default is ``redis://localhost:6379/11``
or ``analysis_engine.consts.WORKER_BROKER_URL``
environment variable)
this is required for distributing algorithms
:param backend_url: Celery backend address
(default is ``redis://localhost:6379/12``
or ``analysis_engine.consts.WORKER_BACKEND_URL``
environment variable)
this is required for distributing algorithms
:param include_tasks: list of modules containing tasks to add
(default is ``analysis_engine.consts.INCLUDE_TASKS``)
:param ssl_options: security options dictionary
(default is ``analysis_engine.consts.SSL_OPTIONS``)
:param trasport_options: transport options dictionary
(default is ``analysis_engine.consts.TRANSPORT_OPTIONS``)
:param path_to_config_module: config module for advanced
Celery worker connectivity requirements
(default is ``analysis_engine.work_tasks.celery_config``
or ``analysis_engine.consts.WORKER_CELERY_CONFIG_MODULE``)
**Load Algorithm-Ready Dataset From Source**
Use these arguments to load algorithm-ready datasets
from supported sources (file, s3 or redis)
:param load_from_s3_bucket: optional - string load the algo from an
a previously-created s3 bucket holding an s3 key with an
algorithm-ready dataset for use with:
``handle_data``
:param load_from_s3_key: optional - string load the algo from an
a previously-created s3 key holding an
algorithm-ready dataset for use with:
``handle_data``
:param load_from_redis_key: optional - string load the algo from a
a previously-created redis key holding an
algorithm-ready dataset for use with:
``handle_data``
:param load_from_file: optional - string path to
a previously-created local file holding an
algorithm-ready dataset for use with:
``handle_data``
:param load_compress: optional - boolean
flag for toggling to decompress
or not when loading an algorithm-ready
dataset (``True`` means the dataset
must be decompressed to load correctly inside
an algorithm to run a backtest)
:param load_publish: boolean - toggle publishing
the load progress to slack, s3, redis or a file
(default is ``True``)
:param load_config: optional - dictionary
for setting member variables to load an
agorithm-ready dataset from
a file, s3 or redis
**Publishing Control Bool Flags**
:param publish_to_s3: optional - boolean for
toggling publishing to s3 on/off
(default is ``True``)
:param publish_to_redis: optional - boolean for
publishing to redis on/off
(default is ``True``)
:param publish_to_slack: optional - boolean for
publishing to slack
(default is ``True``)
**Algorithm Trade History Arguments**
:param history_redis_key: optional - string
where the algorithm trading history will be stored in
an redis key
:param history_s3_bucket: optional - string
where the algorithm trading history will be stored in
an s3 bucket
:param history_s3_key: optional - string
where the algorithm trading history will be stored in
an s3 key
:param history_file: optional - string key
where the algorithm trading history will be stored in
a file serialized as a json-string
:param history_compress: optional - boolean
flag for toggling to decompress
or not when loading an algorithm-ready
dataset (``True`` means the dataset
will be compressed on publish)
:param history_publish: boolean - toggle publishing
the history to s3, redis or a file
(default is ``True``)
:param history_config: optional - dictionary
for setting member variables to publish
an algo ``trade history`` to s3, redis, a file
or slack
**Algorithm Trade Performance Report Arguments (Output Dataset)**
:param report_redis_key: optional - string
where the algorithm ``trading performance report`` (report)
will be stored in an redis key
:param report_s3_bucket: optional - string
where the algorithm report will be stored in
an s3 bucket
:param report_s3_key: optional - string
where the algorithm report will be stored in
an s3 key
:param report_file: optional - string key
where the algorithm report will be stored in
a file serialized as a json-string
:param report_compress: optional - boolean
flag for toggling to decompress
or not when loading an algorithm-ready
dataset (``True`` means the dataset
will be compressed on publish)
:param report_publish: boolean - toggle publishing
the ``trading performance report`` s3, redis or a file
(default is ``True``)
:param report_config: optional - dictionary
for setting member variables to publish
an algo ``trading performance report`` to s3,
redis, a file or slack
**Extract an Algorithm-Ready Dataset Arguments**
:param extract_redis_key: optional - string
where the algorithm report will be stored in
an redis key
:param extract_s3_bucket: optional - string
where the algorithm report will be stored in
an s3 bucket
:param extract_s3_key: optional - string
where the algorithm report will be stored in
an s3 key
:param extract_file: optional - string key
where the algorithm report will be stored in
a file serialized as a json-string
:param extract_save_dir: optional - string path to
auto-generated files from the algo
:param extract_compress: optional - boolean
flag for toggling to decompress
or not when loading an algorithm-ready
dataset (``True`` means the dataset
will be compressed on publish)
:param extract_publish: boolean - toggle publishing
the used ``algorithm-ready dataset`` to s3, redis or a file
(default is ``True``)
:param extract_config: optional - dictionary
for setting member variables to publish
an algo ``trading performance report`` to s3,
redis, a file or slack
**Dataset Arguments**
:param dataset_type: optional - dataset type
(default is ``SA_DATASET_TYPE_ALGO_READY``)
:param serialize_datasets: optional - list of dataset names to
deserialize in the dataset
(default is ``DEFAULT_SERIALIZED_DATASETS``)
:param encoding: optional - string for data encoding
**Publish Algorithm Datasets to S3, Redis or a File**
:param dataset_publish_extract: optional - bool
for publishing the algorithm's
``algorithm-ready``
dataset to: s3, redis or file
:param dataset_publish_history: optional - bool
for publishing the algorithm's
``trading history``
dataset to: s3, redis or file
:param dataset_publish_report: optional - bool
for publishing the algorithm's
``trading performance report``
dataset to: s3, redis or file
**Redis connectivity arguments**
:param redis_enabled: bool - toggle for auto-caching all
datasets in Redis
(default is ``True``)
:param redis_key: string - key to save the data in redis
(default is ``None``)
:param redis_address: Redis connection string format: ``host:port``
(default is ``localhost:6379``)
:param redis_db: Redis db to use
(default is ``0``)
:param redis_password: optional - Redis password
(default is ``None``)
:param redis_expire: optional - Redis expire value
(default is ``None``)
:param redis_serializer: not used yet - support for future
pickle objects in redis
:param redis_encoding: format of the encoded key in redis
**Minio (S3) connectivity arguments**
:param s3_enabled: bool - toggle for auto-archiving on Minio (S3)
(default is ``True``)
:param s3_key: string - key to save the data in redis
(default is ``None``)
:param s3_address: Minio S3 connection string format: ``host:port``
(default is ``localhost:9000``)
:param s3_bucket: S3 Bucket for storing the artifacts
(default is ``dev``) which should be viewable on a browser:
http://localhost:9000/minio/dev/
:param s3_access_key: S3 Access key
(default is ``trexaccesskey``)
:param s3_secret_key: S3 Secret key
(default is ``trex123321``)
:param s3_region_name: S3 region name
(default is ``us-east-1``)
:param s3_secure: Transmit using tls encryption
(default is ``False``)
**Slack arguments**
:param slack_enabled: optional - boolean for
publishing to slack
:param slack_code_block: optional - boolean for
publishing as a code black in slack
:param slack_full_width: optional - boolean for
publishing as a to slack using the full
width allowed
**Debugging arguments**
:param debug: optional - bool for debug tracking
:param verbose: optional - bool for increasing
logging
:param raise_on_err: boolean - set this to ``False`` on prod
to ensure exceptions do not interrupt services.
With the default (``True``) any exceptions from the library
and your own algorithm are sent back out immediately exiting
the backtest.
"""
module_name = 'BaseAlgo'
custom_algo_module = None
new_algo_object = None
use_custom_algo = False
found_algo_module = True
should_publish_extract_dataset = False
should_publish_history_dataset = False
should_publish_report_dataset = False
use_config_file = None
use_config_dict = config_dict
if config_file:
if os.path.exists(config_file):
use_config_file = config_file
if not config_dict:
try:
use_config_dict = json.loads(open(config_file, 'r').read())
except Exception as e:
msg = (f'failed parsing json config_file={config_file} '
f'with ex={e}')
log.error(msg)
raise Exception(msg)
# end of loading the config_file
err = None
if mod_path:
module_name = mod_path.split('/')[-1]
loader = importlib.machinery.SourceFileLoader(module_name, mod_path)
custom_algo_module = types.ModuleType(loader.name)
loader.exec_module(custom_algo_module)
use_custom_algo = True
for member in inspect.getmembers(custom_algo_module):
if module_name in str(member):
found_algo_module = True
break
# for all members in this custom module file
# if loading a custom algorithm module from a file on disk
if not found_algo_module:
err = (f'unable to find custom algorithm module={custom_algo_module}')
if mod_path:
err = (
'analysis_engine.run_custom_algo.run_custom_algo was unable '
f'to find custom algorithm module={custom_algo_module} with '
f'provided path to \n file: {mod_path} \n'
'\n'
'Please confirm '
'that the class inherits from the BaseAlgo class like:\n'
'\n'
'import analysis_engine.algo\n'
'class MyAlgo(analysis_engine.algo.BaseAlgo):\n '
'\n'
'If it is then please file an issue on github:\n '
'https://github.com/AlgoTraders/stock-analysis-engine/'
'issues/new \n\nFor now this error results in a shutdown'
'\n')
# if mod_path set
if verbose or debug:
log.error(err)
return build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
# if not found_algo_module
use_start_date = start_date
use_end_date = end_date
if not use_end_date:
end_date = datetime.datetime.utcnow()
use_end_date = end_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT)
if not use_start_date:
start_date = end_date - datetime.timedelta(days=75)
use_start_date = start_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT)
if verbose:
log.info(
f'{name} {ticker} setting default start_date={use_start_date}')
# Load an algorithm-ready dataset from:
# file, s3, or redis
if not load_config:
load_config = build_publish_request.build_publish_request(
ticker=ticker,
output_file=None,
s3_bucket=None,
s3_key=None,
redis_key=None,
compress=load_compress,
redis_enabled=publish_to_redis,
redis_address=redis_address,
redis_db=redis_db,
redis_password=redis_password,
redis_expire=redis_expire,
redis_serializer=redis_serializer,
redis_encoding=redis_encoding,
s3_enabled=publish_to_s3,
s3_address=s3_address,
s3_access_key=s3_access_key,
s3_secret_key=s3_secret_key,
s3_region_name=s3_region_name,
s3_secure=s3_secure,
slack_enabled=publish_to_slack,
slack_code_block=slack_code_block,
slack_full_width=slack_full_width,
verbose=verbose,
label=f'load-{name}')
if load_from_file:
load_config['output_file'] = load_from_file
if load_from_redis_key:
load_config['redis_key'] = load_from_redis_key
load_config['redis_enabled'] = True
if load_from_s3_bucket and load_from_s3_key:
load_config['s3_bucket'] = load_from_s3_bucket
load_config['s3_key'] = load_from_s3_key
load_config['s3_enabled'] = True
# end of building load_config dictionary if not already set
# Automatically save all datasets to an algorithm-ready:
# file, s3, or redis
if not extract_config:
extract_config = build_publish_request.build_publish_request(
ticker=ticker,
output_file=None,
s3_bucket=None,
s3_key=None,
redis_key=None,
compress=extract_compress,
redis_enabled=publish_to_redis,
redis_address=redis_address,
redis_db=redis_db,
redis_password=redis_password,
redis_expire=redis_expire,
redis_serializer=redis_serializer,
redis_encoding=redis_encoding,
s3_enabled=publish_to_s3,
s3_address=s3_address,
s3_access_key=s3_access_key,
s3_secret_key=s3_secret_key,
s3_region_name=s3_region_name,
s3_secure=s3_secure,
slack_enabled=publish_to_slack,
slack_code_block=slack_code_block,
slack_full_width=slack_full_width,
verbose=verbose,
label=f'extract-{name}')
should_publish_extract_dataset = False
if extract_file:
extract_config['output_file'] = extract_file
should_publish_extract_dataset = True
if extract_redis_key and publish_to_redis:
extract_config['redis_key'] = extract_redis_key
extract_config['redis_enabled'] = True
should_publish_extract_dataset = True
if extract_s3_bucket and extract_s3_key and publish_to_s3:
extract_config['s3_bucket'] = extract_s3_bucket
extract_config['s3_key'] = extract_s3_key
extract_config['s3_enabled'] = True
should_publish_extract_dataset = True
else:
extract_config['s3_enabled'] = False
# end of building extract_config dictionary if not already set
# Automatically save the trading performance report:
# file, s3, or redis
if not report_config:
report_config = build_publish_request.build_publish_request(
ticker=ticker,
output_file=None,
s3_bucket=None,
s3_key=None,
redis_key=None,
compress=report_compress,
redis_enabled=publish_to_redis,
redis_address=redis_address,
redis_db=redis_db,
redis_password=redis_password,
redis_expire=redis_expire,
redis_serializer=redis_serializer,
redis_encoding=redis_encoding,
s3_enabled=publish_to_s3,
s3_address=s3_address,
s3_access_key=s3_access_key,
s3_secret_key=s3_secret_key,
s3_region_name=s3_region_name,
s3_secure=s3_secure,
slack_enabled=publish_to_slack,
slack_code_block=slack_code_block,
slack_full_width=slack_full_width,
verbose=verbose,
label=f'report-{name}')
should_publish_report_dataset = False
if report_file:
report_config['output_file'] = report_file
should_publish_report_dataset = True
if report_redis_key and publish_to_redis:
report_config['redis_key'] = report_redis_key
report_config['redis_enabled'] = True
should_publish_report_dataset = True
if report_s3_bucket and report_s3_key and publish_to_s3:
report_config['s3_bucket'] = report_s3_bucket
report_config['s3_key'] = report_s3_key
report_config['s3_enabled'] = True
should_publish_report_dataset = True
# end of building report_config dictionary if not already set
# Automatically save the trade history:
# file, s3, or redis
if not history_config:
history_config = build_publish_request.build_publish_request(
ticker=ticker,
output_file=None,
s3_bucket=None,
s3_key=None,
redis_key=None,
compress=report_compress,
redis_enabled=publish_to_redis,
redis_address=redis_address,
redis_db=redis_db,
redis_password=redis_password,
redis_expire=redis_expire,
redis_serializer=redis_serializer,
redis_encoding=redis_encoding,
s3_enabled=publish_to_s3,
s3_address=s3_address,
s3_access_key=s3_access_key,
s3_secret_key=s3_secret_key,
s3_region_name=s3_region_name,
s3_secure=s3_secure,
slack_enabled=publish_to_slack,
slack_code_block=slack_code_block,
slack_full_width=slack_full_width,
verbose=verbose,
label=f'history-{name}')
should_publish_history_dataset = False
if history_file:
history_config['output_file'] = history_file
should_publish_history_dataset = True
if history_redis_key and publish_to_redis:
history_config['redis_key'] = history_redis_key
history_config['redis_enabled'] = True
should_publish_history_dataset = True
if history_s3_bucket and history_s3_key and publish_to_s3:
history_config['s3_bucket'] = history_s3_bucket
history_config['s3_key'] = history_s3_key
history_config['s3_enabled'] = True
should_publish_history_dataset = True
# end of building history_config dictionary if not already set
if verbose:
remove_vals = ['s3_access_key', 's3_secret_key', 'redis_password']
debug_extract_config = {}
for k in extract_config:
if k not in remove_vals:
debug_extract_config[k] = extract_config[k]
debug_report_config = {}
for k in report_config:
if k not in remove_vals:
debug_report_config[k] = report_config[k]
debug_history_config = {}
for k in history_config:
if k not in remove_vals:
debug_history_config[k] = history_config[k]
debug_load_config = {}
for k in load_config:
if k not in remove_vals:
debug_load_config[k] = load_config[k]
log.info(f'{name} {ticker} using extract config '
f'{ae_consts.ppj(debug_extract_config)}')
log.info(f'{name} {ticker} using report config '
f'{ae_consts.ppj(debug_report_config)}')
log.info(f'{name} {ticker} using trade history config '
f'{ae_consts.ppj(debug_history_config)}')
log.info(f'{name} {ticker} using load config '
f'{ae_consts.ppj(debug_load_config)}')
log.info(f'{name} {ticker} - building algo request')
# end of verbose
algo_req = build_algo_request.build_algo_request(
ticker=ticker,
balance=balance,
commission=commission,
start_date=use_start_date,
end_date=use_end_date,
timeseries=timeseries,
trade_strategy=trade_strategy,
config_file=use_config_file,
config_dict=use_config_dict,
load_config=load_config,
history_config=history_config,
report_config=report_config,
extract_config=extract_config,
label=name)
algo_req['name'] = name
algo_req['should_publish_extract_dataset'] = should_publish_extract_dataset
algo_req['should_publish_history_dataset'] = should_publish_history_dataset
algo_req['should_publish_report_dataset'] = should_publish_report_dataset
algo_res = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec=None)
if run_on_engine:
rec = {'algo_req': algo_req, 'task_id': None}
task_name = ('analysis_engine.work_tasks.'
'task_run_algo.task_run_algo')
if verbose:
log.info(f'starting distributed algo task={task_name}')
elif debug:
log.info(
'starting distributed algo by publishing to '
f'task={task_name} broker={auth_url} backend={backend_url}')
# Get the Celery app
app = get_celery_app.get_celery_app(
name=__name__,
auth_url=auth_url,
backend_url=backend_url,
path_to_config_module=path_to_config_module,
ssl_options=ssl_options,
transport_options=transport_options,
include_tasks=include_tasks)
if debug:
log.info(f'calling distributed algo task={task_name} '
f'request={ae_consts.ppj(algo_req)}')
elif verbose:
log.info(f'calling distributed algo task={task_name}')
job_id = app.send_task(task_name, (algo_req, ))
if verbose:
log.info(f'calling task={task_name} - success job_id={job_id}')
rec['task_id'] = job_id
algo_res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
return algo_res
# end of run_on_engine
if use_custom_algo:
if verbose:
log.info(
f'inspecting {custom_algo_module} for class {module_name}')
use_class_member_object = None
for member in inspect.getmembers(custom_algo_module):
if module_name in str(member):
if verbose:
log.info(f'start {name} with {member[1]}')
use_class_member_object = member
break
# end of looking over the class definition but did not find it
if use_class_member_object:
new_algo_object = member[1](**algo_req)
else:
err = ('did not find a derived analysis_engine.algo.BaseAlgo '
f'class in the module file={mod_path} '
f'for ticker={ticker} algo_name={name}')
if verbose or debug:
log.error(err)
return build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
# end of finding a valid algorithm object
else:
new_algo_object = ae_algo.BaseAlgo(**algo_req)
# if using a custom module path or the BaseAlgo
if new_algo_object:
# heads up - logging this might have passwords in the algo_req
# log.debug(
# f'{name} algorithm request: {algo_req}')
if verbose:
log.info(f'{name} - run ticker={ticker} from {use_start_date} '
f'to {use_end_date}')
algo_res = run_algo.run_algo(algo=new_algo_object,
raise_on_err=raise_on_err,
**algo_req)
algo_res['algo'] = new_algo_object
if verbose:
log.info(f'{name} - run ticker={ticker} from {use_start_date} '
f'to {use_end_date}')
if custom_algo_module:
if verbose:
log.info(f'{name} - done run_algo '
f'custom_algo_module={custom_algo_module} '
f'module_name={module_name} ticker={ticker} '
f'from {use_start_date} to {use_end_date}')
else:
if verbose:
log.info(f'{name} - done run_algo BaseAlgo ticker={ticker} '
f'from {use_start_date} to {use_end_date}')
else:
err = ('missing a derived analysis_engine.algo.BaseAlgo '
f'class in the module file={mod_path} for ticker={ticker} '
f'algo_name={name}')
return build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
# end of finding a valid algorithm object
algo = algo_res.get('algo', None)
if not algo:
err = (f'failed creating algorithm object - ticker={ticker} '
f'status={ae_consts.get_status(status=algo_res["status"])} '
f'error={algo_res["err"]} algo name={name} '
f'custom_algo_module={custom_algo_module} '
f'module_name={module_name} '
f'from {use_start_date} to {use_end_date}')
return build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
if should_publish_extract_dataset or dataset_publish_extract:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (extract_config['redis_address'] and extract_config['redis_db'] >= 0
and extract_config['redis_key']):
redis_log = (
f'redis://{extract_config["redis_address"]}'
f'@{extract_config["redis_db"]}/{extract_config["redis_key"]}')
use_log += f' {redis_log}'
else:
extract_config['redis_enabled'] = False
if (extract_config['s3_address'] and extract_config['s3_bucket']
and extract_config['s3_key']):
s3_log = (
f's3://{extract_config["s3_address"]}'
f'/{extract_config["s3_bucket"]}/{extract_config["s3_key"]}')
use_log += f' {s3_log}'
else:
extract_config['s3_enabled'] = False
if extract_config['output_file']:
file_log = f'file:{extract_config["output_file"]}'
use_log += f' {file_log}'
if verbose:
log.info(f'{name} - publish - start ticker={ticker} '
f'algorithm-ready {use_log}')
publish_status = algo.publish_input_dataset(**extract_config)
if publish_status != ae_consts.SUCCESS:
msg = (
'failed to publish algorithm-ready datasets '
f'with status {ae_consts.get_status(status=publish_status)} '
f'attempted to {use_log}')
log.error(msg)
return build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
if verbose:
log.info(f'{name} - publish - done ticker={ticker} '
f'algorithm-ready {use_log}')
# if publish the algorithm-ready dataset
if should_publish_history_dataset or dataset_publish_history:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (history_config['redis_address'] and history_config['redis_db'] >= 0
and history_config['redis_key']):
redis_log = (
f'redis://{history_config["redis_address"]}'
f'@{history_config["redis_db"]}/{history_config["redis_key"]}')
use_log += f' {redis_log}'
else:
history_config['redis_enabled'] = False
if (history_config['s3_address'] and history_config['s3_bucket']
and history_config['s3_key']):
s3_log = (
f's3://{history_config["s3_address"]}'
f'/{history_config["s3_bucket"]}/{history_config["s3_key"]}')
use_log += f' {s3_log}'
else:
history_config['s3_enabled'] = False
if history_config['output_file']:
file_log = f'file:{history_config["output_file"]}'
use_log += f' {file_log}'
if verbose:
log.info(f'{name} - publish - start ticker={ticker} trading '
f'history {use_log}')
publish_status = algo.publish_trade_history_dataset(**history_config)
if publish_status != ae_consts.SUCCESS:
msg = (
'failed to publish trading history datasets '
f'with status {ae_consts.get_status(status=publish_status)} '
f'attempted to {use_log}')
log.error(msg)
return build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
if verbose:
log.info(f'{name} - publish - done ticker={ticker} trading '
f'history {use_log}')
# if publish an trading history dataset
if should_publish_report_dataset or dataset_publish_report:
s3_log = ''
redis_log = ''
file_log = ''
use_log = 'publish'
if (report_config['redis_address'] and report_config['redis_db'] >= 0
and report_config['redis_key']):
redis_log = (
f'redis://{report_config["redis_address"]}'
f'@{report_config["redis_db"]}/{report_config["redis_key"]}')
use_log += f' {redis_log}'
else:
report_config['redis_enabled'] = False
if (report_config['s3_address'] and report_config['s3_bucket']
and report_config['s3_key']):
s3_log = (
f's3://{report_config["s3_address"]}'
f'/{report_config["s3_bucket"]}/{report_config["s3_key"]}')
use_log += f' {s3_log}'
else:
report_config['s3_enabled'] = False
if report_config['output_file']:
file_log = f'file:{report_config["output_file"]}'
use_log += f' {file_log}'
if verbose:
log.info(
f'{name} - publishing ticker={ticker} trading performance '
f'report {use_log}')
publish_status = algo.publish_report_dataset(**report_config)
if publish_status != ae_consts.SUCCESS:
msg = (
'failed to publish trading performance report datasets '
f'with status {ae_consts.get_status(status=publish_status)} '
f'attempted to {use_log}')
log.error(msg)
return build_result.build_result(status=ae_consts.ERR,
err=err,
rec=None)
if verbose:
log.info(
f'{name} - publish - done ticker={ticker} trading performance '
f'report {use_log}')
# if publish an trading performance report dataset
if verbose:
log.info(f'{name} - done publishing datasets for ticker={ticker} '
f'from {use_start_date} to {use_end_date}')
return algo_res
def latest(self,
date_str=None,
start_row=-200,
extract_iex=True,
extract_yahoo=False,
extract_td=True,
verbose=False,
**kwargs):
"""latest
Run the algorithm with the latest pricing data. Also
supports running a backtest for a historical date in
the pricing history (format ``YYYY-MM-DD``)
:param date_str: optional - string start date ``YYYY-MM-DD``
default is the latest close date
:param start_row: negative number of rows back
from the end of the list in the data
default is ``-200`` where this means the algorithm
will process the latest 200 rows in the minute
dataset
:param extract_iex: bool flag for extracting from ``IEX``
:param extract_yahoo: bool flag for extracting from ``Yahoo``
which is disabled as of 1/2019
:param extract_td: bool flag for extracting from ``Tradier``
:param verbose: bool flag for logs
:param kwargs: keyword arg dict
"""
use_date_str = date_str
if not use_date_str:
use_date_str = ae_utils.get_last_close_str()
log.info(f'creating algo')
self.algo_obj = base_algo.BaseAlgo(
ticker=self.config_dict['ticker'],
balance=self.config_dict['balance'],
commission=self.config_dict['commission'],
name=self.use_name,
start_date=self.use_start_date,
end_date=self.use_end_date,
auto_fill=self.auto_fill,
config_dict=self.config_dict,
load_from_s3_bucket=self.load_from_s3_bucket,
load_from_s3_key=self.load_from_s3_key,
load_from_redis_key=self.load_from_redis_key,
load_from_file=self.load_from_file,
load_compress=self.load_compress,
load_publish=self.load_publish,
load_config=self.load_config,
report_redis_key=self.report_redis_key,
report_s3_bucket=self.report_s3_bucket,
report_s3_key=self.report_s3_key,
report_file=self.report_file,
report_compress=self.report_compress,
report_publish=self.report_publish,
report_config=self.report_config,
history_redis_key=self.history_redis_key,
history_s3_bucket=self.history_s3_bucket,
history_s3_key=self.history_s3_key,
history_file=self.history_file,
history_compress=self.history_compress,
history_publish=self.history_publish,
history_config=self.history_config,
extract_redis_key=self.extract_redis_key,
extract_s3_bucket=self.extract_s3_bucket,
extract_s3_key=self.extract_s3_key,
extract_file=self.extract_file,
extract_save_dir=self.extract_save_dir,
extract_compress=self.extract_compress,
extract_publish=self.extract_publish,
extract_config=self.extract_config,
publish_to_slack=self.publish_to_slack,
publish_to_s3=self.publish_to_s3,
publish_to_redis=self.publish_to_redis,
dataset_type=self.dataset_type,
serialize_datasets=self.serialize_datasets,
compress=self.compress,
encoding=self.encoding,
redis_enabled=self.redis_enabled,
redis_key=self.redis_key,
redis_address=self.redis_address,
redis_db=self.redis_db,
redis_password=self.redis_password,
redis_expire=self.redis_expire,
redis_serializer=self.redis_serializer,
redis_encoding=self.redis_encoding,
s3_enabled=self.s3_enabled,
s3_key=self.s3_key,
s3_address=self.s3_address,
s3_bucket=self.s3_bucket,
s3_access_key=self.s3_access_key,
s3_secret_key=self.s3_secret_key,
s3_region_name=self.s3_region_name,
s3_secure=self.s3_secure,
slack_enabled=self.slack_enabled,
slack_code_block=self.slack_code_block,
slack_full_width=self.slack_full_width,
dataset_publish_extract=self.extract_publish,
dataset_publish_history=self.history_publish,
dataset_publish_report=self.report_publish,
run_on_engine=self.run_on_engine,
auth_url=self.broker_url,
backend_url=self.backend_url,
include_tasks=self.include_tasks,
ssl_options=self.ssl_options,
transport_options=self.transport_options,
path_to_config_module=self.path_to_config_module,
timeseries=self.timeseries,
trade_strategy=self.trade_strategy,
verbose=False,
raise_on_err=self.raise_on_err)
log.info(f'run latest - start')
ticker = self.config_dict['ticker']
self.common_fetch_vals['base_key'] = f'{ticker}_{use_date_str}'
extract_req = api_requests.get_ds_dict(
ticker=ticker,
base_key=self.common_fetch_vals['base_key'],
ds_id=ticker,
service_dict=self.common_fetch_vals)
node_date_key = use_date_str.replace(f'{ticker}_', '')
req = {
'id': use_date_str,
'ticker': ticker,
'date_key': self.common_fetch_vals['base_key'],
'date': node_date_key,
'req': extract_req
}
# fetch
iex_daily_df = None
iex_minute_df = None
iex_quote_df = None
iex_stats_df = None
iex_peers_df = None
iex_news_df = None
iex_financials_df = None
iex_earnings_df = None
iex_dividends_df = None
iex_company_df = None
yahoo_option_calls_df = None
yahoo_option_puts_df = None
yahoo_pricing_df = None
yahoo_news_df = None
td_calls_df = None
td_puts_df = None
node_date_key = req['date']
dataset_node_id = req['id']
dataset_id = dataset_node_id
label = (f'ticker={ticker} ' f'date={node_date_key}')
if verbose:
log.info(f'{label} - extract - start')
if 'daily' in self.iex_datasets or extract_iex:
iex_daily_status, iex_daily_df = \
iex_extract_utils.extract_daily_dataset(
extract_req)
if iex_daily_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract iex_daily={ticker}')
if 'minute' in self.iex_datasets or extract_iex:
iex_minute_status, iex_minute_df = \
iex_extract_utils.extract_minute_dataset(
extract_req)
if iex_minute_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract iex_minute={ticker}')
if 'quote' in self.iex_datasets or extract_iex:
iex_quote_status, iex_quote_df = \
iex_extract_utils.extract_quote_dataset(
extract_req)
if iex_quote_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract iex_quote={ticker}')
if 'stats' in self.iex_datasets or extract_iex:
iex_stats_status, iex_stats_df = \
iex_extract_utils.extract_stats_dataset(
extract_req)
if iex_stats_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract iex_stats={ticker}')
if 'peers' in self.iex_datasets or extract_iex:
iex_peers_status, iex_peers_df = \
iex_extract_utils.extract_peers_dataset(
extract_req)
if iex_peers_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract iex_peers={ticker}')
if 'news' in self.iex_datasets or extract_iex:
iex_news_status, iex_news_df = \
iex_extract_utils.extract_news_dataset(
extract_req)
if iex_news_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract iex_news={ticker}')
if 'financials' in self.iex_datasets or extract_iex:
iex_financials_status, iex_financials_df = \
iex_extract_utils.extract_financials_dataset(
extract_req)
if iex_financials_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract iex_financials={ticker}')
if 'earnings' in self.iex_datasets or extract_iex:
iex_earnings_status, iex_earnings_df = \
iex_extract_utils.extract_earnings_dataset(
extract_req)
if iex_earnings_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract iex_earnings={ticker}')
if 'dividends' in self.iex_datasets or extract_iex:
iex_dividends_status, iex_dividends_df = \
iex_extract_utils.extract_dividends_dataset(
extract_req)
if iex_dividends_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract iex_dividends={ticker}')
if 'company' in self.iex_datasets or extract_iex:
iex_company_status, iex_company_df = \
iex_extract_utils.extract_company_dataset(
extract_req)
if iex_company_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract iex_company={ticker}')
# end of iex extracts
if extract_yahoo:
yahoo_options_status, yahoo_option_calls_df = \
yahoo_extract_utils.extract_option_calls_dataset(
extract_req)
yahoo_options_status, yahoo_option_puts_df = \
yahoo_extract_utils.extract_option_puts_dataset(
extract_req)
if yahoo_options_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract yahoo_options={ticker}')
yahoo_pricing_status, yahoo_pricing_df = \
yahoo_extract_utils.extract_pricing_dataset(
extract_req)
if yahoo_pricing_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract yahoo_pricing={ticker}')
yahoo_news_status, yahoo_news_df = \
yahoo_extract_utils.extract_yahoo_news_dataset(
extract_req)
if yahoo_news_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract yahoo_news={ticker}')
# end of yahoo extracts
if extract_td:
"""
Debug by setting:
extract_req['verbose_td'] = True
"""
convert_to_datetime = [
'date', 'created', 'ask_date', 'bid_date', 'trade_date'
]
td_calls_status, td_calls_df = \
td_extract_utils.extract_option_calls_dataset(
extract_req)
if td_calls_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract tdcalls={ticker}')
else:
if ae_consts.is_df(df=td_calls_df):
for c in convert_to_datetime:
if c in td_calls_df:
td_calls_df[c] = pd.to_datetime(
td_calls_df[c],
format=ae_consts.COMMON_TICK_DATE_FORMAT)
if 'date' in td_calls_df:
td_calls_df.sort_values('date', ascending=True)
# end of converting dates
td_puts_status, td_puts_df = \
td_extract_utils.extract_option_puts_dataset(
extract_req)
if td_puts_status != ae_consts.SUCCESS:
if verbose:
log.warning(f'unable to extract tdputs={ticker}')
else:
if ae_consts.is_df(df=td_puts_df):
for c in convert_to_datetime:
if c in td_puts_df:
td_puts_df[c] = pd.to_datetime(
td_puts_df[c],
format=ae_consts.COMMON_TICK_DATE_FORMAT)
if 'date' in td_puts_df:
td_puts_df.sort_values('date', ascending=True)
# end of converting dates
# td extracts
# map extracted data to DEFAULT_SERIALIZED_DATASETS
ticker_data = {}
ticker_data['daily'] = iex_daily_df
ticker_data['minute'] = iex_minute_df
ticker_data['quote'] = iex_quote_df
ticker_data['stats'] = iex_stats_df
ticker_data['peers'] = iex_peers_df
ticker_data['news1'] = iex_news_df
ticker_data['financials'] = iex_financials_df
ticker_data['earnings'] = iex_earnings_df
ticker_data['dividends'] = iex_dividends_df
ticker_data['company'] = iex_company_df
ticker_data['calls'] = yahoo_option_calls_df
ticker_data['puts'] = yahoo_option_puts_df
ticker_data['pricing'] = yahoo_pricing_df
ticker_data['news'] = yahoo_news_df
ticker_data['tdcalls'] = td_calls_df
ticker_data['tdputs'] = td_puts_df
algo_data_req = {
ticker: [{
'id': dataset_id, # id is currently the cache key in redis
'date': use_date_str, # used to confirm dates in asc order
'data': ticker_data,
'start_row': start_row
}]
}
if verbose:
log.info(f'extract - {label} '
f'dataset={len(algo_data_req[ticker])}')
# this could be a separate celery task
try:
if verbose:
log.info(f'handle_data START - {label} from '
f'{node_date_key}')
self.algo_obj.handle_data(data=algo_data_req)
if verbose:
log.info(f'handle_data END - {label} from ' f'{node_date_key}')
except Exception as e:
a_name = self.algo_obj.get_name()
a_debug_msg = self.algo_obj.get_debug_msg()
if not a_debug_msg:
a_debug_msg = 'debug message not set'
# a_config_dict = ae_consts.ppj(self.algo_obj.config_dict)
msg = (f'{label} - algo={a_name} '
f'encountered exception in handle_data tickers={ticker} '
f'from {node_date_key} ex={e} '
f'and failed during operation: {a_debug_msg}')
log.critical(f'{msg}')
# end try/ex
log.info(f'run latest - create history')
history_ds = self.algo_obj.create_history_dataset()
self.history_df = pd.DataFrame(history_ds[ticker])
self.determine_latest_times_in_history()
self.num_rows = len(self.history_df.index)
if verbose:
log.info(self.history_df[['minute', 'close']].tail(5))
log.info(f'run latest minute={self.end_date} - '
f'rows={self.num_rows} - done')
return self.get_history()
