def plot_history_from_s3():
"""plot_history_from_s3
Run a derived algorithm with an algorithm config dictionary
:param config_dict: algorithm config dictionary
"""
log.debug('start - plot')
parser = argparse.ArgumentParser(
description=('plot a local algorithm trading history file'))
parser.add_argument('-b',
help=('saved in this s3 bucket'),
required=False,
dest='s3_bucket')
parser.add_argument('-k',
help=('saved in this s3 key'),
required=False,
dest='history_json_file')
parser.add_argument('-d',
help=('debug'),
required=False,
dest='debug',
action='store_true')
args = parser.parse_args()
s3_access_key = consts.S3_ACCESS_KEY
s3_secret_key = consts.S3_SECRET_KEY
s3_region_name = consts.S3_REGION_NAME
s3_address = consts.S3_ADDRESS
s3_secure = consts.S3_SECURE
compress = True
s3_bucket = ('bt-spy-williamsr-2018-12-05-22-44-50-714400')
s3_key = ('-181.55_netgain_9818.45_netvalue_NEGATIVE_'
'10000.0_startbalance_1710.95_endbalance_'
'30_shares_270.25_close_3_buys_0_sells_'
'1_minbuyinds_1_minsellinds_'
'43.52_seconds_'
'trade_history-SPY_williamsr_test_'
'0.73_for_176_of_24000.json')
debug = False
if args.debug:
debug = True
load_res = load_history.load_history_dataset(s3_enabled=True,
s3_key=s3_key,
s3_address=s3_address,
s3_bucket=s3_bucket,
s3_access_key=s3_access_key,
s3_secret_key=s3_secret_key,
s3_region_name=s3_region_name,
s3_secure=s3_secure,
compress=compress)
algo_config = load_res.get('algo_config_dict', None)
algo_name = load_res.get('algo_name', None)
tickers = load_res.get('tickers', [
'SPY',
])
ticker = tickers[0]
log.info(f'found algo: {algo_name}')
log.info(f'config: {consts.ppj(algo_config)}')
history_df = load_res[ticker]
history_df['date'] = pd.to_datetime(history_df['date'])
history_df['minute'] = pd.to_datetime(history_df['minute'])
ticker = history_df['ticker'].iloc[0]
log.info('plotting history')
first_date = history_df['date'].iloc[0]
end_date = history_df['date'].iloc[-1]
title = (f'Trading History {ticker}\n'
f'Backtest dates from {first_date} to {end_date}')
use_xcol = 'date'
use_as_date_format = '%d\n%b'
use_minute = False
if 'minute' in history_df:
found_valid_minute = history_df['minute'].iloc[0]
if found_valid_minute:
use_minute = True
if use_minute:
use_xcol = 'minute'
use_as_date_format = '%d %H:%M:%S\n%b'
xlabel = 'Dates vs Algo values'
ylabel = 'Algo values'
df_filter = (history_df['close'] > 1.00)
# set default hloc columns:
blue = None
green = None
orange = None
red = 'close'
blue = 'balance'
if debug:
for i, r in history_df.iterrows():
log.info(f'{r["minute"]} - {r["close"]}')
# end of debug
show_plot = True
if show_plot:
plot_trading_history.plot_trading_history(
title=title,
df=history_df,
red=red,
blue=blue,
green=green,
orange=orange,
date_col=use_xcol,
date_format=use_as_date_format,
xlabel=xlabel,
ylabel=ylabel,
df_filter=df_filter,
show_plot=True,
dropna_for_all=True)
def backtest_with_runner():
"""backtest_with_runner
build and publish a trading history from an algorithm config.
::
backtest_with_runner.py -t TICKER -c ALGO_CONFIG -s START_DATE
-k S3_KEY -b S3_BUCKET -l
"""
parser = argparse.ArgumentParser(
description=('backtest an algorithm and publish '
'the trading history'))
parser.add_argument('-t',
help=('ticker symbol'),
required=False,
dest='ticker')
parser.add_argument('-k', help=('s3_key'), required=False, dest='s3_key')
parser.add_argument('-b',
help=('s3_bucket'),
required=False,
dest='s3_bucket')
parser.add_argument('-s',
help=('start date format YYYY-MM-DD'),
required=False,
dest='start_date')
parser.add_argument('-c',
help=('algo config file'),
required=False,
dest='algo_config')
parser.add_argument('-l',
help=('run a backtest with the latest '
'pricing data'),
required=False,
dest='latest',
action='store_true')
parser.add_argument('-d',
help='debug',
required=False,
dest='debug',
action='store_true')
args = parser.parse_args()
ticker = 'SPY'
s3_bucket = (f'algohistory')
s3_key = (f'trade_history_{ticker}')
start_date = (f'2019-01-01')
algo_config = (f'/opt/sa/cfg/default_algo.json')
latest = False
show_plot = True
debug = False
if args.ticker:
ticker = args.ticker.upper()
if args.s3_key:
s3_key = args.s3_key
if args.s3_bucket:
s3_bucket = args.s3_bucket
if args.start_date:
start_date = args.start_date
if args.algo_config:
algo_config = args.algo_config
if args.latest:
latest = True
start_date = ae_utils.get_last_close_str()
if args.debug:
debug = True
history_loc = (f's3://{s3_bucket}/{s3_key}')
log.info(f'building {ticker} trade history '
f'start_date={start_date} '
f'config={algo_config} '
f'history_loc={history_loc}')
runner = algo_runner.AlgoRunner(ticker=ticker,
start_date=start_date,
history_loc=history_loc,
algo_config=algo_config,
verbose_algo=debug,
verbose_processor=False,
verbose_indicators=False)
trading_history_df = None
if latest:
trading_history_df = runner.latest()
log.info(f'{ticker} latest:')
print(trading_history_df[['minute', 'close']].tail(5))
log.info(f'Other available columns to plot:')
print(trading_history_df.columns.values)
if show_plot:
plot.plot_trading_history(
title=(f'{ticker} at '
f'${trading_history_df["close"].iloc[-1]} '
f'at: '
f'{trading_history_df["minute"].iloc[-1]}'),
df=trading_history_df,
red='high',
blue='close')
else:
runner.start()
sys.exit(0)
def run_backtest_and_plot_history(config_dict):
"""run_backtest_and_plot_history
Run a derived algorithm with an algorithm config dictionary
:param config_dict: algorithm config dictionary
"""
log.debug('start - sa')
parser = argparse.ArgumentParser(description=('stock analysis tool'))
parser.add_argument('-t', help=('ticker'), required=True, dest='ticker')
parser.add_argument('-e',
help=('file path to extract an '
'algorithm-ready datasets from redis'),
required=False,
dest='algo_extract_loc')
parser.add_argument('-l',
help=('show dataset in this file'),
required=False,
dest='show_from_file')
parser.add_argument('-H',
help=('show trading history dataset in this file'),
required=False,
dest='show_history_from_file')
parser.add_argument(
'-E',
help=('show trading performance report dataset in this file'),
required=False,
dest='show_report_from_file')
parser.add_argument(
'-L',
help=('restore an algorithm-ready dataset file back into redis'),
required=False,
dest='restore_algo_file')
parser.add_argument('-f',
help=('save the trading history dataframe '
'to this file'),
required=False,
dest='history_json_file')
parser.add_argument(
'-J',
help=('plot action - after preparing you can use: '
'-J show to open the image (good for debugging)'),
required=False,
dest='plot_action')
parser.add_argument(
'-b',
help=('run a backtest using the dataset in '
'a file path/s3 key/redis key formats: '
'file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
's3://algoready/SPY-latest.json or '
'redis:SPY-latest'),
required=False,
dest='backtest_loc')
parser.add_argument('-B',
help=('optional - broker url for Celery'),
required=False,
dest='broker_url')
parser.add_argument('-C',
help=('optional - broker url for Celery'),
required=False,
dest='backend_url')
parser.add_argument(
'-w',
help=('optional - flag for publishing an algorithm job '
'using Celery to the ae workers'),
required=False,
dest='run_on_engine',
action='store_true')
parser.add_argument('-k',
help=('optional - s3 access key'),
required=False,
dest='s3_access_key')
parser.add_argument('-K',
help=('optional - s3 secret key'),
required=False,
dest='s3_secret_key')
parser.add_argument('-a',
help=('optional - s3 address format: <host:port>'),
required=False,
dest='s3_address')
parser.add_argument('-Z',
help=('optional - s3 secure: default False'),
required=False,
dest='s3_secure')
parser.add_argument('-s',
help=('optional - start date: YYYY-MM-DD'),
required=False,
dest='start_date')
parser.add_argument('-n',
help=('optional - end date: YYYY-MM-DD'),
required=False,
dest='end_date')
parser.add_argument('-u',
help=('optional - s3 bucket name'),
required=False,
dest='s3_bucket_name')
parser.add_argument('-G',
help=('optional - s3 region name'),
required=False,
dest='s3_region_name')
parser.add_argument(
'-g',
help=('Path to a custom algorithm module file '
'on disk. This module must have a single '
'class that inherits from: '
'https://github.com/AlgoTraders/stock-analysis-engine/'
'blob/master/'
'analysis_engine/algo.py Additionally you '
'can find the Example-Minute-Algorithm here: '
'https://github.com/AlgoTraders/stock-analysis-engine/'
'blob/master/analysis_engine/mocks/'
'example_algo_minute.py'),
required=False,
dest='run_algo_in_file')
parser.add_argument('-p',
help=('optional - s3 bucket/file for trading history'),
required=False,
dest='algo_history_loc')
parser.add_argument(
'-o',
help=('optional - s3 bucket/file for trading performance report'),
required=False,
dest='algo_report_loc')
parser.add_argument('-r',
help=('optional - redis_address format: <host:port>'),
required=False,
dest='redis_address')
parser.add_argument('-R',
help=('optional - redis and s3 key name'),
required=False,
dest='keyname')
parser.add_argument(
'-m',
help=('optional - redis database number (0 by default)'),
required=False,
dest='redis_db')
parser.add_argument('-x',
help=('optional - redis expiration in seconds'),
required=False,
dest='redis_expire')
parser.add_argument(
'-c',
help=('optional - algorithm config_file path for setting '
'up internal algorithm trading strategies and '
'indicators'),
required=False,
dest='config_file')
parser.add_argument('-v',
help=('set the Algorithm to verbose logging'),
required=False,
dest='verbose_algo',
action='store_true')
parser.add_argument(
'-P',
help=('set the Algorithm\'s IndicatorProcessor to verbose logging'),
required=False,
dest='verbose_processor',
action='store_true')
parser.add_argument(
'-I',
help=('set all Algorithm\'s Indicators to verbose logging '
'(note indivdual indicators support a \'verbose\' key '
'that can be set to True to debug just one '
'indicator)'),
required=False,
dest='verbose_indicators',
action='store_true')
parser.add_argument(
'-V',
help=('inspect the datasets an algorithm is processing - this'
'will slow down processing to show debugging'),
required=False,
dest='inspect_datasets',
action='store_true')
parser.add_argument(
'-j',
help=('run the algorithm on just this specific date in the datasets '
'- specify the date in a format: YYYY-MM-DD like: 2018-11-29'),
required=False,
dest='run_this_date')
parser.add_argument('-d',
help=('debug'),
required=False,
dest='debug',
action='store_true')
args = parser.parse_args()
ticker = ae_consts.TICKER
use_balance = 10000.0
use_commission = 6.0
use_start_date = None
use_end_date = None
use_config_file = None
debug = False
verbose_algo = None
verbose_processor = None
verbose_indicators = None
inspect_datasets = None
history_json_file = None
run_this_date = None
s3_access_key = ae_consts.S3_ACCESS_KEY
s3_secret_key = ae_consts.S3_SECRET_KEY
s3_region_name = ae_consts.S3_REGION_NAME
s3_address = ae_consts.S3_ADDRESS
s3_secure = ae_consts.S3_SECURE
redis_address = ae_consts.REDIS_ADDRESS
redis_password = ae_consts.REDIS_PASSWORD
redis_db = ae_consts.REDIS_DB
redis_expire = ae_consts.REDIS_EXPIRE
if args.s3_access_key:
s3_access_key = args.s3_access_key
if args.s3_secret_key:
s3_secret_key = args.s3_secret_key
if args.s3_region_name:
s3_region_name = args.s3_region_name
if args.s3_address:
s3_address = args.s3_address
if args.s3_secure:
s3_secure = args.s3_secure
if args.redis_address:
redis_address = args.redis_address
if args.redis_db:
redis_db = args.redis_db
if args.redis_expire:
redis_expire = args.redis_expire
if args.history_json_file:
history_json_file = args.history_json_file
if args.ticker:
ticker = args.ticker.upper()
if args.debug:
debug = True
if args.verbose_algo:
verbose_algo = True
if args.verbose_processor:
verbose_processor = True
if args.verbose_indicators:
verbose_indicators = True
if args.inspect_datasets:
inspect_datasets = True
if args.run_this_date:
run_this_date = args.run_this_date
if args.start_date:
try:
use_start_date = f'{str(args.start_date)} 00:00:00'
datetime.datetime.strptime(args.start_date,
ae_consts.COMMON_DATE_FORMAT)
except Exception as e:
msg = ('please use a start date formatted as: '
f'{ae_consts.COMMON_DATE_FORMAT}\nerror was: {e}')
log.error(msg)
sys.exit(1)
# end of testing for a valid date
# end of args.start_date
if args.end_date:
try:
use_end_date = f'{str(args.end_date)} 00:00:00'
datetime.datetime.strptime(args.end_date,
ae_consts.COMMON_DATE_FORMAT)
except Exception as e:
msg = ('please use an end date formatted as: '
f'{ae_consts.COMMON_DATE_FORMAT}\nerror was: {e}')
log.error(msg)
sys.exit(1)
# end of testing for a valid date
# end of args.end_date
if args.config_file:
use_config_file = args.config_file
if not os.path.exists(use_config_file):
log.error(
f'Failed: unable to find config file: -c {use_config_file}')
sys.exit(1)
if args.backtest_loc:
backtest_loc = args.backtest_loc
if ('file:/' not in backtest_loc and 's3://' not in backtest_loc
and 'redis://' not in backtest_loc):
log.error('invalid -b <backtest dataset file> specified. '
f'{backtest_loc} '
'please use either: '
'-b file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
'-b s3://algoready/SPY-latest.json or '
'-b redis://SPY-latest')
sys.exit(1)
load_from_s3_bucket = None
load_from_s3_key = None
load_from_redis_key = None
load_from_file = None
if 's3://' in backtest_loc:
load_from_s3_bucket = backtest_loc.split('/')[-2]
load_from_s3_key = backtest_loc.split('/')[-1]
elif 'redis://' in backtest_loc:
load_from_redis_key = backtest_loc.split('/')[-1]
elif 'file:/' in backtest_loc:
load_from_file = backtest_loc.split(':')[-1]
# end of parsing supported transport - loading an algo-ready
load_config = build_publish_request.build_publish_request(
ticker=ticker,
output_file=load_from_file,
s3_bucket=load_from_s3_bucket,
s3_key=load_from_s3_key,
redis_key=load_from_redis_key,
redis_address=redis_address,
redis_db=redis_db,
redis_password=redis_password,
redis_expire=redis_expire,
s3_address=s3_address,
s3_access_key=s3_access_key,
s3_secret_key=s3_secret_key,
s3_region_name=s3_region_name,
s3_secure=s3_secure,
verbose=debug,
label=f'load-{backtest_loc}')
if load_from_file:
load_config['output_file'] = load_from_file
if load_from_redis_key:
load_config['redis_key'] = load_from_redis_key
load_config['redis_enabled'] = True
if load_from_s3_bucket and load_from_s3_key:
load_config['s3_bucket'] = load_from_s3_bucket
load_config['s3_key'] = load_from_s3_key
load_config['s3_enabled'] = True
if debug:
log.info('starting algo')
config_dict['ticker'] = ticker
config_dict['balance'] = use_balance
config_dict['commission'] = use_commission
if verbose_algo:
config_dict['verbose'] = verbose_algo
if verbose_processor:
config_dict['verbose_processor'] = verbose_processor
if verbose_indicators:
config_dict['verbose_indicators'] = verbose_indicators
if inspect_datasets:
config_dict['inspect_datasets'] = inspect_datasets
if run_this_date:
config_dict['run_this_date'] = run_this_date
algo_obj = ExampleCustomAlgo(ticker=config_dict['ticker'],
config_dict=config_dict)
algo_res = run_algo.run_algo(ticker=ticker,
algo=algo_obj,
start_date=use_start_date,
end_date=use_end_date,
raise_on_err=True)
if algo_res['status'] != ae_consts.SUCCESS:
log.error('failed running algo backtest '
f'{algo_obj.get_name()} hit status: '
f'{ae_consts.get_status(status=algo_res["status"])} '
f'error: {algo_res["err"]}')
return
# if not successful
log.info(f'backtest: {algo_obj.get_name()} '
f'{ae_consts.get_status(status=algo_res["status"])}')
trading_history_dict = algo_obj.get_history_dataset()
history_df = trading_history_dict[ticker]
if not hasattr(history_df, 'to_json'):
return
if history_json_file:
log.info(f'saving history to: {history_json_file}')
history_df.to_json(history_json_file,
orient='records',
date_format='iso')
log.info('plotting history')
use_xcol = 'date'
use_as_date_format = '%d\n%b'
xlabel = f'Dates vs {trading_history_dict["algo_name"]} values'
ylabel = f'Algo {trading_history_dict["algo_name"]}\nvalues'
df_filter = (history_df['close'] > 0.01)
first_date = history_df[df_filter]['date'].iloc[0]
end_date = history_df[df_filter]['date'].iloc[-1]
if config_dict['timeseries'] == 'minute':
use_xcol = 'minute'
use_as_date_format = '%d %H:%M:%S\n%b'
first_date = history_df[df_filter]['minute'].iloc[0]
end_date = history_df[df_filter]['minute'].iloc[-1]
title = (f'Trading History {ticker} for Algo '
f'{trading_history_dict["algo_name"]}\n'
f'Backtest dates from {first_date} to {end_date}')
# set default hloc columns:
blue = None
green = None
orange = None
red = 'close'
blue = 'balance'
if debug:
for i, r in history_df.iterrows():
log.debug(f'{r["minute"]} - {r["close"]}')
plot_trading_history.plot_trading_history(title=title,
df=history_df,
red=red,
blue=blue,
green=green,
orange=orange,
date_col=use_xcol,
date_format=use_as_date_format,
xlabel=xlabel,
ylabel=ylabel,
df_filter=df_filter,
show_plot=True,
dropna_for_all=True)
def examine_dataset_in_file(
path_to_file,
compress=False,
encoding='utf-8',
ticker=None,
dataset_type=ae_consts.SA_DATASET_TYPE_ALGO_READY,
serialize_datasets=ae_consts.DEFAULT_SERIALIZED_DATASETS,
):
"""examine_dataset_in_file
Show the internal dataset dictionary structure in dataset file
:param path_to_file: path to file
:param compress: optional - boolean flag for decompressing
the contents of the ``path_to_file`` if necessary
(default is ``False`` and algorithms
use ``zlib`` for compression)
:param encoding: optional - string for data encoding
:param ticker: optional - string ticker symbol
to verify is in the dataset
:param dataset_type: optional - dataset type
(default is ``SA_DATASET_TYPE_ALGO_READY``)
:param serialize_datasets: optional - list of dataset names to
deserialize in the dataset
"""
if dataset_type == ae_consts.SA_DATASET_TYPE_ALGO_READY:
log.info(f'show start - load dataset from file={path_to_file}')
show_dataset.show_dataset(path_to_file=path_to_file,
compress=compress,
encoding=encoding,
dataset_type=dataset_type,
serialize_datasets=serialize_datasets)
log.info(f'show done - dataset in file={path_to_file}')
elif dataset_type == ae_consts.SA_DATASET_TYPE_TRADING_HISTORY:
log.info('load trading history dataset ' f'from file={path_to_file}')
trading_history_dict = load_history.load_history_dataset_from_file(
path_to_file=path_to_file, compress=compress, encoding=encoding)
history_df = trading_history_dict[ticker]
first_date = history_df['date'].iloc[0]
end_date = history_df['date'].iloc[-1]
title = (f'Trading History {ticker} for Algo '
f'{trading_history_dict["algo_name"]}\n'
f'Backtest dates from {first_date} to {end_date}')
xcol = 'date'
xlabel = f'Dates vs {trading_history_dict["algo_name"]} values'
ylabel = (
f'Algo Values from columns:\n{list(history_df.columns.values)}')
df_filter = (history_df['close'] > 0.01)
# set default hloc columns:
red = 'close'
blue = 'low'
green = 'high'
orange = 'open'
log.info('available columns to plot in dataset: '
f'{ae_consts.ppj(list(history_df.columns.values))}')
plot_trading_history.plot_trading_history(title=title,
df=history_df,
red=red,
blue=blue,
green=green,
orange=orange,
date_col=xcol,
xlabel=xlabel,
ylabel=ylabel,
df_filter=df_filter,
show_plot=True,
dropna_for_all=False)
elif dataset_type == ae_consts.SA_DATASET_TYPE_TRADING_REPORT:
log.info('load trading performance report dataset '
f'from file={path_to_file}')
trading_report_dict = load_report.load_report_dataset_from_file(
path_to_file=path_to_file, compress=compress, encoding=encoding)
print(trading_report_dict)
else:
log.error(f'show unsupported dataset type={dataset_type} for '
f'file={path_to_file}')
return
def create_column_dnn(predict_feature='close',
ticker='',
debug=False,
use_epochs=10,
use_batch_size=10,
use_test_size=0.1,
use_random_state=1,
use_seed=7,
use_shuffle=False,
model_verbose=True,
fit_verbose=True,
use_scalers=True,
df=[],
dnn_config={},
compile_config={},
s3_bucket='',
s3_key='',
send_plots_to_slack=False):
"""create_column_dnn
For scaler-normalized datasets this will
compile numeric columns and ignore string/non-numeric
columns as training and test feature columns
:param predict_feature: Column to create DNN with
:param ticker: Ticker being used
:param debug: Debug mode
:param use_epochs: Epochs times to use
:param use_batch_size: Batch size to use
:param use_test_size: Test size to use
:param use_random_state: Random state to train with
:param use_seed: Seed used to build scalar datasets
:param use_shuffle: To shuffle the regression estimator or not
:param model_verbose: To use a verbose Keras regression model or not
:param fit_verbose: To use a verbose fitting of the regression estimator
:param use_scalers: To build using scalars or not
:param df: Ticker dataset
:param dnn_config: Deep Neural Net keras model json to build the model
:param compile_config: Deep Neural Net dictionary of compile options
:param s3_bucket: S3 Bucket
:param s3_key: S3 Key
"""
df_filter = (df[f'{predict_feature}'] >= 0.1)
first_date = df[df_filter]['date'].iloc[0]
end_date = df[df_filter]['date'].iloc[-1]
if 'minute' in df:
found_valid_minute = df['minute'].iloc[0]
if found_valid_minute:
first_date = df[df_filter]['minute'].iloc[0]
end_date = df[df_filter]['minute'].iloc[-1]
num_rows = len(df.index)
log.info(f'prepared training data from '
f'history {s3_bucket}@{s3_key} '
f'rows={num_rows} '
f'dates: {first_date} to {end_date}')
if debug:
for i, r in df.iterrows():
log.info(f'{r["minute"]} - {r["{}".format(predict_feature)]}')
# end of for loop
log.info(f'columns: {df.columns.values}')
log.info(f'rows: {len(df.index)}')
# end of debug
use_all_features = use_scalers
all_features = []
train_features = []
if use_all_features:
for c in df.columns.values:
if (pandas_types.is_numeric_dtype(df[c])
and c not in train_features):
if c != predict_feature:
train_features.append(c)
if c not in all_features:
all_features.append(c)
dnn_config['layers'][-1]['activation'] = ('sigmoid')
else:
temp_choices = choices[:]
temp_choices.remove(predict_feature)
train_features = ['open']
train_features.extend(temp_choices)
all_features = [f'{predict_feature}'] + train_features
num_features = len(train_features)
features_and_minute = ['minute'] + all_features
log.info('converting columns to floats')
timeseries_df = df[df_filter][features_and_minute].fillna(-10000.0)
converted_df = timeseries_df[all_features].astype('float32')
train_df = None
test_df = None
scaler_predictions = None
if use_all_features:
scaler_res = build_scaler_datasets.build_datasets_using_scalers(
train_features=train_features,
test_feature=predict_feature,
df=converted_df,
test_size=use_test_size,
seed=use_seed)
if scaler_res['status'] != ae_consts.SUCCESS:
log.error('failed to build scaler train and test datasets')
return
train_df = scaler_res['scaled_train_df']
test_df = scaler_res['scaled_test_df']
x_train = scaler_res['x_train']
x_test = scaler_res['x_test']
y_train = scaler_res['y_train']
y_test = scaler_res['y_test']
scaler_predictions = scaler_res['scaler_test']
else:
log.info('building train and test dfs from subset of features')
train_df = converted_df[train_features]
test_df = converted_df[[predict_feature]]
log.info(f'splitting {num_rows} into test and training '
f'size={use_test_size}')
(x_train, x_test, y_train,
y_test) = tt_split.train_test_split(train_df,
test_df,
test_size=use_test_size,
random_state=use_random_state)
log.info(f'split breakdown - '
f'x_train={len(x_train)} '
f'x_test={len(x_test)} '
f'y_train={len(y_train)} '
f'y_test={len(y_test)}')
def set_model():
return build_dnn.build_regression_dnn(num_features=num_features,
compile_config=compile_config,
model_config=dnn_config)
estimator = keras_scikit.KerasRegressor(build_fn=set_model,
epochs=use_epochs,
batch_size=use_batch_size,
verbose=model_verbose)
log.info(f'fitting estimator - '
f'predicting={predict_feature} '
f'epochs={use_epochs} '
f'batch={use_batch_size} '
f'test_size={use_test_size} '
f'seed={use_seed}')
history = estimator.fit(x_train,
y_train,
validation_data=(x_train, y_train),
epochs=use_epochs,
batch_size=use_batch_size,
shuffle=use_shuffle,
verbose=fit_verbose)
created_on = (datetime.datetime.now().strftime(
ae_consts.COMMON_TICK_DATE_FORMAT))
plot_fit_history.plot_dnn_fit_history(
df=history.history,
title=(f'DNN Errors Over Training Epochs\n'
f'Training Data: s3://{s3_bucket}/{s3_key}\n'
f'Created: {created_on}'),
red='mean_squared_error',
blue='mean_absolute_error',
green='acc',
orange='cosine_proximity',
send_plots_to_slack=send_plots_to_slack)
# on production use newly fetched pricing data
# not the training data
predict_records = []
if use_all_features:
prediction_res = build_scaler_df.build_scaler_dataset_from_df(
df=converted_df[train_features])
if prediction_res['status'] == ae_consts.SUCCESS:
predict_records = prediction_res['df']
else:
predict_records = converted_df[train_features]
log.info(f'making predictions: {len(predict_records)}')
predictions = estimator.model.predict(predict_records, verbose=True)
np.set_printoptions(threshold=np.nan)
indexes = tf.argmax(predictions, axis=1)
data = {}
data['indexes'] = indexes
price_predictions = []
if use_all_features and scaler_predictions:
price_predictions = [
ae_consts.to_f(x) for x in scaler_predictions.inverse_transform(
predictions.reshape(-1, 1)).reshape(-1)
]
else:
price_predictions = [ae_consts.to_f(x[0]) for x in predictions]
timeseries_df[f'predicted_{predict_feature}'] = price_predictions
timeseries_df['error'] = (timeseries_df[f'{predict_feature}'] -
timeseries_df[f'predicted_{predict_feature}'])
output_features = [
'minute', f'{predict_feature}', f'predicted_{predict_feature}', 'error'
]
date_str = (f'Dates: {timeseries_df["minute"].iloc[0]} '
f'to '
f'{timeseries_df["minute"].iloc[-1]}')
log.info(f'historical {predict_feature} with predicted {predict_feature}: '
f'{timeseries_df[output_features]}')
log.info(date_str)
log.info(f'Columns: {output_features}')
average_error = ae_consts.to_f(timeseries_df['error'].sum() /
len(timeseries_df.index))
log.info(f'Average historical {predict_feature} '
f'vs predicted {predict_feature} error: '
f'{average_error}')
log.info(
f'plotting historical {predict_feature} vs predicted {predict_feature}'
f' from training with columns={num_features}')
ts_filter = (timeseries_df[f'{predict_feature}'] > 0.1)
latest_feature = (timeseries_df[ts_filter][f'{predict_feature}'].iloc[-1])
latest_predicted_feature = (
timeseries_df[ts_filter][f'predicted_{predict_feature}'].iloc[-1])
log.info(f'{end_date} {predict_feature}={latest_feature} '
f'with '
f'predicted_{predict_feature}={latest_predicted_feature}')
plot_trading_history.plot_trading_history(
title=(f'{ticker} - Historical {predict_feature.title()} vs '
f'Predicted {predict_feature.title()}\n'
f'Number of Training Features: {num_features}\n'
f'{date_str}'),
df=timeseries_df,
red=f'{predict_feature}',
blue=f'predicted_{predict_feature}',
green=None,
orange=None,
date_col='minute',
date_format='%d %H:%M:%S\n%b',
xlabel='minute',
ylabel=(f'Historical {predict_feature.title()} vs '
f'Predicted {predict_feature.title()}'),
df_filter=ts_filter,
width=8.0,
height=8.0,
show_plot=True,
dropna_for_all=False,
send_plots_to_slack=send_plots_to_slack)
def plot_local_history_file():
"""plot_local_history_file
Run a derived algorithm with an algorithm config dictionary
:param config_dict: algorithm config dictionary
"""
log.debug('start - plot')
parser = argparse.ArgumentParser(
description=('plot a local algorithm trading history file'))
parser.add_argument('-f',
help=('plot this trading history dataframe '
'saved in this file'),
required=False,
dest='history_json_file')
parser.add_argument('-d',
help=('debug'),
required=False,
dest='debug',
action='store_true')
args = parser.parse_args()
history_json_file = None
debug = False
if args.history_json_file:
history_json_file = args.history_json_file
if args.debug:
debug = True
if not history_json_file:
log.error('usage error - please run with: '
'-f <path to local trading history file>')
return
elif not os.path.exists(history_json_file):
log.error(
'did not find trading history file={}'.format(history_json_file))
return
# end of checking the file arg is set and exists on disk
log.info('plotting history to: {}'.format(history_json_file))
history_df = pd.read_json(history_json_file, orient='records')
history_df['date'] = pd.to_datetime(history_df['date'])
history_df['minute'] = pd.to_datetime(history_df['minute'])
ticker = history_df['ticker'].iloc[0]
log.info('plotting history')
first_date = history_df['date'].iloc[0]
end_date = history_df['date'].iloc[-1]
title = ('Trading History {}\n'
'Backtest dates from {} to {}'.format(ticker, first_date,
end_date))
use_xcol = 'date'
use_as_date_format = '%d\n%b'
use_minute = False
if 'minute' in history_df:
found_valid_minute = history_df['minute'].iloc[0]
if found_valid_minute:
use_minute = True
if use_minute:
use_xcol = 'minute'
use_as_date_format = '%d %H:%M:%S\n%b'
xlabel = 'Dates vs Algo values'
ylabel = 'Algo values'
df_filter = (history_df['close'] > 1.00)
# set default hloc columns:
blue = None
green = None
orange = None
red = 'close'
blue = 'balance'
if debug:
for i, r in history_df.iterrows():
log.info('{} - {}'.format(r['minute'], r['close']))
# end of debug
show_plot = True
if show_plot:
plot_trading_history.plot_trading_history(
title=title,
df=history_df,
red=red,
blue=blue,
green=green,
orange=orange,
date_col=use_xcol,
date_format=use_as_date_format,
xlabel=xlabel,
ylabel=ylabel,
df_filter=df_filter,
show_plot=True,
dropna_for_all=True)
def run_backtest_and_plot_history(config_dict):
"""run_backtest_and_plot_history
Run a derived algorithm with an algorithm config dictionary
:param config_dict: algorithm config dictionary
"""
log.debug('start - sa')
parser = argparse.ArgumentParser(description=('stock analysis tool'))
parser.add_argument('-t', help=('ticker'), required=False, dest='ticker')
parser.add_argument('-e',
help=('file path to extract an '
'algorithm-ready datasets from redis'),
required=False,
dest='algo_extract_loc')
parser.add_argument('-l',
help=('show dataset in this file'),
required=False,
dest='show_from_file')
parser.add_argument('-H',
help=('show trading history dataset in this file'),
required=False,
dest='show_history_from_file')
parser.add_argument(
'-E',
help=('show trading performance report dataset in this file'),
required=False,
dest='show_report_from_file')
parser.add_argument(
'-L',
help=('restore an algorithm-ready dataset file back into redis'),
required=False,
dest='restore_algo_file')
parser.add_argument('-f',
help=('save the trading history dataframe '
'to this file'),
required=False,
dest='history_json_file')
parser.add_argument(
'-J',
help=('plot action - after preparing you can use: '
'-J show to open the image (good for debugging)'),
required=False,
dest='plot_action')
parser.add_argument(
'-b',
help=('run a backtest using the dataset in '
'a file path/s3 key/redis key formats: '
'file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
's3://algoready/SPY-latest.json or '
'redis:SPY-latest'),
required=False,
dest='backtest_loc')
parser.add_argument('-B',
help=('optional - broker url for Celery'),
required=False,
dest='broker_url')
parser.add_argument('-C',
help=('optional - broker url for Celery'),
required=False,
dest='backend_url')
parser.add_argument(
'-w',
help=('optional - flag for publishing an algorithm job '
'using Celery to the analysis_engine workers'),
required=False,
dest='run_on_engine',
action='store_true')
parser.add_argument('-k',
help=('optional - s3 access key'),
required=False,
dest='s3_access_key')
parser.add_argument('-K',
help=('optional - s3 secret key'),
required=False,
dest='s3_secret_key')
parser.add_argument('-a',
help=('optional - s3 address format: <host:port>'),
required=False,
dest='s3_address')
parser.add_argument('-Z',
help=('optional - s3 secure: default False'),
required=False,
dest='s3_secure')
parser.add_argument('-s',
help=('optional - start date: YYYY-MM-DD'),
required=False,
dest='start_date')
parser.add_argument('-n',
help=('optional - end date: YYYY-MM-DD'),
required=False,
dest='end_date')
parser.add_argument('-u',
help=('optional - s3 bucket name'),
required=False,
dest='s3_bucket_name')
parser.add_argument('-G',
help=('optional - s3 region name'),
required=False,
dest='s3_region_name')
parser.add_argument('-g',
help=('Path to a custom algorithm module file '
'on disk. This module must have a single '
'class that inherits from: '
'https://github.com/AlgoTraders/stock-ana'
'lysis-engine/blob/master/'
'analysis_engine/algo.py Additionally you '
'can find the Example-Minute-Algorithm here: '
'https://github.com/AlgoTraders/stock-anal'
'ysis-engine/blob/master/analysis_engine/mocks/'
'example_algo_minute.py'),
required=False,
dest='run_algo_in_file')
parser.add_argument('-p',
help=('optional - s3 bucket/file for trading history'),
required=False,
dest='algo_history_loc')
parser.add_argument(
'-o',
help=('optional - s3 bucket/file for trading performance report'),
required=False,
dest='algo_report_loc')
parser.add_argument('-r',
help=('optional - redis_address format: <host:port>'),
required=False,
dest='redis_address')
parser.add_argument('-R',
help=('optional - redis and s3 key name'),
required=False,
dest='keyname')
parser.add_argument(
'-m',
help=('optional - redis database number (0 by default)'),
required=False,
dest='redis_db')
parser.add_argument('-x',
help=('optional - redis expiration in seconds'),
required=False,
dest='redis_expire')
parser.add_argument(
'-c',
help=('optional - algorithm config_file path for setting '
'up internal algorithm trading strategies and '
'indicators'),
required=False,
dest='config_file')
parser.add_argument('-v',
help=('set the Algorithm to verbose logging'),
required=False,
dest='verbose_algo',
action='store_true')
parser.add_argument(
'-P',
help=('set the Algorithm\'s IndicatorProcessor to verbose logging'),
required=False,
dest='verbose_processor',
action='store_true')
parser.add_argument(
'-I',
help=('set all Algorithm\'s Indicators to verbose logging '
'(note indivdual indicators support a \'verbose\' key '
'that can be set to True to debug just one '
'indicator)'),
required=False,
dest='verbose_indicators',
action='store_true')
parser.add_argument(
'-V',
help=('inspect the datasets an algorithm is processing - this'
'will slow down processing to show debugging'),
required=False,
dest='inspect_datasets',
action='store_true')
parser.add_argument(
'-j',
help=('run the algorithm on just this specific date in the datasets '
'- specify the date in a format: YYYY-MM-DD like: 2018-11-29'),
required=False,
dest='run_this_date')
parser.add_argument('-d',
help=('debug'),
required=False,
dest='debug',
action='store_true')
args = parser.parse_args()
ticker = None
use_balance = 10000.0
use_commission = 6.0
use_start_date = None
use_end_date = None
use_config_file = None
debug = False
verbose_algo = None
verbose_processor = None
verbose_indicators = None
inspect_datasets = None
history_json_file = None
run_this_date = None
algo_obj = None
algo_history_loc = 's3://algohistory'
algo_report_loc = 's3://algoreport'
algo_extract_loc = 's3://algoready'
backtest_loc = None
ssl_options = ae_consts.SSL_OPTIONS
transport_options = ae_consts.TRANSPORT_OPTIONS
broker_url = ae_consts.WORKER_BROKER_URL
backend_url = ae_consts.WORKER_BACKEND_URL
path_to_config_module = ae_consts.WORKER_CELERY_CONFIG_MODULE
include_tasks = ae_consts.INCLUDE_TASKS
load_from_s3_bucket = None
load_from_s3_key = None
load_from_redis_key = None
load_from_file = None
load_compress = False
load_publish = True
load_config = None
report_redis_key = None
report_s3_bucket = None
report_s3_key = None
report_file = None
report_compress = False
report_publish = True
report_config = None
history_redis_key = None
history_s3_bucket = None
history_s3_key = None
history_file = None
history_compress = False
history_publish = True
history_config = None
extract_redis_key = None
extract_s3_bucket = None
extract_s3_key = None
extract_file = None
extract_save_dir = None
extract_compress = False
extract_publish = True
extract_config = None
s3_enabled = True
s3_access_key = ae_consts.S3_ACCESS_KEY
s3_secret_key = ae_consts.S3_SECRET_KEY
s3_region_name = ae_consts.S3_REGION_NAME
s3_address = ae_consts.S3_ADDRESS
s3_bucket_name = ae_consts.S3_BUCKET
s3_key = None
s3_secure = ae_consts.S3_SECURE
redis_enabled = True
redis_address = ae_consts.REDIS_ADDRESS
redis_key = None
redis_password = ae_consts.REDIS_PASSWORD
redis_db = ae_consts.REDIS_DB
redis_expire = ae_consts.REDIS_EXPIRE
redis_serializer = 'json'
redis_encoding = 'utf-8'
publish_to_s3 = True
publish_to_redis = True
publish_to_slack = True
slack_enabled = False
slack_code_block = False
slack_full_width = False
dataset_type = ae_consts.SA_DATASET_TYPE_ALGO_READY
serialize_datasets = ae_consts.DEFAULT_SERIALIZED_DATASETS
compress = False
encoding = 'utf-8'
debug = False
run_on_engine = False
auto_fill = True
timeseries = 'minute'
trade_strategy = 'count'
if args.s3_access_key:
s3_access_key = args.s3_access_key
if args.s3_secret_key:
s3_secret_key = args.s3_secret_key
if args.s3_region_name:
s3_region_name = args.s3_region_name
if args.s3_address:
s3_address = args.s3_address
if args.s3_secure:
s3_secure = args.s3_secure
if args.redis_address:
redis_address = args.redis_address
if args.redis_db:
redis_db = args.redis_db
if args.redis_expire:
redis_expire = args.redis_expire
if args.history_json_file:
history_json_file = args.history_json_file
if args.ticker:
ticker = args.ticker.upper()
if args.debug:
debug = True
if args.verbose_algo:
verbose_algo = True
if args.verbose_processor:
verbose_processor = True
if args.verbose_indicators:
verbose_indicators = True
if args.inspect_datasets:
inspect_datasets = True
if args.run_this_date:
run_this_date = args.run_this_date
if args.start_date:
try:
use_start_date = '{} 00:00:00'.format(str(args.start_date))
datetime.datetime.strptime(args.start_date,
ae_consts.COMMON_DATE_FORMAT)
except Exception as e:
msg = ('please use a start date formatted as: {}'
'\n'
'error was: {}'.format(ae_consts.COMMON_DATE_FORMAT, e))
log.error(msg)
sys.exit(1)
# end of testing for a valid date
# end of args.start_date
if args.end_date:
try:
use_end_date = '{} 00:00:00'.format(str(args.end_date))
datetime.datetime.strptime(args.end_date,
ae_consts.COMMON_DATE_FORMAT)
except Exception as e:
msg = ('please use an end date formatted as: {}'
'\n'
'error was: {}'.format(ae_consts.COMMON_DATE_FORMAT, e))
log.error(msg)
sys.exit(1)
# end of testing for a valid date
# end of args.end_date
algo_mod_path = None
if args.run_algo_in_file:
if not os.path.exists(args.run_algo_in_file):
log.error('missing algorithm module file: {}'.format(
args.run_algo_in_file))
sys.exit(1)
algo_mod_path = args.run_algo_in_file
if args.config_file:
use_config_file = args.config_file
if not os.path.exists(use_config_file):
log.error('Failed: unable to find config file: -c {}'.format(
use_config_file))
sys.exit(1)
config_dict = json.loads(open(use_config_file).read())
algo_mod_path = config_dict.get('algo_path', algo_mod_path)
if not os.path.exists(algo_mod_path):
log.error('missing algorithm module file from config: {}'.format(
algo_mod_path))
sys.exit(1)
"""
Finalize the algo config
"""
if config_dict:
use_balance = float(config_dict.get('balance', use_balance))
use_commission = float(config_dict.get('commission', use_commission))
ticker = str(config_dict.get('ticker', ticker)).upper()
config_dict['ticker'] = ticker
config_dict['balance'] = use_balance
config_dict['commission'] = use_commission
else:
if not ticker:
ticker = str(config_dict.get('ticker', ae_consts.TICKER)).upper()
if not ticker:
log.error('usage error: please set a ticker with -t <TICKER>')
sys.exit(1)
if verbose_algo:
config_dict['verbose'] = verbose_algo
if verbose_processor:
config_dict['verbose_processor'] = verbose_processor
if verbose_indicators:
config_dict['verbose_indicators'] = verbose_indicators
if inspect_datasets:
config_dict['inspect_datasets'] = inspect_datasets
if run_this_date:
config_dict['run_this_date'] = run_this_date
"""
Run a custom algo module from disk
"""
if algo_mod_path:
if args.backtest_loc:
backtest_loc = args.backtest_loc
if ('file:/' not in backtest_loc and 's3://' not in backtest_loc
and 'redis://' not in backtest_loc):
log.error(
'invalid -b <backtest dataset file> specified. '
'{} '
'please use either: '
'-b file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
'-b s3://algoready/SPY-latest.json or '
'-b redis://SPY-latest'.format(backtest_loc))
sys.exit(1)
if 's3://' in backtest_loc:
load_from_s3_bucket = backtest_loc.split('/')[-2]
load_from_s3_key = backtest_loc.split('/')[-1]
elif 'redis://' in backtest_loc:
load_from_redis_key = backtest_loc.split('/')[-1]
elif 'file:/' in backtest_loc:
load_from_file = backtest_loc.split(':')[-1]
load_publish = True
# end of parsing supported transport - loading an algo-ready
if args.algo_history_loc:
algo_history_loc = args.algo_history_loc
if ('file:/' not in algo_history_loc
and 's3://' not in algo_history_loc
and 'redis://' not in algo_history_loc):
log.error(
'invalid -b <backtest dataset file> specified. '
'{} '
'please use either: '
'-p file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
'-p s3://algoready/SPY-latest.json or '
'-p redis://SPY-latest'.format(algo_history_loc))
sys.exit(1)
if 's3://' in algo_history_loc:
history_s3_bucket = algo_history_loc.split('/')[-2]
history_s3_key = algo_history_loc.split('/')[-1]
elif 'redis://' in algo_history_loc:
history_redis_key = algo_history_loc.split('/')[-1]
elif 'file:/' in algo_history_loc:
history_file = algo_history_loc.split(':')[-1]
history_publish = True
# end of parsing supported transport - trading history
if args.algo_report_loc:
algo_report_loc = args.algo_report_loc
if ('file:/' not in algo_report_loc
and 's3://' not in algo_report_loc
and 'redis://' not in algo_report_loc):
log.error(
'invalid -b <backtest dataset file> specified. '
'{} '
'please use either: '
'-o file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
'-o s3://algoready/SPY-latest.json or '
'-o redis://SPY-latest'.format(algo_report_loc))
sys.exit(1)
if 's3://' in algo_report_loc:
report_s3_bucket = algo_report_loc.split('/')[-2]
report_s3_key = algo_report_loc.split('/')[-1]
elif 'redis://' in algo_report_loc:
report_redis_key = algo_report_loc.split('/')[-1]
elif 'file:/' in algo_report_loc:
report_file = algo_report_loc.split(':')[-1]
report_publish = True
# end of parsing supported transport - trading performance report
if args.algo_extract_loc:
algo_extract_loc = args.algo_extract_loc
if ('file:/' not in algo_extract_loc
and 's3://' not in algo_extract_loc
and 'redis://' not in algo_extract_loc):
log.error(
'invalid -b <backtest dataset file> specified. '
'{} '
'please use either: '
'-e file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
'-e s3://algoready/SPY-latest.json or '
'-e redis://SPY-latest'.format(algo_extract_loc))
sys.exit(1)
if 's3://' in algo_extract_loc:
extract_s3_bucket = algo_extract_loc.split('/')[-2]
extract_s3_key = algo_extract_loc.split('/')[-1]
elif 'redis://' in algo_extract_loc:
extract_redis_key = algo_extract_loc.split('/')[-1]
elif 'file:/' in algo_extract_loc:
extract_file = algo_extract_loc.split(':')[-1]
extract_publish = True
# end of parsing supported transport - extract algorithm-ready
if args.run_on_engine:
run_on_engine = True
if verbose_algo:
log.info('starting algo on the engine')
use_name = config_dict.get('name', 'missing-algo-name')
auto_fill = config_dict.get('auto_fill', auto_fill)
timeseries = config_dict.get('timeseries', timeseries)
trade_strategy = config_dict.get('trade_strategy', trade_strategy)
algo_res = run_custom_algo.run_custom_algo(
mod_path=algo_mod_path,
ticker=config_dict['ticker'],
balance=config_dict['balance'],
commission=config_dict['commission'],
name=use_name,
start_date=use_start_date,
end_date=use_end_date,
auto_fill=auto_fill,
config_dict=config_dict,
load_from_s3_bucket=load_from_s3_bucket,
load_from_s3_key=load_from_s3_key,
load_from_redis_key=load_from_redis_key,
load_from_file=load_from_file,
load_compress=load_compress,
load_publish=load_publish,
load_config=load_config,
report_redis_key=report_redis_key,
report_s3_bucket=report_s3_bucket,
report_s3_key=report_s3_key,
report_file=report_file,
report_compress=report_compress,
report_publish=report_publish,
report_config=report_config,
history_redis_key=history_redis_key,
history_s3_bucket=history_s3_bucket,
history_s3_key=history_s3_key,
history_file=history_file,
history_compress=history_compress,
history_publish=history_publish,
history_config=history_config,
extract_redis_key=extract_redis_key,
extract_s3_bucket=extract_s3_bucket,
extract_s3_key=extract_s3_key,
extract_file=extract_file,
extract_save_dir=extract_save_dir,
extract_compress=extract_compress,
extract_publish=extract_publish,
extract_config=extract_config,
publish_to_slack=publish_to_slack,
publish_to_s3=publish_to_s3,
publish_to_redis=publish_to_redis,
dataset_type=dataset_type,
serialize_datasets=serialize_datasets,
compress=compress,
encoding=encoding,
redis_enabled=redis_enabled,
redis_key=redis_key,
redis_address=redis_address,
redis_db=redis_db,
redis_password=redis_password,
redis_expire=redis_expire,
redis_serializer=redis_serializer,
redis_encoding=redis_encoding,
s3_enabled=s3_enabled,
s3_key=s3_key,
s3_address=s3_address,
s3_bucket=s3_bucket_name,
s3_access_key=s3_access_key,
s3_secret_key=s3_secret_key,
s3_region_name=s3_region_name,
s3_secure=s3_secure,
slack_enabled=slack_enabled,
slack_code_block=slack_code_block,
slack_full_width=slack_full_width,
dataset_publish_extract=extract_publish,
dataset_publish_history=history_publish,
dataset_publish_report=report_publish,
run_on_engine=run_on_engine,
auth_url=broker_url,
backend_url=backend_url,
include_tasks=include_tasks,
ssl_options=ssl_options,
transport_options=transport_options,
path_to_config_module=path_to_config_module,
timeseries=timeseries,
trade_strategy=trade_strategy,
verbose=verbose_algo)
show_label = 'algo.name={}'.format(use_name)
show_extract = '{}'.format(algo_extract_loc)
show_history = '{}'.format(algo_history_loc)
show_report = '{}'.format(algo_report_loc)
base_label = ('load={} '
'extract={} '
'history={} '
'report={}'.format(args.run_algo_in_file, show_extract,
show_history, show_report))
algo_obj = algo_res.get('algo', None)
if not algo_obj:
log.error(
'{} - failed creating algorithm object'.format(show_label))
sys.exit(1)
if not run_on_engine:
algo_trade_history_recs = algo_res['rec'].get('history', [])
show_label = ('{} algo.name={} {} trade_history_len={}'.format(
ticker, use_name, base_label, len(algo_trade_history_recs)))
if args.debug:
log.info('algo_res={}'.format(algo_res))
if algo_res['status'] == ae_consts.SUCCESS:
log.info('{} - done running {}'.format(
ae_consts.get_status(status=algo_res['status']),
show_label))
else:
log.error('{} - done running {}'.format(
ae_consts.get_status(status=algo_res['status']),
show_label))
else:
if algo_res['status'] == ae_consts.SUCCESS:
log.info('{} - done running {}'.format(
ae_consts.get_status(status=algo_res['status']),
show_label))
else:
log.error('run_custom_algo returned error: {}'.format(
algo_res['err']))
sys.exit(1)
# end of running the custom algo handler
# end if running a custom algorithm module
else:
if args.backtest_loc:
backtest_loc = args.backtest_loc
if ('file:/' not in backtest_loc and 's3://' not in backtest_loc
and 'redis://' not in backtest_loc):
log.error(
'invalid -b <backtest dataset file> specified. '
'{} '
'please use either: '
'-b file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
'-b s3://algoready/SPY-latest.json or '
'-b redis://SPY-latest'.format(backtest_loc))
sys.exit(1)
load_from_s3_bucket = None
load_from_s3_key = None
load_from_redis_key = None
load_from_file = None
if 's3://' in backtest_loc:
load_from_s3_bucket = backtest_loc.split('/')[-2]
load_from_s3_key = backtest_loc.split('/')[-1]
elif 'redis://' in backtest_loc:
load_from_redis_key = backtest_loc.split('/')[-1]
elif 'file:/' in backtest_loc:
load_from_file = backtest_loc.split(':')[-1]
# end of parsing supported transport - loading an algo-ready
# end of backtest_loc
load_config = build_publish_request.build_publish_request(
ticker=ticker,
output_file=load_from_file,
s3_bucket=load_from_s3_bucket,
s3_key=load_from_s3_key,
redis_key=load_from_redis_key,
redis_address=redis_address,
redis_db=redis_db,
redis_password=redis_password,
redis_expire=redis_expire,
s3_address=s3_address,
s3_access_key=s3_access_key,
s3_secret_key=s3_secret_key,
s3_region_name=s3_region_name,
s3_secure=s3_secure,
verbose=debug,
label='load-{}'.format(backtest_loc))
if load_from_file:
load_config['output_file'] = load_from_file
if load_from_redis_key:
load_config['redis_key'] = load_from_redis_key
load_config['redis_enabled'] = True
if load_from_s3_bucket and load_from_s3_key:
load_config['s3_bucket'] = load_from_s3_bucket
load_config['s3_key'] = load_from_s3_key
load_config['s3_enabled'] = True
log.info('starting algo')
algo_obj = ExampleCustomAlgo(ticker=config_dict['ticker'],
config_dict=config_dict)
algo_res = run_algo.run_algo(ticker=ticker,
algo=algo_obj,
start_date=use_start_date,
end_date=use_end_date,
raise_on_err=True)
if algo_res['status'] != ae_consts.SUCCESS:
log.error('failed running algo backtest '
'{} hit status: {} error: {}'.format(
algo_obj.get_name(),
ae_consts.get_status(status=algo_res['status']),
algo_res['err']))
return
# if not successful
log.info('backtest: {} {}'.format(
algo_obj.get_name(),
ae_consts.get_status(status=algo_res['status'])))
# end of use custom algo or not
if algo_obj:
trading_history_dict = algo_obj.get_history_dataset()
history_df = trading_history_dict[ticker]
if not hasattr(history_df, 'to_json'):
return
if history_json_file:
log.info('saving history to: {}'.format(history_json_file))
history_df.to_json(history_json_file,
orient='records',
date_format='iso')
log.info('plotting history')
first_date = history_df['date'].iloc[0]
end_date = history_df['date'].iloc[-1]
title = ('Trading History {} for Algo {}\n'
'Backtest dates from {} to {}'.format(
ticker, trading_history_dict['algo_name'], first_date,
end_date))
use_xcol = 'date'
use_as_date_format = '%d\n%b'
if config_dict['timeseries'] == 'minute':
use_xcol = 'minute'
use_as_date_format = '%d %H:%M:%S\n%b'
xlabel = 'Dates vs {} values'.format(trading_history_dict['algo_name'])
ylabel = 'Algo {}\nvalues'.format(trading_history_dict['algo_name'])
df_filter = (history_df['close'] > 0.01)
# set default hloc columns:
blue = None
green = None
orange = None
red = 'close'
blue = 'balance'
if debug:
for i, r in history_df.iterrows():
log.debug('{} - {}'.format(r['minute'], r['close']))
plot_trading_history.plot_trading_history(
title=title,
df=history_df,
red=red,
blue=blue,
green=green,
orange=orange,
date_col=use_xcol,
date_format=use_as_date_format,
xlabel=xlabel,
ylabel=ylabel,
df_filter=df_filter,
show_plot=True,
dropna_for_all=True)
