def train_and_predict_from_history_in_s3(): """train_and_predict_from_history_in_s3 Run a derived algorithm with an algorithm config dictionary :param config_dict: algorithm config dictionary """ log.debug('start - plot') parser = argparse.ArgumentParser( description=('train a dnn to predict a column from a' 'a trading history file in s3')) parser.add_argument('-b', help=('s3 bucket'), required=False, dest='s3_bucket') parser.add_argument('-k', help=('s3 key'), required=False, dest='s3_key') parser.add_argument('-q', help=('disable scaler normalization and ' 'only use 2 of close, high, or low with open to ' 'predict the remainder of close, high, or low'), required=False, dest='disable_scaler', action='store_true') parser.add_argument('-c', help=('column(s) to predict'), required=False, dest='predict_features', choices=choices, nargs='*') parser.add_argument('-n', help=('number of DNNs to create'), required=False, dest='number_of_dnns', default=1, type=int) parser.add_argument('-s', help=('send plots to slack'), required=False, dest='send_plots_to_slack', action='store_true') parser.add_argument('-d', help=('debug'), required=False, dest='debug', action='store_true') args = parser.parse_args() use_scalers = True s3_access_key = ae_consts.S3_ACCESS_KEY s3_secret_key = ae_consts.S3_SECRET_KEY s3_region_name = ae_consts.S3_REGION_NAME s3_address = ae_consts.S3_ADDRESS s3_secure = ae_consts.S3_SECURE compress = True s3_bucket = ('algohistory') s3_key = ('algo_training_SPY.json') predict_features = ['close'] number_of_dnns = 1 send_plots_to_slack = False debug = False if args.s3_bucket: s3_bucket = args.s3_bucket if args.s3_key: s3_key = args.s3_key if args.disable_scaler: use_scalers = False if args.debug: debug = True if args.predict_features and len(args.predict_features): predict_features = set(args.predict_features) # remove any duplicates if args.number_of_dnns != number_of_dnns: number_of_dnns = args.number_of_dnns if args.send_plots_to_slack: send_plots_to_slack = args.send_plots_to_slack load_res = load_history.load_history_dataset(s3_enabled=True, s3_key=s3_key, s3_address=s3_address, s3_bucket=s3_bucket, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, compress=compress) algo_config = load_res.get('algo_config_dict', None) algo_name = load_res.get('algo_name', None) tickers = load_res.get('tickers', [ 'SPY', ]) ticker = tickers[0] log.info(f'found algo: {algo_name}') if debug: log.info(f'config: {ae_consts.ppj(algo_config)}') df = load_res[ticker] df['date'] = pd.to_datetime(df['date']) df['minute'] = pd.to_datetime(df['minute']) ticker = df['ticker'].iloc[0] dnn_config = { 'layers': [{ 'num_neurons': 150, 'init': 'uniform', 'activation': 'relu' }, { 'num_neurons': 100, 'init': 'uniform', 'activation': 'relu' }, { 'num_neurons': 50, 'init': 'uniform', 'activation': 'relu' }, { 'num_neurons': 1, 'init': 'uniform', 'activation': 'relu' }] } compile_config = { 'loss': 'mse', 'optimizer': 'adam', 'metrics': ['accuracy', 'mse', 'mae', 'mape', 'cosine'] } for predict_feature in predict_features: for index in range(number_of_dnns): log.info(f'Creating DNN-{index+1} for column: {predict_feature}') create_column_dnn(predict_feature=predict_feature, ticker=ticker, debug=debug, use_scalers=use_scalers, df=deepcopy(df), dnn_config=deepcopy(dnn_config), compile_config=compile_config, s3_bucket=s3_bucket, s3_key=s3_key, send_plots_to_slack=send_plots_to_slack)
def plot_history_from_s3(): """plot_history_from_s3 Run a derived algorithm with an algorithm config dictionary :param config_dict: algorithm config dictionary """ log.debug('start - plot') parser = argparse.ArgumentParser( description=('plot a local algorithm trading history file')) parser.add_argument('-b', help=('saved in this s3 bucket'), required=False, dest='s3_bucket') parser.add_argument('-k', help=('saved in this s3 key'), required=False, dest='history_json_file') parser.add_argument('-d', help=('debug'), required=False, dest='debug', action='store_true') args = parser.parse_args() s3_access_key = consts.S3_ACCESS_KEY s3_secret_key = consts.S3_SECRET_KEY s3_region_name = consts.S3_REGION_NAME s3_address = consts.S3_ADDRESS s3_secure = consts.S3_SECURE compress = True s3_bucket = ('bt-spy-williamsr-2018-12-05-22-44-50-714400') s3_key = ('-181.55_netgain_9818.45_netvalue_NEGATIVE_' '10000.0_startbalance_1710.95_endbalance_' '30_shares_270.25_close_3_buys_0_sells_' '1_minbuyinds_1_minsellinds_' '43.52_seconds_' 'trade_history-SPY_williamsr_test_' '0.73_for_176_of_24000.json') debug = False if args.debug: debug = True load_res = load_history.load_history_dataset(s3_enabled=True, s3_key=s3_key, s3_address=s3_address, s3_bucket=s3_bucket, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, compress=compress) algo_config = load_res.get('algo_config_dict', None) algo_name = load_res.get('algo_name', None) tickers = load_res.get('tickers', [ 'SPY', ]) ticker = tickers[0] log.info(f'found algo: {algo_name}') log.info(f'config: {consts.ppj(algo_config)}') history_df = load_res[ticker] history_df['date'] = pd.to_datetime(history_df['date']) history_df['minute'] = pd.to_datetime(history_df['minute']) ticker = history_df['ticker'].iloc[0] log.info('plotting history') first_date = history_df['date'].iloc[0] end_date = history_df['date'].iloc[-1] title = (f'Trading History {ticker}\n' f'Backtest dates from {first_date} to {end_date}') use_xcol = 'date' use_as_date_format = '%d\n%b' use_minute = False if 'minute' in history_df: found_valid_minute = history_df['minute'].iloc[0] if found_valid_minute: use_minute = True if use_minute: use_xcol = 'minute' use_as_date_format = '%d %H:%M:%S\n%b' xlabel = 'Dates vs Algo values' ylabel = 'Algo values' df_filter = (history_df['close'] > 1.00) # set default hloc columns: blue = None green = None orange = None red = 'close' blue = 'balance' if debug: for i, r in history_df.iterrows(): log.info(f'{r["minute"]} - {r["close"]}') # end of debug show_plot = True if show_plot: plot_trading_history.plot_trading_history( title=title, df=history_df, red=red, blue=blue, green=green, orange=orange, date_col=use_xcol, date_format=use_as_date_format, xlabel=xlabel, ylabel=ylabel, df_filter=df_filter, show_plot=True, dropna_for_all=True)
def start_algo(): """start_algo Run a derived algorithm with an algorithm config dictionary :param config_dict: algorithm config dictionary """ log.debug('start - ae') parser = argparse.ArgumentParser(description=('stock analysis tool')) parser.add_argument('-t', help=('ticker'), required=False, dest='ticker') parser.add_argument('-e', help=('file path to extract an ' 'algorithm-ready datasets from redis'), required=False, dest='algo_extract_loc') parser.add_argument('-l', help=('show dataset in this file'), required=False, dest='show_from_file') parser.add_argument('-H', help=('show trading history dataset in this file'), required=False, dest='show_history_from_file') parser.add_argument( '-E', help=('show trading performance report dataset in this file'), required=False, dest='show_report_from_file') parser.add_argument( '-L', help=('restore an algorithm-ready dataset file back into redis'), required=False, dest='restore_algo_file') parser.add_argument('-f', help=('save the trading history dataframe ' 'to this file'), required=False, dest='history_json_file') parser.add_argument( '-J', help=('plot action - after preparing you can use: ' '-J show to open the image (good for debugging)'), required=False, dest='plot_action') parser.add_argument( '-b', help=('run a backtest using the dataset in ' 'a file path/s3 key/redis key formats: ' 'file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' 's3://algoready/SPY-latest.json or ' 'redis:SPY-latest'), required=False, dest='backtest_loc') parser.add_argument('-B', help=('optional - broker url for Celery'), required=False, dest='broker_url') parser.add_argument('-C', help=('optional - broker url for Celery'), required=False, dest='backend_url') parser.add_argument( '-w', help=('optional - flag for publishing an algorithm job ' 'using Celery to the ae workers'), required=False, dest='run_on_engine', action='store_true') parser.add_argument('-k', help=('optional - s3 access key'), required=False, dest='s3_access_key') parser.add_argument('-K', help=('optional - s3 secret key'), required=False, dest='s3_secret_key') parser.add_argument('-a', help=('optional - s3 address format: <host:port>'), required=False, dest='s3_address') parser.add_argument('-Z', help=('optional - s3 secure: default False'), required=False, dest='s3_secure') parser.add_argument('-s', help=('optional - start date: YYYY-MM-DD'), required=False, dest='start_date') parser.add_argument('-n', help=('optional - end date: YYYY-MM-DD'), required=False, dest='end_date') parser.add_argument('-u', help=('optional - s3 bucket name'), required=False, dest='s3_bucket_name') parser.add_argument('-G', help=('optional - s3 region name'), required=False, dest='s3_region_name') parser.add_argument( '-g', help=('Path to a custom algorithm module file ' 'on disk. This module must have a single ' 'class that inherits from: ' 'https://github.com/AlgoTraders/stock-analysis-engine/' 'blob/master/' 'analysis_engine/algo.py Additionally you ' 'can find the Example-Minute-Algorithm here: ' 'https://github.com/AlgoTraders/stock-analysis-engine/' 'blob/master/analysis_engine/mocks/' 'example_algo_minute.py'), required=False, dest='run_algo_in_file') parser.add_argument('-p', help=('optional - s3 bucket/file for trading history'), required=False, dest='algo_history_loc') parser.add_argument( '-o', help=('optional - s3 bucket/file for trading performance report'), required=False, dest='algo_report_loc') parser.add_argument('-r', help=('optional - redis_address format: <host:port>'), required=False, dest='redis_address') parser.add_argument('-R', help=('optional - redis and s3 key name'), required=False, dest='keyname') parser.add_argument( '-m', help=('optional - redis database number (0 by default)'), required=False, dest='redis_db') parser.add_argument('-x', help=('optional - redis expiration in seconds'), required=False, dest='redis_expire') parser.add_argument( '-c', help=('optional - algorithm config_file path for setting ' 'up internal algorithm trading strategies and ' 'indicators'), required=False, dest='config_file') parser.add_argument('-v', help=('set the Algorithm to verbose logging'), required=False, dest='verbose_algo', action='store_true') parser.add_argument( '-P', help=('set the Algorithm\'s IndicatorProcessor to verbose logging'), required=False, dest='verbose_processor', action='store_true') parser.add_argument( '-I', help=('set all Algorithm\'s Indicators to verbose logging ' '(note indivdual indicators support a \'verbose\' key ' 'that can be set to True to debug just one ' 'indicator)'), required=False, dest='verbose_indicators', action='store_true') parser.add_argument( '-V', help=('inspect the datasets an algorithm is processing - this' 'will slow down processing to show debugging'), required=False, dest='inspect_datasets', action='store_true') parser.add_argument( '-j', help=('run the algorithm on just this specific date in the datasets ' '- specify the date in a format: YYYY-MM-DD like: 2018-11-29'), required=False, dest='run_this_date') parser.add_argument('-d', help=('debug'), required=False, dest='debug', action='store_true') args = parser.parse_args() ticker = None use_config_file = '/opt/sa/cfg/default_algo.json' algo_mod_path = '/opt/sa/analysis_engine/algo.py' use_balance = 10000.0 use_commission = 6.0 use_start_date = None use_end_date = None debug = False verbose_algo = None verbose_processor = None verbose_indicators = None inspect_datasets = None history_json_file = None run_this_date = None algo_obj = None algo_history_loc = None algo_report_loc = None algo_extract_loc = None backtest_loc = None ssl_options = ae_consts.SSL_OPTIONS transport_options = ae_consts.TRANSPORT_OPTIONS broker_url = ae_consts.WORKER_BROKER_URL backend_url = ae_consts.WORKER_BACKEND_URL path_to_config_module = ae_consts.WORKER_CELERY_CONFIG_MODULE include_tasks = ae_consts.INCLUDE_TASKS load_from_s3_bucket = None load_from_s3_key = None load_from_redis_key = None load_from_file = None load_compress = True load_publish = True load_config = None report_redis_key = None report_s3_bucket = None report_s3_key = None report_file = None report_compress = True report_publish = False report_config = None history_redis_key = None history_s3_bucket = None history_s3_key = None history_file = None history_compress = True history_publish = True history_config = None extract_redis_key = None extract_s3_bucket = None extract_s3_key = None extract_file = None extract_save_dir = None extract_compress = False extract_publish = False extract_config = None s3_enabled = True s3_access_key = ae_consts.S3_ACCESS_KEY s3_secret_key = ae_consts.S3_SECRET_KEY s3_region_name = ae_consts.S3_REGION_NAME s3_address = ae_consts.S3_ADDRESS s3_bucket_name = ae_consts.S3_BUCKET s3_key = None s3_secure = ae_consts.S3_SECURE redis_enabled = True redis_address = ae_consts.REDIS_ADDRESS redis_key = None redis_password = ae_consts.REDIS_PASSWORD redis_db = ae_consts.REDIS_DB redis_expire = ae_consts.REDIS_EXPIRE redis_serializer = 'json' redis_encoding = 'utf-8' publish_to_s3 = True publish_to_redis = False publish_to_slack = False slack_enabled = False slack_code_block = False slack_full_width = False dataset_type = ae_consts.SA_DATASET_TYPE_ALGO_READY serialize_datasets = ae_consts.DEFAULT_SERIALIZED_DATASETS compress = False encoding = 'utf-8' debug = False run_on_engine = False auto_fill = True timeseries = 'minute' trade_strategy = 'count' if args.s3_access_key: s3_access_key = args.s3_access_key if args.s3_secret_key: s3_secret_key = args.s3_secret_key if args.s3_region_name: s3_region_name = args.s3_region_name if args.s3_address: s3_address = args.s3_address if args.s3_secure: s3_secure = args.s3_secure if args.redis_address: redis_address = args.redis_address if args.redis_db: redis_db = args.redis_db if args.redis_expire: redis_expire = args.redis_expire if args.history_json_file: history_json_file = args.history_json_file if args.ticker: ticker = args.ticker.upper() if args.debug: debug = True if args.verbose_algo: verbose_algo = True if args.verbose_processor: verbose_processor = True if args.verbose_indicators: verbose_indicators = True if args.inspect_datasets: inspect_datasets = True if args.run_this_date: run_this_date = args.run_this_date if args.start_date: try: use_start_date = f'{str(args.start_date)} 00:00:00' datetime.datetime.strptime(args.start_date, ae_consts.COMMON_DATE_FORMAT) except Exception as e: msg = ('please use a start date formatted as: ' f'{ae_consts.COMMON_DATE_FORMAT}\n' f'error was: {e}') log.error(msg) sys.exit(1) # end of testing for a valid date # end of args.start_date if args.end_date: try: use_end_date = f'{str(args.end_date)} 00:00:00' datetime.datetime.strptime(args.end_date, ae_consts.COMMON_DATE_FORMAT) except Exception as e: msg = ('please use an end date formatted as: ' f'{ae_consts.COMMON_DATE_FORMAT}\n' f'error was: {e}') log.error(msg) sys.exit(1) # end of testing for a valid date # end of args.end_date config_dict = None if args.run_algo_in_file: if not os.path.exists(args.run_algo_in_file): log.error( f'missing algorithm module file: {args.run_algo_in_file}') sys.exit(1) algo_mod_path = args.run_algo_in_file if args.config_file: use_config_file = args.config_file if not os.path.exists(use_config_file): log.error(f'Failed: unable to find config file: -c {use_config_file}') sys.exit(1) config_dict = json.loads(open(use_config_file).read()) algo_mod_path = config_dict.get('algo_path', algo_mod_path) if not os.path.exists(algo_mod_path): log.error( f'missing algorithm module file from config: {algo_mod_path}') sys.exit(1) """ Finalize the algo config """ if config_dict: use_balance = float(config_dict.get('balance', use_balance)) use_commission = float(config_dict.get('commission', use_commission)) ticker = str(config_dict.get('ticker', ticker)).upper() config_dict['ticker'] = ticker config_dict['balance'] = use_balance config_dict['commission'] = use_commission else: if not ticker: ticker = str(config_dict.get('ticker', ae_consts.TICKER)).upper() if not ticker: log.error('usage error: please set a ticker with -t <TICKER>') sys.exit(1) if verbose_algo: config_dict['verbose'] = verbose_algo if verbose_processor: config_dict['verbose_processor'] = verbose_processor if verbose_indicators: config_dict['verbose_indicators'] = verbose_indicators if inspect_datasets: config_dict['inspect_datasets'] = inspect_datasets if run_this_date: config_dict['run_this_date'] = run_this_date log.info('starting') """ Run a custom algo module from disk """ if algo_mod_path: if args.backtest_loc: backtest_loc = args.backtest_loc if ('file:/' not in backtest_loc and 's3://' not in backtest_loc and 'redis://' not in backtest_loc): log.error( 'invalid -b <backtest dataset file> specified. ' f'{backtest_loc} ' 'please use either: ' '-b file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-b s3://algoready/SPY-latest.json or ' '-b redis://SPY-latest') sys.exit(1) if 's3://' in backtest_loc: load_from_s3_bucket = backtest_loc.split('/')[-2] load_from_s3_key = backtest_loc.split('/')[-1] elif 'redis://' in backtest_loc: load_from_redis_key = backtest_loc.split('/')[-1] elif 'file:/' in backtest_loc: load_from_file = backtest_loc.split(':')[-1] load_publish = True # end of parsing supported transport - loading an algo-ready if args.algo_history_loc: algo_history_loc = args.algo_history_loc if ('file:/' not in algo_history_loc and 's3://' not in algo_history_loc and 'redis://' not in algo_history_loc): log.error( 'invalid -p <backtest dataset file> specified. ' f'{algo_history_loc} ' 'please use either: ' '-p file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-p s3://algoready/SPY-latest.json or ' '-p redis://SPY-latest') sys.exit(1) if 's3://' in algo_history_loc: history_s3_bucket = algo_history_loc.split('/')[-2] history_s3_key = algo_history_loc.split('/')[-1] elif 'redis://' in algo_history_loc: history_redis_key = algo_history_loc.split('/')[-1] elif 'file:/' in algo_history_loc: history_file = algo_history_loc.split(':')[-1] history_publish = True # end of parsing supported transport - trading history if args.algo_report_loc: algo_report_loc = args.algo_report_loc if ('file:/' not in algo_report_loc and 's3://' not in algo_report_loc and 'redis://' not in algo_report_loc): log.error( 'invalid -o <backtest dataset file> specified. ' f'{algo_report_loc} ' 'please use either: ' '-o file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-o s3://algoready/SPY-latest.json or ' '-o redis://SPY-latest') sys.exit(1) if 's3://' in algo_report_loc: report_s3_bucket = algo_report_loc.split('/')[-2] report_s3_key = algo_report_loc.split('/')[-1] elif 'redis://' in algo_report_loc: report_redis_key = algo_report_loc.split('/')[-1] elif 'file:/' in algo_report_loc: report_file = algo_report_loc.split(':')[-1] report_publish = True # end of parsing supported transport - trading performance report if args.algo_extract_loc: algo_extract_loc = args.algo_extract_loc if ('file:/' not in algo_extract_loc and 's3://' not in algo_extract_loc and 'redis://' not in algo_extract_loc): log.error( 'invalid -e <backtest dataset file> specified. ' f'{algo_extract_loc} ' 'please use either: ' '-e file:/opt/sa/tests/datasets/algo/SPY-latest.json or ' '-e s3://algoready/SPY-latest.json or ' '-e redis://SPY-latest') sys.exit(1) if 's3://' in algo_extract_loc: extract_s3_bucket = algo_extract_loc.split('/')[-2] extract_s3_key = algo_extract_loc.split('/')[-1] elif 'redis://' in algo_extract_loc: extract_redis_key = algo_extract_loc.split('/')[-1] elif 'file:/' in algo_extract_loc: extract_file = algo_extract_loc.split(':')[-1] extract_publish = True # end of parsing supported transport - extract algorithm-ready if args.run_on_engine: run_on_engine = True if verbose_algo: log.info('starting algo on the engine') use_name = config_dict.get('name', 'missing-algo-name') auto_fill = config_dict.get('auto_fill', auto_fill) timeseries = config_dict.get('timeseries', timeseries) trade_strategy = config_dict.get('trade_strategy', trade_strategy) algo_res = run_custom_algo.run_custom_algo( mod_path=algo_mod_path, ticker=config_dict['ticker'], balance=config_dict['balance'], commission=config_dict['commission'], name=use_name, start_date=use_start_date, end_date=use_end_date, auto_fill=auto_fill, config_dict=config_dict, load_from_s3_bucket=load_from_s3_bucket, load_from_s3_key=load_from_s3_key, load_from_redis_key=load_from_redis_key, load_from_file=load_from_file, load_compress=load_compress, load_publish=load_publish, load_config=load_config, report_redis_key=report_redis_key, report_s3_bucket=report_s3_bucket, report_s3_key=report_s3_key, report_file=report_file, report_compress=report_compress, report_publish=report_publish, report_config=report_config, history_redis_key=history_redis_key, history_s3_bucket=history_s3_bucket, history_s3_key=history_s3_key, history_file=history_file, history_compress=history_compress, history_publish=history_publish, history_config=history_config, extract_redis_key=extract_redis_key, extract_s3_bucket=extract_s3_bucket, extract_s3_key=extract_s3_key, extract_file=extract_file, extract_save_dir=extract_save_dir, extract_compress=extract_compress, extract_publish=extract_publish, extract_config=extract_config, publish_to_slack=publish_to_slack, publish_to_s3=publish_to_s3, publish_to_redis=publish_to_redis, dataset_type=dataset_type, serialize_datasets=serialize_datasets, compress=compress, encoding=encoding, redis_enabled=redis_enabled, redis_key=redis_key, redis_address=redis_address, redis_db=redis_db, redis_password=redis_password, redis_expire=redis_expire, redis_serializer=redis_serializer, redis_encoding=redis_encoding, s3_enabled=s3_enabled, s3_key=s3_key, s3_address=s3_address, s3_bucket=s3_bucket_name, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, slack_enabled=slack_enabled, slack_code_block=slack_code_block, slack_full_width=slack_full_width, dataset_publish_extract=extract_publish, dataset_publish_history=history_publish, dataset_publish_report=report_publish, run_on_engine=run_on_engine, auth_url=broker_url, backend_url=backend_url, include_tasks=include_tasks, ssl_options=ssl_options, transport_options=transport_options, path_to_config_module=path_to_config_module, timeseries=timeseries, trade_strategy=trade_strategy, verbose=verbose_algo) show_label = f'algo.name={use_name}' show_extract = f'{algo_extract_loc}' show_history = f'{algo_history_loc}' show_report = f'{algo_report_loc}' base_label = (f'load={args.run_algo_in_file} extract={show_extract} ' f'history={show_history} report={show_report}') task_id = None if run_on_engine: task_id = algo_res.get('rec', {}).get('task_id', None) if task_id: log.info(f'waiting on task_id={task_id} to finish') res = task_id.get() history_config = res.get('algo_req', {}).get('history_config', None) s3_bucket = history_config.get('s3_bucket', None) s3_key = history_config.get('s3_key', None) load_res = load_history_utils.load_history_dataset( s3_bucket=s3_bucket, s3_key=s3_key) history_df = load_res[ticker] log.info('plotting history df with columns: ' f'{history_df.columns.values}') history_df['date'] = pd.to_datetime(history_df['date']) history_df['minute'] = pd.to_datetime(history_df['minute']) ticker = history_df['ticker'].iloc[0] first_date = history_df['date'].iloc[0] end_date = history_df['date'].iloc[-1] title = (f'Trading History {ticker}\n' f'Backtest dates from {first_date} to {end_date}') use_xcol = 'date' use_as_date_format = '%d\n%b' use_minute = False if 'minute' in history_df: found_valid_minute = history_df['minute'].iloc[0] if found_valid_minute: use_minute = True if use_minute: use_xcol = 'minute' use_as_date_format = '%d %H:%M:%S\n%b' xlabel = 'Dates vs Algo values' ylabel = 'Algo values' df_filter = (history_df['close'] > 1.00) # set default columns: blue = None green = None orange = None red = 'balance' blue = 'close' plot_trading_history.plot_trading_history( title=title, df=history_df, red=red, blue=blue, green=green, orange=orange, date_col=use_xcol, date_format=use_as_date_format, xlabel=xlabel, ylabel=ylabel, df_filter=df_filter, show_plot=True, dropna_for_all=True) return else: algo_obj = algo_res.get('algo', None) # end of getting the algo results from s3 if not algo_obj: log.error(f'{show_label} - did not create algorithm object') sys.exit(1) if not run_on_engine: algo_trade_history_recs = algo_res['rec'].get('history', []) show_label = (f'{ticker} algo.name={use_name} {base_label} ' f'trade_history_len={len(algo_trade_history_recs)}') if args.debug: log.info(f'algo_res={algo_res}') if algo_res['status'] == ae_consts.SUCCESS: log.info(f'''{ae_consts.get_status( status=algo_res['status'])} - ''' f'done running {show_label}') else: log.error(f'''{ae_consts.get_status( status=algo_res['status'])} - ''' f'done running {show_label}') else: if algo_res['status'] == ae_consts.SUCCESS: log.info(f'''{ae_consts.get_status( status=algo_res['status'])} - ''' f'done running {show_label}') else: log.error(f'run_custom_algo returned error: {algo_res["err"]}') sys.exit(1) # end of running the custom algo handler else: log.error(f'missing an algorithm mod path: {algo_mod_path}') sys.exit(1) # end if running a custom algorithm module if algo_obj: trading_history_dict = algo_obj.get_history_dataset() history_df = trading_history_dict[ticker] if not hasattr(history_df, 'to_json'): return if history_json_file: log.info(f'saving history to: {history_json_file}') history_df.to_json(history_json_file, orient='records', date_format='iso') log.info('plotting history') first_date = history_df['date'].iloc[0] end_date = history_df['date'].iloc[-1] title = (f'Trading History {ticker} for Algo ' f'{trading_history_dict["algo_name"]}\n' f'Backtest dates from {first_date} to {end_date}') use_xcol = 'date' use_as_date_format = '%d\n%b' if config_dict['timeseries'] == 'minute': use_xcol = 'minute' use_as_date_format = '%d %H:%M:%S\n%b' xlabel = f'Dates vs {trading_history_dict["algo_name"]} values' ylabel = f'Algo {trading_history_dict["algo_name"]}\nvalues' df_filter = (history_df['close'] > 0.01) # set default hloc columns: blue = None green = None orange = None red = 'close' blue = 'balance' if debug: for i, r in history_df.iterrows(): log.info(f'{r["minute"]} - {r["close"]}') plot_trading_history.plot_trading_history( title=title, df=history_df, red=red, blue=blue, green=green, orange=orange, date_col=use_xcol, date_format=use_as_date_format, xlabel=xlabel, ylabel=ylabel, df_filter=df_filter, show_plot=True, dropna_for_all=True)