def test_integration_daily_indicator_with_algo_config(self): """test_integration_daily_indicator_with_algo_config""" if ae_consts.ev('INT_TESTS', '0') == '0': return algo = base_algo.BaseAlgo( ticker=self.ticker, balance=self.balance, start_date_str=self.start_date_str, end_date_str=self.end_date_str, config_dict=self.algo_config_dict) self.assertEqual( algo.name, self.algo_config_dict['name']) self.assertEqual( algo.tickers, [self.ticker]) algo.handle_data( data=self.data) res = algo.get_result() print(ae_consts.ppj(res)) self.assertTrue( res['history'][0]['total_sells'] >= 1) self.assertTrue( res['history'][0]['total_buys'] == 0)
def test_run_daily_indicator_with_algo_config_sell(self): """test_run_daily_indicator_with_algo_config_sell""" algo = base_algo.BaseAlgo( ticker=self.ticker, balance=self.balance, start_date_str=self.start_date_str, end_date_str=self.end_date_str, config_dict=self.algo_config_dict) self.assertEqual( algo.name, self.algo_config_dict['name']) self.assertEqual( algo.tickers, [self.ticker]) algo.handle_data( data=self.data) res = algo.get_result() print(ae_consts.ppj(res)) self.assertEqual( res['history'][0]['total_sells'], 1) self.assertEqual( res['history'][0]['total_buys'], 0)
def run_distributed_algorithm(self, algo_req): """run_distributed_algorithm Process an Algorithm using a Celery task that is processed by a Celery worker :param algo_req: dictionary for key/values for running an algorithm using Celery workers """ label = algo_req.get('name', 'ae-algo') verbose = algo_req.get('verbose_task', False) debug = algo_req.get('debug', False) # please be careful logging prod passwords: if debug: log.info(f'task - {label} - start algo_req={algo_req}') elif verbose: log.info(f'task - {label} - start ') # end of start log rec = {} res = build_result.build_result(status=ae_consts.NOT_RUN, err=None, rec=rec) created_algo_object = None custom_algo_module = None new_algo_object = None use_custom_algo = False found_algo_module = True # assume the BaseAlgo should_publish_extract_dataset = False should_publish_history_dataset = False should_publish_report_dataset = False ticker = algo_req.get('ticker', 'SPY') num_days_back = algo_req.get('num_days_back', 75) name = algo_req.get('name', 'ae-algo') algo_module_path = algo_req.get('mod_path', None) module_name = algo_req.get('module_name', 'BaseAlgo') custom_algo_module = algo_req.get('custom_algo_module', None) new_algo_object = algo_req.get('new_algo_object', None) use_custom_algo = algo_req.get('use_custom_algo', False) should_publish_extract_dataset = algo_req.get( 'should_publish_extract_dataset', False) should_publish_history_dataset = algo_req.get( 'should_publish_history_dataset', False) should_publish_report_dataset = algo_req.get( 'should_publish_report_dataset', False) start_date = algo_req.get('start_date', None) end_date = algo_req.get('end_date', None) raise_on_err = algo_req.get('raise_on_err', True) report_config = algo_req.get('report_config', None) history_config = algo_req.get('history_config', None) extract_config = algo_req.get('extract_config', None) err = None if algo_module_path: found_algo_module = False module_name = algo_module_path.split('/')[-1] loader = importlib.machinery.SourceFileLoader(module_name, algo_module_path) custom_algo_module = types.ModuleType(loader.name) loader.exec_module(custom_algo_module) use_custom_algo = True for member in inspect.getmembers(custom_algo_module): if module_name in str(member): found_algo_module = True break # for all members in this custom module file # if loading a custom algorithm module from a file on disk if not found_algo_module: err = (f'{label} - unable to find custom algorithm ' f'module={custom_algo_module} module_path={algo_module_path}') if algo_module_path: err = ( f'{label} - analysis_engine.' 'work_tasks.run_distributed_algorithm was unable ' f'to find custom algorithm module={custom_algo_module} with ' f'provided path to \n file: {algo_module_path} \n' '\n' 'Please confirm ' 'that the class inherits from the BaseAlgo class like:\n' '\n' 'import analysis_engine.algo\n' 'class MyAlgo(analysis_engine.algo.BaseAlgo):\n ' '\n' 'If it is then please file an issue on github:\n ' 'https://github.com/AlgoTraders/stock-analysis-engine/' 'issues/new \n\nFor now this error results in a shutdown' '\n') # if algo_module_path set log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) task_result = { 'status': res['status'], 'err': res['err'], 'algo_req': algo_req, 'rec': rec } return task_result # if not found_algo_module use_start_date = start_date use_end_date = end_date if not use_end_date: end_date = datetime.datetime.utcnow() use_end_date = end_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT) if not use_start_date: start_date = end_date - datetime.timedelta(days=num_days_back) use_start_date = start_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT) dataset_publish_extract = algo_req.get('dataset_publish_extract', False) dataset_publish_history = algo_req.get('dataset_publish_history', False) dataset_publish_report = algo_req.get('dataset_publish_report', False) try: if use_custom_algo: if verbose: log.info( f'inspecting {custom_algo_module} for class {module_name}') use_class_member_object = None for member in inspect.getmembers(custom_algo_module): if module_name in str(member): if verbose: log.info(f'start {name} with {member[1]}') use_class_member_object = member break # end of looking over the class definition but did not find it if use_class_member_object: if algo_req.get('backtest', False): new_algo_object = member[1](ticker=algo_req['ticker'], config_dict=algo_req) else: new_algo_object = member[1](**algo_req) else: err = (f'{label} - did not find a derived ' 'analysis_engine.algo.BaseAlgo ' f'class in the module file={algo_module_path} ' f'for ticker={ticker} algo_name={name}') log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) task_result = { 'status': res['status'], 'err': res['err'], 'algo_req': algo_req, 'rec': rec } return task_result # end of finding a valid algorithm object else: new_algo_object = ae_algo.BaseAlgo(**algo_req) # if using a custom module path or the BaseAlgo if new_algo_object: # heads up - logging this might have passwords in the algo_req # log.debug(f'{name} algorithm request: {algo_req}') if verbose: log.info(f'{name} - run START ticker={ticker} ' f'from {use_start_date} to {use_end_date}') if algo_req.get('backtest', False): algo_res = run_algo.run_algo(algo=new_algo_object, config_dict=algo_req) created_algo_object = new_algo_object else: algo_res = run_algo.run_algo(algo=new_algo_object, **algo_req) created_algo_object = new_algo_object if verbose: log.info(f'{name} - run DONE ticker={ticker} ' f'from {use_start_date} to {use_end_date}') if debug: if custom_algo_module: log.info(f'{name} - done run_algo ' f'custom_algo_module={custom_algo_module} ' f'module_name={module_name} ticker={ticker} ' f'from {use_start_date} to {use_end_date}') else: log.info( f'{name} - done run_algo BaseAlgo ticker={ticker} ' f'from {use_start_date} to {use_end_date}') else: err = ( f'{label} - missing a derived analysis_engine.algo.BaseAlgo ' f'class in the module file={algo_module_path} for ' f'ticker={ticker} algo_name={name}') log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) task_result = { 'status': res['status'], 'err': res['err'], 'algo_req': algo_req, 'rec': rec } return task_result # end of finding a valid algorithm object if not created_algo_object: err = (f'{label} - failed creating algorithm object - ' f'ticker={ticker} ' f'status={ae_consts.get_status(status=algo_res["status"])} ' f'error={algo_res["err"]} algo name={name} ' f'custom_algo_module={custom_algo_module} ' f'module_name={module_name} ' f'from {use_start_date} to {use_end_date}') res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) task_result = { 'status': res['status'], 'err': res['err'], 'algo_req': algo_req, 'rec': rec } return task_result # end of stop early if should_publish_extract_dataset or dataset_publish_extract: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (extract_config['redis_address'] and extract_config['redis_db'] and extract_config['redis_key']): redis_log = (f'redis://{extract_config["redis_address"]}' f'@{extract_config["redis_db"]}' f'/{extract_config["redis_key"]}') use_log += f' {redis_log}' else: extract_config['redis_enabled'] = False if (extract_config['s3_address'] and extract_config['s3_bucket'] and extract_config['s3_key']): s3_log = (f's3://{extract_config["s3_address"]}' f'/{extract_config["s3_bucket"]}' f'/{extract_config["s3_key"]}') use_log += f' {s3_log}' else: extract_config['s3_enabled'] = False if extract_config['output_file']: file_log = f'file:{extract_config["output_file"]}' use_log += f' {file_log}' if verbose: log.info(f'{name} - publish - start ticker={ticker} ' f'algorithm-ready {use_log}') publish_status = created_algo_object.publish_input_dataset( **extract_config) if publish_status != ae_consts.SUCCESS: msg = ('failed to publish algorithm-ready datasets with ' f'status {ae_consts.get_status(status=publish_status)} ' f'attempted to {use_log}') log.error(msg) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) task_result = { 'status': res['status'], 'err': res['err'], 'algo_req': algo_req, 'rec': rec } return task_result # end of stop early if verbose: log.info(f'{name} - publish - done ticker={ticker} ' f'algorithm-ready {use_log}') # if publish the algorithm-ready dataset if should_publish_history_dataset or dataset_publish_history: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (history_config['redis_address'] and history_config['redis_db'] and history_config['redis_key']): redis_log = (f'redis://{history_config["redis_address"]}' f'@{history_config["redis_db"]}' f'/{history_config["redis_key"]}') use_log += f' {redis_log}' if (history_config['s3_address'] and history_config['s3_bucket'] and history_config['s3_key']): s3_log = (f's3://{history_config["s3_address"]}' f'/{history_config["s3_bucket"]}' f'/{history_config["s3_key"]}') use_log += f' {s3_log}' if history_config['output_file']: file_log = f'file:{history_config["output_file"]}' use_log += f' {file_log}' if verbose: log.info(f'{name} - publish - start ticker={ticker} trading ' f'history {use_log}') publish_status = \ created_algo_object.publish_trade_history_dataset( **history_config) if publish_status != ae_consts.SUCCESS: msg = ('failed to publish trading history datasets with ' f'status {ae_consts.get_status(status=publish_status)} ' f'attempted to {use_log}') log.error(msg) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) task_result = { 'status': res['status'], 'err': res['err'], 'algo_req': algo_req, 'rec': rec } return task_result # end of stop early if verbose: log.info(f'{name} - publish - done ticker={ticker} trading ' f'history {use_log}') # if publish an trading history dataset if should_publish_report_dataset or dataset_publish_report: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (report_config['redis_address'] and report_config['redis_db'] and report_config['redis_key']): redis_log = (f'redis://{report_config["redis_address"]}' f'@{report_config["redis_db"]}' f'/{report_config["redis_key"]}') use_log += f' {redis_log}' if (report_config['s3_address'] and report_config['s3_bucket'] and report_config['s3_key']): s3_log = (f's3://{report_config["s3_address"]}' f'/{report_config["s3_bucket"]}' f'/{report_config["s3_key"]}') use_log += f' {s3_log}' if report_config['output_file']: file_log = f' file:{report_config["output_file"]}' use_log += f' {file_log}' if verbose: log.info( f'{name} - publishing ticker={ticker} trading performance ' f'report {use_log}') publish_status = created_algo_object.publish_report_dataset( **report_config) if publish_status != ae_consts.SUCCESS: msg = ('failed to publish trading performance ' 'report datasets with ' f'status {ae_consts.get_status(status=publish_status)} ' f'attempted to {use_log}') log.error(msg) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) task_result = { 'status': res['status'], 'err': res['err'], 'algo_req': algo_req, 'rec': rec } return task_result # end of stop early if verbose: log.info(f'{name} - publish - done ticker={ticker} trading ' f'performance report {use_log}') # if publish an trading performance report dataset if verbose: log.info(f'{name} - done publishing datasets for ticker={ticker} ' f'from {use_start_date} to {use_end_date}') rec['history_config'] = history_config rec['report_config'] = report_config res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) except Exception as e: res = build_result.build_result( status=ae_consts.ERR, err=('failed - run_distributed_algorithm ' f'dict={algo_req} with ex={e}'), rec=rec) if raise_on_err: raise e else: log.error(f'{label} - {res["err"]}') # end of try/ex if verbose: log.info('task - run_distributed_algorithm done - ' f'{label} - status={ae_consts.get_status(res["status"])}') task_result = { 'status': res['status'], 'err': res['err'], 'algo_req': algo_req, 'rec': rec } return task_result
def latest(self, date_str=None, start_row=-200, extract_iex=True, extract_yahoo=False, extract_td=True, verbose=False, **kwargs): """latest Run the algorithm with the latest pricing data. Also supports running a backtest for a historical date in the pricing history (format ``YYYY-MM-DD``) :param date_str: optional - string start date ``YYYY-MM-DD`` default is the latest close date :param start_row: negative number of rows back from the end of the list in the data default is ``-200`` where this means the algorithm will process the latest 200 rows in the minute dataset :param extract_iex: bool flag for extracting from ``IEX`` :param extract_yahoo: bool flag for extracting from ``Yahoo`` which is disabled as of 1/2019 :param extract_td: bool flag for extracting from ``Tradier`` :param verbose: bool flag for logs :param kwargs: keyword arg dict """ use_date_str = date_str if not use_date_str: use_date_str = ae_utils.get_last_close_str() log.info('creating algo') self.algo_obj = base_algo.BaseAlgo( ticker=self.config_dict['ticker'], balance=self.config_dict['balance'], commission=self.config_dict['commission'], name=self.use_name, start_date=self.use_start_date, end_date=self.use_end_date, auto_fill=self.auto_fill, config_dict=self.config_dict, load_from_s3_bucket=self.load_from_s3_bucket, load_from_s3_key=self.load_from_s3_key, load_from_redis_key=self.load_from_redis_key, load_from_file=self.load_from_file, load_compress=self.load_compress, load_publish=self.load_publish, load_config=self.load_config, report_redis_key=self.report_redis_key, report_s3_bucket=self.report_s3_bucket, report_s3_key=self.report_s3_key, report_file=self.report_file, report_compress=self.report_compress, report_publish=self.report_publish, report_config=self.report_config, history_redis_key=self.history_redis_key, history_s3_bucket=self.history_s3_bucket, history_s3_key=self.history_s3_key, history_file=self.history_file, history_compress=self.history_compress, history_publish=self.history_publish, history_config=self.history_config, extract_redis_key=self.extract_redis_key, extract_s3_bucket=self.extract_s3_bucket, extract_s3_key=self.extract_s3_key, extract_file=self.extract_file, extract_save_dir=self.extract_save_dir, extract_compress=self.extract_compress, extract_publish=self.extract_publish, extract_config=self.extract_config, publish_to_slack=self.publish_to_slack, publish_to_s3=self.publish_to_s3, publish_to_redis=self.publish_to_redis, dataset_type=self.dataset_type, serialize_datasets=self.serialize_datasets, compress=self.compress, encoding=self.encoding, redis_enabled=self.redis_enabled, redis_key=self.redis_key, redis_address=self.redis_address, redis_db=self.redis_db, redis_password=self.redis_password, redis_expire=self.redis_expire, redis_serializer=self.redis_serializer, redis_encoding=self.redis_encoding, s3_enabled=self.s3_enabled, s3_key=self.s3_key, s3_address=self.s3_address, s3_bucket=self.s3_bucket, s3_access_key=self.s3_access_key, s3_secret_key=self.s3_secret_key, s3_region_name=self.s3_region_name, s3_secure=self.s3_secure, slack_enabled=self.slack_enabled, slack_code_block=self.slack_code_block, slack_full_width=self.slack_full_width, dataset_publish_extract=self.extract_publish, dataset_publish_history=self.history_publish, dataset_publish_report=self.report_publish, run_on_engine=self.run_on_engine, auth_url=self.broker_url, backend_url=self.backend_url, include_tasks=self.include_tasks, ssl_options=self.ssl_options, transport_options=self.transport_options, path_to_config_module=self.path_to_config_module, timeseries=self.timeseries, trade_strategy=self.trade_strategy, verbose=False, raise_on_err=self.raise_on_err) log.info('run latest - start') ticker = self.config_dict['ticker'] dataset_id = f'{ticker}_{use_date_str}' self.common_fetch_vals['base_key'] = dataset_id verbose_extract = self.config_dict.get('verbose_extract', False) indicator_datasets = self.algo_obj.get_indicator_datasets() ticker_data = build_dataset_node.build_dataset_node( ticker=ticker, date=use_date_str, datasets=indicator_datasets, service_dict=self.common_fetch_vals, verbose=verbose_extract) algo_data_req = { ticker: [{ 'id': dataset_id, # id is currently the cache key in redis 'date': use_date_str, # used to confirm dates in asc order 'data': ticker_data, 'start_row': start_row }] } if verbose: log.info(f'extract - {dataset_id} ' f'dataset={len(algo_data_req[ticker])}') # this could be a separate celery task try: if verbose: log.info(f'handle_data START - {dataset_id}') self.algo_obj.handle_data(data=algo_data_req) if verbose: log.info(f'handle_data END - {dataset_id}') except Exception as e: a_name = self.algo_obj.get_name() a_debug_msg = self.algo_obj.get_debug_msg() if not a_debug_msg: a_debug_msg = 'debug message not set' # a_config_dict = ae_consts.ppj(self.algo_obj.config_dict) msg = (f'{dataset_id} - algo={a_name} ' f'encountered exception in handle_data tickers={ticker} ' f'from ex={e} ' f'and failed during operation: {a_debug_msg}') log.critical(f'{msg}') # end try/ex log.info('run latest - create history') history_ds = self.algo_obj.create_history_dataset() self.history_df = pd.DataFrame(history_ds[ticker]) self.determine_latest_times_in_history() self.num_rows = len(self.history_df.index) if verbose: log.info(self.history_df[['minute', 'close']].tail(5)) log.info(f'run latest minute={self.end_date} - ' f'rows={self.num_rows} - done') return self.get_history()
def run_algo( ticker=None, tickers=None, algo=None, # optional derived ``analysis_engine.algo.Algo`` instance balance=None, # float starting base capital commission=None, # float for single trade commission for buy or sell start_date=None, # string YYYY-MM-DD HH:MM:SS end_date=None, # string YYYY-MM-DD HH:MM:SS datasets=None, # string list of identifiers num_owned_dict=None, # not supported cache_freq='daily', # 'minute' not supported auto_fill=True, load_config=None, report_config=None, history_config=None, extract_config=None, use_key=None, extract_mode='all', iex_datasets=None, redis_enabled=True, redis_address=None, redis_db=None, redis_password=None, redis_expire=None, redis_key=None, s3_enabled=True, s3_address=None, s3_bucket=None, s3_access_key=None, s3_secret_key=None, s3_region_name=None, s3_secure=False, s3_key=None, celery_disabled=True, broker_url=None, result_backend=None, label=None, name=None, timeseries=None, trade_strategy=None, verbose=False, publish_to_slack=True, publish_to_s3=True, publish_to_redis=True, extract_datasets=None, config_file=None, config_dict=None, version=1, raise_on_err=True, **kwargs): """run_algo Run an algorithm with steps: 1) Extract redis keys between dates 2) Compile a data pipeline dictionary (call it ``data``) 3) Call algorithm's ``myalgo.handle_data(data=data)`` .. note:: If no ``algo`` is set, the ``analysis_engine.algo.BaseAlgo`` algorithm is used. .. note:: Please ensure Redis and Minio are running before trying to extract tickers **Stock tickers to extract** :param ticker: single stock ticker/symbol/ETF to extract :param tickers: optional - list of tickers to extract :param use_key: optional - extract historical key from Redis **Algo Configuration** :param algo: derived instance of ``analysis_engine.algo.Algo`` object :param balance: optional - float balance parameter can also be set on the ``algo`` object if not set on the args :param commission: float for single trade commission for buy or sell. can also be set on the ``algo`` objet :param start_date: string ``YYYY-MM-DD_HH:MM:SS`` cache value :param end_date: string ``YYYY-MM-DD_HH:MM:SS`` cache value :param dataset_types: list of strings that are ``iex`` or ``yahoo`` datasets that are cached. :param cache_freq: optional - depending on if you are running data feeds on a ``daily`` cron (default) vs every ``minute`` (or faster) :param num_owned_dict: not supported yet :param auto_fill: optional - boolean for auto filling buy/sell orders for backtesting (default is ``True``) :param trading_calendar: ``trading_calendar.TradingCalendar`` object, by default ``analysis_engine.calendars. always_open.AlwaysOpen`` trading calendar # TradingCalendar by ``TFSExchangeCalendar`` :param config_file: path to a json file containing custom algorithm object member values (like indicator configuration and predict future date units ahead for a backtest) :param config_dict: optional - dictionary that can be passed to derived class implementations of: ``def load_from_config(config_dict=config_dict)`` **Timeseries** :param timeseries: optional - string to set ``day`` or ``minute`` backtesting or live trading (default is ``minute``) **Trading Strategy** :param trade_strategy: optional - string to set the type of ``Trading Strategy`` for backtesting or live trading (default is ``count``) **Algorithm Dataset Loading, Extracting, Reporting and Trading History arguments** :param load_config: optional - dictionary for setting member variables to load an agorithm-ready dataset from a file, s3 or redis :param report_config: optional - dictionary for setting member variables to publish an algo ``trading performance report`` to s3, redis, a file or slack :param history_config: optional - dictionary for setting member variables to publish an algo ``trade history`` to s3, redis, a file or slack :param extract_config: optional - dictionary for setting member variables to publish an algo ``trading performance report`` to s3, redis, a file or slack **(Optional) Data sources, datafeeds and datasets to gather** :param iex_datasets: list of strings for gathering specific `IEX datasets <https://iexcloud.io/>`__ which are set as consts: ``analysis_engine.iex.consts.FETCH_*``. **(Optional) Redis connectivity arguments** :param redis_enabled: bool - toggle for auto-caching all datasets in Redis (default is ``True``) :param redis_address: Redis connection string format is ``host:port`` (default is ``localhost:6379``) :param redis_db: Redis db to use (default is ``0``) :param redis_password: optional - Redis password (default is ``None``) :param redis_expire: optional - Redis expire value (default is ``None``) :param redis_key: optional - redis key not used (default is ``None``) **(Optional) Minio (S3) connectivity arguments** :param s3_enabled: bool - toggle for auto-archiving on Minio (S3) (default is ``True``) :param s3_address: Minio S3 connection string format ``host:port`` (default is ``localhost:9000``) :param s3_bucket: S3 Bucket for storing the artifacts (default is ``dev``) which should be viewable on a browser: http://localhost:9000/minio/dev/ :param s3_access_key: S3 Access key (default is ``trexaccesskey``) :param s3_secret_key: S3 Secret key (default is ``trex123321``) :param s3_region_name: S3 region name (default is ``us-east-1``) :param s3_secure: Transmit using tls encryption (default is ``False``) :param s3_key: optional s3 key not used (default is ``None``) **(Optional) Celery worker broker connectivity arguments** :param celery_disabled: bool - toggle synchronous mode or publish to an engine connected to the `Celery broker and backend <https://github.com/celery/celery#transports-and-backends>`__ (default is ``True`` - synchronous mode without an engine or need for a broker or backend for Celery) :param broker_url: Celery broker url (default is ``redis://0.0.0.0:6379/13``) :param result_backend: Celery backend url (default is ``redis://0.0.0.0:6379/14``) :param label: tracking log label :param publish_to_slack: optional - boolean for publishing to slack (coming soon) :param publish_to_s3: optional - boolean for publishing to s3 (coming soon) :param publish_to_redis: optional - boolean for publishing to redis (coming soon) **(Optional) Debugging** :param verbose: bool - show extract warnings and other debug logging (default is False) :param raise_on_err: optional - boolean for unittests and developing algorithms with the ``analysis_engine.run_algo.run_algo`` helper. When set to ``True`` exceptions will are raised to the calling functions :param kwargs: keyword arguments dictionary """ # dictionary structure with a list sorted on: ascending dates # algo_data_req[ticker][list][dataset] = pd.DataFrame algo_data_req = {} extract_requests = [] return_algo = False # return created algo objects for use by caller rec = {} msg = None use_tickers = tickers use_balance = balance use_commission = commission if ticker: use_tickers = [ticker] else: if not use_tickers: use_tickers = [] # if these are not set as args, but the algo object # has them, use them instead: if algo: if len(use_tickers) == 0: use_tickers = algo.get_tickers() if not use_balance: use_balance = algo.get_balance() if not use_commission: use_commission = algo.get_commission() default_iex_datasets = [ 'daily', 'minute', 'quote', 'stats', 'peers', 'news', 'financials', 'earnings', 'dividends', 'company' ] if not iex_datasets: iex_datasets = default_iex_datasets if redis_enabled: if not redis_address: redis_address = os.getenv('REDIS_ADDRESS', 'localhost:6379') if not redis_password: redis_password = os.getenv('REDIS_PASSWORD', None) if not redis_db: redis_db = int(os.getenv('REDIS_DB', '0')) if not redis_expire: redis_expire = os.getenv('REDIS_EXPIRE', None) if s3_enabled: if not s3_address: s3_address = os.getenv('S3_ADDRESS', 'localhost:9000') if not s3_access_key: s3_access_key = os.getenv('AWS_ACCESS_KEY_ID', 'trexaccesskey') if not s3_secret_key: s3_secret_key = os.getenv('AWS_SECRET_ACCESS_KEY', 'trex123321') if not s3_region_name: s3_region_name = os.getenv('AWS_DEFAULT_REGION', 'us-east-1') if not s3_secure: s3_secure = os.getenv('S3_SECURE', '0') == '1' if not s3_bucket: s3_bucket = os.getenv('S3_BUCKET', 'dev') if not broker_url: broker_url = os.getenv('WORKER_BROKER_URL', 'redis://0.0.0.0:6379/11') if not result_backend: result_backend = os.getenv('WORKER_BACKEND_URL', 'redis://0.0.0.0:6379/12') if not label: label = 'run-algo' num_tickers = len(use_tickers) last_close_str = ae_utils.get_last_close_str() if iex_datasets: if verbose: log.info(f'{label} - tickers={num_tickers} ' f'iex={json.dumps(iex_datasets)}') else: if verbose: log.info(f'{label} - tickers={num_tickers}') ticker_key = use_key if not ticker_key: ticker_key = f'{ticker}_{last_close_str}' if not algo: algo = base_algo.BaseAlgo(ticker=None, tickers=use_tickers, balance=use_balance, commission=use_commission, config_dict=config_dict, name=label, auto_fill=auto_fill, timeseries=timeseries, trade_strategy=trade_strategy, publish_to_slack=publish_to_slack, publish_to_s3=publish_to_s3, publish_to_redis=publish_to_redis, raise_on_err=raise_on_err) return_algo = True # the algo object is stored # in the result at: res['rec']['algo'] if not algo: msg = f'{label} - missing algo object' log.error(msg) return build_result.build_result(status=ae_consts.EMPTY, err=msg, rec=rec) if raise_on_err: log.debug(f'{label} - enabling algo exception raises') algo.raise_on_err = True indicator_datasets = algo.get_indicator_datasets() if len(indicator_datasets) == 0: indicator_datasets = ae_consts.BACKUP_DATASETS log.info(f'using all datasets={indicator_datasets}') verbose_extract = False if config_dict: verbose_extract = config_dict.get('verbose_extract', False) common_vals = {} common_vals['base_key'] = ticker_key common_vals['celery_disabled'] = celery_disabled common_vals['ticker'] = ticker common_vals['label'] = label common_vals['iex_datasets'] = iex_datasets common_vals['s3_enabled'] = s3_enabled common_vals['s3_bucket'] = s3_bucket common_vals['s3_address'] = s3_address common_vals['s3_secure'] = s3_secure common_vals['s3_region_name'] = s3_region_name common_vals['s3_access_key'] = s3_access_key common_vals['s3_secret_key'] = s3_secret_key common_vals['s3_key'] = ticker_key common_vals['redis_enabled'] = redis_enabled common_vals['redis_address'] = redis_address common_vals['redis_password'] = redis_password common_vals['redis_db'] = redis_db common_vals['redis_key'] = ticker_key common_vals['redis_expire'] = redis_expire use_start_date_str = start_date use_end_date_str = end_date last_close_date = ae_utils.last_close() end_date_val = None cache_freq_fmt = ae_consts.COMMON_TICK_DATE_FORMAT if not use_end_date_str: use_end_date_str = last_close_date.strftime(cache_freq_fmt) end_date_val = ae_utils.get_date_from_str(date_str=use_end_date_str, fmt=cache_freq_fmt) start_date_val = None if not use_start_date_str: start_date_val = end_date_val - datetime.timedelta(days=60) use_start_date_str = start_date_val.strftime(cache_freq_fmt) else: start_date_val = datetime.datetime.strptime( use_start_date_str, ae_consts.COMMON_TICK_DATE_FORMAT) total_dates = (end_date_val - start_date_val).days if end_date_val < start_date_val: msg = ( f'{label} - invalid dates - start_date={start_date_val} is after ' f'end_date={end_date_val}') raise Exception(msg) if verbose: log.info(f'{label} - days={total_dates} ' f'start={use_start_date_str} ' f'end={use_end_date_str} ' f'datasets={indicator_datasets}') for ticker in use_tickers: req = algo_utils.build_algo_request(ticker=ticker, use_key=use_key, start_date=use_start_date_str, end_date=use_end_date_str, datasets=datasets, balance=use_balance, cache_freq=cache_freq, timeseries=timeseries, trade_strategy=trade_strategy, label=label) ticker_key = f'{ticker}_{last_close_str}' common_vals['ticker'] = ticker common_vals['base_key'] = ticker_key common_vals['redis_key'] = ticker_key common_vals['s3_key'] = ticker_key for date_key in req['extract_datasets']: date_req = api_requests.get_ds_dict(ticker=ticker, base_key=date_key, ds_id=label, service_dict=common_vals) node_date_key = date_key.replace(f'{ticker}_', '') extract_requests.append({ 'id': date_key, 'ticker': ticker, 'date_key': date_key, 'date': node_date_key, 'req': date_req }) # end of for all ticker in use_tickers first_extract_date = None last_extract_date = None total_extract_requests = len(extract_requests) cur_idx = 1 for idx, extract_node in enumerate(extract_requests): extract_ticker = extract_node['ticker'] extract_date = extract_node['date'] ds_node_id = extract_node['id'] if not first_extract_date: first_extract_date = extract_date last_extract_date = extract_date perc_progress = ae_consts.get_percent_done( progress=cur_idx, total=total_extract_requests) percent_label = (f'{label} ' f'ticker={extract_ticker} ' f'date={extract_date} ' f'{perc_progress} ' f'{idx}/{total_extract_requests} ' f'{indicator_datasets}') if verbose: log.info(f'extracting - {percent_label}') ticker_bt_data = build_ds_node.build_dataset_node( ticker=extract_ticker, date=extract_date, service_dict=common_vals, datasets=indicator_datasets, log_label=label, verbose=verbose_extract) if ticker not in algo_data_req: algo_data_req[ticker] = [] algo_data_req[ticker].append({ 'id': ds_node_id, # id is currently the cache key in redis 'date': extract_date, # used to confirm dates in asc order 'data': ticker_bt_data }) if verbose: log.info(f'extract - {percent_label} ' f'dataset={len(algo_data_req[ticker])}') cur_idx += 1 # end of for service_dict in extract_requests # this could be a separate celery task status = ae_consts.NOT_RUN if len(algo_data_req) == 0: msg = (f'{label} - nothing to test - no data found for ' f'tickers={use_tickers} ' f'between {first_extract_date} and {last_extract_date}') log.info(msg) return build_result.build_result(status=ae_consts.EMPTY, err=msg, rec=rec) # this could be a separate celery task try: if verbose: log.info(f'handle_data START - {percent_label} from ' f'{first_extract_date} to {last_extract_date}') algo.handle_data(data=algo_data_req) if verbose: log.info(f'handle_data END - {percent_label} from ' f'{first_extract_date} to {last_extract_date}') except Exception as e: a_name = algo.get_name() a_debug_msg = algo.get_debug_msg() if not a_debug_msg: a_debug_msg = 'debug message not set' a_config_dict = ae_consts.ppj(algo.config_dict) msg = (f'{percent_label} - algo={a_name} ' f'encountered exception in handle_data tickers={use_tickers} ' f'from {first_extract_date} to {last_extract_date} ex={e} ' f'and failed during operation: {a_debug_msg}') if raise_on_err: if algo: try: ind_obj = \ algo.get_indicator_process_last_indicator() if ind_obj: ind_obj_path = ind_obj.get_path_to_module() ind_obj_config = ae_consts.ppj(ind_obj.get_config()) found_error_hint = False if hasattr(ind_obj.use_df, 'to_json'): if len(ind_obj.use_df.index) == 0: log.critical( f'indicator failure report for ' f'last module: ' f'{ind_obj_path} ' f'indicator={ind_obj.get_name()} ' f'config={ind_obj_config} ' f'dataset={ind_obj.use_df.head(5)} ' f'name_of_dataset={ind_obj.uses_data}') log.critical( '--------------------------------------' '--------------------------------------') log.critical('Please check if this indicator: ' f'{ind_obj_path} ' 'supports Empty Dataframes') log.critical( '--------------------------------------' '--------------------------------------') found_error_hint = True # indicator error hints if not found_error_hint: log.critical( f'indicator failure report for last module: ' f'{ind_obj_path} ' f'indicator={ind_obj.get_name()} ' f'config={ind_obj_config} ' f'dataset={ind_obj.use_df.head(5)} ' f'name_of_dataset={ind_obj.uses_data}') except Exception as f: log.critical(f'failed to pull indicator processor ' f'last indicator for debugging ' f'from ex={e} with parsing ex={f}') # end of ignoring non-supported ways of creating # indicator processors log.error(msg) log.error(f'algo failure report: ' f'algo={a_name} handle_data() ' f'config={a_config_dict} ') log.critical(f'algo failed during operation: {a_debug_msg}') raise e else: log.error(msg) return build_result.build_result(status=ae_consts.ERR, err=msg, rec=rec) # end of try/ex # this could be a separate celery task try: if verbose: log.info(f'get_result START - {percent_label} from ' f'{first_extract_date} to {last_extract_date}') rec = algo.get_result() status = ae_consts.SUCCESS if verbose: log.info(f'get_result END - {percent_label} from ' f'{first_extract_date} to {last_extract_date}') except Exception as e: msg = ( f'{percent_label} - algo={algo.get_name()} encountered exception ' f'in get_result tickers={use_tickers} from ' f'{first_extract_date} to {last_extract_date} ex={e}') if raise_on_err: if algo: log.error(f'algo={algo.get_name()} failed in get_result with ' f'debug_msg={algo.get_debug_msg()}') log.error(msg) raise e else: log.error(msg) return build_result.build_result(status=ae_consts.ERR, err=msg, rec=rec) # end of try/ex if return_algo: rec['algo'] = algo return build_result.build_result(status=status, err=msg, rec=rec)
def run_distributed_algorithm(self, algo_req): """run_distributed_algorithm Process a distributed Algorithm :param algo_req: dictionary for key/values for running an algorithm using Celery workers """ label = algo_req.get('name', 'ae-algo') verbose = algo_req.get('verbose', False) debug = algo_req.get('debug', False) # please be careful logging prod passwords: if verbose or debug: log.info('task - {} - start ' 'algo_req={}'.format(label, algo_req)) else: log.info('task - {} - start '.format(label)) # end of start log rec = {} res = build_result.build_result(status=ae_consts.NOT_RUN, err=None, rec=rec) created_algo_object = None custom_algo_module = None new_algo_object = None use_custom_algo = False found_algo_module = True # assume the BaseAlgo should_publish_extract_dataset = False should_publish_history_dataset = False should_publish_report_dataset = False ticker = algo_req.get('ticker', 'SPY') num_days_back = algo_req.get('num_days_back', 75) name = algo_req.get('name', 'ae-algo') algo_module_path = algo_req.get('mod_path', None) module_name = algo_req.get('module_name', 'BaseAlgo') custom_algo_module = algo_req.get('custom_algo_module', None) new_algo_object = algo_req.get('new_algo_object', None) use_custom_algo = algo_req.get('use_custom_algo', False) should_publish_extract_dataset = algo_req.get( 'should_publish_extract_dataset', False) should_publish_history_dataset = algo_req.get( 'should_publish_history_dataset', False) should_publish_report_dataset = algo_req.get( 'should_publish_report_dataset', False) start_date = algo_req.get('start_date', None) end_date = algo_req.get('end_date', None) raise_on_err = algo_req.get('raise_on_err', False) report_config = algo_req.get('report_config', None) history_config = algo_req.get('history_config', None) extract_config = algo_req.get('extract_config', None) err = None if algo_module_path: found_algo_module = False module_name = algo_module_path.split('/')[-1] loader = importlib.machinery.SourceFileLoader(module_name, algo_module_path) custom_algo_module = types.ModuleType(loader.name) loader.exec_module(custom_algo_module) use_custom_algo = True for member in inspect.getmembers(custom_algo_module): if module_name in str(member): found_algo_module = True break # for all members in this custom module file # if loading a custom algorithm module from a file on disk if not found_algo_module: err = ('{} - unable to find custom algorithm module={} ' 'module_path={}'.format(label, custom_algo_module, algo_module_path)) if algo_module_path: err = ( '{} - analysis_engine.work_tasks.run_distributed_algorithm ' 'was unable ' 'to find custom algorithm module={} with provided path to \n ' 'file: {} \n' '\n' 'Please confirm ' 'that the class inherits from the BaseAlgo class like:\n' '\n' 'import analysis_engine.algo\n' 'class MyAlgo(analysis_engine.algo.BaseAlgo):\n ' '\n' 'If it is then please file an issue on github:\n ' 'https://github.com/AlgoTraders/stock-analysis-engine/' 'issues/new \n\nFor now this error results in a shutdown' '\n'.format(label, custom_algo_module, algo_module_path)) # if algo_module_path set log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # if not found_algo_module use_start_date = start_date use_end_date = end_date if not use_end_date: end_date = datetime.datetime.utcnow() use_end_date = end_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT) if not use_start_date: start_date = end_date - datetime.timedelta(days=num_days_back) use_start_date = start_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT) dataset_publish_extract = algo_req.get('dataset_publish_extract', False) dataset_publish_history = algo_req.get('dataset_publish_history', False) dataset_publish_report = algo_req.get('dataset_publish_report', False) try: if use_custom_algo: log.info('inspecting {} for class {}'.format( custom_algo_module, module_name)) use_class_member_object = None for member in inspect.getmembers(custom_algo_module): if module_name in str(member): log.info('start {} with {}'.format(name, member[1])) use_class_member_object = member break # end of looking over the class definition but did not find it if use_class_member_object: new_algo_object = member[1](**algo_req) else: err = ('{} - did not find a derived ' 'analysis_engine.algo.BaseAlgo ' 'class in the module file={} ' 'for ticker={} algo_name={}'.format( label, algo_module_path, ticker, name)) log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # end of finding a valid algorithm object else: new_algo_object = ae_algo.BaseAlgo(**algo_req) # if using a custom module path or the BaseAlgo if new_algo_object: # heads up - logging this might have passwords in the algo_req # log.debug( # '{} algorithm request: {}'.format( # name, # algo_req)) log.info('{} - run ticker={} from {} to {}'.format( name, ticker, use_start_date, use_end_date)) algo_res = run_algo.run_algo(algo=new_algo_object, raise_on_err=raise_on_err, **algo_req) created_algo_object = new_algo_object log.info('{} - run ticker={} from {} to {}'.format( name, ticker, use_start_date, use_end_date)) if custom_algo_module: log.info( '{} - done run_algo custom_algo_module={} module_name={} ' 'ticker={} from {} to {}'.format(name, custom_algo_module, module_name, ticker, use_start_date, use_end_date)) else: log.info('{} - done run_algo BaseAlgo ticker={} from {} ' 'to {}'.format(name, ticker, use_start_date, use_end_date)) else: err = ('{} - missing a derived analysis_engine.algo.BaseAlgo ' 'class in the module file={} for ' 'ticker={} algo_name={}'.format(label, algo_module_path, ticker, name)) log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # end of finding a valid algorithm object if not created_algo_object: err = ('{} - failed creating algorithm object - ' 'ticker={} status={} error={}' 'algo name={} custom_algo_module={} module_name={} ' 'from {} to {}'.format( label, ticker, ae_consts.get_status(status=algo_res['status']), algo_res['err'], name, custom_algo_module, module_name, use_start_date, use_end_date)) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # end of stop early if should_publish_extract_dataset or dataset_publish_extract: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (extract_config['redis_address'] and extract_config['redis_db'] and extract_config['redis_key']): redis_log = 'redis://{}@{}/{}'.format( extract_config['redis_address'], extract_config['redis_db'], extract_config['redis_key']) use_log += ' {}'.format(redis_log) else: extract_config['redis_enabled'] = False if (extract_config['s3_address'] and extract_config['s3_bucket'] and extract_config['s3_key']): s3_log = 's3://{}/{}/{}'.format(extract_config['s3_address'], extract_config['s3_bucket'], extract_config['s3_key']) use_log += ' {}'.format(s3_log) else: extract_config['s3_enabled'] = False if extract_config['output_file']: file_log = 'file:{}'.format(extract_config['output_file']) use_log += ' {}'.format(file_log) log.info('{} - publish - start ticker={} algorithm-ready {}' ''.format(name, ticker, use_log)) publish_status = created_algo_object.publish_input_dataset( **extract_config) if publish_status != ae_consts.SUCCESS: msg = ('failed to publish algorithm-ready datasets ' 'with status {} attempted to {}'.format( ae_consts.get_status(status=publish_status), use_log)) log.error(msg) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # end of stop early log.info('{} - publish - done ticker={} algorithm-ready {}' ''.format(name, ticker, use_log)) # if publish the algorithm-ready dataset if should_publish_history_dataset or dataset_publish_history: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (history_config['redis_address'] and history_config['redis_db'] and history_config['redis_key']): redis_log = 'redis://{}@{}/{}'.format( history_config['redis_address'], history_config['redis_db'], history_config['redis_key']) use_log += ' {}'.format(redis_log) if (history_config['s3_address'] and history_config['s3_bucket'] and history_config['s3_key']): s3_log = 's3://{}/{}/{}'.format(history_config['s3_address'], history_config['s3_bucket'], history_config['s3_key']) use_log += ' {}'.format(s3_log) if history_config['output_file']: file_log = 'file:{}'.format(history_config['output_file']) use_log += ' {}'.format(file_log) log.info('{} - publish - start ticker={} trading history {}' ''.format(name, ticker, use_log)) publish_status = \ created_algo_object.publish_trade_history_dataset( **history_config) if publish_status != ae_consts.SUCCESS: msg = ('failed to publish trading history datasets ' 'with status {} attempted to {}'.format( ae_consts.get_status(status=publish_status), use_log)) log.error(msg) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # end of stop early log.info('{} - publish - done ticker={} trading history {}' ''.format(name, ticker, use_log)) # if publish an trading history dataset if should_publish_report_dataset or dataset_publish_report: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (report_config['redis_address'] and report_config['redis_db'] and report_config['redis_key']): redis_log = 'redis://{}@{}/{}'.format( report_config['redis_address'], report_config['redis_db'], report_config['redis_key']) use_log += ' {}'.format(redis_log) if (report_config['s3_address'] and report_config['s3_bucket'] and report_config['s3_key']): s3_log = 's3://{}/{}/{}'.format(report_config['s3_address'], report_config['s3_bucket'], report_config['s3_key']) use_log += ' {}'.format(s3_log) if report_config['output_file']: file_log = ' file:{}'.format(report_config['output_file']) use_log += ' {}'.format(file_log) log.info('{} - publishing ticker={} trading performance report {}' ''.format(name, ticker, use_log)) publish_status = created_algo_object.publish_report_dataset( **report_config) if publish_status != ae_consts.SUCCESS: msg = ('failed to publish trading performance report datasets ' 'with status {} attempted to {}'.format( ae_consts.get_status(status=publish_status), use_log)) log.error(msg) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # end of stop early log.info( '{} - publish - done ticker={} trading performance report {}' ''.format(name, ticker, use_log)) # if publish an trading performance report dataset log.info( '{} - done publishing datasets for ticker={} from {} to {}'.format( name, ticker, use_start_date, use_end_date)) res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) except Exception as e: res = build_result.build_result( status=ae_consts.ERR, err=('failed - run_distributed_algorithm ' 'dict={} with ex={}').format(algo_req, e), rec=rec) log.error('{} - {}'.format(label, res['err'])) # end of try/ex log.info('task - run_distributed_algorithm done - ' '{} - status={}'.format(label, ae_consts.get_status(res['status']))) return get_task_results.get_task_results(work_dict=algo_req, result=res)
def run_custom_algo( mod_path, ticker='SPY', balance=50000, commission=6.0, start_date=None, end_date=None, name='myalgo', auto_fill=True, config_file=None, config_dict=None, load_from_s3_bucket=None, load_from_s3_key=None, load_from_redis_key=None, load_from_file=None, load_compress=False, load_publish=True, load_config=None, report_redis_key=None, report_s3_bucket=None, report_s3_key=None, report_file=None, report_compress=False, report_publish=True, report_config=None, history_redis_key=None, history_s3_bucket=None, history_s3_key=None, history_file=None, history_compress=False, history_publish=True, history_config=None, extract_redis_key=None, extract_s3_bucket=None, extract_s3_key=None, extract_file=None, extract_save_dir=None, extract_compress=False, extract_publish=True, extract_config=None, publish_to_s3=True, publish_to_redis=True, publish_to_slack=True, dataset_type=ae_consts.SA_DATASET_TYPE_ALGO_READY, serialize_datasets=ae_consts.DEFAULT_SERIALIZED_DATASETS, compress=False, encoding='utf-8', redis_enabled=True, redis_key=None, redis_address=None, redis_db=None, redis_password=None, redis_expire=None, redis_serializer='json', redis_encoding='utf-8', s3_enabled=True, s3_key=None, s3_address=None, s3_bucket=None, s3_access_key=None, s3_secret_key=None, s3_region_name=None, s3_secure=False, slack_enabled=False, slack_code_block=False, slack_full_width=False, timeseries=None, trade_strategy=None, verbose=False, debug=False, dataset_publish_extract=False, dataset_publish_history=False, dataset_publish_report=False, run_on_engine=False, auth_url=ae_consts.WORKER_BROKER_URL, backend_url=ae_consts.WORKER_BACKEND_URL, include_tasks=ae_consts.INCLUDE_TASKS, ssl_options=ae_consts.SSL_OPTIONS, transport_options=ae_consts.TRANSPORT_OPTIONS, path_to_config_module=ae_consts.WORKER_CELERY_CONFIG_MODULE, raise_on_err=True): """run_custom_algo Run a custom algorithm that derives the ``analysis_engine.algo.BaseAlgo`` class .. note:: Make sure to only have **1** class defined in an algo module. Imports from other modules should work just fine. **Algorithm arguments** :param mod_path: file path to custom algorithm class module :param ticker: ticker symbol :param balance: float - starting balance capital for creating buys and sells :param commission: float - cost pet buy or sell :param name: string - name for tracking algorithm in the logs :param start_date: string - start date for backtest with format ``YYYY-MM-DD HH:MM:SS`` :param end_date: end date for backtest with format ``YYYY-MM-DD HH:MM:SS`` :param auto_fill: optional - boolean for auto filling buy and sell orders for backtesting (default is ``True``) :param config_file: path to a json file containing custom algorithm object member values (like indicator configuration and predict future date units ahead for a backtest) :param config_dict: optional - dictionary that can be passed to derived class implementations of: ``def load_from_config(config_dict=config_dict)`` **Timeseries** :param timeseries: optional - string to set ``day`` or ``minute`` backtesting or live trading (default is ``minute``) **Trading Strategy** :param trade_strategy: optional - string to set the type of ``Trading Strategy`` for backtesting or live trading (default is ``count``) **Running Distributed Algorithms on the Engine Workers** :param run_on_engine: optional - boolean flag for publishing custom algorithms to Celery ae workers for distributing algorithm workloads (default is ``False`` which will run algos locally) this is required for distributing algorithms :param auth_url: Celery broker address (default is ``redis://localhost:6379/11`` or ``analysis_engine.consts.WORKER_BROKER_URL`` environment variable) this is required for distributing algorithms :param backend_url: Celery backend address (default is ``redis://localhost:6379/12`` or ``analysis_engine.consts.WORKER_BACKEND_URL`` environment variable) this is required for distributing algorithms :param include_tasks: list of modules containing tasks to add (default is ``analysis_engine.consts.INCLUDE_TASKS``) :param ssl_options: security options dictionary (default is ``analysis_engine.consts.SSL_OPTIONS``) :param trasport_options: transport options dictionary (default is ``analysis_engine.consts.TRANSPORT_OPTIONS``) :param path_to_config_module: config module for advanced Celery worker connectivity requirements (default is ``analysis_engine.work_tasks.celery_config`` or ``analysis_engine.consts.WORKER_CELERY_CONFIG_MODULE``) **Load Algorithm-Ready Dataset From Source** Use these arguments to load algorithm-ready datasets from supported sources (file, s3 or redis) :param load_from_s3_bucket: optional - string load the algo from an a previously-created s3 bucket holding an s3 key with an algorithm-ready dataset for use with: ``handle_data`` :param load_from_s3_key: optional - string load the algo from an a previously-created s3 key holding an algorithm-ready dataset for use with: ``handle_data`` :param load_from_redis_key: optional - string load the algo from a a previously-created redis key holding an algorithm-ready dataset for use with: ``handle_data`` :param load_from_file: optional - string path to a previously-created local file holding an algorithm-ready dataset for use with: ``handle_data`` :param load_compress: optional - boolean flag for toggling to decompress or not when loading an algorithm-ready dataset (``True`` means the dataset must be decompressed to load correctly inside an algorithm to run a backtest) :param load_publish: boolean - toggle publishing the load progress to slack, s3, redis or a file (default is ``True``) :param load_config: optional - dictionary for setting member variables to load an agorithm-ready dataset from a file, s3 or redis **Publishing Control Bool Flags** :param publish_to_s3: optional - boolean for toggling publishing to s3 on/off (default is ``True``) :param publish_to_redis: optional - boolean for publishing to redis on/off (default is ``True``) :param publish_to_slack: optional - boolean for publishing to slack (default is ``True``) **Algorithm Trade History Arguments** :param history_redis_key: optional - string where the algorithm trading history will be stored in an redis key :param history_s3_bucket: optional - string where the algorithm trading history will be stored in an s3 bucket :param history_s3_key: optional - string where the algorithm trading history will be stored in an s3 key :param history_file: optional - string key where the algorithm trading history will be stored in a file serialized as a json-string :param history_compress: optional - boolean flag for toggling to decompress or not when loading an algorithm-ready dataset (``True`` means the dataset will be compressed on publish) :param history_publish: boolean - toggle publishing the history to s3, redis or a file (default is ``True``) :param history_config: optional - dictionary for setting member variables to publish an algo ``trade history`` to s3, redis, a file or slack **Algorithm Trade Performance Report Arguments (Output Dataset)** :param report_redis_key: optional - string where the algorithm ``trading performance report`` (report) will be stored in an redis key :param report_s3_bucket: optional - string where the algorithm report will be stored in an s3 bucket :param report_s3_key: optional - string where the algorithm report will be stored in an s3 key :param report_file: optional - string key where the algorithm report will be stored in a file serialized as a json-string :param report_compress: optional - boolean flag for toggling to decompress or not when loading an algorithm-ready dataset (``True`` means the dataset will be compressed on publish) :param report_publish: boolean - toggle publishing the ``trading performance report`` s3, redis or a file (default is ``True``) :param report_config: optional - dictionary for setting member variables to publish an algo ``trading performance report`` to s3, redis, a file or slack **Extract an Algorithm-Ready Dataset Arguments** :param extract_redis_key: optional - string where the algorithm report will be stored in an redis key :param extract_s3_bucket: optional - string where the algorithm report will be stored in an s3 bucket :param extract_s3_key: optional - string where the algorithm report will be stored in an s3 key :param extract_file: optional - string key where the algorithm report will be stored in a file serialized as a json-string :param extract_save_dir: optional - string path to auto-generated files from the algo :param extract_compress: optional - boolean flag for toggling to decompress or not when loading an algorithm-ready dataset (``True`` means the dataset will be compressed on publish) :param extract_publish: boolean - toggle publishing the used ``algorithm-ready dataset`` to s3, redis or a file (default is ``True``) :param extract_config: optional - dictionary for setting member variables to publish an algo ``trading performance report`` to s3, redis, a file or slack **Dataset Arguments** :param dataset_type: optional - dataset type (default is ``SA_DATASET_TYPE_ALGO_READY``) :param serialize_datasets: optional - list of dataset names to deserialize in the dataset (default is ``DEFAULT_SERIALIZED_DATASETS``) :param encoding: optional - string for data encoding **Publish Algorithm Datasets to S3, Redis or a File** :param dataset_publish_extract: optional - bool for publishing the algorithm's ``algorithm-ready`` dataset to: s3, redis or file :param dataset_publish_history: optional - bool for publishing the algorithm's ``trading history`` dataset to: s3, redis or file :param dataset_publish_report: optional - bool for publishing the algorithm's ``trading performance report`` dataset to: s3, redis or file **Redis connectivity arguments** :param redis_enabled: bool - toggle for auto-caching all datasets in Redis (default is ``True``) :param redis_key: string - key to save the data in redis (default is ``None``) :param redis_address: Redis connection string format: ``host:port`` (default is ``localhost:6379``) :param redis_db: Redis db to use (default is ``0``) :param redis_password: optional - Redis password (default is ``None``) :param redis_expire: optional - Redis expire value (default is ``None``) :param redis_serializer: not used yet - support for future pickle objects in redis :param redis_encoding: format of the encoded key in redis **Minio (S3) connectivity arguments** :param s3_enabled: bool - toggle for auto-archiving on Minio (S3) (default is ``True``) :param s3_key: string - key to save the data in redis (default is ``None``) :param s3_address: Minio S3 connection string format: ``host:port`` (default is ``localhost:9000``) :param s3_bucket: S3 Bucket for storing the artifacts (default is ``dev``) which should be viewable on a browser: http://localhost:9000/minio/dev/ :param s3_access_key: S3 Access key (default is ``trexaccesskey``) :param s3_secret_key: S3 Secret key (default is ``trex123321``) :param s3_region_name: S3 region name (default is ``us-east-1``) :param s3_secure: Transmit using tls encryption (default is ``False``) **Slack arguments** :param slack_enabled: optional - boolean for publishing to slack :param slack_code_block: optional - boolean for publishing as a code black in slack :param slack_full_width: optional - boolean for publishing as a to slack using the full width allowed **Debugging arguments** :param debug: optional - bool for debug tracking :param verbose: optional - bool for increasing logging :param raise_on_err: boolean - set this to ``False`` on prod to ensure exceptions do not interrupt services. With the default (``True``) any exceptions from the library and your own algorithm are sent back out immediately exiting the backtest. """ module_name = 'BaseAlgo' custom_algo_module = None new_algo_object = None use_custom_algo = False found_algo_module = True should_publish_extract_dataset = False should_publish_history_dataset = False should_publish_report_dataset = False use_config_file = None use_config_dict = config_dict if config_file: if os.path.exists(config_file): use_config_file = config_file if not config_dict: try: use_config_dict = json.loads(open(config_file, 'r').read()) except Exception as e: msg = (f'failed parsing json config_file={config_file} ' f'with ex={e}') log.error(msg) raise Exception(msg) # end of loading the config_file err = None if mod_path: module_name = mod_path.split('/')[-1] loader = importlib.machinery.SourceFileLoader(module_name, mod_path) custom_algo_module = types.ModuleType(loader.name) loader.exec_module(custom_algo_module) use_custom_algo = True for member in inspect.getmembers(custom_algo_module): if module_name in str(member): found_algo_module = True break # for all members in this custom module file # if loading a custom algorithm module from a file on disk if not found_algo_module: err = (f'unable to find custom algorithm module={custom_algo_module}') if mod_path: err = ( 'analysis_engine.run_custom_algo.run_custom_algo was unable ' f'to find custom algorithm module={custom_algo_module} with ' f'provided path to \n file: {mod_path} \n' '\n' 'Please confirm ' 'that the class inherits from the BaseAlgo class like:\n' '\n' 'import analysis_engine.algo\n' 'class MyAlgo(analysis_engine.algo.BaseAlgo):\n ' '\n' 'If it is then please file an issue on github:\n ' 'https://github.com/AlgoTraders/stock-analysis-engine/' 'issues/new \n\nFor now this error results in a shutdown' '\n') # if mod_path set if verbose or debug: log.error(err) return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) # if not found_algo_module use_start_date = start_date use_end_date = end_date if not use_end_date: end_date = datetime.datetime.utcnow() use_end_date = end_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT) if not use_start_date: start_date = end_date - datetime.timedelta(days=75) use_start_date = start_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT) if verbose: log.info( f'{name} {ticker} setting default start_date={use_start_date}') # Load an algorithm-ready dataset from: # file, s3, or redis if not load_config: load_config = build_publish_request.build_publish_request( ticker=ticker, output_file=None, s3_bucket=None, s3_key=None, redis_key=None, compress=load_compress, redis_enabled=publish_to_redis, redis_address=redis_address, redis_db=redis_db, redis_password=redis_password, redis_expire=redis_expire, redis_serializer=redis_serializer, redis_encoding=redis_encoding, s3_enabled=publish_to_s3, s3_address=s3_address, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, slack_enabled=publish_to_slack, slack_code_block=slack_code_block, slack_full_width=slack_full_width, verbose=verbose, label=f'load-{name}') if load_from_file: load_config['output_file'] = load_from_file if load_from_redis_key: load_config['redis_key'] = load_from_redis_key load_config['redis_enabled'] = True if load_from_s3_bucket and load_from_s3_key: load_config['s3_bucket'] = load_from_s3_bucket load_config['s3_key'] = load_from_s3_key load_config['s3_enabled'] = True # end of building load_config dictionary if not already set # Automatically save all datasets to an algorithm-ready: # file, s3, or redis if not extract_config: extract_config = build_publish_request.build_publish_request( ticker=ticker, output_file=None, s3_bucket=None, s3_key=None, redis_key=None, compress=extract_compress, redis_enabled=publish_to_redis, redis_address=redis_address, redis_db=redis_db, redis_password=redis_password, redis_expire=redis_expire, redis_serializer=redis_serializer, redis_encoding=redis_encoding, s3_enabled=publish_to_s3, s3_address=s3_address, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, slack_enabled=publish_to_slack, slack_code_block=slack_code_block, slack_full_width=slack_full_width, verbose=verbose, label=f'extract-{name}') should_publish_extract_dataset = False if extract_file: extract_config['output_file'] = extract_file should_publish_extract_dataset = True if extract_redis_key and publish_to_redis: extract_config['redis_key'] = extract_redis_key extract_config['redis_enabled'] = True should_publish_extract_dataset = True if extract_s3_bucket and extract_s3_key and publish_to_s3: extract_config['s3_bucket'] = extract_s3_bucket extract_config['s3_key'] = extract_s3_key extract_config['s3_enabled'] = True should_publish_extract_dataset = True else: extract_config['s3_enabled'] = False # end of building extract_config dictionary if not already set # Automatically save the trading performance report: # file, s3, or redis if not report_config: report_config = build_publish_request.build_publish_request( ticker=ticker, output_file=None, s3_bucket=None, s3_key=None, redis_key=None, compress=report_compress, redis_enabled=publish_to_redis, redis_address=redis_address, redis_db=redis_db, redis_password=redis_password, redis_expire=redis_expire, redis_serializer=redis_serializer, redis_encoding=redis_encoding, s3_enabled=publish_to_s3, s3_address=s3_address, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, slack_enabled=publish_to_slack, slack_code_block=slack_code_block, slack_full_width=slack_full_width, verbose=verbose, label=f'report-{name}') should_publish_report_dataset = False if report_file: report_config['output_file'] = report_file should_publish_report_dataset = True if report_redis_key and publish_to_redis: report_config['redis_key'] = report_redis_key report_config['redis_enabled'] = True should_publish_report_dataset = True if report_s3_bucket and report_s3_key and publish_to_s3: report_config['s3_bucket'] = report_s3_bucket report_config['s3_key'] = report_s3_key report_config['s3_enabled'] = True should_publish_report_dataset = True # end of building report_config dictionary if not already set # Automatically save the trade history: # file, s3, or redis if not history_config: history_config = build_publish_request.build_publish_request( ticker=ticker, output_file=None, s3_bucket=None, s3_key=None, redis_key=None, compress=report_compress, redis_enabled=publish_to_redis, redis_address=redis_address, redis_db=redis_db, redis_password=redis_password, redis_expire=redis_expire, redis_serializer=redis_serializer, redis_encoding=redis_encoding, s3_enabled=publish_to_s3, s3_address=s3_address, s3_access_key=s3_access_key, s3_secret_key=s3_secret_key, s3_region_name=s3_region_name, s3_secure=s3_secure, slack_enabled=publish_to_slack, slack_code_block=slack_code_block, slack_full_width=slack_full_width, verbose=verbose, label=f'history-{name}') should_publish_history_dataset = False if history_file: history_config['output_file'] = history_file should_publish_history_dataset = True if history_redis_key and publish_to_redis: history_config['redis_key'] = history_redis_key history_config['redis_enabled'] = True should_publish_history_dataset = True if history_s3_bucket and history_s3_key and publish_to_s3: history_config['s3_bucket'] = history_s3_bucket history_config['s3_key'] = history_s3_key history_config['s3_enabled'] = True should_publish_history_dataset = True # end of building history_config dictionary if not already set if verbose: remove_vals = ['s3_access_key', 's3_secret_key', 'redis_password'] debug_extract_config = {} for k in extract_config: if k not in remove_vals: debug_extract_config[k] = extract_config[k] debug_report_config = {} for k in report_config: if k not in remove_vals: debug_report_config[k] = report_config[k] debug_history_config = {} for k in history_config: if k not in remove_vals: debug_history_config[k] = history_config[k] debug_load_config = {} for k in load_config: if k not in remove_vals: debug_load_config[k] = load_config[k] log.info(f'{name} {ticker} using extract config ' f'{ae_consts.ppj(debug_extract_config)}') log.info(f'{name} {ticker} using report config ' f'{ae_consts.ppj(debug_report_config)}') log.info(f'{name} {ticker} using trade history config ' f'{ae_consts.ppj(debug_history_config)}') log.info(f'{name} {ticker} using load config ' f'{ae_consts.ppj(debug_load_config)}') log.info(f'{name} {ticker} - building algo request') # end of verbose algo_req = build_algo_request.build_algo_request( ticker=ticker, balance=balance, commission=commission, start_date=use_start_date, end_date=use_end_date, timeseries=timeseries, trade_strategy=trade_strategy, config_file=use_config_file, config_dict=use_config_dict, load_config=load_config, history_config=history_config, report_config=report_config, extract_config=extract_config, label=name) algo_req['name'] = name algo_req['should_publish_extract_dataset'] = should_publish_extract_dataset algo_req['should_publish_history_dataset'] = should_publish_history_dataset algo_req['should_publish_report_dataset'] = should_publish_report_dataset algo_res = build_result.build_result(status=ae_consts.NOT_RUN, err=None, rec=None) if run_on_engine: rec = {'algo_req': algo_req, 'task_id': None} task_name = ('analysis_engine.work_tasks.' 'task_run_algo.task_run_algo') if verbose: log.info(f'starting distributed algo task={task_name}') elif debug: log.info( 'starting distributed algo by publishing to ' f'task={task_name} broker={auth_url} backend={backend_url}') # Get the Celery app app = get_celery_app.get_celery_app( name=__name__, auth_url=auth_url, backend_url=backend_url, path_to_config_module=path_to_config_module, ssl_options=ssl_options, transport_options=transport_options, include_tasks=include_tasks) if debug: log.info(f'calling distributed algo task={task_name} ' f'request={ae_consts.ppj(algo_req)}') elif verbose: log.info(f'calling distributed algo task={task_name}') job_id = app.send_task(task_name, (algo_req, )) if verbose: log.info(f'calling task={task_name} - success job_id={job_id}') rec['task_id'] = job_id algo_res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) return algo_res # end of run_on_engine if use_custom_algo: if verbose: log.info( f'inspecting {custom_algo_module} for class {module_name}') use_class_member_object = None for member in inspect.getmembers(custom_algo_module): if module_name in str(member): if verbose: log.info(f'start {name} with {member[1]}') use_class_member_object = member break # end of looking over the class definition but did not find it if use_class_member_object: new_algo_object = member[1](**algo_req) else: err = ('did not find a derived analysis_engine.algo.BaseAlgo ' f'class in the module file={mod_path} ' f'for ticker={ticker} algo_name={name}') if verbose or debug: log.error(err) return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) # end of finding a valid algorithm object else: new_algo_object = ae_algo.BaseAlgo(**algo_req) # if using a custom module path or the BaseAlgo if new_algo_object: # heads up - logging this might have passwords in the algo_req # log.debug( # f'{name} algorithm request: {algo_req}') if verbose: log.info(f'{name} - run ticker={ticker} from {use_start_date} ' f'to {use_end_date}') algo_res = run_algo.run_algo(algo=new_algo_object, raise_on_err=raise_on_err, **algo_req) algo_res['algo'] = new_algo_object if verbose: log.info(f'{name} - run ticker={ticker} from {use_start_date} ' f'to {use_end_date}') if custom_algo_module: if verbose: log.info(f'{name} - done run_algo ' f'custom_algo_module={custom_algo_module} ' f'module_name={module_name} ticker={ticker} ' f'from {use_start_date} to {use_end_date}') else: if verbose: log.info(f'{name} - done run_algo BaseAlgo ticker={ticker} ' f'from {use_start_date} to {use_end_date}') else: err = ('missing a derived analysis_engine.algo.BaseAlgo ' f'class in the module file={mod_path} for ticker={ticker} ' f'algo_name={name}') return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) # end of finding a valid algorithm object algo = algo_res.get('algo', None) if not algo: err = (f'failed creating algorithm object - ticker={ticker} ' f'status={ae_consts.get_status(status=algo_res["status"])} ' f'error={algo_res["err"]} algo name={name} ' f'custom_algo_module={custom_algo_module} ' f'module_name={module_name} ' f'from {use_start_date} to {use_end_date}') return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) if should_publish_extract_dataset or dataset_publish_extract: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (extract_config['redis_address'] and extract_config['redis_db'] >= 0 and extract_config['redis_key']): redis_log = ( f'redis://{extract_config["redis_address"]}' f'@{extract_config["redis_db"]}/{extract_config["redis_key"]}') use_log += f' {redis_log}' else: extract_config['redis_enabled'] = False if (extract_config['s3_address'] and extract_config['s3_bucket'] and extract_config['s3_key']): s3_log = ( f's3://{extract_config["s3_address"]}' f'/{extract_config["s3_bucket"]}/{extract_config["s3_key"]}') use_log += f' {s3_log}' else: extract_config['s3_enabled'] = False if extract_config['output_file']: file_log = f'file:{extract_config["output_file"]}' use_log += f' {file_log}' if verbose: log.info(f'{name} - publish - start ticker={ticker} ' f'algorithm-ready {use_log}') publish_status = algo.publish_input_dataset(**extract_config) if publish_status != ae_consts.SUCCESS: msg = ( 'failed to publish algorithm-ready datasets ' f'with status {ae_consts.get_status(status=publish_status)} ' f'attempted to {use_log}') log.error(msg) return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) if verbose: log.info(f'{name} - publish - done ticker={ticker} ' f'algorithm-ready {use_log}') # if publish the algorithm-ready dataset if should_publish_history_dataset or dataset_publish_history: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (history_config['redis_address'] and history_config['redis_db'] >= 0 and history_config['redis_key']): redis_log = ( f'redis://{history_config["redis_address"]}' f'@{history_config["redis_db"]}/{history_config["redis_key"]}') use_log += f' {redis_log}' else: history_config['redis_enabled'] = False if (history_config['s3_address'] and history_config['s3_bucket'] and history_config['s3_key']): s3_log = ( f's3://{history_config["s3_address"]}' f'/{history_config["s3_bucket"]}/{history_config["s3_key"]}') use_log += f' {s3_log}' else: history_config['s3_enabled'] = False if history_config['output_file']: file_log = f'file:{history_config["output_file"]}' use_log += f' {file_log}' if verbose: log.info(f'{name} - publish - start ticker={ticker} trading ' f'history {use_log}') publish_status = algo.publish_trade_history_dataset(**history_config) if publish_status != ae_consts.SUCCESS: msg = ( 'failed to publish trading history datasets ' f'with status {ae_consts.get_status(status=publish_status)} ' f'attempted to {use_log}') log.error(msg) return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) if verbose: log.info(f'{name} - publish - done ticker={ticker} trading ' f'history {use_log}') # if publish an trading history dataset if should_publish_report_dataset or dataset_publish_report: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (report_config['redis_address'] and report_config['redis_db'] >= 0 and report_config['redis_key']): redis_log = ( f'redis://{report_config["redis_address"]}' f'@{report_config["redis_db"]}/{report_config["redis_key"]}') use_log += f' {redis_log}' else: report_config['redis_enabled'] = False if (report_config['s3_address'] and report_config['s3_bucket'] and report_config['s3_key']): s3_log = ( f's3://{report_config["s3_address"]}' f'/{report_config["s3_bucket"]}/{report_config["s3_key"]}') use_log += f' {s3_log}' else: report_config['s3_enabled'] = False if report_config['output_file']: file_log = f'file:{report_config["output_file"]}' use_log += f' {file_log}' if verbose: log.info( f'{name} - publishing ticker={ticker} trading performance ' f'report {use_log}') publish_status = algo.publish_report_dataset(**report_config) if publish_status != ae_consts.SUCCESS: msg = ( 'failed to publish trading performance report datasets ' f'with status {ae_consts.get_status(status=publish_status)} ' f'attempted to {use_log}') log.error(msg) return build_result.build_result(status=ae_consts.ERR, err=err, rec=None) if verbose: log.info( f'{name} - publish - done ticker={ticker} trading performance ' f'report {use_log}') # if publish an trading performance report dataset if verbose: log.info(f'{name} - done publishing datasets for ticker={ticker} ' f'from {use_start_date} to {use_end_date}') return algo_res
def latest(self, date_str=None, start_row=-200, extract_iex=True, extract_yahoo=False, extract_td=True, verbose=False, **kwargs): """latest Run the algorithm with the latest pricing data. Also supports running a backtest for a historical date in the pricing history (format ``YYYY-MM-DD``) :param date_str: optional - string start date ``YYYY-MM-DD`` default is the latest close date :param start_row: negative number of rows back from the end of the list in the data default is ``-200`` where this means the algorithm will process the latest 200 rows in the minute dataset :param extract_iex: bool flag for extracting from ``IEX`` :param extract_yahoo: bool flag for extracting from ``Yahoo`` which is disabled as of 1/2019 :param extract_td: bool flag for extracting from ``Tradier`` :param verbose: bool flag for logs :param kwargs: keyword arg dict """ use_date_str = date_str if not use_date_str: use_date_str = ae_utils.get_last_close_str() log.info(f'creating algo') self.algo_obj = base_algo.BaseAlgo( ticker=self.config_dict['ticker'], balance=self.config_dict['balance'], commission=self.config_dict['commission'], name=self.use_name, start_date=self.use_start_date, end_date=self.use_end_date, auto_fill=self.auto_fill, config_dict=self.config_dict, load_from_s3_bucket=self.load_from_s3_bucket, load_from_s3_key=self.load_from_s3_key, load_from_redis_key=self.load_from_redis_key, load_from_file=self.load_from_file, load_compress=self.load_compress, load_publish=self.load_publish, load_config=self.load_config, report_redis_key=self.report_redis_key, report_s3_bucket=self.report_s3_bucket, report_s3_key=self.report_s3_key, report_file=self.report_file, report_compress=self.report_compress, report_publish=self.report_publish, report_config=self.report_config, history_redis_key=self.history_redis_key, history_s3_bucket=self.history_s3_bucket, history_s3_key=self.history_s3_key, history_file=self.history_file, history_compress=self.history_compress, history_publish=self.history_publish, history_config=self.history_config, extract_redis_key=self.extract_redis_key, extract_s3_bucket=self.extract_s3_bucket, extract_s3_key=self.extract_s3_key, extract_file=self.extract_file, extract_save_dir=self.extract_save_dir, extract_compress=self.extract_compress, extract_publish=self.extract_publish, extract_config=self.extract_config, publish_to_slack=self.publish_to_slack, publish_to_s3=self.publish_to_s3, publish_to_redis=self.publish_to_redis, dataset_type=self.dataset_type, serialize_datasets=self.serialize_datasets, compress=self.compress, encoding=self.encoding, redis_enabled=self.redis_enabled, redis_key=self.redis_key, redis_address=self.redis_address, redis_db=self.redis_db, redis_password=self.redis_password, redis_expire=self.redis_expire, redis_serializer=self.redis_serializer, redis_encoding=self.redis_encoding, s3_enabled=self.s3_enabled, s3_key=self.s3_key, s3_address=self.s3_address, s3_bucket=self.s3_bucket, s3_access_key=self.s3_access_key, s3_secret_key=self.s3_secret_key, s3_region_name=self.s3_region_name, s3_secure=self.s3_secure, slack_enabled=self.slack_enabled, slack_code_block=self.slack_code_block, slack_full_width=self.slack_full_width, dataset_publish_extract=self.extract_publish, dataset_publish_history=self.history_publish, dataset_publish_report=self.report_publish, run_on_engine=self.run_on_engine, auth_url=self.broker_url, backend_url=self.backend_url, include_tasks=self.include_tasks, ssl_options=self.ssl_options, transport_options=self.transport_options, path_to_config_module=self.path_to_config_module, timeseries=self.timeseries, trade_strategy=self.trade_strategy, verbose=False, raise_on_err=self.raise_on_err) log.info(f'run latest - start') ticker = self.config_dict['ticker'] self.common_fetch_vals['base_key'] = f'{ticker}_{use_date_str}' extract_req = api_requests.get_ds_dict( ticker=ticker, base_key=self.common_fetch_vals['base_key'], ds_id=ticker, service_dict=self.common_fetch_vals) node_date_key = use_date_str.replace(f'{ticker}_', '') req = { 'id': use_date_str, 'ticker': ticker, 'date_key': self.common_fetch_vals['base_key'], 'date': node_date_key, 'req': extract_req } # fetch iex_daily_df = None iex_minute_df = None iex_quote_df = None iex_stats_df = None iex_peers_df = None iex_news_df = None iex_financials_df = None iex_earnings_df = None iex_dividends_df = None iex_company_df = None yahoo_option_calls_df = None yahoo_option_puts_df = None yahoo_pricing_df = None yahoo_news_df = None td_calls_df = None td_puts_df = None node_date_key = req['date'] dataset_node_id = req['id'] dataset_id = dataset_node_id label = (f'ticker={ticker} ' f'date={node_date_key}') if verbose: log.info(f'{label} - extract - start') if 'daily' in self.iex_datasets or extract_iex: iex_daily_status, iex_daily_df = \ iex_extract_utils.extract_daily_dataset( extract_req) if iex_daily_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract iex_daily={ticker}') if 'minute' in self.iex_datasets or extract_iex: iex_minute_status, iex_minute_df = \ iex_extract_utils.extract_minute_dataset( extract_req) if iex_minute_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract iex_minute={ticker}') if 'quote' in self.iex_datasets or extract_iex: iex_quote_status, iex_quote_df = \ iex_extract_utils.extract_quote_dataset( extract_req) if iex_quote_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract iex_quote={ticker}') if 'stats' in self.iex_datasets or extract_iex: iex_stats_status, iex_stats_df = \ iex_extract_utils.extract_stats_dataset( extract_req) if iex_stats_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract iex_stats={ticker}') if 'peers' in self.iex_datasets or extract_iex: iex_peers_status, iex_peers_df = \ iex_extract_utils.extract_peers_dataset( extract_req) if iex_peers_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract iex_peers={ticker}') if 'news' in self.iex_datasets or extract_iex: iex_news_status, iex_news_df = \ iex_extract_utils.extract_news_dataset( extract_req) if iex_news_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract iex_news={ticker}') if 'financials' in self.iex_datasets or extract_iex: iex_financials_status, iex_financials_df = \ iex_extract_utils.extract_financials_dataset( extract_req) if iex_financials_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract iex_financials={ticker}') if 'earnings' in self.iex_datasets or extract_iex: iex_earnings_status, iex_earnings_df = \ iex_extract_utils.extract_earnings_dataset( extract_req) if iex_earnings_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract iex_earnings={ticker}') if 'dividends' in self.iex_datasets or extract_iex: iex_dividends_status, iex_dividends_df = \ iex_extract_utils.extract_dividends_dataset( extract_req) if iex_dividends_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract iex_dividends={ticker}') if 'company' in self.iex_datasets or extract_iex: iex_company_status, iex_company_df = \ iex_extract_utils.extract_company_dataset( extract_req) if iex_company_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract iex_company={ticker}') # end of iex extracts if extract_yahoo: yahoo_options_status, yahoo_option_calls_df = \ yahoo_extract_utils.extract_option_calls_dataset( extract_req) yahoo_options_status, yahoo_option_puts_df = \ yahoo_extract_utils.extract_option_puts_dataset( extract_req) if yahoo_options_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract yahoo_options={ticker}') yahoo_pricing_status, yahoo_pricing_df = \ yahoo_extract_utils.extract_pricing_dataset( extract_req) if yahoo_pricing_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract yahoo_pricing={ticker}') yahoo_news_status, yahoo_news_df = \ yahoo_extract_utils.extract_yahoo_news_dataset( extract_req) if yahoo_news_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract yahoo_news={ticker}') # end of yahoo extracts if extract_td: """ Debug by setting: extract_req['verbose_td'] = True """ convert_to_datetime = [ 'date', 'created', 'ask_date', 'bid_date', 'trade_date' ] td_calls_status, td_calls_df = \ td_extract_utils.extract_option_calls_dataset( extract_req) if td_calls_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract tdcalls={ticker}') else: if ae_consts.is_df(df=td_calls_df): for c in convert_to_datetime: if c in td_calls_df: td_calls_df[c] = pd.to_datetime( td_calls_df[c], format=ae_consts.COMMON_TICK_DATE_FORMAT) if 'date' in td_calls_df: td_calls_df.sort_values('date', ascending=True) # end of converting dates td_puts_status, td_puts_df = \ td_extract_utils.extract_option_puts_dataset( extract_req) if td_puts_status != ae_consts.SUCCESS: if verbose: log.warning(f'unable to extract tdputs={ticker}') else: if ae_consts.is_df(df=td_puts_df): for c in convert_to_datetime: if c in td_puts_df: td_puts_df[c] = pd.to_datetime( td_puts_df[c], format=ae_consts.COMMON_TICK_DATE_FORMAT) if 'date' in td_puts_df: td_puts_df.sort_values('date', ascending=True) # end of converting dates # td extracts # map extracted data to DEFAULT_SERIALIZED_DATASETS ticker_data = {} ticker_data['daily'] = iex_daily_df ticker_data['minute'] = iex_minute_df ticker_data['quote'] = iex_quote_df ticker_data['stats'] = iex_stats_df ticker_data['peers'] = iex_peers_df ticker_data['news1'] = iex_news_df ticker_data['financials'] = iex_financials_df ticker_data['earnings'] = iex_earnings_df ticker_data['dividends'] = iex_dividends_df ticker_data['company'] = iex_company_df ticker_data['calls'] = yahoo_option_calls_df ticker_data['puts'] = yahoo_option_puts_df ticker_data['pricing'] = yahoo_pricing_df ticker_data['news'] = yahoo_news_df ticker_data['tdcalls'] = td_calls_df ticker_data['tdputs'] = td_puts_df algo_data_req = { ticker: [{ 'id': dataset_id, # id is currently the cache key in redis 'date': use_date_str, # used to confirm dates in asc order 'data': ticker_data, 'start_row': start_row }] } if verbose: log.info(f'extract - {label} ' f'dataset={len(algo_data_req[ticker])}') # this could be a separate celery task try: if verbose: log.info(f'handle_data START - {label} from ' f'{node_date_key}') self.algo_obj.handle_data(data=algo_data_req) if verbose: log.info(f'handle_data END - {label} from ' f'{node_date_key}') except Exception as e: a_name = self.algo_obj.get_name() a_debug_msg = self.algo_obj.get_debug_msg() if not a_debug_msg: a_debug_msg = 'debug message not set' # a_config_dict = ae_consts.ppj(self.algo_obj.config_dict) msg = (f'{label} - algo={a_name} ' f'encountered exception in handle_data tickers={ticker} ' f'from {node_date_key} ex={e} ' f'and failed during operation: {a_debug_msg}') log.critical(f'{msg}') # end try/ex log.info(f'run latest - create history') history_ds = self.algo_obj.create_history_dataset() self.history_df = pd.DataFrame(history_ds[ticker]) self.determine_latest_times_in_history() self.num_rows = len(self.history_df.index) if verbose: log.info(self.history_df[['minute', 'close']].tail(5)) log.info(f'run latest minute={self.end_date} - ' f'rows={self.num_rows} - done') return self.get_history()