def prepare_pricing_dataset( self, work_dict): """prepare_pricing_dataset Prepare dataset for analysis. Supports loading dataset from s3 if not found in redis. Outputs prepared artifact as a csv to s3 and redis. :param work_dict: dictionary for key/values """ label = 'prepare' log.info( 'task - {} - start ' 'work_dict={}'.format( label, work_dict)) initial_data = None ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID rec = { 'ticker': None, 'ticker_id': None, 's3_enabled': True, 'redis_enabled': True, 's3_bucket': None, 's3_key': None, 'redis_key': None, 'prepared_s3_key': None, 'prepared_s3_bucket': None, 'prepared_redis_key': None, 'prepared_data': None, 'prepared_size': None, 'initial_data': None, 'initial_size': None, 'ignore_columns': None, 'updated': None } res = build_result.build_result( status=ae_consts.NOT_RUN, err=None, rec=rec) try: ticker = work_dict.get( 'ticker', ae_consts.TICKER) ticker_id = int(work_dict.get( 'ticker_id', ae_consts.TICKER_ID)) if not ticker: res = build_result.build_result( status=ae_consts.ERR, err='missing ticker', rec=rec) return res label = work_dict.get( 'label', label) s3_key = work_dict.get( 's3_key', None) s3_bucket_name = work_dict.get( 's3_bucket', 'pricing') s3_access_key = work_dict.get( 's3_access_key', ae_consts.S3_ACCESS_KEY) s3_secret_key = work_dict.get( 's3_secret_key', ae_consts.S3_SECRET_KEY) s3_region_name = work_dict.get( 's3_region_name', ae_consts.S3_REGION_NAME) s3_address = work_dict.get( 's3_address', ae_consts.S3_ADDRESS) s3_secure = work_dict.get( 's3_secure', ae_consts.S3_SECURE) == '1' redis_address = work_dict.get( 'redis_address', ae_consts.REDIS_ADDRESS) redis_key = work_dict.get( 'redis_key', ae_consts.REDIS_KEY) redis_password = work_dict.get( 'redis_password', ae_consts.REDIS_PASSWORD) redis_db = work_dict.get( 'redis_db', None) if not redis_db: redis_db = ae_consts.REDIS_DB redis_expire = None if 'redis_expire' in work_dict: redis_expire = work_dict.get( 'redis_expire', ae_consts.REDIS_EXPIRE) updated = work_dict.get( 'updated', datetime.datetime.utcnow().strftime( '%Y_%m_%d_%H_%M_%S')) prepared_s3_key = work_dict.get( 'prepared_s3_key', '{}_{}.csv'.format( ticker, updated)) prepared_s3_bucket = work_dict.get( 'prepared_s3_bucket', 'prepared') prepared_redis_key = work_dict.get( 'prepared_redis_key', 'prepared') ignore_columns = work_dict.get( 'ignore_columns', None) log.info( '{} redis enabled address={}@{} ' 'key={} prepare_s3={}:{} prepare_redis={} ' 'ignore_columns={}'.format( label, redis_address, redis_db, redis_key, prepared_s3_bucket, prepared_s3_key, prepared_redis_key, ignore_columns)) redis_host = redis_address.split(':')[0] redis_port = redis_address.split(':')[1] enable_s3 = True enable_redis_publish = True rec['ticker'] = ticker rec['ticker_id'] = ticker_id rec['s3_bucket'] = s3_bucket_name rec['s3_key'] = s3_key rec['redis_key'] = redis_key rec['prepared_s3_key'] = prepared_s3_key rec['prepared_s3_bucket'] = prepared_s3_bucket rec['prepared_redis_key'] = prepared_redis_key rec['updated'] = updated rec['s3_enabled'] = enable_s3 rec['redis_enabled'] = enable_redis_publish try: log.info( '{} connecting redis={}:{} ' 'db={} key={} ' 'updated={} expire={}'.format( label, redis_host, redis_port, redis_db, redis_key, updated, redis_expire)) rc = redis.Redis( host=redis_host, port=redis_port, password=redis_password, db=redis_db) except Exception as e: err = ( '{} failed - redis connection to address={}@{} ' 'key={} ex={}'.format( label, redis_address, redis_key, redis_db, e)) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res # end of try/ex for connecting to redis initial_data_res = redis_get.get_data_from_redis_key( label=label, client=rc, key=redis_key) log.info( '{} get redis key={} status={} err={}'.format( label, redis_key, ae_consts.get_status(initial_data_res['status']), initial_data_res['err'])) initial_data = initial_data_res['rec'].get( 'data', None) if enable_s3 and not initial_data: log.info( '{} failed to find redis_key={} trying s3 ' 'from s3_key={} s3_bucket={} s3_address={}'.format( label, redis_key, s3_key, s3_bucket_name, s3_address)) get_from_s3_req = \ api_requests.build_publish_from_s3_to_redis_request() get_from_s3_req['s3_enabled'] = enable_s3 get_from_s3_req['s3_access_key'] = s3_access_key get_from_s3_req['s3_secret_key'] = s3_secret_key get_from_s3_req['s3_region_name'] = s3_region_name get_from_s3_req['s3_address'] = s3_address get_from_s3_req['s3_secure'] = s3_secure get_from_s3_req['s3_key'] = s3_key get_from_s3_req['s3_bucket'] = s3_bucket_name get_from_s3_req['redis_key'] = redis_key get_from_s3_req['label'] = ( '{}-run_publish_from_s3_to_redis'.format( label)) log.info( '{} load from s3={} to ' 'redis={}'.format( label, s3_key, redis_key)) try: # run in synchronous mode: get_from_s3_req['celery_disabled'] = True task_res = s3_to_redis.run_publish_from_s3_to_redis( get_from_s3_req) if task_res.get( 'status', ae_consts.ERR) == ae_consts.SUCCESS: log.info( '{} loaded s3={}:{} ' 'to redis={} retrying'.format( label, s3_bucket_name, s3_key, redis_key)) initial_data_res = redis_get.get_data_from_redis_key( label=label, client=rc, key=redis_key) log.info( '{} get redis try=2 key={} status={} err={}'.format( label, redis_key, ae_consts.get_status(initial_data_res['status']), initial_data_res['err'])) initial_data = initial_data_res['rec'].get( 'data', None) else: err = ( '{} ERR failed loading from bucket={} ' 's3_key={} to redis_key={} with res={}'.format( label, s3_bucket_name, s3_key, redis_key, task_res)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res except Exception as e: err = ( '{} extract from s3 and publish to redis failed loading ' 'data from bucket={} in ' 's3_key={} with publish to redis_key={} ' 'with ex={}'.format( label, s3_bucket_name, s3_key, redis_key, e)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res # end of try/ex for publishing from s3->redis # end of if enable_s3 if not initial_data: err = ( '{} did not find any data to prepare in redis_key={} or ' 's3_key={} in bucket={}'.format( label, redis_key, s3_key, s3_bucket_name)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res initial_data_num_chars = len(str(initial_data)) initial_size_value = None initial_size_str = None if initial_data_num_chars < ae_consts.PREPARE_DATA_MIN_SIZE: err = ( '{} not enough data={} in redis_key={} or ' 's3_key={} in bucket={}'.format( label, initial_data_num_chars, redis_key, s3_key, s3_bucket_name)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res else: initial_size_value = initial_data_num_chars / 1024000 initial_size_str = ae_consts.to_f(initial_size_value) if ae_consts.ev('DEBUG_PREPARE', '0') == '1': log.info( '{} initial - redis_key={} data={}'.format( label, redis_key, str(initial_data))) else: log.info( '{} initial - redis_key={} data size={} MB'.format( label, redis_key, initial_size_str)) # end of trying to get initial_data rec['initial_data'] = initial_data rec['initial_size'] = initial_data_num_chars prepare_data = None try: if ae_consts.ev('DEBUG_PREPARE', '0') == '1': log.info( '{} data={} - flatten - {} MB from ' 'redis_key={}'.format( label, ae_consts.ppj(initial_data), initial_size_str, redis_key)) else: log.info( '{} flatten - {} MB from ' 'redis_key={}'.format( label, initial_size_str, redis_key)) prepare_data = dict_to_csv.flatten_dict( data=initial_data) except Exception as e: prepare_data = None err = ( '{} flatten - convert to csv failed with ex={} ' 'redis_key={}'.format( label, e, redis_key)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res # end of try/ex if not prepare_data: err = ( '{} flatten - did not return any data from redis_key={} ' 'or s3_key={} in bucket={}'.format( label, redis_key, s3_key, s3_bucket_name)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res # end of prepare_data prepare_data_num_chars = len(str(prepare_data)) prepare_size_value = None if prepare_data_num_chars < ae_consts.PREPARE_DATA_MIN_SIZE: err = ( '{} prepare - there is not enough data={} in redis_key={}' ''.format( label, prepare_data_num_chars, redis_key)) log.error(err) res = build_result.build_result( status=ae_consts.ERR, err=err, rec=rec) return res else: prepare_size_value = prepare_data_num_chars / 1024000 prepare_size_str = ae_consts.to_f(prepare_size_value) if ae_consts.ev('DEBUG_PREPARE', '0') == '1': log.info( '{} data={} - prepare - redis_key={}'.format( label, redis_key, ae_consts.ppj(prepare_data))) else: log.info( '{} prepare - redis_key={} data size={} MB'.format( label, redis_key, prepare_size_str)) # end of trying to the size of the prepared data rec['prepared_data'] = prepare_data rec['prepared_size'] = prepare_data_num_chars res = build_result.build_result( status=ae_consts.SUCCESS, err=None, rec=rec) rc = None except Exception as e: res = build_result.build_result( status=ae_consts.ERR, err=( 'failed - prepare_pricing_dataset ' 'dict={} with ex={}').format( work_dict, e), rec=rec) log.error( '{} - {}'.format( label, res['err'])) # end of try/ex log.info( 'task - prepare_pricing_dataset done - ' '{} - status={}'.format( label, ae_consts.get_status(res['status']))) return get_task_results.get_task_results( work_dict=work_dict, result=res)
def publish_ticker_aggregate_from_s3(self, work_dict): """publish_ticker_aggregate_from_s3 Publish Aggregated Ticker Data from S3 to Redis :param work_dict: dictionary for key/values """ label = 'pub-tic-agg-s3-to-redis' log.info(f'task - {label} - start work_dict={work_dict}') ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID rec = { 'ticker': None, 'ticker_id': None, 's3_read_enabled': True, 's3_upload_enabled': True, 'redis_enabled': True, 's3_bucket': None, 's3_compiled_bucket': None, 's3_key': None, 'redis_key': None, 'updated': None } res = build_result.build_result(status=ae_consts.NOT_RUN, err=None, rec=rec) try: ticker = work_dict.get('ticker', ae_consts.TICKER) ticker_id = int(work_dict.get('ticker_id', ae_consts.TICKER_ID)) if not ticker: res = build_result.build_result(status=ae_consts.ERR, err='missing ticker', rec=rec) return res label = work_dict.get('label', label) s3_key = work_dict.get('s3_key', None) s3_bucket_name = work_dict.get('s3_bucket', 'pricing') s3_compiled_bucket_name = work_dict.get('s3_compiled_bucket', 'compileddatasets') redis_key = work_dict.get('redis_key', None) updated = work_dict.get('updated', None) enable_s3_upload = work_dict.get('s3_upload_enabled', ae_consts.ENABLED_S3_UPLOAD) enable_redis_publish = work_dict.get('redis_enabled', ae_consts.ENABLED_REDIS_PUBLISH) serializer = work_dict.get('serializer', 'json') encoding = work_dict.get('encoding', 'utf-8') enable_s3_read = True rec['ticker'] = ticker rec['ticker_id'] = ticker_id rec['s3_bucket'] = s3_bucket_name rec['s3_compiled_bucket'] = s3_compiled_bucket_name rec['s3_key'] = s3_key rec['redis_key'] = redis_key rec['updated'] = updated rec['s3_read_enabled'] = enable_s3_read rec['s3_upload_enabled'] = enable_s3_upload rec['redis_enabled'] = enable_redis_publish if enable_s3_read: log.info(f'{label} parsing s3 values') access_key = work_dict.get('s3_access_key', ae_consts.S3_ACCESS_KEY) secret_key = work_dict.get('s3_secret_key', ae_consts.S3_SECRET_KEY) region_name = work_dict.get('s3_region_name', ae_consts.S3_REGION_NAME) service_address = work_dict.get('s3_address', ae_consts.S3_ADDRESS) secure = work_dict.get('s3_secure', ae_consts.S3_SECURE) == '1' endpoint_url = f'http{"s" if secure else ""}://{service_address}' log.info(f'{label} building s3 endpoint_url={endpoint_url} ' f'region={region_name}') s3 = boto3.resource( 's3', endpoint_url=endpoint_url, aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region_name, config=boto3.session.Config(signature_version='s3v4')) try: log.info(f'{label} checking bucket={s3_bucket_name} exists') if s3.Bucket(s3_bucket_name) not in s3.buckets.all(): log.info(f'{label} creating bucket={s3_bucket_name}') s3.create_bucket(Bucket=s3_bucket_name) except Exception as e: log.info(f'{label} failed creating bucket={s3_bucket_name} ' f'with ex={e}') # end of try/ex for creating bucket try: log.info(f'{label} checking bucket={s3_bucket_name} keys') date_keys = [] keys = [] # {TICKER}_YYYY-DD-MM regex reg = r'^.*_\d{4}-(0?[1-9]|1[012])-(0?[1-9]|[12][0-9]|3[01])$' for bucket in s3.buckets.all(): for key in bucket.objects.all(): if (ticker.lower() in key.key.lower() and bool(re.compile(reg).search(key.key))): keys.append(key.key) date_keys.append(key.key.split(f'{ticker}_')[1]) except Exception as e: log.info(f'{label} failed to get bucket={s3_bucket_name} ' f'keys with ex={e}') # end of try/ex for getting bucket keys if keys: data = [] for idx, key in enumerate(keys): try: log.info( f'{label} reading to s3={s3_bucket_name}/{key} ' f'updated={updated}') loop_data = s3_read_contents_from_key.\ s3_read_contents_from_key( s3=s3, s3_bucket_name=s3_bucket_name, s3_key=key, encoding=encoding, convert_as_json=True) initial_size_value = \ len(str(loop_data)) / 1024000 initial_size_str = ae_consts.to_f(initial_size_value) if ae_consts.ev('DEBUG_S3', '0') == '1': log.info(f'{label} read s3={s3_bucket_name}/{key} ' f'data={ae_consts.ppj(loop_data)}') else: log.info( f'{label} read s3={s3_bucket_name}/{key} data ' f'size={initial_size_str} MB') data.append({f'{date_keys[idx]}': loop_data}) except Exception as e: err = ( f'{label} failed reading bucket={s3_bucket_name} ' f'key={key} ex={e}') log.error(err) res = build_result.build_result( status=ae_consts.NOT_RUN, err=err, rec=rec) # end of try/ex for creating bucket else: log.info(f'{label} No keys found in S3 ' f'bucket={s3_bucket_name} for ticker={ticker}') else: log.info(f'{label} SKIP S3 read bucket={s3_bucket_name} ' f'ticker={ticker}') # end of if enable_s3_read if data and enable_s3_upload: try: log.info(f'{label} checking bucket={s3_compiled_bucket_name} ' 'exists') if s3.Bucket(s3_compiled_bucket_name) not in s3.buckets.all(): log.info( f'{label} creating bucket={s3_compiled_bucket_name}') s3.create_bucket(Bucket=s3_compiled_bucket_name) except Exception as e: log.info(f'{label} failed creating ' f'bucket={s3_compiled_bucket_name} with ex={e}') # end of try/ex for creating bucket try: cmpr_data = zlib.compress(json.dumps(data).encode(encoding), 9) if ae_consts.ev('DEBUG_S3', '0') == '1': log.info( f'{label} uploading to ' f's3={s3_compiled_bucket_name}/{s3_key} ' f'data={ae_consts.ppj(loop_data)} updated={updated}') else: sizes = { 'MB': 1024000, 'GB': 1024000000, 'TB': 1024000000000, 'PB': 1024000000000000 } initial_size_value = len(str(data)) org_data_size = 'MB' for key in sizes.keys(): size = float(initial_size_value) / float(sizes[key]) if size > 1024: continue org_data_size = key initial_size_value = size break initial_size_str = ae_consts.to_f(initial_size_value) cmpr_data_size_value = len(cmpr_data) cmpr_data_size = 'MB' for key in sizes.keys(): size = float(cmpr_data_size_value) / float(sizes[key]) if size > 1024: continue cmpr_data_size = key cmpr_data_size_value = size break cmpr_size_str = ae_consts.to_f(cmpr_data_size_value) log.info( f'{label} uploading to ' f's3={s3_compiled_bucket_name}/{s3_key} data ' f'original_size={initial_size_str} {org_data_size} ' f'compressed_size={cmpr_size_str} {cmpr_data_size} ' f'updated={updated}') s3.Bucket(s3_compiled_bucket_name).put_object(Key=s3_key, Body=cmpr_data) except Exception as e: log.error(f'{label} failed ' f'uploading bucket={s3_compiled_bucket_name} ' f'key={s3_key} ex={e}') # end of try/ex for creating bucket else: log.info( f'{label} SKIP S3 upload bucket={s3_bucket_name} key={s3_key}') # end of if enable_s3_upload if data and enable_redis_publish: redis_address = work_dict.get('redis_address', ae_consts.REDIS_ADDRESS) redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY) redis_password = work_dict.get('redis_password', ae_consts.REDIS_PASSWORD) redis_db = work_dict.get('redis_db', None) if not redis_db: redis_db = ae_consts.REDIS_DB redis_expire = None if 'redis_expire' in work_dict: redis_expire = work_dict.get('redis_expire', ae_consts.REDIS_EXPIRE) log.info(f'redis enabled address={redis_address}@{redis_db} ' f'key={redis_key}') redis_host = redis_address.split(':')[0] redis_port = redis_address.split(':')[1] try: if ae_consts.ev('DEBUG_REDIS', '0') == '1': log.info( f'{label} publishing redis={redis_host}:{redis_port} ' f'db={redis_db} key={redis_key} updated={updated} ' f'expire={redis_expire} data={ae_consts.ppj(data)}') else: log.info( f'{label} publishing redis={redis_host}:{redis_port} ' f'db={redis_db} key={redis_key} ' f'updated={updated} expire={redis_expire}') # end of if/else rc = redis.Redis(host=redis_host, port=redis_port, password=redis_password, db=redis_db) redis_set_res = redis_set.set_data_in_redis_key( label=label, client=rc, key=redis_key, data=data, serializer=serializer, encoding=encoding, expire=redis_expire, px=None, nx=False, xx=False) log.info( f'{label} redis_set ' f'status={ae_consts.get_status(redis_set_res["status"])} ' f'err={redis_set_res["err"]}') except Exception as e: log.error(f'{label} failed - redis publish to ' f'key={redis_key} ex={e}') # end of try/ex for creating bucket else: log.info(f'{label} SKIP REDIS publish key={redis_key}') # end of if enable_redis_publish res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) except Exception as e: res = build_result.build_result( status=ae_consts.ERR, err=(f'failed - publish_from_s3 dict={work_dict} with ex={e}'), rec=rec) log.error(f'{label} - {res["err"]}') # end of try/ex log.info('task - publish_from_s3 done - ' f'{label} - status={ae_consts.get_status(res["status"])}') return get_task_results.get_task_results(work_dict=work_dict, result=res)
def run_distributed_algorithm(self, algo_req): """run_distributed_algorithm Process a distributed Algorithm :param algo_req: dictionary for key/values for running an algorithm using Celery workers """ label = algo_req.get('name', 'ae-algo') verbose = algo_req.get('verbose', False) debug = algo_req.get('debug', False) # please be careful logging prod passwords: if verbose or debug: log.info('task - {} - start ' 'algo_req={}'.format(label, algo_req)) else: log.info('task - {} - start '.format(label)) # end of start log rec = {} res = build_result.build_result(status=ae_consts.NOT_RUN, err=None, rec=rec) created_algo_object = None custom_algo_module = None new_algo_object = None use_custom_algo = False found_algo_module = True # assume the BaseAlgo should_publish_extract_dataset = False should_publish_history_dataset = False should_publish_report_dataset = False ticker = algo_req.get('ticker', 'SPY') num_days_back = algo_req.get('num_days_back', 75) name = algo_req.get('name', 'ae-algo') algo_module_path = algo_req.get('mod_path', None) module_name = algo_req.get('module_name', 'BaseAlgo') custom_algo_module = algo_req.get('custom_algo_module', None) new_algo_object = algo_req.get('new_algo_object', None) use_custom_algo = algo_req.get('use_custom_algo', False) should_publish_extract_dataset = algo_req.get( 'should_publish_extract_dataset', False) should_publish_history_dataset = algo_req.get( 'should_publish_history_dataset', False) should_publish_report_dataset = algo_req.get( 'should_publish_report_dataset', False) start_date = algo_req.get('start_date', None) end_date = algo_req.get('end_date', None) raise_on_err = algo_req.get('raise_on_err', False) report_config = algo_req.get('report_config', None) history_config = algo_req.get('history_config', None) extract_config = algo_req.get('extract_config', None) err = None if algo_module_path: found_algo_module = False module_name = algo_module_path.split('/')[-1] loader = importlib.machinery.SourceFileLoader(module_name, algo_module_path) custom_algo_module = types.ModuleType(loader.name) loader.exec_module(custom_algo_module) use_custom_algo = True for member in inspect.getmembers(custom_algo_module): if module_name in str(member): found_algo_module = True break # for all members in this custom module file # if loading a custom algorithm module from a file on disk if not found_algo_module: err = ('{} - unable to find custom algorithm module={} ' 'module_path={}'.format(label, custom_algo_module, algo_module_path)) if algo_module_path: err = ( '{} - analysis_engine.work_tasks.run_distributed_algorithm ' 'was unable ' 'to find custom algorithm module={} with provided path to \n ' 'file: {} \n' '\n' 'Please confirm ' 'that the class inherits from the BaseAlgo class like:\n' '\n' 'import analysis_engine.algo\n' 'class MyAlgo(analysis_engine.algo.BaseAlgo):\n ' '\n' 'If it is then please file an issue on github:\n ' 'https://github.com/AlgoTraders/stock-analysis-engine/' 'issues/new \n\nFor now this error results in a shutdown' '\n'.format(label, custom_algo_module, algo_module_path)) # if algo_module_path set log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # if not found_algo_module use_start_date = start_date use_end_date = end_date if not use_end_date: end_date = datetime.datetime.utcnow() use_end_date = end_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT) if not use_start_date: start_date = end_date - datetime.timedelta(days=num_days_back) use_start_date = start_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT) dataset_publish_extract = algo_req.get('dataset_publish_extract', False) dataset_publish_history = algo_req.get('dataset_publish_history', False) dataset_publish_report = algo_req.get('dataset_publish_report', False) try: if use_custom_algo: log.info('inspecting {} for class {}'.format( custom_algo_module, module_name)) use_class_member_object = None for member in inspect.getmembers(custom_algo_module): if module_name in str(member): log.info('start {} with {}'.format(name, member[1])) use_class_member_object = member break # end of looking over the class definition but did not find it if use_class_member_object: new_algo_object = member[1](**algo_req) else: err = ('{} - did not find a derived ' 'analysis_engine.algo.BaseAlgo ' 'class in the module file={} ' 'for ticker={} algo_name={}'.format( label, algo_module_path, ticker, name)) log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # end of finding a valid algorithm object else: new_algo_object = ae_algo.BaseAlgo(**algo_req) # if using a custom module path or the BaseAlgo if new_algo_object: # heads up - logging this might have passwords in the algo_req # log.debug( # '{} algorithm request: {}'.format( # name, # algo_req)) log.info('{} - run ticker={} from {} to {}'.format( name, ticker, use_start_date, use_end_date)) algo_res = run_algo.run_algo(algo=new_algo_object, raise_on_err=raise_on_err, **algo_req) created_algo_object = new_algo_object log.info('{} - run ticker={} from {} to {}'.format( name, ticker, use_start_date, use_end_date)) if custom_algo_module: log.info( '{} - done run_algo custom_algo_module={} module_name={} ' 'ticker={} from {} to {}'.format(name, custom_algo_module, module_name, ticker, use_start_date, use_end_date)) else: log.info('{} - done run_algo BaseAlgo ticker={} from {} ' 'to {}'.format(name, ticker, use_start_date, use_end_date)) else: err = ('{} - missing a derived analysis_engine.algo.BaseAlgo ' 'class in the module file={} for ' 'ticker={} algo_name={}'.format(label, algo_module_path, ticker, name)) log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # end of finding a valid algorithm object if not created_algo_object: err = ('{} - failed creating algorithm object - ' 'ticker={} status={} error={}' 'algo name={} custom_algo_module={} module_name={} ' 'from {} to {}'.format( label, ticker, ae_consts.get_status(status=algo_res['status']), algo_res['err'], name, custom_algo_module, module_name, use_start_date, use_end_date)) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # end of stop early if should_publish_extract_dataset or dataset_publish_extract: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (extract_config['redis_address'] and extract_config['redis_db'] and extract_config['redis_key']): redis_log = 'redis://{}@{}/{}'.format( extract_config['redis_address'], extract_config['redis_db'], extract_config['redis_key']) use_log += ' {}'.format(redis_log) else: extract_config['redis_enabled'] = False if (extract_config['s3_address'] and extract_config['s3_bucket'] and extract_config['s3_key']): s3_log = 's3://{}/{}/{}'.format(extract_config['s3_address'], extract_config['s3_bucket'], extract_config['s3_key']) use_log += ' {}'.format(s3_log) else: extract_config['s3_enabled'] = False if extract_config['output_file']: file_log = 'file:{}'.format(extract_config['output_file']) use_log += ' {}'.format(file_log) log.info('{} - publish - start ticker={} algorithm-ready {}' ''.format(name, ticker, use_log)) publish_status = created_algo_object.publish_input_dataset( **extract_config) if publish_status != ae_consts.SUCCESS: msg = ('failed to publish algorithm-ready datasets ' 'with status {} attempted to {}'.format( ae_consts.get_status(status=publish_status), use_log)) log.error(msg) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # end of stop early log.info('{} - publish - done ticker={} algorithm-ready {}' ''.format(name, ticker, use_log)) # if publish the algorithm-ready dataset if should_publish_history_dataset or dataset_publish_history: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (history_config['redis_address'] and history_config['redis_db'] and history_config['redis_key']): redis_log = 'redis://{}@{}/{}'.format( history_config['redis_address'], history_config['redis_db'], history_config['redis_key']) use_log += ' {}'.format(redis_log) if (history_config['s3_address'] and history_config['s3_bucket'] and history_config['s3_key']): s3_log = 's3://{}/{}/{}'.format(history_config['s3_address'], history_config['s3_bucket'], history_config['s3_key']) use_log += ' {}'.format(s3_log) if history_config['output_file']: file_log = 'file:{}'.format(history_config['output_file']) use_log += ' {}'.format(file_log) log.info('{} - publish - start ticker={} trading history {}' ''.format(name, ticker, use_log)) publish_status = \ created_algo_object.publish_trade_history_dataset( **history_config) if publish_status != ae_consts.SUCCESS: msg = ('failed to publish trading history datasets ' 'with status {} attempted to {}'.format( ae_consts.get_status(status=publish_status), use_log)) log.error(msg) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # end of stop early log.info('{} - publish - done ticker={} trading history {}' ''.format(name, ticker, use_log)) # if publish an trading history dataset if should_publish_report_dataset or dataset_publish_report: s3_log = '' redis_log = '' file_log = '' use_log = 'publish' if (report_config['redis_address'] and report_config['redis_db'] and report_config['redis_key']): redis_log = 'redis://{}@{}/{}'.format( report_config['redis_address'], report_config['redis_db'], report_config['redis_key']) use_log += ' {}'.format(redis_log) if (report_config['s3_address'] and report_config['s3_bucket'] and report_config['s3_key']): s3_log = 's3://{}/{}/{}'.format(report_config['s3_address'], report_config['s3_bucket'], report_config['s3_key']) use_log += ' {}'.format(s3_log) if report_config['output_file']: file_log = ' file:{}'.format(report_config['output_file']) use_log += ' {}'.format(file_log) log.info('{} - publishing ticker={} trading performance report {}' ''.format(name, ticker, use_log)) publish_status = created_algo_object.publish_report_dataset( **report_config) if publish_status != ae_consts.SUCCESS: msg = ('failed to publish trading performance report datasets ' 'with status {} attempted to {}'.format( ae_consts.get_status(status=publish_status), use_log)) log.error(msg) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=None) return get_task_results.get_task_results(work_dict=algo_req, result=res) # end of stop early log.info( '{} - publish - done ticker={} trading performance report {}' ''.format(name, ticker, use_log)) # if publish an trading performance report dataset log.info( '{} - done publishing datasets for ticker={} from {} to {}'.format( name, ticker, use_start_date, use_end_date)) res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) except Exception as e: res = build_result.build_result( status=ae_consts.ERR, err=('failed - run_distributed_algorithm ' 'dict={} with ex={}').format(algo_req, e), rec=rec) log.error('{} - {}'.format(label, res['err'])) # end of try/ex log.info('task - run_distributed_algorithm done - ' '{} - status={}'.format(label, ae_consts.get_status(res['status']))) return get_task_results.get_task_results(work_dict=algo_req, result=res)
def get_new_pricing_data(self, work_dict): """get_new_pricing_data Get Ticker information on: - prices - turn off with ``work_dict.get_pricing = False`` - news - turn off with ``work_dict.get_news = False`` - options - turn off with ``work_dict.get_options = False`` :param work_dict: dictionary for key/values """ label = 'get_new_pricing_data' log.debug(f'task - {label} - start ' f'work_dict={work_dict}') num_success = 0 ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID rec = { 'pricing': None, 'options': None, 'calls': None, 'puts': None, 'news': None, 'daily': None, 'minute': None, 'quote': None, 'stats': None, 'peers': None, 'iex_news': None, 'financials': None, 'earnings': None, 'dividends': None, 'company': None, 'exp_date': None, 'publish_pricing_update': None, 'num_success': num_success, 'date': ae_utils.utc_now_str(), 'updated': None, 'version': ae_consts.DATASET_COLLECTION_VERSION } res = {'status': ae_consts.NOT_RUN, 'err': None, 'rec': rec} try: ticker = work_dict.get('ticker', ticker) ticker_id = work_dict.get('ticker_id', ae_consts.TICKER_ID) s3_bucket = work_dict.get('s3_bucket', ae_consts.S3_BUCKET) s3_key = work_dict.get('s3_key', ae_consts.S3_KEY) redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY) exp_date = work_dict.get('exp_date', None) cur_date = ae_utils.last_close() cur_strike = work_dict.get('strike', None) contract_type = str(work_dict.get('contract', 'C')).upper() label = work_dict.get('label', label) iex_datasets = work_dict.get('iex_datasets', iex_consts.DEFAULT_FETCH_DATASETS) td_datasets = work_dict.get('td_datasets', td_consts.DEFAULT_FETCH_DATASETS_TD) fetch_mode = work_dict.get('fetch_mode', ae_consts.FETCH_MODE_ALL) iex_token = work_dict.get('iex_token', iex_consts.IEX_TOKEN) td_token = work_dict.get('td_token', td_consts.TD_TOKEN) str_fetch_mode = str(fetch_mode).lower() # control flags to deal with feed issues: get_iex_data = True get_td_data = True if (fetch_mode == ae_consts.FETCH_MODE_ALL or str_fetch_mode == 'initial'): get_iex_data = True get_td_data = True iex_datasets = ae_consts.IEX_INITIAL_DATASETS elif (fetch_mode == ae_consts.FETCH_MODE_ALL or str_fetch_mode == 'all'): get_iex_data = True get_td_data = True iex_datasets = ae_consts.IEX_DATASETS_DEFAULT elif (fetch_mode == ae_consts.FETCH_MODE_YHO or str_fetch_mode == 'yahoo'): get_iex_data = False get_td_data = False elif (fetch_mode == ae_consts.FETCH_MODE_IEX or str_fetch_mode == 'iex-all'): get_iex_data = True get_td_data = False iex_datasets = ae_consts.IEX_DATASETS_DEFAULT elif (fetch_mode == ae_consts.FETCH_MODE_IEX or str_fetch_mode == 'iex'): get_iex_data = True get_td_data = False iex_datasets = ae_consts.IEX_INTRADAY_DATASETS elif (fetch_mode == ae_consts.FETCH_MODE_INTRADAY or str_fetch_mode == 'intra'): get_iex_data = True get_td_data = True iex_datasets = ae_consts.IEX_INTRADAY_DATASETS elif (fetch_mode == ae_consts.FETCH_MODE_DAILY or str_fetch_mode == 'daily'): get_iex_data = True get_td_data = False iex_datasets = ae_consts.IEX_DAILY_DATASETS elif (fetch_mode == ae_consts.FETCH_MODE_WEEKLY or str_fetch_mode == 'weekly'): get_iex_data = True get_td_data = False iex_datasets = ae_consts.IEX_WEEKLY_DATASETS elif (fetch_mode == ae_consts.FETCH_MODE_TD or str_fetch_mode == 'td'): get_iex_data = False get_td_data = True else: get_iex_data = False get_td_data = False fetch_arr = str_fetch_mode.split(',') found_fetch = False iex_datasets = [] for fetch_name in fetch_arr: if fetch_name not in iex_datasets: if fetch_name == 'iex_min': iex_datasets.append('minute') elif fetch_name == 'iex_day': iex_datasets.append('daily') elif fetch_name == 'iex_quote': iex_datasets.append('quote') elif fetch_name == 'iex_stats': iex_datasets.append('stats') elif fetch_name == 'iex_peers': iex_datasets.append('peers') elif fetch_name == 'iex_news': iex_datasets.append('news') elif fetch_name == 'iex_fin': iex_datasets.append('financials') elif fetch_name == 'iex_earn': iex_datasets.append('earnings') elif fetch_name == 'iex_div': iex_datasets.append('dividends') elif fetch_name == 'iex_comp': iex_datasets.append('company') elif fetch_name == 'td': get_td_data = True else: log.warn('unsupported IEX dataset ' f'{fetch_name}') found_fetch = (len(iex_datasets) != 0) if not found_fetch: log.error(f'{label} - unsupported ' f'fetch_mode={fetch_mode} value') else: get_iex_data = True log.debug(f'{label} - ' f'fetching={len(iex_datasets)} ' f'{iex_datasets} ' f'fetch_mode={fetch_mode}') # end of screening custom fetch_mode settings num_tokens = 0 if get_iex_data: if not iex_token: log.warn(f'{label} - ' 'please set a valid IEX Cloud Account token (' 'https://iexcloud.io/cloud-login/#/register' ') to fetch data from IEX Cloud. It must be ' 'set as an environment variable like: ' 'export IEX_TOKEN=<token>') get_iex_data = False else: num_tokens += 1 # sanity check - disable IEX fetch if the token is not set if get_td_data: missing_td_token = [ 'MISSING_TD_TOKEN', 'SETYOURTDTOKEN', 'SETYOURTRADIERTOKENHERE' ] if td_token in missing_td_token: log.warn(f'{label} - ' 'please set a valid Tradier Account token (' 'https://developer.tradier.com/user/sign_up' ') to fetch pricing data from Tradier. It must be ' 'set as an environment variable like: ' 'export TD_TOKEN=<token>') get_td_data = False else: num_tokens += 1 # sanity check - disable Tradier fetch if the token is not set """ as of Thursday, Jan. 3, 2019: https://developer.yahoo.com/yql/ Important EOL Notice: As of Thursday, Jan. 3, 2019 the YQL service at query.yahooapis.com will be retired """ get_yahoo_data = False if (not get_iex_data and not get_td_data and not get_yahoo_data): err = None if num_tokens == 0: res['status'] = ae_consts.MISSING_TOKEN err = (f'Please set a valid IEX_TOKEN or TD_TOKEN ' f'environment variable') else: err = (f'Please set at least one supported datafeed from ' f'either: ' f'IEX Cloud (fetch -t TICKER -g iex) or ' f'Tradier (fetch -t TICKER -g td) ' f'for ' f'ticker={ticker} ' f'cur_date={cur_date} ' f'IEX enabled={get_iex_data} ' f'TD enabled={get_td_data} ' f'YHO enabled={get_yahoo_data}') res['status'] = ae_consts.ERR res['err'] = err return get_task_results.get_task_results(work_dict=work_dict, result=res) # end of checking that there is at least 1 feed on if not exp_date: exp_date = opt_dates.option_expiration(date=exp_date) else: exp_date = datetime.datetime.strptime(exp_date, '%Y-%m-%d') rec['updated'] = cur_date.strftime('%Y-%m-%d %H:%M:%S') log.debug(f'{label} getting pricing for ticker={ticker} ' f'cur_date={cur_date} exp_date={exp_date} ' f'IEX={get_iex_data} ' f'TD={get_td_data} ' f'YHO={get_yahoo_data}') yahoo_rec = { 'ticker': ticker, 'pricing': None, 'options': None, 'calls': None, 'puts': None, 'news': None, 'exp_date': None, 'publish_pricing_update': None, 'date': None, 'updated': None } # disabled on 2019-01-03 if get_yahoo_data: log.debug(f'{label} YHO ticker={ticker}') yahoo_res = yahoo_data.get_data_from_yahoo(work_dict=work_dict) status_str = ae_consts.get_status(status=yahoo_res['status']) if yahoo_res['status'] == ae_consts.SUCCESS: yahoo_rec = yahoo_res['rec'] msg = (f'{label} YHO ticker={ticker} ' f'status={status_str} err={yahoo_res["err"]}') if ae_consts.ev('SHOW_SUCCESS', '0') == '1': log.info(msg) else: log.debug(msg) rec['pricing'] = yahoo_rec.get('pricing', '{}') rec['news'] = yahoo_rec.get('news', '{}') rec['options'] = yahoo_rec.get('options', '{}') rec['calls'] = rec['options'].get('calls', ae_consts.EMPTY_DF_STR) rec['puts'] = rec['options'].get('puts', ae_consts.EMPTY_DF_STR) num_success += 1 else: log.error(f'{label} failed YHO ticker={ticker} ' f'status={status_str} err={yahoo_res["err"]}') # end of get from yahoo if get_iex_data: num_iex_ds = len(iex_datasets) log.debug(f'{label} IEX datasets={num_iex_ds}') for idx, ft_type in enumerate(iex_datasets): dataset_field = iex_consts.get_ft_str(ft_type=ft_type) log.debug(f'{label} IEX={idx}/{num_iex_ds} ' f'field={dataset_field} ticker={ticker}') iex_label = f'{label}-{dataset_field}' iex_req = copy.deepcopy(work_dict) iex_req['label'] = iex_label iex_req['ft_type'] = ft_type iex_req['field'] = dataset_field iex_req['ticker'] = ticker iex_res = iex_data.get_data_from_iex(work_dict=iex_req) status_str = (ae_consts.get_status(status=iex_res['status'])) if iex_res['status'] == ae_consts.SUCCESS: iex_rec = iex_res['rec'] msg = (f'{label} IEX ticker={ticker} ' f'field={dataset_field} ' f'status={status_str} ' f'err={iex_res["err"]}') if ae_consts.ev('SHOW_SUCCESS', '0') == '1': log.info(msg) else: log.debug(msg) if dataset_field == 'news': rec['iex_news'] = iex_rec['data'] else: rec[dataset_field] = iex_rec['data'] num_success += 1 else: log.debug(f'{label} failed IEX ticker={ticker} ' f'field={dataset_field} ' f'status={status_str} err={iex_res["err"]}') # end of if/else succcess # end idx, ft_type in enumerate(iex_datasets): # end of if get_iex_data if get_td_data: num_td_ds = len(td_datasets) log.debug(f'{label} TD datasets={num_td_ds}') for idx, ft_type in enumerate(td_datasets): dataset_field = td_consts.get_ft_str_td(ft_type=ft_type) log.debug(f'{label} TD={idx}/{num_td_ds} ' f'field={dataset_field} ticker={ticker}') td_label = (f'{label}-{dataset_field}') td_req = copy.deepcopy(work_dict) td_req['label'] = td_label td_req['ft_type'] = ft_type td_req['field'] = dataset_field td_req['ticker'] = ticker td_res = td_data.get_data_from_td(work_dict=td_req) status_str = (ae_consts.get_status(status=td_res['status'])) if td_res['status'] == ae_consts.SUCCESS: td_rec = td_res['rec'] msg = (f'{label} TD ticker={ticker} ' f'field={dataset_field} ' f'status={status_str} ' f'err={td_res["err"]}') if ae_consts.ev('SHOW_SUCCESS', '0') == '1': log.info(msg) else: log.debug(msg) if dataset_field == 'tdcalls': rec['tdcalls'] = td_rec['data'] if dataset_field == 'tdputs': rec['tdputs'] = td_rec['data'] else: rec[dataset_field] = td_rec['data'] num_success += 1 else: log.critical(f'{label} failed TD ticker={ticker} ' f'field={dataset_field} ' f'status={status_str} err={td_res["err"]}') # end of if/else succcess # end idx, ft_type in enumerate(td_datasets): # end of if get_td_data rec['num_success'] = num_success update_req = {'data': rec} update_req['ticker'] = ticker update_req['ticker_id'] = ticker_id update_req['strike'] = cur_strike update_req['contract'] = contract_type update_req['s3_enabled'] = work_dict.get('s3_enabled', ae_consts.ENABLED_S3_UPLOAD) update_req['redis_enabled'] = work_dict.get( 'redis_enabled', ae_consts.ENABLED_REDIS_PUBLISH) update_req['s3_bucket'] = s3_bucket update_req['s3_key'] = s3_key update_req['s3_access_key'] = work_dict.get('s3_access_key', ae_consts.S3_ACCESS_KEY) update_req['s3_secret_key'] = work_dict.get('s3_secret_key', ae_consts.S3_SECRET_KEY) update_req['s3_region_name'] = work_dict.get('s3_region_name', ae_consts.S3_REGION_NAME) update_req['s3_address'] = work_dict.get('s3_address', ae_consts.S3_ADDRESS) update_req['s3_secure'] = work_dict.get('s3_secure', ae_consts.S3_SECURE) update_req['redis_key'] = redis_key update_req['redis_address'] = work_dict.get('redis_address', ae_consts.REDIS_ADDRESS) update_req['redis_password'] = work_dict.get('redis_password', ae_consts.REDIS_PASSWORD) update_req['redis_db'] = int( work_dict.get('redis_db', ae_consts.REDIS_DB)) update_req['redis_expire'] = work_dict.get('redis_expire', ae_consts.REDIS_EXPIRE) update_req['updated'] = rec['updated'] update_req['label'] = label update_req['celery_disabled'] = True update_status = ae_consts.NOT_SET try: update_res = publisher.run_publish_pricing_update( work_dict=update_req) update_status = update_res.get('status', ae_consts.NOT_SET) status_str = ae_consts.get_status(status=update_status) if ae_consts.ev('DEBUG_RESULTS', '0') == '1': log.debug(f'{label} update_res ' f'status={status_str} ' f'data={ae_consts.ppj(update_res)}') else: log.debug(f'{label} run_publish_pricing_update ' f'status={status_str}') # end of if/else rec['publish_pricing_update'] = update_res res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) except Exception as f: err = (f'{label} publisher.run_publish_pricing_update failed ' f'with ex={f}') log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=rec) # end of trying to publish results to connected services except Exception as e: res = build_result.build_result(status=ae_consts.ERR, err=('failed - get_new_pricing_data ' f'dict={work_dict} with ex={e}'), rec=rec) log.error(f'{label} - {res["err"]}') # end of try/ex if ae_consts.ev('DATASET_COLLECTION_SLACK_ALERTS', '0') == '1': env_name = 'DEV' if ae_consts.ev('PROD_SLACK_ALERTS', '1') == '1': env_name = 'PROD' done_msg = (f'Dataset collected ticker=*{ticker}* on ' f'env=*{env_name}* ' f'redis_key={redis_key} s3_key={s3_key} ' f'IEX={get_iex_data} ' f'TD={get_td_data} ' f'YHO={get_yahoo_data}') log.debug(f'{label} sending slack msg={done_msg}') if res['status'] == ae_consts.SUCCESS: slack_utils.post_success(msg=done_msg, block=False, jupyter=True) else: slack_utils.post_failure(msg=done_msg, block=False, jupyter=True) # end of if/else success # end of publishing to slack log.debug('task - get_new_pricing_data done - ' f'{label} - status={ae_consts.get_status(res["status"])}') return get_task_results.get_task_results(work_dict=work_dict, result=res)
def task_screener_analysis(self, work_dict): """task_screener_analysis :param work_dict: task dictionary """ label = work_dict.get('label', 'screener') log.info('{} - start'.format(label)) rec = {} res = build_result.build_result(status=ae_consts.NOT_RUN, err=None, rec=rec) """ Input - Set up dataset sources to collect """ ticker = work_dict.get('ticker', None) org_tickers = work_dict.get('tickers', None) if not ticker and not org_tickers: res = build_result.build_result(status=ae_consts.ERR, err='missing ticker or tickers', rec=rec) tickers = [] if not org_tickers: if ticker: tickers = [ticker] else: for t in org_tickers: upper_cased_ticker = str(t).upper() if upper_cased_ticker not in tickers: tickers.append(upper_cased_ticker) # build a unique ticker list # end of ensuring tickers is a unique list of # upper-cased ticker symbol strings # fetch from: 'all', 'iex' or 'yahoo' fetch_mode = work_dict.get('fetch_mode', os.getenv('FETCH_MODE', 'iex')) iex_datasets = work_dict.get( 'iex_datasets', os.getenv('IEX_DATASETS_DEFAULT', ae_consts.IEX_DATASETS_DEFAULT)) # if defined, these are task functions for # calling customiized determine Celery tasks determine_sells_callback = work_dict.get('determine_sells', None) determine_buys_callback = work_dict.get('determine_buys', None) try: log.info('{} fetch={} tickers={} ' 'iex_datasets={} ' 'sell_task={} ' 'buy_task={}'.format(label, fetch_mode, tickers, iex_datasets, determine_sells_callback, determine_buys_callback)) """ Input - Set up required urls for building buckets """ fv_urls = work_dict.get('urls', ) if not fv_urls: res = build_result.build_result( status=ae_consts.ERR, err='missing required urls list of screeners', rec=rec) # stop if something errored out with the # celery helper for turning off celery to debug # without an engine running if res['err']: log.error('{} - tickers={} fetch={} iex_datasets={} ' 'hit validation err={}'.format(label, tickers, fetch_mode, iex_datasets, res['err'])) return get_task_results.get_task_results(work_dict=work_dict, result=res) # end of input validation checks num_urls = len(fv_urls) log.info('{} - running urls={}'.format(label, fv_urls)) fv_dfs = [] for uidx, url in enumerate(fv_urls): log.info('{} - url={}/{} url={}'.format(label, uidx, num_urls, url)) fv_res = finviz_utils.fetch_tickers_from_screener(url=url) if fv_res['status'] == ae_consts.SUCCESS: fv_dfs.append(fv_res['rec']['data']) for ft_tick in fv_res['rec']['tickers']: upper_ft_ticker = ft_tick.upper() if upper_ft_ticker not in tickers: tickers.append(upper_ft_ticker) # end of for all found tickers else: log.error('{} - failed url={}/{} url={}'.format( label, uidx, num_urls, url)) # if success vs log the error # end of urls to get pandas.DataFrame and unique tickers """ Find tickers in screens """ num_tickers = len(tickers) log.info('{} - fetching tickers={} from urls={}'.format( label, num_tickers, num_urls)) """ pull ticker data """ fetch_recs = fetch_utils.fetch(tickers=tickers, fetch_mode=fetch_mode, iex_datasets=iex_datasets) if fetch_recs: rec = fetch_recs """ Output - Where is data getting cached and archived? (this helps to retroactively evaluate trading performance) """ res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) else: err = ('{} - tickers={} failed fetch={} ' 'iex_datasets={}'.format(label, tickers, fetch_mode, iex_datasets)) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=rec) log.info('{} - done'.format(label)) except Exception as e: err = ('{} - tickers={} fetch={} hit ex={} '.format( label, tickers, fetch_mode, e)) log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=rec) # end of try/ex return get_task_results.get_task_results(work_dict=work_dict, result=res)
def get_new_pricing_data(self, work_dict): """get_new_pricing_data Get Ticker information on: - prices - turn off with ``work_dict.get_pricing = False`` - news - turn off with ``work_dict.get_news = False`` - options - turn off with ``work_dict.get_options = False`` :param work_dict: dictionary for key/values """ label = 'get_new_pricing_data' log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict)) ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID rec = { 'pricing': None, 'options': None, 'calls': None, 'puts': None, 'news': None, 'daily': None, 'minute': None, 'quote': None, 'stats': None, 'peers': None, 'iex_news': None, 'financials': None, 'earnings': None, 'dividends': None, 'company': None, 'exp_date': None, 'publish_pricing_update': None, 'date': ae_utils.utc_now_str(), 'updated': None, 'version': ae_consts.DATASET_COLLECTION_VERSION } res = {'status': ae_consts.NOT_RUN, 'err': None, 'rec': rec} try: ticker = work_dict.get('ticker', ticker) ticker_id = work_dict.get('ticker_id', ae_consts.TICKER_ID) s3_bucket = work_dict.get('s3_bucket', ae_consts.S3_BUCKET) s3_key = work_dict.get('s3_key', ae_consts.S3_KEY) redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY) exp_date = work_dict.get('exp_date', None) cur_date = datetime.datetime.utcnow() cur_strike = work_dict.get('strike', None) contract_type = str(work_dict.get('contract', 'C')).upper() label = work_dict.get('label', label) iex_datasets = work_dict.get('iex_datasets', iex_consts.DEFAULT_FETCH_DATASETS) td_datasets = work_dict.get('td_datasets', td_consts.DEFAULT_FETCH_DATASETS_TD) fetch_mode = work_dict.get('fetch_mode', ae_consts.FETCH_MODE_ALL) # control flags to deal with feed issues: get_iex_data = True get_td_data = True if (fetch_mode == ae_consts.FETCH_MODE_ALL or str(fetch_mode).lower() == 'all'): get_iex_data = True get_td_data = True elif (fetch_mode == ae_consts.FETCH_MODE_YHO or str(fetch_mode).lower() == 'yahoo'): get_iex_data = False get_td_data = False elif (fetch_mode == ae_consts.FETCH_MODE_IEX or str(fetch_mode).lower() == 'iex'): get_iex_data = True get_td_data = False elif (fetch_mode == ae_consts.FETCH_MODE_TD or str(fetch_mode).lower() == 'td'): get_iex_data = False get_td_data = True else: log.debug('{} - unsupported fetch_mode={} value'.format( label, fetch_mode)) """ as of Thursday, Jan. 3, 2019: https://developer.yahoo.com/yql/ Important EOL Notice: As of Thursday, Jan. 3, 2019 the YQL service at query.yahooapis.com will be retired """ get_yahoo_data = False if not exp_date: exp_date = opt_dates.option_expiration(date=exp_date) else: exp_date = datetime.datetime.strptime(exp_date, '%Y-%m-%d') rec['updated'] = cur_date.strftime('%Y-%m-%d %H:%M:%S') log.info('{} getting pricing for ticker={} ' 'cur_date={} exp_date={} ' 'yahoo={} iex={}'.format(label, ticker, cur_date, exp_date, get_yahoo_data, get_iex_data)) yahoo_rec = { 'ticker': ticker, 'pricing': None, 'options': None, 'calls': None, 'puts': None, 'news': None, 'exp_date': None, 'publish_pricing_update': None, 'date': None, 'updated': None } # disabled on 2019-01-03 if get_yahoo_data: log.info('{} yahoo ticker={}'.format(label, ticker)) yahoo_res = yahoo_data.get_data_from_yahoo(work_dict=work_dict) if yahoo_res['status'] == ae_consts.SUCCESS: yahoo_rec = yahoo_res['rec'] log.info('{} yahoo ticker={} ' 'status={} err={}'.format( label, ticker, ae_consts.get_status(status=yahoo_res['status']), yahoo_res['err'])) rec['pricing'] = yahoo_rec.get('pricing', '{}') rec['news'] = yahoo_rec.get('news', '{}') rec['options'] = yahoo_rec.get('options', '{}') rec['calls'] = rec['options'].get('calls', ae_consts.EMPTY_DF_STR) rec['puts'] = rec['options'].get('puts', ae_consts.EMPTY_DF_STR) else: log.error('{} failed YAHOO ticker={} ' 'status={} err={}'.format( label, ticker, ae_consts.get_status(status=yahoo_res['status']), yahoo_res['err'])) # end of get from yahoo if get_iex_data: num_iex_ds = len(iex_datasets) log.debug('{} iex datasets={}'.format(label, num_iex_ds)) for idx, ft_type in enumerate(iex_datasets): dataset_field = iex_consts.get_ft_str(ft_type=ft_type) log.info('{} iex={}/{} field={} ticker={}'.format( label, idx, num_iex_ds, dataset_field, ticker)) iex_label = '{}-{}'.format(label, dataset_field) iex_req = copy.deepcopy(work_dict) iex_req['label'] = iex_label iex_req['ft_type'] = ft_type iex_req['field'] = dataset_field iex_req['ticker'] = ticker iex_res = iex_data.get_data_from_iex(work_dict=iex_req) if iex_res['status'] == ae_consts.SUCCESS: iex_rec = iex_res['rec'] log.info( '{} iex ticker={} field={} ' 'status={} err={}'.format( label, ticker, dataset_field, ae_consts.get_status(status=iex_res['status']), iex_res['err'])) if dataset_field == 'news': rec['iex_news'] = iex_rec['data'] else: rec[dataset_field] = iex_rec['data'] else: log.debug( '{} failed IEX ticker={} field={} ' 'status={} err={}'.format( label, ticker, dataset_field, ae_consts.get_status(status=iex_res['status']), iex_res['err'])) # end of if/else succcess # end idx, ft_type in enumerate(iex_datasets): # end of if get_iex_data if get_td_data: num_td_ds = len(td_datasets) log.debug('{} td datasets={}'.format(label, num_td_ds)) for idx, ft_type in enumerate(td_datasets): dataset_field = td_consts.get_ft_str_td(ft_type=ft_type) log.info('{} td={}/{} field={} ticker={}'.format( label, idx, num_td_ds, dataset_field, ticker)) td_label = '{}-{}'.format(label, dataset_field) td_req = copy.deepcopy(work_dict) td_req['label'] = td_label td_req['ft_type'] = ft_type td_req['field'] = dataset_field td_req['ticker'] = ticker td_res = td_data.get_data_from_td(work_dict=td_req) if td_res['status'] == ae_consts.SUCCESS: td_rec = td_res['rec'] log.info('{} td ticker={} field={} ' 'status={} err={}'.format( label, ticker, dataset_field, ae_consts.get_status(status=td_res['status']), td_res['err'])) if dataset_field == 'tdcalls': rec['tdcalls'] = td_rec['data'] if dataset_field == 'tdputs': rec['tdputs'] = td_rec['data'] else: rec[dataset_field] = td_rec['data'] else: log.critical( '{} failed TD ticker={} field={} ' 'status={} err={}'.format( label, ticker, dataset_field, ae_consts.get_status(status=td_res['status']), td_res['err'])) # end of if/else succcess # end idx, ft_type in enumerate(td_datasets): # end of if get_td_data update_req = {'data': rec} update_req['ticker'] = ticker update_req['ticker_id'] = ticker_id update_req['strike'] = cur_strike update_req['contract'] = contract_type update_req['s3_enabled'] = work_dict.get('s3_enabled', ae_consts.ENABLED_S3_UPLOAD) update_req['redis_enabled'] = work_dict.get( 'redis_enabled', ae_consts.ENABLED_REDIS_PUBLISH) update_req['s3_bucket'] = s3_bucket update_req['s3_key'] = s3_key update_req['s3_access_key'] = work_dict.get('s3_access_key', ae_consts.S3_ACCESS_KEY) update_req['s3_secret_key'] = work_dict.get('s3_secret_key', ae_consts.S3_SECRET_KEY) update_req['s3_region_name'] = work_dict.get('s3_region_name', ae_consts.S3_REGION_NAME) update_req['s3_address'] = work_dict.get('s3_address', ae_consts.S3_ADDRESS) update_req['s3_secure'] = work_dict.get('s3_secure', ae_consts.S3_SECURE) update_req['redis_key'] = redis_key update_req['redis_address'] = work_dict.get('redis_address', ae_consts.REDIS_ADDRESS) update_req['redis_password'] = work_dict.get('redis_password', ae_consts.REDIS_PASSWORD) update_req['redis_db'] = int( work_dict.get('redis_db', ae_consts.REDIS_DB)) update_req['redis_expire'] = work_dict.get('redis_expire', ae_consts.REDIS_EXPIRE) update_req['updated'] = rec['updated'] update_req['label'] = label update_req['celery_disabled'] = True update_status = ae_consts.NOT_SET try: update_res = publisher.run_publish_pricing_update( work_dict=update_req) update_status = update_res.get('status', ae_consts.NOT_SET) if ae_consts.ev('DEBUG_RESULTS', '0') == '1': log.info('{} update_res status={} data={}'.format( label, ae_consts.get_status(status=update_status), ae_consts.ppj(update_res))) else: log.info('{} run_publish_pricing_update status={}'.format( label, ae_consts.get_status(status=update_status))) # end of if/else rec['publish_pricing_update'] = update_res res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) except Exception as f: err = ('{} publisher.run_publish_pricing_update failed ' 'with ex={}'.format(label, f)) log.error(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=rec) # end of trying to publish results to connected services except Exception as e: res = build_result.build_result(status=ae_consts.ERR, err=('failed - get_new_pricing_data ' 'dict={} with ex={}').format( work_dict, e), rec=rec) log.error('{} - {}'.format(label, res['err'])) # end of try/ex if ae_consts.ev('DATASET_COLLECTION_SLACK_ALERTS', '0') == '1': env_name = 'DEV' if ae_consts.ev('PROD_SLACK_ALERTS', '1') == '1': env_name = 'PROD' done_msg = ('Dataset collected ticker=*{}* on env=*{}* ' 'redis_key={} s3_key={} iex={} yahoo={}'.format( ticker, env_name, redis_key, s3_key, get_iex_data, get_yahoo_data)) log.debug('{} sending slack msg={}'.format(label, done_msg)) if res['status'] == ae_consts.SUCCESS: slack_utils.post_success(msg=done_msg, block=False, jupyter=True) else: slack_utils.post_failure(msg=done_msg, block=False, jupyter=True) # end of if/else success # end of publishing to slack log.info('task - get_new_pricing_data done - ' '{} - status={}'.format(label, ae_consts.get_status(res['status']))) return get_task_results.get_task_results(work_dict=work_dict, result=res)
def publish_from_s3_to_redis(self, work_dict): """publish_from_s3_to_redis Publish Ticker Data from S3 to Redis :param work_dict: dictionary for key/values """ label = 'pub-s3-to-redis' log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict)) ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID rec = { 'ticker': None, 'ticker_id': None, 's3_enabled': True, 'redis_enabled': True, 's3_bucket': None, 's3_key': None, 'redis_key': None, 'updated': None } res = build_result.build_result(status=ae_consts.NOT_RUN, err=None, rec=rec) try: ticker = work_dict.get('ticker', ae_consts.TICKER) ticker_id = int(work_dict.get('ticker_id', ae_consts.TICKER_ID)) if not ticker: res = build_result.build_result(status=ae_consts.ERR, err='missing ticker', rec=rec) return res s3_key = work_dict.get('s3_key', None) s3_bucket_name = work_dict.get('s3_bucket', 'pricing') redis_key = work_dict.get('redis_key', None) updated = work_dict.get('updated', None) serializer = work_dict.get('serializer', 'json') encoding = work_dict.get('encoding', 'utf-8') label = work_dict.get('label', label) enable_s3_read = True enable_redis_publish = True rec['ticker'] = ticker rec['ticker_id'] = ticker_id rec['s3_bucket'] = s3_bucket_name rec['s3_key'] = s3_key rec['redis_key'] = redis_key rec['updated'] = updated rec['s3_enabled'] = enable_s3_read rec['redis_enabled'] = enable_redis_publish data = None if enable_s3_read: log.info('{} parsing s3 values'.format(label)) access_key = work_dict.get('s3_access_key', ae_consts.S3_ACCESS_KEY) secret_key = work_dict.get('s3_secret_key', ae_consts.S3_SECRET_KEY) region_name = work_dict.get('s3_region_name', ae_consts.S3_REGION_NAME) service_address = work_dict.get('s3_address', ae_consts.S3_ADDRESS) secure = work_dict.get('s3_secure', ae_consts.S3_SECURE) == '1' endpoint_url = 'http://{}'.format(service_address) if secure: endpoint_url = 'https://{}'.format(service_address) log.info('{} building s3 endpoint_url={} ' 'region={}'.format(label, endpoint_url, region_name)) s3 = boto3.resource( 's3', endpoint_url=endpoint_url, aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region_name, config=boto3.session.Config(signature_version='s3v4')) try: log.info('{} checking bucket={} exists'.format( label, s3_bucket_name)) if s3.Bucket(s3_bucket_name) not in s3.buckets.all(): log.info('{} creating bucket={}'.format( label, s3_bucket_name)) s3.create_bucket(Bucket=s3_bucket_name) except Exception as e: log.info('{} failed creating bucket={} ' 'with ex={}'.format(label, s3_bucket_name, e)) # end of try/ex for creating bucket try: log.info('{} reading to s3={}/{} ' 'updated={}'.format(label, s3_bucket_name, s3_key, updated)) data = s3_read_contents_from_key.s3_read_contents_from_key( s3=s3, s3_bucket_name=s3_bucket_name, s3_key=s3_key, encoding=encoding, convert_as_json=True) initial_size_value = \ len(str(data)) / 1024000 initial_size_str = ae_consts.to_f(initial_size_value) if ae_consts.ev('DEBUG_S3', '0') == '1': log.info('{} read s3={}/{} data={}'.format( label, s3_bucket_name, s3_key, ae_consts.ppj(data))) else: log.info('{} read s3={}/{} data size={} MB'.format( label, s3_bucket_name, s3_key, initial_size_str)) except Exception as e: err = ('{} failed reading bucket={} ' 'key={} ex={}').format(label, s3_bucket_name, s3_key, e) log.error(err) res = build_result.build_result(status=ae_consts.NOT_RUN, err=err, rec=rec) # end of try/ex for creating bucket else: log.info('{} SKIP S3 read bucket={} ' 'key={}'.format(label, s3_bucket_name, s3_key)) # end of if enable_s3_read if enable_redis_publish: redis_address = work_dict.get('redis_address', ae_consts.REDIS_ADDRESS) redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY) redis_password = work_dict.get('redis_password', ae_consts.REDIS_PASSWORD) redis_db = work_dict.get('redis_db', None) if not redis_db: redis_db = ae_consts.REDIS_DB redis_expire = None if 'redis_expire' in work_dict: redis_expire = work_dict.get('redis_expire', ae_consts.REDIS_EXPIRE) log.info('redis enabled address={}@{} ' 'key={}'.format(redis_address, redis_db, redis_key)) redis_host = redis_address.split(':')[0] redis_port = redis_address.split(':')[1] try: if ae_consts.ev('DEBUG_REDIS', '0') == '1': log.info('{} publishing redis={}:{} ' 'db={} key={} ' 'updated={} expire={} ' 'data={}'.format(label, redis_host, redis_port, redis_db, redis_key, updated, redis_expire, ae_consts.ppj(data))) else: log.info('{} publishing redis={}:{} ' 'db={} key={} ' 'updated={} expire={}'.format( label, redis_host, redis_port, redis_db, redis_key, updated, redis_expire)) # end of if/else rc = redis.Redis(host=redis_host, port=redis_port, password=redis_password, db=redis_db) redis_set_res = redis_set.set_data_in_redis_key( label=label, client=rc, key=redis_key, data=data, serializer=serializer, encoding=encoding, expire=redis_expire, px=None, nx=False, xx=False) log.info('{} redis_set status={} err={}'.format( label, ae_consts.get_status(redis_set_res['status']), redis_set_res['err'])) except Exception as e: log.error('{} failed - redis publish to ' 'key={} ex={}'.format(label, redis_key, e)) # end of try/ex for creating bucket else: log.info('{} SKIP REDIS publish ' 'key={}'.format(label, redis_key)) # end of if enable_redis_publish res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) except Exception as e: res = build_result.build_result( status=ae_consts.ERR, err=('failed - publish_from_s3_to_redis ' 'dict={} with ex={}').format(work_dict, e), rec=rec) log.error('{} - {}'.format(label, res['err'])) # end of try/ex log.info('task - publish_from_s3_to_redis done - ' '{} - status={}'.format(label, ae_consts.get_status(res['status']))) return get_task_results.get_task_results(work_dict=work_dict, result=res)
def publish_pricing_update(self, work_dict): """publish_pricing_update Publish Ticker Data to S3 and Redis - prices - turn off with ``work_dict.get_pricing = False`` - news - turn off with ``work_dict.get_news = False`` - options - turn off with ``work_dict.get_options = False`` :param work_dict: dictionary for key/values """ label = 'publish_pricing' log.debug(f'task - {label} - start') ticker = ae_consts.TICKER ticker_id = ae_consts.TICKER_ID rec = { 'ticker': None, 'ticker_id': None, 's3_enabled': False, 'redis_enabled': False, 's3_bucket': None, 's3_key': None, 'redis_key': None, 'updated': None } res = build_result.build_result(status=ae_consts.NOT_RUN, err=None, rec=rec) try: ticker = work_dict.get('ticker', ae_consts.TICKER) ticker_id = int(work_dict.get('ticker_id', ae_consts.TICKER_ID)) if not ticker: res = build_result.build_result(status=ae_consts.ERR, err='missing ticker', rec=rec) return res label = work_dict.get('label', label) s3_key = work_dict.get('s3_key', None) s3_bucket_name = work_dict.get('s3_bucket', 'pricing') redis_key = work_dict.get('redis_key', None) data = work_dict.get('data', None) updated = work_dict.get('updated', None) enable_s3_upload = work_dict.get('s3_enabled', ae_consts.ENABLED_S3_UPLOAD) enable_redis_publish = work_dict.get('redis_enabled', ae_consts.ENABLED_REDIS_PUBLISH) serializer = work_dict.get('serializer', 'json') encoding = work_dict.get('encoding', 'utf-8') rec['ticker'] = ticker rec['ticker_id'] = ticker_id rec['s3_bucket'] = s3_bucket_name rec['s3_key'] = s3_key rec['redis_key'] = redis_key rec['updated'] = updated rec['s3_enabled'] = enable_s3_upload rec['redis_enabled'] = enable_redis_publish if (enable_s3_upload and s3_bucket_name != 'MISSING_AN_S3_BUCKET'): access_key = work_dict.get('s3_access_key', ae_consts.S3_ACCESS_KEY) secret_key = work_dict.get('s3_secret_key', ae_consts.S3_SECRET_KEY) region_name = work_dict.get('s3_region_name', ae_consts.S3_REGION_NAME) service_address = work_dict.get('s3_address', ae_consts.S3_ADDRESS) secure = work_dict.get('s3_secure', ae_consts.S3_SECURE) == '1' endpoint_url = f'http{"s" if secure else ""}://{service_address}' log.debug(f'{label} building s3 endpoint_url={endpoint_url} ' f'region={region_name}') s3 = boto3.resource( 's3', endpoint_url=endpoint_url, aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region_name, config=boto3.session.Config(signature_version='s3v4')) try: log.debug(f'{label} checking bucket={s3_bucket_name} exists') if s3.Bucket(s3_bucket_name) not in s3.buckets.all(): log.debug(f'{label} creating bucket={s3_bucket_name}') s3.create_bucket(Bucket=s3_bucket_name) except Exception as e: log.debug(f'{label} failed creating bucket={s3_bucket_name} ' f'with ex={e}') # end of try/ex for creating bucket try: log.debug(f'{label} uploading to s3={s3_bucket_name}/{s3_key} ' f'updated={updated}') s3.Bucket(s3_bucket_name).put_object( Key=s3_key, Body=json.dumps(data).encode(encoding)) except Exception as e: log.error(f'{label} failed uploading bucket={s3_bucket_name} ' f'key={s3_key} ex={e}') # end of try/ex for creating bucket else: log.debug(f'{label} SKIP S3 upload bucket={s3_bucket_name} ' f'key={s3_key}') # end of if enable_s3_upload if enable_redis_publish: redis_address = work_dict.get('redis_address', ae_consts.REDIS_ADDRESS) redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY) redis_password = work_dict.get('redis_password', ae_consts.REDIS_PASSWORD) redis_db = work_dict.get('redis_db', ae_consts.REDIS_DB) redis_expire = None if 'redis_expire' in work_dict: redis_expire = work_dict.get('redis_expire', ae_consts.REDIS_EXPIRE) log.debug(f'redis enabled address={redis_address}@{redis_db} ' f'key={redis_key}') redis_host = None redis_port = None try: redis_host = redis_address.split(':')[0] redis_port = redis_address.split(':')[1] except Exception as c: err = (f'{label} failed parsing redis_address={redis_address} ' f'with ex={c} ' 'please set one with the format: ' '<hostname>:<port>') log.critical(err) res = build_result.build_result(status=ae_consts.ERR, err=err, rec=rec) return res # end of checking that redis_address is valid try: log.debug( f'{label} publishing redis={redis_host}:{redis_port} ' f'db={redis_db} key={redis_key} ' f'updated={updated} expire={redis_expire}') rc = redis.Redis(host=redis_host, port=redis_port, password=redis_password, db=redis_db) already_compressed = False uses_data = data try: uses_data = zlib.compress( json.dumps(data).encode(encoding), 9) already_compressed = True except Exception as p: log.critical('failed to compress dataset for ' f'redis_key={redis_key} with ex={p}') redis_set_res = redis_set.set_data_in_redis_key( label=label, client=rc, key=redis_key, data=uses_data, already_compressed=already_compressed, serializer=serializer, encoding=encoding, expire=redis_expire, px=None, nx=False, xx=False) log.debug( f'{label} redis_set ' f'status={ae_consts.get_status(redis_set_res["status"])} ' f'err={redis_set_res["err"]}') except Exception as e: log.error(f'{label} failed - redis publish to ' f'key={redis_key} ex={e}') # end of try/ex for creating bucket else: log.debug(f'{label} SKIP REDIS publish key={redis_key}') # end of if enable_redis_publish res = build_result.build_result(status=ae_consts.SUCCESS, err=None, rec=rec) except Exception as e: res = build_result.build_result(status=ae_consts.ERR, err=('failed - publish_pricing_update ' f'dict={work_dict} with ex={e}'), rec=rec) log.error(f'{label} - {res["err"]}') # end of try/ex log.debug(f'task - publish_pricing_update done - {label} - ' f'status={ae_consts.get_status(res["status"])}') return get_task_results.get_task_results(work_dict=work_dict, result=res)
def handle_pricing_update_task( self, work_dict): """handle_pricing_update_task Writes pricing updates to S3 and Redis :param work_dict: dictionary for key/values """ label = 'update_prices' log.info( 'task - {} - start'.format( label)) ticker = ae_consts.TICKER ticker_id = 1 rec = { 'ticker': None, 'ticker_id': None, 'pricing_s3_bucket': None, 'pricing_s3_key': None, 'pricing_size': None, 'pricing_redis_key': None, 'news_s3_bucket': None, 'news_s3_key': None, 'news_size': None, 'news_redis_key': None, 'options_s3_bucket': None, 'options_s3_key': None, 'options_size': None, 'options_redis_key': None } res = build_result.build_result( status=ae_consts.NOT_RUN, err=None, rec=rec) try: ticker = work_dict.get( 'ticker', ae_consts.TICKER) ticker_id = int(work_dict.get( 'ticker_id', 1)) rec['ticker'] = ticker rec['ticker_id'] = ticker_id pricing_data = work_dict['pricing'] news_data = work_dict['news'] options_data = work_dict['options'] calls_data = options_data.get( 'calls', ae_consts.EMPTY_DF_STR) puts_data = options_data.get( 'puts', ae_consts.EMPTY_DF_STR) updated = work_dict['updated'] label = work_dict.get( 'label', label) cur_date = datetime.datetime.utcnow() cur_date_str = cur_date.strftime( '%Y_%m_%d_%H_%M_%S') pricing_s3_key = work_dict.get( 'pricing_s3_key', 'pricing_ticker_{}_id_{}_date_{}'.format( ticker, ticker_id, cur_date_str)) news_s3_key = work_dict.get( 'news_s3_key', 'news_ticker_{}_id_{}_date_{}'.format( ticker, ticker_id, cur_date_str)) options_s3_key = work_dict.get( 'options_s3_key', 'options_ticker_{}_id_{}_date_{}'.format( ticker, ticker_id, cur_date_str)) calls_s3_key = work_dict.get( 'calls_s3_key', 'calls_ticker_{}_id_{}_date_{}'.format( ticker, ticker_id, cur_date_str)) puts_s3_key = work_dict.get( 'puts_s3_key', 'puts_ticker_{}_id_{}_date_{}'.format( ticker, ticker_id, cur_date_str)) pricing_s3_bucket = work_dict.get( 'pricing_s3_bucket', 'pricing') news_s3_bucket = work_dict.get( 'news_s3_bucket', 'news') options_s3_bucket = work_dict.get( 'options_s3_bucket', 'options') pricing_by_ticker_redis_key = work_dict.get( 'pricing_redis_key', 'price_{}'.format( ticker)) news_by_ticker_redis_key = work_dict.get( 'news_redis_key', 'news_{}'.format( ticker)) options_by_ticker_redis_key = work_dict.get( 'options_redis_key', 'options_{}'.format( ticker)) calls_by_ticker_redis_key = work_dict.get( 'calls_redis_key', 'calls_{}'.format( ticker)) puts_by_ticker_redis_key = work_dict.get( 'puts_redis_key', 'puts_{}'.format( ticker)) pricing_size = len(str( pricing_data)) news_size = len(str( news_data)) options_size = len(str( options_data)) calls_size = len(str( calls_data)) puts_size = len(str( puts_data)) payloads_to_publish = [ { 'ticker': ticker, 'ticker_id': ticker_id, 's3_bucket': pricing_s3_bucket, 's3_key': pricing_s3_key, 'data': pricing_data, 'redis_key': pricing_by_ticker_redis_key, 'size': pricing_size, 'updated': updated, 'label': label }, { 'ticker': ticker, 'ticker_id': ticker_id, 's3_bucket': options_s3_bucket, 's3_key': options_s3_key, 'data': options_data, 'redis_key': options_by_ticker_redis_key, 'size': options_size, 'updated': updated, 'label': label }, { 'ticker': ticker, 'ticker_id': ticker_id, 's3_bucket': options_s3_bucket, 's3_key': calls_s3_key, 'data': calls_data, 'redis_key': calls_by_ticker_redis_key, 'size': calls_size, 'updated': updated, 'label': label }, { 'ticker': ticker, 'ticker_id': ticker_id, 's3_bucket': options_s3_bucket, 's3_key': puts_s3_key, 'data': puts_data, 'redis_key': puts_by_ticker_redis_key, 'size': puts_size, 'updated': updated, 'label': label }, { 'ticker': ticker, 'ticker_id': ticker_id, 's3_bucket': news_s3_bucket, 's3_key': news_s3_key, 'data': news_data, 'redis_key': news_by_ticker_redis_key, 'size': news_size, 'updated': updated, 'label': label } ] rec['pricing_s3_bucket'] = pricing_s3_bucket rec['pricing_s3_key'] = pricing_s3_key rec['pricing_redis_key'] = pricing_by_ticker_redis_key rec['news_s3_bucket'] = news_s3_bucket rec['news_s3_key'] = news_s3_key rec['news_redis_key'] = news_by_ticker_redis_key rec['options_s3_bucket'] = options_s3_bucket rec['options_s3_key'] = options_s3_bucket rec['options_redis_key'] = options_by_ticker_redis_key total_payloads = len(payloads_to_publish) log.info( '{} ticker={} processing payloads={}'.format( label, ticker, total_payloads)) for ridx, r in enumerate(payloads_to_publish): log.info( '{} ticker={} update={}/{} key={} redis_key={}'.format( label, ticker, ridx, total_payloads, r['s3_key'], r['redis_key'])) r['celery_disabled'] = False r['label'] = 'handle_pricing_update_task-{}'.format( label) payload_res = \ publisher.task_publish_pricing_update( work_dict=r) log.info( '{} ticker={} update={}/{} status={} ' 's3_key={} redis_key={}'.format( label, ticker, ridx, total_payloads, ae_consts.get_status(status=payload_res['status']), r['s3_key'], r['redis_key'])) # end of for all payloads to publish res = build_result.build_result( status=ae_consts.SUCCESS, err=None, rec=rec) except Exception as e: res = build_result.build_result( status=ae_consts.ERR, err=( 'failed - handle_pricing_update_task ' 'dict={} with ex={}').format( work_dict, e), rec=rec) log.error( '{} - {}'.format( label, res['err'])) # end of try/ex log.info( 'task - handle_pricing_update_task done - ' '{} - status={}'.format( label, ae_consts.get_status(res['status']))) return get_task_results.get_task_results( work_dict=work_dict, result=res)