def test_run_daily_indicator_with_algo_config_sell(self):
"""test_run_daily_indicator_with_algo_config_sell"""
algo = base_algo.BaseAlgo(
ticker=self.ticker,
balance=self.balance,
start_date_str=self.start_date_str,
end_date_str=self.end_date_str,
config_dict=self.algo_config_dict)
self.assertEqual(
algo.name,
self.algo_config_dict['name'])
self.assertEqual(
algo.tickers,
[self.ticker])
algo.handle_data(
data=self.data)
res = algo.get_result()
print(ae_consts.ppj(res))
self.assertEqual(
res['history'][0]['total_sells'],
1)
self.assertEqual(
res['history'][0]['total_buys'],
0)
def test_integration_daily_indicator_with_algo_config(self):
"""test_integration_daily_indicator_with_algo_config"""
if ae_consts.ev('INT_TESTS', '0') == '0':
return
algo = base_algo.BaseAlgo(
ticker=self.ticker,
balance=self.balance,
start_date_str=self.start_date_str,
end_date_str=self.end_date_str,
config_dict=self.algo_config_dict)
self.assertEqual(
algo.name,
self.algo_config_dict['name'])
self.assertEqual(
algo.tickers,
[self.ticker])
algo.handle_data(
data=self.data)
res = algo.get_result()
print(ae_consts.ppj(res))
self.assertTrue(
res['history'][0]['total_sells'] >= 1)
self.assertTrue(
res['history'][0]['total_buys'] == 0)
def run_screener_analysis(work_dict):
"""run_screener_analysis
Celery wrapper for running without celery
:param work_dict: task data
"""
fn_name = 'run_screener_analysis'
label = '{} - {}'.format(fn_name, work_dict.get('label', ''))
log.info('{} - start'.format(label))
response = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec={})
task_res = {}
# allow running without celery
if ae_consts.is_celery_disabled(work_dict=work_dict):
work_dict['celery_disabled'] = True
task_res = task_screener_analysis(work_dict)
if task_res:
response = task_res.get('result', task_res)
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
response_details = response
try:
response_details = ae_consts.ppj(response)
except Exception:
response_details = response
log.info('{} task result={}'.format(label, response_details))
else:
log.error('{} celery was disabled but the task={} '
'did not return anything'.format(label, response))
# end of if response
else:
task_res = task_screener_analysis.delay(work_dict=work_dict)
rec = {'task_id': task_res}
response = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
# if celery enabled
if response:
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
log.info('{} - done '
'status={} err={} rec={}'.format(
label, ae_consts.get_status(response['status']),
response['err'], response['rec']))
else:
log.info('{} - done '
'status={} err={}'.format(
label, ae_consts.get_status(response['status']),
response['err']))
else:
log.info('{} - done ' 'no response'.format(label))
# end of if/else response
return response
def run_screener_analysis(work_dict):
"""run_screener_analysis
Celery wrapper for running without celery
:param work_dict: task data
"""
fn_name = 'run_screener_analysis'
label = f'''{fn_name} - {work_dict.get(
'label',
'')}'''
log.info(f'{label} - start')
response = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec={})
task_res = {}
# allow running without celery
if ae_consts.is_celery_disabled(work_dict=work_dict):
work_dict['celery_disabled'] = True
task_res = task_screener_analysis(work_dict)
if task_res:
response = task_res.get('result', task_res)
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
response_details = response
try:
response_details = ae_consts.ppj(response)
except Exception:
response_details = response
log.info(f'{label} task result={response_details}')
else:
log.error(f'{label} celery was disabled but the task={response} '
'did not return anything')
# end of if response
else:
task_res = task_screener_analysis.delay(work_dict=work_dict)
rec = {'task_id': task_res}
response = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
# if celery enabled
if response:
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
log.info(f'{label} - done '
f'status={ae_consts.get_status(response["status"])} '
f'err={response["err"]} rec={response["rec"]}')
else:
log.info(f'{label} - done '
f'status={ae_consts.get_status(response["status"])} '
f'err={response["err"]}')
else:
log.info(f'{label} - done no response')
# end of if/else response
return response
def run_get_new_pricing_data(work_dict):
"""run_get_new_pricing_data
Celery wrapper for running without celery
:param work_dict: task data
"""
label = work_dict.get('label', '')
log.debug(f'run_get_new_pricing_data - {label} - start')
response = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec={})
task_res = {}
# allow running without celery
if ae_consts.is_celery_disabled(work_dict=work_dict):
work_dict['celery_disabled'] = True
task_res = get_new_pricing_data(work_dict)
if task_res:
response = task_res.get('result', task_res)
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
response_details = response
try:
response_details = ae_consts.ppj(response)
except Exception:
response_details = response
log.debug(f'{label} task result={response_details}')
else:
log.error(f'{label} celery was disabled but the task={response} '
'did not return anything')
# end of if response
else:
task_res = get_new_pricing_data.delay(work_dict=work_dict)
rec = {'task_id': task_res}
response = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
# if celery enabled
if response:
status_str = ae_consts.get_status(response['status'])
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
log.debug(f'run_get_new_pricing_data - {label} - done '
f'status={status_str} '
f'err={response["err"]} '
f'rec={response["rec"]}')
else:
log.debug(f'run_get_new_pricing_data - {label} - done '
f'status={status_str} '
f'rec={response["rec"]}')
else:
log.debug(f'run_get_new_pricing_data - {label} - done ' 'no response')
# end of if/else response
return response
def run_publish_ticker_aggregate_from_s3(work_dict):
"""run_publish_ticker_aggregate_from_s3
Celery wrapper for running without celery
:param work_dict: task data
"""
label = work_dict.get('label', '')
log.info('run_publish_ticker_aggregate_from_s3 - {} - start'.format(label))
response = build_result.build_result(status=NOT_RUN, err=None, rec={})
task_res = {}
# allow running without celery
if is_celery_disabled(work_dict=work_dict):
work_dict['celery_disabled'] = True
task_res = publish_ticker_aggregate_from_s3(work_dict=work_dict)
if task_res:
response = task_res.get('result', task_res)
if ev('DEBUG_RESULTS', '0') == '1':
response_details = response
try:
response_details = ppj(response)
except Exception:
response_details = response
log.info('{} task result={}'.format(label, response_details))
else:
log.error('{} celery was disabled but the task={} '
'did not return anything'.format(label, response))
# end of if response
else:
task_res = publish_ticker_aggregate_from_s3.delay(work_dict=work_dict)
rec = {'task_id': task_res}
response = build_result.build_result(status=SUCCESS, err=None, rec=rec)
# if celery enabled
if response:
if ev('DEBUG_RESULTS', '0') == '1':
log.info('run_publish_ticker_aggregate_from_s3 - {} - done '
'status={} err={} rec={}'.format(
label, get_status(response['status']),
response['err'], response['rec']))
else:
log.info('run_publish_ticker_aggregate_from_s3 - {} - done '
'status={} err={}'.format(label,
get_status(response['status']),
response['err']))
else:
log.info('run_publish_ticker_aggregate_from_s3 - {} - done '
'no response'.format(label))
# end of if/else response
return response
def build_entry_call_spread_details(
ticker,
close,
num_contracts,
low_strike,
low_ask,
low_bid,
high_strike,
high_ask,
high_bid):
"""build_entry_call_spread_details
Calculate pricing information for
buying into ``Vertical Bull Call Option Spread`` contracts
:param ticker: string ticker symbol
:param num_contracts: integer number of contracts
:param low_strike: float - strike for
the low leg of the spread
:param low_ask: float - ask price for
the low leg of the spread
:param low_bid: float - bid price for
the low leg of the spread
:param high_strike: float - strike for
the high leg of the spread
:param high_ask: float - ask price for
the high leg of the spread
:param high_bid: float - bid price for
the high leg of the spread
"""
spread_details = spread_utils.build_option_spread_details(
trade_type=TRADE_ENTRY,
spread_type=SPREAD_VERTICAL_BULL,
option_type=OPTION_CALL,
close=close,
num_contracts=num_contracts,
low_strike=low_strike,
low_ask=low_ask,
low_bid=low_bid,
high_strike=high_strike,
high_ask=high_ask,
high_bid=high_bid)
log.debug(
'{} type={} spread={} option={} close={} spread={}'.format(
ticker,
get_status(status=spread_details['trade_type']),
get_status(status=spread_details['spread_type']),
get_status(status=spread_details['option_type']),
close,
ppj(spread_details)))
return spread_details
def get_configurables(self, **kwargs):
"""get_configurables
**Derive this in your indicators**
This is used as a helper for setting up algorithm
configs for this indicator and to programmatically set
the values based off the domain rules
:param kwargs: optional keyword args
"""
self.ind_confs = []
self.lg('configurables={} for class={}'.format(
ae_consts.ppj(self.ind_confs), self.__class__.__name__))
return self.ind_confs
def build_exit_put_spread_details(ticker, close, num_contracts, low_strike,
low_ask, low_bid, high_strike, high_ask,
high_bid):
"""build_exit_put_spread_details
Calculate pricing information for
selling (closing-out) ``Vertical Bear Put Option Spread`` contracts
:param ticker: string ticker name
:param num_contracts: integer number of contracts
:param low_strike: float - strike for
the low leg of the spread
:param low_ask: float - ask price for
the low leg of the spread
:param low_bid: float - bid price for
the low leg of the spread
:param high_strike: float - strike for
the high leg of the spread
:param high_ask: float - ask price for
the high leg of the spread
:param high_bid: float - bid price for
the high leg of the spread
"""
spread_details = spread_utils.build_option_spread_details(
trade_type=ae_consts.TRADE_EXIT,
spread_type=ae_consts.SPREAD_VERTICAL_BEAR,
option_type=ae_consts.OPTION_PUT,
close=close,
num_contracts=num_contracts,
low_strike=low_strike,
low_ask=low_ask,
low_bid=low_bid,
high_strike=high_strike,
high_ask=high_ask,
high_bid=high_bid)
log.debug('{} type={} spread={} option={} close={} spread={}'.format(
ticker, ae_consts.get_status(status=spread_details['trade_type']),
ae_consts.get_status(status=spread_details['spread_type']),
ae_consts.get_status(status=spread_details['option_type']), close,
ae_consts.ppj(spread_details)))
return spread_details
def test_integration_extract_option_calls(self):
"""test_integration_extract_option_calls"""
if ae_consts.ev('INT_TESTS', '0') == '0':
return
# build dataset cache dictionary
work = api_requests.get_ds_dict(
ticker='SPY',
base_key='SPY_2018-12-31',
label='test_integration_extract_option_calls')
status, df = yahoo_extract.extract_option_calls_dataset(work_dict=work)
if status == ae_consts.SUCCESS:
self.assertIsNotNone(df)
self.debug_df(df=df)
self.assertTrue(ae_consts.is_df(df=df))
for i, r in df.iterrows():
print(ae_consts.ppj(json.loads(r.to_json())))
log.info('done printing option call data')
else:
log.critical('Yahoo Option Calls are missing in redis '
f'for ticker={work["ticker"]} '
f'status={ae_consts.get_status(status=status)}')
def get_data_from_yahoo(work_dict):
"""get_data_from_yahoo
Get data from yahoo
:param work_dict: request dictionary
"""
label = 'get_data_from_yahoo'
log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict))
num_news_rec = 0
num_option_calls = 0
num_option_puts = 0
cur_high = -1
cur_low = -1
cur_open = -1
cur_close = -1
cur_volume = -1
rec = {
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'exp_date': None,
'publish_pricing_update': None,
'date': None,
'updated': None
}
res = {'status': NOT_RUN, 'err': None, 'rec': rec}
try:
ticker = work_dict.get('ticker', TICKER)
exp_date = work_dict.get('exp_date', None)
cur_strike = work_dict.get('strike', None)
contract_type = str(work_dict.get('contract', 'C')).upper()
get_pricing = work_dict.get('get_pricing', True)
get_news = work_dict.get('get_news', True)
get_options = work_dict.get('get_options', True)
orient = work_dict.get('orient', 'records')
label = work_dict.get('label', label)
ticker_results = pinance.Pinance(ticker)
num_news_rec = 0
use_date = exp_date
if not exp_date:
exp_date = opt_dates.option_expiration(date=exp_date)
use_date = exp_date.strftime('%Y-%m-%d')
"""
Debug control flags
Quickly turn specific fetches off:
get_news = False
get_pricing = False
get_options = False
"""
if get_pricing:
log.info('{} getting ticker={} pricing'.format(label, ticker))
ticker_results.get_quotes()
if ticker_results.quotes_data:
pricing_dict = ticker_results.quotes_data
cur_high = pricing_dict.get('regularMarketDayHigh', None)
cur_low = pricing_dict.get('regularMarketDayLow', None)
cur_open = pricing_dict.get('regularMarketOpen', None)
cur_close = pricing_dict.get('regularMarketPreviousClose',
None)
cur_volume = pricing_dict.get('regularMarketVolume', None)
pricing_dict['high'] = cur_high
pricing_dict['low'] = cur_low
pricing_dict['open'] = cur_open
pricing_dict['close'] = cur_close
pricing_dict['volume'] = cur_volume
pricing_dict['date'] = get_last_close_str()
if 'regularMarketTime' in pricing_dict:
pricing_dict['market_time'] = \
datetime.datetime.fromtimestamp(
pricing_dict['regularMarketTime']).strftime(
COMMON_TICK_DATE_FORMAT)
if 'postMarketTime' in pricing_dict:
pricing_dict['post_market_time'] = \
datetime.datetime.fromtimestamp(
pricing_dict['postMarketTime']).strftime(
COMMON_TICK_DATE_FORMAT)
log.info('{} ticker={} converting pricing to '
'df orient={}'.format(label, ticker, orient))
try:
rec['pricing'] = pricing_dict
except Exception as f:
rec['pricing'] = '{}'
log.info('{} ticker={} failed converting pricing '
'data={} to df ex={}'.format(
label, ticker, ppj(pricing_dict), f))
# try/ex
log.info('{} ticker={} done converting pricing to '
'df orient={}'.format(label, ticker, orient))
else:
log.error('{} ticker={} missing quotes_data={}'.format(
label, ticker, ticker_results.quotes_data))
# end of if ticker_results.quotes_data
log.info('{} ticker={} close={} vol={}'.format(
label, ticker, cur_close, cur_volume))
else:
log.info('{} skip - getting ticker={} pricing'.format(
label, ticker, get_pricing))
# if get_pricing
if get_news:
log.info('{} getting ticker={} news'.format(label, ticker))
ticker_results.get_news()
if ticker_results.news_data:
news_list = None
try:
news_list = ticker_results.news_data
log.info('{} ticker={} converting news to '
'df orient={}'.format(label, ticker, orient))
num_news_rec = len(news_list)
rec['news'] = news_list
except Exception as f:
rec['news'] = '{}'
log.info('{} ticker={} failed converting news '
'data={} to df ex={}'.format(
label, ticker, news_list, f))
# try/ex
log.info('{} ticker={} done converting news to '
'df orient={}'.format(label, ticker, orient))
else:
log.info('{} ticker={} Yahoo NO news={}'.format(
label, ticker, ticker_results.news_data))
# end of if ticker_results.news_data
else:
log.info('{} skip - getting ticker={} news'.format(label, ticker))
# end if get_news
if get_options:
get_all_strikes = True
if get_all_strikes:
cur_strike = None
else:
if cur_close:
cur_strike = int(cur_close)
if not cur_strike:
cur_strike = 287
log.info('{} ticker={} num_news={} get options close={} '
'exp_date={} contract={} strike={}'.format(
label, ticker, num_news_rec, cur_close, use_date,
contract_type, cur_strike))
options_dict = \
yahoo_get_pricing.get_options(
ticker=ticker,
exp_date_str=use_date,
contract_type=contract_type,
strike=cur_strike)
rec['options'] = '{}'
try:
log.info('{} ticker={} converting options to '
'df orient={}'.format(label, ticker, orient))
num_option_calls = options_dict.get('num_calls', None)
num_option_puts = options_dict.get('num_puts', None)
rec['options'] = {
'exp_date': options_dict.get('exp_date', None),
'calls': options_dict.get('calls', None),
'puts': options_dict.get('puts', None),
'num_calls': num_option_calls,
'num_puts': num_option_puts
}
rec['calls'] = rec['options'].get('calls', EMPTY_DF_STR)
rec['puts'] = rec['options'].get('puts', EMPTY_DF_STR)
except Exception as f:
rec['options'] = '{}'
log.info('{} ticker={} failed converting options '
'data={} to df ex={}'.format(label, ticker,
options_dict, f))
# try/ex
log.info('{} ticker={} done converting options to '
'df orient={} num_calls={} num_puts={}'.format(
label, ticker, orient, num_option_calls,
num_option_puts))
else:
log.info('{} skip - getting ticker={} options'.format(
label, ticker))
# end of if get_options
log.info('{} yahoo pricing for ticker={} close={} '
'num_calls={} num_puts={} news={}'.format(
label, ticker, cur_close, num_option_calls,
num_option_puts, num_news_rec))
fields_to_upload = ['pricing', 'options', 'calls', 'puts', 'news']
for field_name in fields_to_upload:
upload_and_cache_req = copy.deepcopy(work_dict)
upload_and_cache_req['celery_disabled'] = True
upload_and_cache_req['data'] = rec[field_name]
if not upload_and_cache_req['data']:
upload_and_cache_req['data'] = '{}'
if 'redis_key' in work_dict:
upload_and_cache_req['redis_key'] = '{}_{}'.format(
work_dict.get('redis_key',
'{}_{}'.format(ticker, field_name)),
field_name)
if 's3_key' in work_dict:
upload_and_cache_req['s3_key'] = '{}_{}'.format(
work_dict.get('s3_key', '{}_{}'.format(ticker,
field_name)),
field_name)
try:
update_res = publisher.run_publish_pricing_update(
work_dict=upload_and_cache_req)
update_status = update_res.get('status', NOT_SET)
log.info('{} publish update status={} data={}'.format(
label, get_status(status=update_status), update_res))
except Exception as f:
err = ('{} - failed to upload YAHOO data={} to '
'to s3_key={} and redis_key={}'.format(
label, upload_and_cache_req,
upload_and_cache_req['s3_key'],
upload_and_cache_req['redis_key']))
log.error(err)
# end of try/ex to upload and cache
if not rec[field_name]:
log.debug('{} - ticker={} no data from YAHOO for '
'field_name={}'.format(label, ticker, field_name))
# end of for all fields
res = build_result.build_result(status=SUCCESS, err=None, rec=rec)
except Exception as e:
res = build_result.build_result(status=ERR,
err=('failed - get_data_from_yahoo '
'dict={} with ex={}').format(
work_dict, e),
rec=rec)
log.error('{} - {}'.format(label, res['err']))
# end of try/ex
log.info('task - get_data_from_yahoo done - '
'{} - status={}'.format(label, get_status(res['status'])))
return res
def get_data_from_redis_key(label=None,
client=None,
host=None,
port=None,
password=None,
db=None,
key=None,
expire=None,
serializer='json',
encoding='utf-8'):
"""get_data_from_redis_key
:param label: log tracking label
:param client: initialized redis client
:param host: not used yet - redis host
:param port: not used yet - redis port
:param password: not used yet - redis password
:param db: not used yet - redis db
:param key: not used yet - redis key
:param expire: not used yet - redis expire
:param serializer: not used yet - support for future
pickle objects in redis
:param encoding: format of the encoded key in redis
"""
decoded_data = None
data = None
rec = {'data': data}
res = build_result.build_result(status=NOT_RUN, err=None, rec=rec)
log_id = label if label else 'get-data'
try:
use_client = client
if not use_client:
log.debug('{} get key={} new client={}:{}@{}'.format(
log_id, key, host, port, db))
use_client = redis.Redis(host=host,
port=port,
password=password,
db=db)
else:
log.debug('{} get key={} client'.format(log_id, key))
# create Redis client if not set
# https://redis-py.readthedocs.io/en/latest/index.html#redis.StrictRedis.get # noqa
raw_data = use_client.get(name=key)
if raw_data:
log.debug('{} decoding key={} encoding={}'.format(
log_id, key, encoding))
decoded_data = raw_data.decode(encoding)
log.debug('{} deserial key={} serializer={}'.format(
log_id, key, serializer))
if serializer == 'json':
data = json.loads(decoded_data)
elif serializer == 'df':
data = decoded_data
else:
data = decoded_data
if data:
if ev('DEBUG_REDIS', '0') == '1':
log.info('{} - found key={} data={}'.format(
log_id, key, ppj(data)))
else:
log.debug('{} - found key={}'.format(log_id, key))
# log snippet - if data
rec['data'] = data
return build_result.build_result(status=SUCCESS, err=None, rec=rec)
else:
log.debug('{} no data key={}'.format(log_id, key))
return build_result.build_result(status=SUCCESS, err=None, rec=rec)
except Exception as e:
err = ('{} failed - redis get from decoded={} data={} '
'key={} ex={}'.format(log_id, decoded_data, data, key, e))
log.error(err)
res = build_result.build_result(status=ERR, err=err, rec=rec)
# end of try/ex for getting redis data
return res
def publish_ticker_aggregate_from_s3():
"""publish_ticker_aggregate_from_s3
Download all ticker data from S3 and publish it's contents
to Redis and back to S3
"""
log.info('start - publish_ticker_aggregate_from_s3')
parser = argparse.ArgumentParser(
description=('Download and aggregated all ticker data, '
'and store it in S3 and Redis. '))
parser.add_argument('-t', help=('ticker'), required=True, dest='ticker')
parser.add_argument('-i',
help=('optional - ticker id '
'not used without a database'),
required=False,
dest='ticker_id')
parser.add_argument('-l',
help=('optional - path to the log config file'),
required=False,
dest='log_config_path')
parser.add_argument('-b',
help=('optional - broker url for Celery'),
required=False,
dest='broker_url')
parser.add_argument('-B',
help=('optional - backend url for Celery'),
required=False,
dest='backend_url')
parser.add_argument('-k',
help=('optional - s3 access key'),
required=False,
dest='s3_access_key')
parser.add_argument('-s',
help=('optional - s3 secret key'),
required=False,
dest='s3_secret_key')
parser.add_argument('-a',
help=('optional - s3 address format: <host:port>'),
required=False,
dest='s3_address')
parser.add_argument('-S',
help=('optional - s3 ssl or not'),
required=False,
dest='s3_secure')
parser.add_argument('-u',
help=('optional - s3 bucket name'),
required=False,
dest='s3_bucket_name')
parser.add_argument('-c',
help=('optional - s3 compiled bucket name'),
required=False,
dest='s3_compiled_bucket_name')
parser.add_argument('-g',
help=('optional - s3 region name'),
required=False,
dest='s3_region_name')
parser.add_argument('-p',
help=('optional - redis_password'),
required=False,
dest='redis_password')
parser.add_argument('-r',
help=('optional - redis_address format: <host:port>'),
required=False,
dest='redis_address')
parser.add_argument('-n',
help=('optional - redis and s3 key name'),
required=False,
dest='keyname')
parser.add_argument(
'-m',
help=('optional - redis database number (0 by default)'),
required=False,
dest='redis_db')
parser.add_argument('-x',
help=('optional - redis expiration in seconds'),
required=False,
dest='redis_expire')
parser.add_argument('-d',
help=('debug'),
required=False,
dest='debug',
action='store_true')
args = parser.parse_args()
ticker = TICKER
ticker_id = TICKER_ID
ssl_options = SSL_OPTIONS
transport_options = TRANSPORT_OPTIONS
broker_url = WORKER_BROKER_URL
backend_url = WORKER_BACKEND_URL
celery_config_module = WORKER_CELERY_CONFIG_MODULE
include_tasks = INCLUDE_TASKS
s3_access_key = S3_ACCESS_KEY
s3_secret_key = S3_SECRET_KEY
s3_region_name = S3_REGION_NAME
s3_address = S3_ADDRESS
s3_secure = S3_SECURE
s3_bucket_name = S3_BUCKET
s3_compiled_bucket_name = S3_COMPILED_BUCKET
s3_key = S3_KEY
redis_address = REDIS_ADDRESS
redis_key = REDIS_KEY
redis_password = REDIS_PASSWORD
redis_db = REDIS_DB
redis_expire = REDIS_EXPIRE
debug = False
if args.ticker:
ticker = args.ticker.upper()
if args.ticker_id:
ticker = args.ticker_id
if args.broker_url:
broker_url = args.broker_url
if args.backend_url:
backend_url = args.backend_url
if args.s3_access_key:
s3_access_key = args.s3_access_key
if args.s3_secret_key:
s3_secret_key = args.s3_secret_key
if args.s3_region_name:
s3_region_name = args.s3_region_name
if args.s3_address:
s3_address = args.s3_address
if args.s3_secure:
s3_secure = args.s3_secure
if args.s3_bucket_name:
s3_bucket_name = args.s3_bucket_name
if args.s3_compiled_bucket_name:
s3_compiled_bucket_name = args.s3_compiled_bucket_name
if args.keyname:
s3_key = args.keyname
redis_key = args.keyname
if args.redis_address:
redis_address = args.redis_address
if args.redis_password:
redis_password = args.redis_password
if args.redis_db:
redis_db = args.redis_db
if args.redis_expire:
redis_expire = args.redis_expire
if args.debug:
debug = True
work = build_publish_ticker_aggregate_from_s3_request()
work['ticker'] = ticker
work['ticker_id'] = ticker_id
work['s3_bucket'] = s3_bucket_name
work['s3_compiled_bucket'] = s3_compiled_bucket_name
if args.keyname:
work['s3_key'] = s3_key
work['redis_key'] = redis_key
work['s3_access_key'] = s3_access_key
work['s3_secret_key'] = s3_secret_key
work['s3_region_name'] = s3_region_name
work['s3_address'] = s3_address
work['s3_secure'] = s3_secure
work['redis_address'] = redis_address
work['redis_password'] = redis_password
work['redis_db'] = redis_db
work['redis_expire'] = redis_expire
work['debug'] = debug
work['label'] = 'ticker={}'.format(ticker)
path_to_tasks = 'analysis_engine.work_tasks'
task_name = ('{}.publish_ticker_aggregate_from_s3.'
'publish_ticker_aggregate_from_s3'.format(path_to_tasks))
task_res = None
if is_celery_disabled():
work['celery_disabled'] = True
log.debug('starting without celery work={}'.format(ppj(work)))
task_res = task_publisher.publish_ticker_aggregate_from_s3(
work_dict=work)
if debug:
log.info('done - result={} '
'task={} status={} '
'err={} label={}'.format(
ppj(task_res), task_name,
get_status(status=task_res['status']),
task_res['err'], work['label']))
else:
log.info('done - result '
'task={} status={} '
'err={} label={}'.format(
task_name, get_status(status=task_res['status']),
task_res['err'], work['label']))
# if/else debug
else:
log.info('connecting to broker={} backend={}'.format(
broker_url, backend_url))
# Get the Celery app
app = get_celery_app(name=__name__,
auth_url=broker_url,
backend_url=backend_url,
path_to_config_module=celery_config_module,
ssl_options=ssl_options,
transport_options=transport_options,
include_tasks=include_tasks)
log.info('calling task={} - work={}'.format(task_name, ppj(work)))
job_id = app.send_task(task_name, (work, ))
log.info('calling task={} - success job_id={}'.format(
task_name, job_id))
def build_publish_request(
ticker=None,
tickers=None,
convert_to_json=False,
output_file=None,
compress=False,
redis_enabled=ae_consts.ENABLED_REDIS_PUBLISH,
redis_key=None,
redis_address=ae_consts.REDIS_ADDRESS,
redis_db=ae_consts.REDIS_DB,
redis_password=ae_consts.REDIS_PASSWORD,
redis_expire=ae_consts.REDIS_EXPIRE,
redis_serializer='json',
redis_encoding='utf-8',
s3_enabled=ae_consts.ENABLED_S3_UPLOAD,
s3_key=None,
s3_address=ae_consts.S3_ADDRESS,
s3_bucket=ae_consts.S3_BUCKET,
s3_access_key=ae_consts.S3_ACCESS_KEY,
s3_secret_key=ae_consts.S3_SECRET_KEY,
s3_region_name=ae_consts.S3_REGION_NAME,
s3_secure=ae_consts.S3_SECURE,
slack_enabled=False,
slack_code_block=False,
slack_full_width=False,
verbose=False,
label='publisher'):
"""build_publish_request
Build a dictionary for helping to quickly publish
to multiple optional endpoints:
- a local file path (``output_file``)
- minio (``s3_bucket`` and ``s3_key``)
- redis (``redis_key``)
- slack
:param ticker: ticker
:param tickers: optional - list of tickers
:param label: optional - algo log tracking name
:param output_file: path to save the data
to a file
:param compress: optional - compress before publishing
:param verbose: optional - boolean to log output
:param kwargs: optional - future argument support
**(Optional) Redis connectivity arguments**
:param redis_enabled: bool - toggle for auto-caching all
datasets in Redis
(default is ``ENABLED_REDIS_PUBLISH``)
:param redis_key: string - key to save the data in redis
(default is ``None``)
:param redis_address: Redis connection string format: ``host:port``
(default is ``REDIS_ADDRESS``)
:param redis_db: Redis db to use
(default is ``REDIS_DB``)
:param redis_password: optional - Redis password
(default is ``REDIS_PASSWORD``)
:param redis_expire: optional - Redis expire value
(default is ``REDIS_EXPIRE``)
:param redis_serializer: not used yet - support for future
pickle objects in redis (default is ``json``)
:param redis_encoding: format of the encoded key in redis
(default is ``utf-8``)
**(Optional) Minio (S3) connectivity arguments**
:param s3_enabled: bool - toggle for auto-archiving on Minio (S3)
(default is ``ENABLED_S3_UPLOAD``)
:param s3_key: string - key to save the data in redis
(default is ``None``)
:param s3_address: Minio S3 connection string format: ``host:port``
(default is ``S3_ADDRESS``)
:param s3_bucket: S3 Bucket for storing the artifacts
(default is ``S3_BUCKET``) which should be viewable on a browser:
http://localhost:9000/minio/dev/
:param s3_access_key: S3 Access key
(default is ``S3_ACCESS_KEY``)
:param s3_secret_key: S3 Secret key
(default is ``S3_SECRET_KEY``)
:param s3_region_name: S3 region name
(default is ``S3_REGION_NAME``)
:param s3_secure: Transmit using tls encryption
(default is ``S3_SECURE``)
**(Optional) Slack arguments**
:param slack_enabled: optional - boolean for
publishing to slack
:param slack_code_block: optional - boolean for
publishing as a code black in slack
:param slack_full_width: optional - boolean for
publishing as a to slack using the full
width allowed
"""
use_tickers = []
if ticker:
use_tickers = [
ticker.upper()
]
if tickers:
for t in tickers:
if t not in use_tickers:
use_tickers.append(t.upper())
work = {
'tickers': use_tickers,
'label': label,
'convert_to_json': convert_to_json,
'output_file': output_file,
'compress': compress,
'redis_enabled': redis_enabled,
'redis_key': redis_key,
'redis_address': redis_address,
'redis_db': redis_db,
'redis_password': redis_password,
'redis_expire': redis_expire,
'redis_serializer': redis_serializer,
'redis_encoding': redis_encoding,
's3_enabled': s3_enabled,
's3_key': s3_key,
's3_address': s3_address,
's3_bucket': s3_bucket,
's3_access_key': s3_access_key,
's3_secret_key': s3_secret_key,
's3_region_name': s3_region_name,
's3_secure': s3_secure,
'slack_enabled': slack_enabled,
'slack_code_block': slack_code_block,
'slack_full_width': slack_full_width,
'verbose': verbose,
'version': 1,
'label': label
}
log.debug(
'created publish_request={}'.format(
ae_consts.ppj(work)))
return work
def get_task_results(work_dict=None, result=None, **kwargs):
"""get_task_results
If celery is disabled by the
environment key ``export CELERY_DISABLED=1``
or requested in the ``work_dict['celery_disabled'] = True`` then
return the task result dictionary, otherwise
return ``None``.
This method is useful for allowing tests
to override the returned payloads during task chaining
using ``@mock.patch``.
:param work_dict: task work dictionary
:param result: task result dictionary
:param kwargs: keyword arguments
"""
send_results_back = None
cel_disabled = False
if work_dict:
if ae_consts.is_celery_disabled(work_dict=work_dict):
send_results_back = result
cel_disabled = True
# end of sending back results if told to do so
if ae_consts.ev('DEBUG_TASK', '0') == '1':
status = ae_consts.NOT_SET
err = None
record = None
label = None
if result:
status = result.get('status', ae_consts.NOT_SET)
err = result.get('err', None)
record = result.get('rec', None)
if work_dict:
label = work_dict.get('label', None)
log_id = 'get_task_results'
if label:
log_id = '{} - get_task_results'.format(label)
result_details = record
if record:
result_details = ae_consts.ppj(record)
status_details = status
if status:
status_details = ae_consts.get_status(status=status)
work_details = work_dict
if work_dict:
work_details = ae_consts.ppj(work_dict)
if status == ae_consts.SUCCESS:
log.info('{} celery_disabled={} '
'status={} err={} work_dict={} result={}'.format(
log_id, cel_disabled, status_details, err,
work_details, result_details))
else:
if cel_disabled:
log.error('{} celery_disabled={} '
'status={} err={} work_dict={} result={}'.format(
log_id, cel_disabled, status_details, err,
work_details, result_details))
else:
log.info('{} celery_disabled={} '
'status={} err={} work_dict={} result={}'.format(
log_id, cel_disabled, status_details, err,
work_details, result_details))
# end of if debugging the task results
return send_results_back
def build_df_from_redis(label=None,
client=None,
address=None,
host=None,
port=None,
password=None,
db=None,
key=None,
expire=None,
serializer='json',
encoding='utf-8',
orient='records'):
"""build_df_from_redis
:param label: log tracking label
:param client: initialized redis client
:param address: redis address: <host:port>
:param host: redis host
:param port: redis port
:param password: redis password
:param db: redis db
:param key: redis key
:param expire: not used yet - redis expire
:param serializer: support for future
pickle objects in redis
:param encoding: format of the encoded key in redis
:param orient: use the same orient value as
the ``to_json(orient='records')`` used
to deserialize the DataFrame correctly.
"""
data = None
valid_df = False
df = None
rec = {'valid_df': valid_df, 'data': data}
res = build_result.build_result(status=NOT_RUN, err=None, rec=rec)
log_id = label if label else 'build-df'
try:
log.debug('{} calling get redis key={}'.format(log_id, key))
use_host = host
use_port = port
if not use_host and not use_port:
if address:
use_host = address.split(':')[0]
use_port = int(address.split(':')[1])
use_client = client
if not use_client:
log.debug('{} connecting to redis={}:{}@{}'.format(
log_id, use_host, use_port, db))
use_client = redis.Redis(host=use_host,
port=use_port,
password=password,
db=db)
redis_res = redis_get.get_data_from_redis_key(label=log_id,
client=use_client,
host=use_host,
port=use_port,
password=password,
db=db,
key=key,
expire=expire,
serializer='json',
encoding=encoding)
valid_df = False
if redis_res['status'] == SUCCESS:
data = redis_res['rec'].get('data', None)
if data:
if ev('DEBUG_REDIS', '0') == '1':
log.info('{} - found key={} data={}'.format(
log_id, key, ppj(data)))
else:
log.debug('{} - loading df from key={}'.format(
log_id, key))
df = pd.read_json(data, orient='records')
valid_df = True
else:
log.debug('{} key={} no data'.format(log_id, key))
# if data
rec['data'] = df
rec['valid_df'] = valid_df
res = build_result.build_result(status=SUCCESS, err=None, rec=rec)
return res
else:
log.debug('{} no data key={}'.format(log_id, key))
res = build_result.build_result(status=SUCCESS, err=None, rec=rec)
return res
except Exception as e:
err = ('{} failed - build_df_from_redis data={} '
'key={} ex={}'.format(log_id, (data == '0'), key, e))
log.error(err)
res = build_result.build_result(status=ERR, err=err, rec=rec)
# end of try/ex for getting redis data
return res
def run_algo(
ticker=None,
tickers=None,
algo=None, # optional derived ``analysis_engine.algo.Algo`` instance
balance=None, # float starting base capital
commission=None, # float for single trade commission for buy or sell
start_date=None, # string YYYY-MM-DD HH:MM:SS
end_date=None, # string YYYY-MM-DD HH:MM:SS
datasets=None, # string list of identifiers
num_owned_dict=None, # not supported
cache_freq='daily', # 'minute' not supported
auto_fill=True,
load_config=None,
report_config=None,
history_config=None,
extract_config=None,
use_key=None,
extract_mode='all',
iex_datasets=None,
redis_enabled=True,
redis_address=None,
redis_db=None,
redis_password=None,
redis_expire=None,
redis_key=None,
s3_enabled=True,
s3_address=None,
s3_bucket=None,
s3_access_key=None,
s3_secret_key=None,
s3_region_name=None,
s3_secure=False,
s3_key=None,
celery_disabled=True,
broker_url=None,
result_backend=None,
label=None,
name=None,
timeseries=None,
trade_strategy=None,
verbose=False,
publish_to_slack=True,
publish_to_s3=True,
publish_to_redis=True,
extract_datasets=None,
config_file=None,
config_dict=None,
version=1,
raise_on_err=True,
**kwargs):
"""run_algo
Run an algorithm with steps:
1) Extract redis keys between dates
2) Compile a data pipeline dictionary (call it ``data``)
3) Call algorithm's ``myalgo.handle_data(data=data)``
.. note:: If no ``algo`` is set, the
``analysis_engine.algo.BaseAlgo`` algorithm
is used.
.. note:: Please ensure Redis and Minio are running
before trying to extract tickers
**Stock tickers to extract**
:param ticker: single stock ticker/symbol/ETF to extract
:param tickers: optional - list of tickers to extract
:param use_key: optional - extract historical key from Redis
**Algo Configuration**
:param algo: derived instance of ``analysis_engine.algo.Algo`` object
:param balance: optional - float balance parameter
can also be set on the ``algo`` object if not
set on the args
:param commission: float for single trade commission for
buy or sell. can also be set on the ``algo`` objet
:param start_date: string ``YYYY-MM-DD_HH:MM:SS`` cache value
:param end_date: string ``YYYY-MM-DD_HH:MM:SS`` cache value
:param dataset_types: list of strings that are ``iex`` or ``yahoo``
datasets that are cached.
:param cache_freq: optional - depending on if you are running data feeds
on a ``daily`` cron (default) vs every ``minute`` (or faster)
:param num_owned_dict: not supported yet
:param auto_fill: optional - boolean for auto filling
buy/sell orders for backtesting (default is
``True``)
:param trading_calendar: ``trading_calendar.TradingCalendar``
object, by default ``analysis_engine.calendars.
always_open.AlwaysOpen`` trading calendar
# TradingCalendar by ``TFSExchangeCalendar``
:param config_file: path to a json file
containing custom algorithm object
member values (like indicator configuration and
predict future date units ahead for a backtest)
:param config_dict: optional - dictionary that
can be passed to derived class implementations
of: ``def load_from_config(config_dict=config_dict)``
**Timeseries**
:param timeseries: optional - string to
set ``day`` or ``minute`` backtesting
or live trading
(default is ``minute``)
**Trading Strategy**
:param trade_strategy: optional - string to
set the type of ``Trading Strategy``
for backtesting or live trading
(default is ``count``)
**Algorithm Dataset Loading, Extracting, Reporting
and Trading History arguments**
:param load_config: optional - dictionary
for setting member variables to load an
agorithm-ready dataset from
a file, s3 or redis
:param report_config: optional - dictionary
for setting member variables to publish
an algo ``trading performance report`` to s3,
redis, a file or slack
:param history_config: optional - dictionary
for setting member variables to publish
an algo ``trade history`` to s3, redis, a file
or slack
:param extract_config: optional - dictionary
for setting member variables to publish
an algo ``trading performance report`` to s3,
redis, a file or slack
**(Optional) Data sources, datafeeds and datasets to gather**
:param iex_datasets: list of strings for gathering specific `IEX
datasets <https://iexcloud.io/>`__
which are set as consts: ``analysis_engine.iex.consts.FETCH_*``.
**(Optional) Redis connectivity arguments**
:param redis_enabled: bool - toggle for auto-caching all
datasets in Redis
(default is ``True``)
:param redis_address: Redis connection string
format is ``host:port``
(default is ``localhost:6379``)
:param redis_db: Redis db to use
(default is ``0``)
:param redis_password: optional - Redis password
(default is ``None``)
:param redis_expire: optional - Redis expire value
(default is ``None``)
:param redis_key: optional - redis key not used
(default is ``None``)
**(Optional) Minio (S3) connectivity arguments**
:param s3_enabled: bool - toggle for auto-archiving on Minio (S3)
(default is ``True``)
:param s3_address: Minio S3 connection string
format ``host:port``
(default is ``localhost:9000``)
:param s3_bucket: S3 Bucket for storing the artifacts
(default is ``dev``) which should be viewable on a browser:
http://localhost:9000/minio/dev/
:param s3_access_key: S3 Access key
(default is ``trexaccesskey``)
:param s3_secret_key: S3 Secret key
(default is ``trex123321``)
:param s3_region_name: S3 region name
(default is ``us-east-1``)
:param s3_secure: Transmit using tls encryption
(default is ``False``)
:param s3_key: optional s3 key not used
(default is ``None``)
**(Optional) Celery worker broker connectivity arguments**
:param celery_disabled: bool - toggle synchronous mode or publish
to an engine connected to the `Celery broker and backend
<https://github.com/celery/celery#transports-and-backends>`__
(default is ``True`` - synchronous mode without an engine
or need for a broker or backend for Celery)
:param broker_url: Celery broker url
(default is ``redis://0.0.0.0:6379/13``)
:param result_backend: Celery backend url
(default is ``redis://0.0.0.0:6379/14``)
:param label: tracking log label
:param publish_to_slack: optional - boolean for
publishing to slack (coming soon)
:param publish_to_s3: optional - boolean for
publishing to s3 (coming soon)
:param publish_to_redis: optional - boolean for
publishing to redis (coming soon)
**(Optional) Debugging**
:param verbose: bool - show extract warnings
and other debug logging (default is False)
:param raise_on_err: optional - boolean for
unittests and developing algorithms with the
``analysis_engine.run_algo.run_algo`` helper.
When set to ``True`` exceptions will
are raised to the calling functions
:param kwargs: keyword arguments dictionary
"""
# dictionary structure with a list sorted on: ascending dates
# algo_data_req[ticker][list][dataset] = pd.DataFrame
algo_data_req = {}
extract_requests = []
return_algo = False # return created algo objects for use by caller
rec = {}
msg = None
use_tickers = tickers
use_balance = balance
use_commission = commission
if ticker:
use_tickers = [ticker]
else:
if not use_tickers:
use_tickers = []
# if these are not set as args, but the algo object
# has them, use them instead:
if algo:
if len(use_tickers) == 0:
use_tickers = algo.get_tickers()
if not use_balance:
use_balance = algo.get_balance()
if not use_commission:
use_commission = algo.get_commission()
default_iex_datasets = [
'daily', 'minute', 'quote', 'stats', 'peers', 'news', 'financials',
'earnings', 'dividends', 'company'
]
if not iex_datasets:
iex_datasets = default_iex_datasets
if redis_enabled:
if not redis_address:
redis_address = os.getenv('REDIS_ADDRESS', 'localhost:6379')
if not redis_password:
redis_password = os.getenv('REDIS_PASSWORD', None)
if not redis_db:
redis_db = int(os.getenv('REDIS_DB', '0'))
if not redis_expire:
redis_expire = os.getenv('REDIS_EXPIRE', None)
if s3_enabled:
if not s3_address:
s3_address = os.getenv('S3_ADDRESS', 'localhost:9000')
if not s3_access_key:
s3_access_key = os.getenv('AWS_ACCESS_KEY_ID', 'trexaccesskey')
if not s3_secret_key:
s3_secret_key = os.getenv('AWS_SECRET_ACCESS_KEY', 'trex123321')
if not s3_region_name:
s3_region_name = os.getenv('AWS_DEFAULT_REGION', 'us-east-1')
if not s3_secure:
s3_secure = os.getenv('S3_SECURE', '0') == '1'
if not s3_bucket:
s3_bucket = os.getenv('S3_BUCKET', 'dev')
if not broker_url:
broker_url = os.getenv('WORKER_BROKER_URL', 'redis://0.0.0.0:6379/11')
if not result_backend:
result_backend = os.getenv('WORKER_BACKEND_URL',
'redis://0.0.0.0:6379/12')
if not label:
label = 'run-algo'
num_tickers = len(use_tickers)
last_close_str = ae_utils.get_last_close_str()
if iex_datasets:
if verbose:
log.info(f'{label} - tickers={num_tickers} '
f'iex={json.dumps(iex_datasets)}')
else:
if verbose:
log.info(f'{label} - tickers={num_tickers}')
ticker_key = use_key
if not ticker_key:
ticker_key = f'{ticker}_{last_close_str}'
if not algo:
algo = base_algo.BaseAlgo(ticker=None,
tickers=use_tickers,
balance=use_balance,
commission=use_commission,
config_dict=config_dict,
name=label,
auto_fill=auto_fill,
timeseries=timeseries,
trade_strategy=trade_strategy,
publish_to_slack=publish_to_slack,
publish_to_s3=publish_to_s3,
publish_to_redis=publish_to_redis,
raise_on_err=raise_on_err)
return_algo = True
# the algo object is stored
# in the result at: res['rec']['algo']
if not algo:
msg = f'{label} - missing algo object'
log.error(msg)
return build_result.build_result(status=ae_consts.EMPTY,
err=msg,
rec=rec)
if raise_on_err:
log.debug(f'{label} - enabling algo exception raises')
algo.raise_on_err = True
indicator_datasets = algo.get_indicator_datasets()
if len(indicator_datasets) == 0:
indicator_datasets = ae_consts.BACKUP_DATASETS
log.info(f'using all datasets={indicator_datasets}')
verbose_extract = False
if config_dict:
verbose_extract = config_dict.get('verbose_extract', False)
common_vals = {}
common_vals['base_key'] = ticker_key
common_vals['celery_disabled'] = celery_disabled
common_vals['ticker'] = ticker
common_vals['label'] = label
common_vals['iex_datasets'] = iex_datasets
common_vals['s3_enabled'] = s3_enabled
common_vals['s3_bucket'] = s3_bucket
common_vals['s3_address'] = s3_address
common_vals['s3_secure'] = s3_secure
common_vals['s3_region_name'] = s3_region_name
common_vals['s3_access_key'] = s3_access_key
common_vals['s3_secret_key'] = s3_secret_key
common_vals['s3_key'] = ticker_key
common_vals['redis_enabled'] = redis_enabled
common_vals['redis_address'] = redis_address
common_vals['redis_password'] = redis_password
common_vals['redis_db'] = redis_db
common_vals['redis_key'] = ticker_key
common_vals['redis_expire'] = redis_expire
use_start_date_str = start_date
use_end_date_str = end_date
last_close_date = ae_utils.last_close()
end_date_val = None
cache_freq_fmt = ae_consts.COMMON_TICK_DATE_FORMAT
if not use_end_date_str:
use_end_date_str = last_close_date.strftime(cache_freq_fmt)
end_date_val = ae_utils.get_date_from_str(date_str=use_end_date_str,
fmt=cache_freq_fmt)
start_date_val = None
if not use_start_date_str:
start_date_val = end_date_val - datetime.timedelta(days=60)
use_start_date_str = start_date_val.strftime(cache_freq_fmt)
else:
start_date_val = datetime.datetime.strptime(
use_start_date_str, ae_consts.COMMON_TICK_DATE_FORMAT)
total_dates = (end_date_val - start_date_val).days
if end_date_val < start_date_val:
msg = (
f'{label} - invalid dates - start_date={start_date_val} is after '
f'end_date={end_date_val}')
raise Exception(msg)
if verbose:
log.info(f'{label} - days={total_dates} '
f'start={use_start_date_str} '
f'end={use_end_date_str} '
f'datasets={indicator_datasets}')
for ticker in use_tickers:
req = algo_utils.build_algo_request(ticker=ticker,
use_key=use_key,
start_date=use_start_date_str,
end_date=use_end_date_str,
datasets=datasets,
balance=use_balance,
cache_freq=cache_freq,
timeseries=timeseries,
trade_strategy=trade_strategy,
label=label)
ticker_key = f'{ticker}_{last_close_str}'
common_vals['ticker'] = ticker
common_vals['base_key'] = ticker_key
common_vals['redis_key'] = ticker_key
common_vals['s3_key'] = ticker_key
for date_key in req['extract_datasets']:
date_req = api_requests.get_ds_dict(ticker=ticker,
base_key=date_key,
ds_id=label,
service_dict=common_vals)
node_date_key = date_key.replace(f'{ticker}_', '')
extract_requests.append({
'id': date_key,
'ticker': ticker,
'date_key': date_key,
'date': node_date_key,
'req': date_req
})
# end of for all ticker in use_tickers
first_extract_date = None
last_extract_date = None
total_extract_requests = len(extract_requests)
cur_idx = 1
for idx, extract_node in enumerate(extract_requests):
extract_ticker = extract_node['ticker']
extract_date = extract_node['date']
ds_node_id = extract_node['id']
if not first_extract_date:
first_extract_date = extract_date
last_extract_date = extract_date
perc_progress = ae_consts.get_percent_done(
progress=cur_idx, total=total_extract_requests)
percent_label = (f'{label} '
f'ticker={extract_ticker} '
f'date={extract_date} '
f'{perc_progress} '
f'{idx}/{total_extract_requests} '
f'{indicator_datasets}')
if verbose:
log.info(f'extracting - {percent_label}')
ticker_bt_data = build_ds_node.build_dataset_node(
ticker=extract_ticker,
date=extract_date,
service_dict=common_vals,
datasets=indicator_datasets,
log_label=label,
verbose=verbose_extract)
if ticker not in algo_data_req:
algo_data_req[ticker] = []
algo_data_req[ticker].append({
'id': ds_node_id, # id is currently the cache key in redis
'date': extract_date, # used to confirm dates in asc order
'data': ticker_bt_data
})
if verbose:
log.info(f'extract - {percent_label} '
f'dataset={len(algo_data_req[ticker])}')
cur_idx += 1
# end of for service_dict in extract_requests
# this could be a separate celery task
status = ae_consts.NOT_RUN
if len(algo_data_req) == 0:
msg = (f'{label} - nothing to test - no data found for '
f'tickers={use_tickers} '
f'between {first_extract_date} and {last_extract_date}')
log.info(msg)
return build_result.build_result(status=ae_consts.EMPTY,
err=msg,
rec=rec)
# this could be a separate celery task
try:
if verbose:
log.info(f'handle_data START - {percent_label} from '
f'{first_extract_date} to {last_extract_date}')
algo.handle_data(data=algo_data_req)
if verbose:
log.info(f'handle_data END - {percent_label} from '
f'{first_extract_date} to {last_extract_date}')
except Exception as e:
a_name = algo.get_name()
a_debug_msg = algo.get_debug_msg()
if not a_debug_msg:
a_debug_msg = 'debug message not set'
a_config_dict = ae_consts.ppj(algo.config_dict)
msg = (f'{percent_label} - algo={a_name} '
f'encountered exception in handle_data tickers={use_tickers} '
f'from {first_extract_date} to {last_extract_date} ex={e} '
f'and failed during operation: {a_debug_msg}')
if raise_on_err:
if algo:
try:
ind_obj = \
algo.get_indicator_process_last_indicator()
if ind_obj:
ind_obj_path = ind_obj.get_path_to_module()
ind_obj_config = ae_consts.ppj(ind_obj.get_config())
found_error_hint = False
if hasattr(ind_obj.use_df, 'to_json'):
if len(ind_obj.use_df.index) == 0:
log.critical(
f'indicator failure report for '
f'last module: '
f'{ind_obj_path} '
f'indicator={ind_obj.get_name()} '
f'config={ind_obj_config} '
f'dataset={ind_obj.use_df.head(5)} '
f'name_of_dataset={ind_obj.uses_data}')
log.critical(
'--------------------------------------'
'--------------------------------------')
log.critical('Please check if this indicator: '
f'{ind_obj_path} '
'supports Empty Dataframes')
log.critical(
'--------------------------------------'
'--------------------------------------')
found_error_hint = True
# indicator error hints
if not found_error_hint:
log.critical(
f'indicator failure report for last module: '
f'{ind_obj_path} '
f'indicator={ind_obj.get_name()} '
f'config={ind_obj_config} '
f'dataset={ind_obj.use_df.head(5)} '
f'name_of_dataset={ind_obj.uses_data}')
except Exception as f:
log.critical(f'failed to pull indicator processor '
f'last indicator for debugging '
f'from ex={e} with parsing ex={f}')
# end of ignoring non-supported ways of creating
# indicator processors
log.error(msg)
log.error(f'algo failure report: '
f'algo={a_name} handle_data() '
f'config={a_config_dict} ')
log.critical(f'algo failed during operation: {a_debug_msg}')
raise e
else:
log.error(msg)
return build_result.build_result(status=ae_consts.ERR,
err=msg,
rec=rec)
# end of try/ex
# this could be a separate celery task
try:
if verbose:
log.info(f'get_result START - {percent_label} from '
f'{first_extract_date} to {last_extract_date}')
rec = algo.get_result()
status = ae_consts.SUCCESS
if verbose:
log.info(f'get_result END - {percent_label} from '
f'{first_extract_date} to {last_extract_date}')
except Exception as e:
msg = (
f'{percent_label} - algo={algo.get_name()} encountered exception '
f'in get_result tickers={use_tickers} from '
f'{first_extract_date} to {last_extract_date} ex={e}')
if raise_on_err:
if algo:
log.error(f'algo={algo.get_name()} failed in get_result with '
f'debug_msg={algo.get_debug_msg()}')
log.error(msg)
raise e
else:
log.error(msg)
return build_result.build_result(status=ae_consts.ERR,
err=msg,
rec=rec)
# end of try/ex
if return_algo:
rec['algo'] = algo
return build_result.build_result(status=status, err=msg, rec=rec)
def get_new_pricing_data(self, work_dict):
"""get_new_pricing_data
Get Ticker information on:
- prices - turn off with ``work_dict.get_pricing = False``
- news - turn off with ``work_dict.get_news = False``
- options - turn off with ``work_dict.get_options = False``
:param work_dict: dictionary for key/values
"""
label = 'get_new_pricing_data'
log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict))
ticker = ae_consts.TICKER
ticker_id = ae_consts.TICKER_ID
rec = {
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'daily': None,
'minute': None,
'quote': None,
'stats': None,
'peers': None,
'iex_news': None,
'financials': None,
'earnings': None,
'dividends': None,
'company': None,
'exp_date': None,
'publish_pricing_update': None,
'date': ae_utils.utc_now_str(),
'updated': None,
'version': ae_consts.DATASET_COLLECTION_VERSION
}
res = {'status': ae_consts.NOT_RUN, 'err': None, 'rec': rec}
try:
ticker = work_dict.get('ticker', ticker)
ticker_id = work_dict.get('ticker_id', ae_consts.TICKER_ID)
s3_bucket = work_dict.get('s3_bucket', ae_consts.S3_BUCKET)
s3_key = work_dict.get('s3_key', ae_consts.S3_KEY)
redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY)
exp_date = work_dict.get('exp_date', None)
cur_date = datetime.datetime.utcnow()
cur_strike = work_dict.get('strike', None)
contract_type = str(work_dict.get('contract', 'C')).upper()
label = work_dict.get('label', label)
iex_datasets = work_dict.get('iex_datasets',
iex_consts.DEFAULT_FETCH_DATASETS)
td_datasets = work_dict.get('td_datasets',
td_consts.DEFAULT_FETCH_DATASETS_TD)
fetch_mode = work_dict.get('fetch_mode', ae_consts.FETCH_MODE_ALL)
# control flags to deal with feed issues:
get_iex_data = True
get_td_data = True
if (fetch_mode == ae_consts.FETCH_MODE_ALL
or str(fetch_mode).lower() == 'all'):
get_iex_data = True
get_td_data = True
elif (fetch_mode == ae_consts.FETCH_MODE_YHO
or str(fetch_mode).lower() == 'yahoo'):
get_iex_data = False
get_td_data = False
elif (fetch_mode == ae_consts.FETCH_MODE_IEX
or str(fetch_mode).lower() == 'iex'):
get_iex_data = True
get_td_data = False
elif (fetch_mode == ae_consts.FETCH_MODE_TD
or str(fetch_mode).lower() == 'td'):
get_iex_data = False
get_td_data = True
else:
log.debug('{} - unsupported fetch_mode={} value'.format(
label, fetch_mode))
"""
as of Thursday, Jan. 3, 2019:
https://developer.yahoo.com/yql/
Important EOL Notice: As of Thursday, Jan. 3, 2019
the YQL service at query.yahooapis.com will be retired
"""
get_yahoo_data = False
if not exp_date:
exp_date = opt_dates.option_expiration(date=exp_date)
else:
exp_date = datetime.datetime.strptime(exp_date, '%Y-%m-%d')
rec['updated'] = cur_date.strftime('%Y-%m-%d %H:%M:%S')
log.info('{} getting pricing for ticker={} '
'cur_date={} exp_date={} '
'yahoo={} iex={}'.format(label, ticker, cur_date, exp_date,
get_yahoo_data, get_iex_data))
yahoo_rec = {
'ticker': ticker,
'pricing': None,
'options': None,
'calls': None,
'puts': None,
'news': None,
'exp_date': None,
'publish_pricing_update': None,
'date': None,
'updated': None
}
# disabled on 2019-01-03
if get_yahoo_data:
log.info('{} yahoo ticker={}'.format(label, ticker))
yahoo_res = yahoo_data.get_data_from_yahoo(work_dict=work_dict)
if yahoo_res['status'] == ae_consts.SUCCESS:
yahoo_rec = yahoo_res['rec']
log.info('{} yahoo ticker={} '
'status={} err={}'.format(
label, ticker,
ae_consts.get_status(status=yahoo_res['status']),
yahoo_res['err']))
rec['pricing'] = yahoo_rec.get('pricing', '{}')
rec['news'] = yahoo_rec.get('news', '{}')
rec['options'] = yahoo_rec.get('options', '{}')
rec['calls'] = rec['options'].get('calls',
ae_consts.EMPTY_DF_STR)
rec['puts'] = rec['options'].get('puts',
ae_consts.EMPTY_DF_STR)
else:
log.error('{} failed YAHOO ticker={} '
'status={} err={}'.format(
label, ticker,
ae_consts.get_status(status=yahoo_res['status']),
yahoo_res['err']))
# end of get from yahoo
if get_iex_data:
num_iex_ds = len(iex_datasets)
log.debug('{} iex datasets={}'.format(label, num_iex_ds))
for idx, ft_type in enumerate(iex_datasets):
dataset_field = iex_consts.get_ft_str(ft_type=ft_type)
log.info('{} iex={}/{} field={} ticker={}'.format(
label, idx, num_iex_ds, dataset_field, ticker))
iex_label = '{}-{}'.format(label, dataset_field)
iex_req = copy.deepcopy(work_dict)
iex_req['label'] = iex_label
iex_req['ft_type'] = ft_type
iex_req['field'] = dataset_field
iex_req['ticker'] = ticker
iex_res = iex_data.get_data_from_iex(work_dict=iex_req)
if iex_res['status'] == ae_consts.SUCCESS:
iex_rec = iex_res['rec']
log.info(
'{} iex ticker={} field={} '
'status={} err={}'.format(
label, ticker, dataset_field,
ae_consts.get_status(status=iex_res['status']),
iex_res['err']))
if dataset_field == 'news':
rec['iex_news'] = iex_rec['data']
else:
rec[dataset_field] = iex_rec['data']
else:
log.debug(
'{} failed IEX ticker={} field={} '
'status={} err={}'.format(
label, ticker, dataset_field,
ae_consts.get_status(status=iex_res['status']),
iex_res['err']))
# end of if/else succcess
# end idx, ft_type in enumerate(iex_datasets):
# end of if get_iex_data
if get_td_data:
num_td_ds = len(td_datasets)
log.debug('{} td datasets={}'.format(label, num_td_ds))
for idx, ft_type in enumerate(td_datasets):
dataset_field = td_consts.get_ft_str_td(ft_type=ft_type)
log.info('{} td={}/{} field={} ticker={}'.format(
label, idx, num_td_ds, dataset_field, ticker))
td_label = '{}-{}'.format(label, dataset_field)
td_req = copy.deepcopy(work_dict)
td_req['label'] = td_label
td_req['ft_type'] = ft_type
td_req['field'] = dataset_field
td_req['ticker'] = ticker
td_res = td_data.get_data_from_td(work_dict=td_req)
if td_res['status'] == ae_consts.SUCCESS:
td_rec = td_res['rec']
log.info('{} td ticker={} field={} '
'status={} err={}'.format(
label, ticker, dataset_field,
ae_consts.get_status(status=td_res['status']),
td_res['err']))
if dataset_field == 'tdcalls':
rec['tdcalls'] = td_rec['data']
if dataset_field == 'tdputs':
rec['tdputs'] = td_rec['data']
else:
rec[dataset_field] = td_rec['data']
else:
log.critical(
'{} failed TD ticker={} field={} '
'status={} err={}'.format(
label, ticker, dataset_field,
ae_consts.get_status(status=td_res['status']),
td_res['err']))
# end of if/else succcess
# end idx, ft_type in enumerate(td_datasets):
# end of if get_td_data
update_req = {'data': rec}
update_req['ticker'] = ticker
update_req['ticker_id'] = ticker_id
update_req['strike'] = cur_strike
update_req['contract'] = contract_type
update_req['s3_enabled'] = work_dict.get('s3_enabled',
ae_consts.ENABLED_S3_UPLOAD)
update_req['redis_enabled'] = work_dict.get(
'redis_enabled', ae_consts.ENABLED_REDIS_PUBLISH)
update_req['s3_bucket'] = s3_bucket
update_req['s3_key'] = s3_key
update_req['s3_access_key'] = work_dict.get('s3_access_key',
ae_consts.S3_ACCESS_KEY)
update_req['s3_secret_key'] = work_dict.get('s3_secret_key',
ae_consts.S3_SECRET_KEY)
update_req['s3_region_name'] = work_dict.get('s3_region_name',
ae_consts.S3_REGION_NAME)
update_req['s3_address'] = work_dict.get('s3_address',
ae_consts.S3_ADDRESS)
update_req['s3_secure'] = work_dict.get('s3_secure',
ae_consts.S3_SECURE)
update_req['redis_key'] = redis_key
update_req['redis_address'] = work_dict.get('redis_address',
ae_consts.REDIS_ADDRESS)
update_req['redis_password'] = work_dict.get('redis_password',
ae_consts.REDIS_PASSWORD)
update_req['redis_db'] = int(
work_dict.get('redis_db', ae_consts.REDIS_DB))
update_req['redis_expire'] = work_dict.get('redis_expire',
ae_consts.REDIS_EXPIRE)
update_req['updated'] = rec['updated']
update_req['label'] = label
update_req['celery_disabled'] = True
update_status = ae_consts.NOT_SET
try:
update_res = publisher.run_publish_pricing_update(
work_dict=update_req)
update_status = update_res.get('status', ae_consts.NOT_SET)
if ae_consts.ev('DEBUG_RESULTS', '0') == '1':
log.info('{} update_res status={} data={}'.format(
label, ae_consts.get_status(status=update_status),
ae_consts.ppj(update_res)))
else:
log.info('{} run_publish_pricing_update status={}'.format(
label, ae_consts.get_status(status=update_status)))
# end of if/else
rec['publish_pricing_update'] = update_res
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as f:
err = ('{} publisher.run_publish_pricing_update failed '
'with ex={}'.format(label, f))
log.error(err)
res = build_result.build_result(status=ae_consts.ERR,
err=err,
rec=rec)
# end of trying to publish results to connected services
except Exception as e:
res = build_result.build_result(status=ae_consts.ERR,
err=('failed - get_new_pricing_data '
'dict={} with ex={}').format(
work_dict, e),
rec=rec)
log.error('{} - {}'.format(label, res['err']))
# end of try/ex
if ae_consts.ev('DATASET_COLLECTION_SLACK_ALERTS', '0') == '1':
env_name = 'DEV'
if ae_consts.ev('PROD_SLACK_ALERTS', '1') == '1':
env_name = 'PROD'
done_msg = ('Dataset collected ticker=*{}* on env=*{}* '
'redis_key={} s3_key={} iex={} yahoo={}'.format(
ticker, env_name, redis_key, s3_key, get_iex_data,
get_yahoo_data))
log.debug('{} sending slack msg={}'.format(label, done_msg))
if res['status'] == ae_consts.SUCCESS:
slack_utils.post_success(msg=done_msg, block=False, jupyter=True)
else:
slack_utils.post_failure(msg=done_msg, block=False, jupyter=True)
# end of if/else success
# end of publishing to slack
log.info('task - get_new_pricing_data done - '
'{} - status={}'.format(label,
ae_consts.get_status(res['status'])))
return get_task_results.get_task_results(work_dict=work_dict, result=res)
daily_df = res['SPY']['daily']
minute_df = res['SPY']['minute']
out_dir = '/opt/sa/tests/datasets'
if not os.path.exists(out_dir):
print(f'missing output dir: {out_dir}')
sys.exit(1)
daily_file = f'{out_dir}/{ticker.lower()}-daily.json'
minute_file = f'{out_dir}/{ticker.lower()}-minute.json'
print('converting dates')
print(f'converting to pretty printed json file={daily_file}')
daily_out_json = ppj(
json.loads(daily_df.iloc[-100:-1].to_json(orient='records',
date_format='iso')))
print(f'writing to file daily_file={daily_file}')
with open(daily_file, 'w') as f:
f.write(daily_out_json)
if not os.path.exists(daily_file):
print(f'failed creating daily ticker={ticker} daily_file={daily_file}')
print(f'converting to pretty printed json file={minute_file}')
minute_out_json = ppj(
json.loads(minute_df.iloc[-100:-1].to_json(orient='records',
date_format='iso')))
print(f'writing to file minute_file={minute_file}')
with open(minute_file, 'w') as f:
f.write(minute_out_json)
def publish_from_s3_to_redis(self, work_dict):
"""publish_from_s3_to_redis
Publish Ticker Data from S3 to Redis
:param work_dict: dictionary for key/values
"""
label = 'pub-s3-to-redis'
log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict))
ticker = ae_consts.TICKER
ticker_id = ae_consts.TICKER_ID
rec = {
'ticker': None,
'ticker_id': None,
's3_enabled': True,
'redis_enabled': True,
's3_bucket': None,
's3_key': None,
'redis_key': None,
'updated': None
}
res = build_result.build_result(status=ae_consts.NOT_RUN,
err=None,
rec=rec)
try:
ticker = work_dict.get('ticker', ae_consts.TICKER)
ticker_id = int(work_dict.get('ticker_id', ae_consts.TICKER_ID))
if not ticker:
res = build_result.build_result(status=ae_consts.ERR,
err='missing ticker',
rec=rec)
return res
s3_key = work_dict.get('s3_key', None)
s3_bucket_name = work_dict.get('s3_bucket', 'pricing')
redis_key = work_dict.get('redis_key', None)
updated = work_dict.get('updated', None)
serializer = work_dict.get('serializer', 'json')
encoding = work_dict.get('encoding', 'utf-8')
label = work_dict.get('label', label)
enable_s3_read = True
enable_redis_publish = True
rec['ticker'] = ticker
rec['ticker_id'] = ticker_id
rec['s3_bucket'] = s3_bucket_name
rec['s3_key'] = s3_key
rec['redis_key'] = redis_key
rec['updated'] = updated
rec['s3_enabled'] = enable_s3_read
rec['redis_enabled'] = enable_redis_publish
data = None
if enable_s3_read:
log.info('{} parsing s3 values'.format(label))
access_key = work_dict.get('s3_access_key',
ae_consts.S3_ACCESS_KEY)
secret_key = work_dict.get('s3_secret_key',
ae_consts.S3_SECRET_KEY)
region_name = work_dict.get('s3_region_name',
ae_consts.S3_REGION_NAME)
service_address = work_dict.get('s3_address', ae_consts.S3_ADDRESS)
secure = work_dict.get('s3_secure', ae_consts.S3_SECURE) == '1'
endpoint_url = 'http://{}'.format(service_address)
if secure:
endpoint_url = 'https://{}'.format(service_address)
log.info('{} building s3 endpoint_url={} '
'region={}'.format(label, endpoint_url, region_name))
s3 = boto3.resource(
's3',
endpoint_url=endpoint_url,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region_name,
config=boto3.session.Config(signature_version='s3v4'))
try:
log.info('{} checking bucket={} exists'.format(
label, s3_bucket_name))
if s3.Bucket(s3_bucket_name) not in s3.buckets.all():
log.info('{} creating bucket={}'.format(
label, s3_bucket_name))
s3.create_bucket(Bucket=s3_bucket_name)
except Exception as e:
log.info('{} failed creating bucket={} '
'with ex={}'.format(label, s3_bucket_name, e))
# end of try/ex for creating bucket
try:
log.info('{} reading to s3={}/{} '
'updated={}'.format(label, s3_bucket_name, s3_key,
updated))
data = s3_read_contents_from_key.s3_read_contents_from_key(
s3=s3,
s3_bucket_name=s3_bucket_name,
s3_key=s3_key,
encoding=encoding,
convert_as_json=True)
initial_size_value = \
len(str(data)) / 1024000
initial_size_str = ae_consts.to_f(initial_size_value)
if ae_consts.ev('DEBUG_S3', '0') == '1':
log.info('{} read s3={}/{} data={}'.format(
label, s3_bucket_name, s3_key, ae_consts.ppj(data)))
else:
log.info('{} read s3={}/{} data size={} MB'.format(
label, s3_bucket_name, s3_key, initial_size_str))
except Exception as e:
err = ('{} failed reading bucket={} '
'key={} ex={}').format(label, s3_bucket_name, s3_key, e)
log.error(err)
res = build_result.build_result(status=ae_consts.NOT_RUN,
err=err,
rec=rec)
# end of try/ex for creating bucket
else:
log.info('{} SKIP S3 read bucket={} '
'key={}'.format(label, s3_bucket_name, s3_key))
# end of if enable_s3_read
if enable_redis_publish:
redis_address = work_dict.get('redis_address',
ae_consts.REDIS_ADDRESS)
redis_key = work_dict.get('redis_key', ae_consts.REDIS_KEY)
redis_password = work_dict.get('redis_password',
ae_consts.REDIS_PASSWORD)
redis_db = work_dict.get('redis_db', None)
if not redis_db:
redis_db = ae_consts.REDIS_DB
redis_expire = None
if 'redis_expire' in work_dict:
redis_expire = work_dict.get('redis_expire',
ae_consts.REDIS_EXPIRE)
log.info('redis enabled address={}@{} '
'key={}'.format(redis_address, redis_db, redis_key))
redis_host = redis_address.split(':')[0]
redis_port = redis_address.split(':')[1]
try:
if ae_consts.ev('DEBUG_REDIS', '0') == '1':
log.info('{} publishing redis={}:{} '
'db={} key={} '
'updated={} expire={} '
'data={}'.format(label, redis_host, redis_port,
redis_db, redis_key,
updated, redis_expire,
ae_consts.ppj(data)))
else:
log.info('{} publishing redis={}:{} '
'db={} key={} '
'updated={} expire={}'.format(
label, redis_host, redis_port, redis_db,
redis_key, updated, redis_expire))
# end of if/else
rc = redis.Redis(host=redis_host,
port=redis_port,
password=redis_password,
db=redis_db)
redis_set_res = redis_set.set_data_in_redis_key(
label=label,
client=rc,
key=redis_key,
data=data,
serializer=serializer,
encoding=encoding,
expire=redis_expire,
px=None,
nx=False,
xx=False)
log.info('{} redis_set status={} err={}'.format(
label, ae_consts.get_status(redis_set_res['status']),
redis_set_res['err']))
except Exception as e:
log.error('{} failed - redis publish to '
'key={} ex={}'.format(label, redis_key, e))
# end of try/ex for creating bucket
else:
log.info('{} SKIP REDIS publish '
'key={}'.format(label, redis_key))
# end of if enable_redis_publish
res = build_result.build_result(status=ae_consts.SUCCESS,
err=None,
rec=rec)
except Exception as e:
res = build_result.build_result(
status=ae_consts.ERR,
err=('failed - publish_from_s3_to_redis '
'dict={} with ex={}').format(work_dict, e),
rec=rec)
log.error('{} - {}'.format(label, res['err']))
# end of try/ex
log.info('task - publish_from_s3_to_redis done - '
'{} - status={}'.format(label,
ae_consts.get_status(res['status'])))
return get_task_results.get_task_results(work_dict=work_dict, result=res)
def build_algo_request(ticker=None,
tickers=None,
use_key=None,
start_date=None,
end_date=None,
datasets=None,
balance=None,
commission=None,
num_shares=None,
config_file=None,
config_dict=None,
load_config=None,
history_config=None,
report_config=None,
extract_config=None,
timeseries=None,
trade_strategy=None,
cache_freq='daily',
label='algo'):
"""build_algo_request
Create a dictionary for building an algorithm. This is
opinionated to how the underlying date-based caching
strategy is running per day. Each business day becomes
a possible dataset to process with an algorithm.
:param ticker: ticker
:param tickers: optional - list of tickers
:param use_key: redis and s3 to store the algo result
:param start_date: string date format ``YYYY-MM-DD HH:MM:SS``
:param end_date: string date format ``YYYY-MM-DD HH:MM:SS``
:param datasets: list of string dataset types
:param balance: starting capital balance
:param commission: commission for buy or sell
:param num_shares: optional - integer number of starting shares
:param cache_freq: optional - cache frequency
(``daily`` is default)
:param label: optional - algo log tracking name
:param config_file: path to a json file
containing custom algorithm object
member values (like indicator configuration and
predict future date units ahead for a backtest)
:param config_dict: optional - dictionary that
can be passed to derived class implementations
of: ``def load_from_config(config_dict=config_dict)``
**Timeseries**
:param timeseries: optional - string to
set ``day`` or ``minute`` backtesting
or live trading
(default is ``minute``)
**Trading Strategy**
:param trade_strategy: optional - string to
set the type of ``Trading Strategy``
for backtesting or live trading
(default is ``count``)
**Algorithm Dataset Extraction, Loading and Publishing arguments**
:param load_config: optional - dictionary
for setting member variables to load an
agorithm-ready dataset from
a file, s3 or redis
:param history_config: optional - dictionary
for setting member variables to publish
an algo ``trade history`` to s3, redis, a file
or slack
:param report_config: optional - dictionary
for setting member variables to publish
an algo ``trading performance report`` to s3,
redis, a file or slack
:param extract_config: optional - dictionary
for setting member variables to publish
an algo ``trading performance report`` to s3,
redis, a file or slack
"""
use_tickers = []
if ticker:
use_tickers = [ticker.upper()]
if tickers:
for t in tickers:
if t not in use_tickers:
use_tickers.append(t.upper())
s3_bucket_name = ae_consts.ALGO_RESULT_S3_BUCKET_NAME
s3_key = use_key
redis_key = use_key
s3_enabled = True
redis_enabled = True
work = {
'tickers': use_tickers,
's3_bucket': s3_bucket_name,
's3_key': s3_key,
'redis_key': redis_key,
's3_enabled': s3_enabled,
'redis_enabled': redis_enabled,
'extract_datasets': [],
'cache_freq': cache_freq,
'config_file': config_file,
'config_dict': config_dict,
'balance': balance,
'commission': commission,
'load_config': load_config,
'history_config': history_config,
'report_config': report_config,
'extract_config': extract_config,
'start_date': None,
'end_date': None,
'timeseries': timeseries,
'trade_strategy': trade_strategy,
'version': 1,
'label': label
}
start_date_val = ae_utils.get_date_from_str(start_date)
end_date_val = ae_utils.get_date_from_str(end_date)
if start_date_val > end_date_val:
raise Exception(
'Invalid start_date={} must be less than end_date={}'.format(
start_date, end_date))
use_dates = []
new_dataset = None
cur_date = start_date_val
if not work['start_date']:
work['start_date'] = start_date_val.strftime(
ae_consts.COMMON_TICK_DATE_FORMAT)
if not work['end_date']:
work['end_date'] = end_date_val.strftime(
ae_consts.COMMON_TICK_DATE_FORMAT)
while cur_date <= end_date_val:
if cur_date.weekday() < 5:
for t in use_tickers:
if cache_freq == 'daily':
new_dataset = '{}_{}'.format(
t, cur_date.strftime(ae_consts.COMMON_DATE_FORMAT))
else:
new_dataset = '{}_{}'.format(
t,
cur_date.strftime(ae_consts.COMMON_TICK_DATE_FORMAT))
if new_dataset:
use_dates.append(new_dataset)
new_dataset = None
# end for all tickers
# end of valid days M-F
if cache_freq == 'daily':
cur_date += datetime.timedelta(days=1)
else:
cur_date += datetime.timedelta(minute=1)
# end of walking all dates to add
if len(use_dates) > 0:
work['extract_datasets'] = use_dates
log.debug('tickers={} balance={} start={} end={} '
'cache_freq={} request={}'.format(
work['tickers'], work['balance'],
work['extract_datasets'][0],
work['extract_datasets'][-1], cache_freq,
ae_consts.ppj(work)))
else:
log.error('there are not enough dates to test between start={} end={} '
'tickers={} request={}'.format(start_date_val, end_date_val,
work['tickers'], cache_freq,
ae_consts.ppj(work)))
return work
def show_dataset(algo_dataset=None,
dataset_type=ae_consts.SA_DATASET_TYPE_ALGO_READY,
serialize_datasets=ae_consts.DEFAULT_SERIALIZED_DATASETS,
path_to_file=None,
compress=False,
encoding='utf-8',
redis_enabled=True,
redis_key=None,
redis_address=None,
redis_db=None,
redis_password=None,
redis_expire=None,
redis_serializer='json',
redis_encoding='utf-8',
s3_enabled=True,
s3_key=None,
s3_address=None,
s3_bucket=None,
s3_access_key=None,
s3_secret_key=None,
s3_region_name=None,
s3_secure=False,
slack_enabled=False,
slack_code_block=False,
slack_full_width=False,
verbose=False):
"""show_dataset
Show a supported dataset's internal structure and preview some
of the values to debug mapping, serialization issues
:param algo_dataset: optional - already loaded algorithm-ready dataset
:param dataset_type: optional - dataset type
(default is ``SA_DATASET_TYPE_ALGO_READY``)
:param serialize_datasets: optional - list of dataset names to
deserialize in the dataset
:param path_to_file: optional - path to an algorithm-ready dataset
in a file
:param compress: optional - boolean flag for decompressing
the contents of the ``path_to_file`` if necessary
(default is ``False`` and algorithms
use ``zlib`` for compression)
:param encoding: optional - string for data encoding
**(Optional) Redis connectivity arguments**
:param redis_enabled: bool - toggle for auto-caching all
datasets in Redis
(default is ``True``)
:param redis_key: string - key to save the data in redis
(default is ``None``)
:param redis_address: Redis connection string format: ``host:port``
(default is ``localhost:6379``)
:param redis_db: Redis db to use
(default is ``0``)
:param redis_password: optional - Redis password
(default is ``None``)
:param redis_expire: optional - Redis expire value
(default is ``None``)
:param redis_serializer: not used yet - support for future
pickle objects in redis
:param redis_encoding: format of the encoded key in redis
**(Optional) Minio (S3) connectivity arguments**
:param s3_enabled: bool - toggle for auto-archiving on Minio (S3)
(default is ``True``)
:param s3_key: string - key to save the data in redis
(default is ``None``)
:param s3_address: Minio S3 connection string format: ``host:port``
(default is ``localhost:9000``)
:param s3_bucket: S3 Bucket for storing the artifacts
(default is ``dev``) which should be viewable on a browser:
http://localhost:9000/minio/dev/
:param s3_access_key: S3 Access key
(default is ``trexaccesskey``)
:param s3_secret_key: S3 Secret key
(default is ``trex123321``)
:param s3_region_name: S3 region name
(default is ``us-east-1``)
:param s3_secure: Transmit using tls encryption
(default is ``False``)
**(Optional) Slack arguments**
:param slack_enabled: optional - boolean for
publishing to slack
:param slack_code_block: optional - boolean for
publishing as a code black in slack
:param slack_full_width: optional - boolean for
publishing as a to slack using the full
width allowed
Additonal arguments
:param verbose: optional - bool for increasing
logging
"""
use_ds = algo_dataset
if not use_ds:
log.info('loading from file={} s3={} redis={}'.format(
path_to_file, s3_key, redis_key))
use_ds = load_dataset.load_dataset(
dataset_type=dataset_type,
compress=compress,
encoding=redis_encoding,
path_to_file=path_to_file,
s3_key=s3_key,
s3_address=s3_address,
s3_bucket=s3_bucket,
s3_access_key=s3_access_key,
s3_secret_key=s3_secret_key,
s3_region_name=s3_region_name,
s3_secure=s3_secure,
redis_key=redis_key,
redis_address=redis_address,
redis_db=redis_db,
redis_password=redis_password,
redis_expire=redis_expire,
redis_serializer=redis_serializer,
serialize_datasets=serialize_datasets)
if not use_ds:
log.error('unable to load a dataset from file={} '
's3={} redis={}'.format(path_to_file, s3_key, redis_key))
return None
# load if not created
if dataset_type == ae_consts.SA_DATASET_TYPE_ALGO_READY:
print('-----------------------------------')
print('dates found in dataset')
root_keys = []
for root_key in use_ds:
print(root_key)
root_keys.append(root_key)
all_dates = []
all_ids = []
for root_key in root_keys:
second_layer = use_ds[root_key]
for ds in second_layer:
if 'date' in ds:
if len(all_dates) == 0:
print('')
print('dates found in dataset')
cur_date = ds.get('date', None)
if cur_date:
print(cur_date)
all_dates.append(cur_date)
first_node = None
last_node = None
end_nodes = []
for root_key in root_keys:
second_layer = use_ds[root_key]
for ds in second_layer:
if not first_node:
first_node = ds
end_nodes.append(ds)
last_node = ds
if 'id' in ds:
if len(all_ids) == 0:
print('')
print('ids in the file')
cur_id = ds.get('id', None)
if cur_id:
print(cur_id)
all_ids.append(cur_id)
if first_node and last_node:
show_first = {}
for ds_key in first_node:
if ds_key == 'data':
show_first[ds_key] = {}
for ds_name in first_node[ds_key]:
print('first_node has dataset with name: {}'.format(
ds_name))
show_first[ds_key][ds_name] = 'EMPTY_DF'
if hasattr(first_node[ds_key][ds_name], 'index'):
show_first[ds_key][ds_name] = (
'pd.DataFrame() rows={}'.format(
len(first_node[ds_key][ds_name].index)))
else:
show_first[ds_key] = first_node[ds_key]
print('')
print('first node:')
print(ae_consts.ppj(show_first))
print('')
num_records = len(all_ids)
cur_cell = num_records - 4
for cur_node in end_nodes[-5:]:
show_node = {}
for ds_key in cur_node:
if ds_key == 'data':
show_node[ds_key] = {}
for ds_name in cur_node[ds_key]:
show_node[ds_key][ds_name] = 'EMPTY_DF'
if hasattr(cur_node[ds_key][ds_name], 'index'):
show_node[ds_key][ds_name] = (
'pd.DataFrame() rows={}'.format(
len(cur_node[ds_key][ds_name].index)))
else:
show_node[ds_key] = cur_node[ds_key]
# end of show cur_node
print('node={}/{} values:'.format(cur_cell, num_records))
print(ae_consts.ppj(show_node))
print('')
cur_cell += 1
# end of end_nodes
else:
if not first_node:
print('missing first node in dataset')
if not last_node:
print('missing last node in dataset')
if len(all_dates) > 0:
print('root_keys={} from {} to {}'.format(root_keys, all_dates[0],
all_dates[-1]))
else:
print('root_keys={} missing dates'.format(root_keys))
print('-----------------------------------')
return use_ds
def fetch_new_stock_datasets():
"""fetch_new_stock_datasets
Collect all datasets for the ticker **SPY**:
::
fetch_new_stock_datasets.py -t SPY
.. note:: This requires the following services are listening on:
- redis ``localhost:6379``
- minio ``localhost:9000``
"""
log.info('start - fetch_new_stock_datasets')
parser = argparse.ArgumentParser(
description=('Download and store the latest stock pricing, '
'news, and options chain data '
'and store it in Minio (S3) and Redis. '
'Also includes support for getting FinViz '
'screener tickers'))
parser.add_argument('-t', help=('ticker'), required=False, dest='ticker')
parser.add_argument('-g',
help=('optional - fetch mode: '
'all = fetch from all data sources (default), '
'td = fetch from just Tradier sources, '
'iex = fetch from just IEX sources'),
required=False,
dest='fetch_mode')
parser.add_argument('-i',
help=('optional - ticker id '
'not used without a database'),
required=False,
dest='ticker_id')
parser.add_argument('-e',
help=('optional - options expiration date'),
required=False,
dest='exp_date_str')
parser.add_argument('-l',
help=('optional - path to the log config file'),
required=False,
dest='log_config_path')
parser.add_argument('-b',
help=('optional - broker url for Celery'),
required=False,
dest='broker_url')
parser.add_argument('-B',
help=('optional - backend url for Celery'),
required=False,
dest='backend_url')
parser.add_argument('-k',
help=('optional - s3 access key'),
required=False,
dest='s3_access_key')
parser.add_argument('-s',
help=('optional - s3 secret key'),
required=False,
dest='s3_secret_key')
parser.add_argument('-a',
help=('optional - s3 address format: <host:port>'),
required=False,
dest='s3_address')
parser.add_argument('-S',
help=('optional - s3 ssl or not'),
required=False,
dest='s3_secure')
parser.add_argument('-u',
help=('optional - s3 bucket name'),
required=False,
dest='s3_bucket_name')
parser.add_argument('-G',
help=('optional - s3 region name'),
required=False,
dest='s3_region_name')
parser.add_argument('-p',
help=('optional - redis_password'),
required=False,
dest='redis_password')
parser.add_argument('-r',
help=('optional - redis_address format: <host:port>'),
required=False,
dest='redis_address')
parser.add_argument('-n',
help=('optional - redis and s3 key name'),
required=False,
dest='keyname')
parser.add_argument(
'-m',
help=('optional - redis database number (0 by default)'),
required=False,
dest='redis_db')
parser.add_argument('-x',
help=('optional - redis expiration in seconds'),
required=False,
dest='redis_expire')
parser.add_argument('-z',
help=('optional - strike price'),
required=False,
dest='strike')
parser.add_argument(
'-c',
help=('optional - contract type "C" for calls "P" for puts'),
required=False,
dest='contract_type')
parser.add_argument(
'-P',
help=('optional - get pricing data if "1" or "0" disabled'),
required=False,
dest='get_pricing')
parser.add_argument(
'-N',
help=('optional - get news data if "1" or "0" disabled'),
required=False,
dest='get_news')
parser.add_argument(
'-O',
help=('optional - get options data if "1" or "0" disabled'),
required=False,
dest='get_options')
parser.add_argument('-U',
help=('optional - s3 enabled for publishing if "1" or '
'"0" is disabled'),
required=False,
dest='s3_enabled')
parser.add_argument(
'-R',
help=('optional - redis enabled for publishing if "1" or '
'"0" is disabled'),
required=False,
dest='redis_enabled')
parser.add_argument('-A',
help=('optional - run an analysis '
'supported modes: scn'),
required=False,
dest='analysis_type')
parser.add_argument('-L',
help=('optional - screener urls to pull '
'tickers for analysis'),
required=False,
dest='urls')
parser.add_argument(
'-Z',
help=('disable run without an engine for local testing and demos'),
required=False,
dest='celery_enabled',
action='store_true')
parser.add_argument('-d',
help=('debug'),
required=False,
dest='debug',
action='store_true')
args = parser.parse_args()
run_offline = True
ticker = ae_consts.TICKER
ticker_id = ae_consts.TICKER_ID
fetch_mode = 'all'
exp_date_str = ae_consts.NEXT_EXP_STR
ssl_options = ae_consts.SSL_OPTIONS
transport_options = ae_consts.TRANSPORT_OPTIONS
broker_url = ae_consts.WORKER_BROKER_URL
backend_url = ae_consts.WORKER_BACKEND_URL
celery_config_module = ae_consts.WORKER_CELERY_CONFIG_MODULE
include_tasks = ae_consts.INCLUDE_TASKS
s3_access_key = ae_consts.S3_ACCESS_KEY
s3_secret_key = ae_consts.S3_SECRET_KEY
s3_region_name = ae_consts.S3_REGION_NAME
s3_address = ae_consts.S3_ADDRESS
s3_secure = ae_consts.S3_SECURE
s3_bucket_name = ae_consts.S3_BUCKET
s3_key = ae_consts.S3_KEY
redis_address = ae_consts.REDIS_ADDRESS
redis_key = ae_consts.REDIS_KEY
redis_password = ae_consts.REDIS_PASSWORD
redis_db = ae_consts.REDIS_DB
redis_expire = ae_consts.REDIS_EXPIRE
strike = None
contract_type = None
get_pricing = True
get_news = True
get_options = True
s3_enabled = True
redis_enabled = True
analysis_type = None
debug = False
if args.ticker:
ticker = args.ticker.upper()
if args.ticker_id:
ticker_id = args.ticker_id
if args.exp_date_str:
exp_date_str = ae_consts.NEXT_EXP_STR
if args.broker_url:
broker_url = args.broker_url
if args.backend_url:
backend_url = args.backend_url
if args.s3_access_key:
s3_access_key = args.s3_access_key
if args.s3_secret_key:
s3_secret_key = args.s3_secret_key
if args.s3_region_name:
s3_region_name = args.s3_region_name
if args.s3_address:
s3_address = args.s3_address
if args.s3_secure:
s3_secure = args.s3_secure
if args.s3_bucket_name:
s3_bucket_name = args.s3_bucket_name
if args.keyname:
s3_key = args.keyname
redis_key = args.keyname
if args.redis_address:
redis_address = args.redis_address
if args.redis_password:
redis_password = args.redis_password
if args.redis_db:
redis_db = args.redis_db
if args.redis_expire:
redis_expire = args.redis_expire
if args.strike:
strike = args.strike
if args.contract_type:
contract_type = args.contract_type
if args.get_pricing:
get_pricing = args.get_pricing == '1'
if args.get_news:
get_news = args.get_news == '1'
if args.get_options:
get_options = args.get_options == '1'
if args.s3_enabled:
s3_enabled = args.s3_enabled == '1'
if args.redis_enabled:
redis_enabled = args.redis_enabled == '1'
if args.fetch_mode:
fetch_mode = str(args.fetch_mode).lower()
if args.analysis_type:
analysis_type = str(args.analysis_type).lower()
if args.celery_enabled:
run_offline = False
if args.debug:
debug = True
work = api_requests.build_get_new_pricing_request()
work['ticker'] = ticker
work['ticker_id'] = ticker_id
work['s3_bucket'] = s3_bucket_name
work['s3_key'] = s3_key
work['redis_key'] = redis_key
work['strike'] = strike
work['contract'] = contract_type
work['exp_date'] = exp_date_str
work['s3_access_key'] = s3_access_key
work['s3_secret_key'] = s3_secret_key
work['s3_region_name'] = s3_region_name
work['s3_address'] = s3_address
work['s3_secure'] = s3_secure
work['redis_address'] = redis_address
work['redis_password'] = redis_password
work['redis_db'] = redis_db
work['redis_expire'] = redis_expire
work['get_pricing'] = get_pricing
work['get_news'] = get_news
work['get_options'] = get_options
work['s3_enabled'] = s3_enabled
work['redis_enabled'] = redis_enabled
work['fetch_mode'] = fetch_mode
work['analysis_type'] = analysis_type
work['iex_datasets'] = iex_consts.DEFAULT_FETCH_DATASETS
work['debug'] = debug
work['label'] = 'ticker={}'.format(ticker)
if analysis_type == 'scn':
label = 'screener={}'.format(work['ticker'])
fv_urls = []
if args.urls:
fv_urls = str(args.urls).split('|')
if len(fv_urls) == 0:
fv_urls = os.getenv('SCREENER_URLS', []).split('|')
screener_req = api_requests.build_screener_analysis_request(
ticker=ticker, fv_urls=fv_urls, label=label)
work.update(screener_req)
start_screener_analysis(req=work)
# end of analysis_type
else:
if not args.keyname:
last_close_date = ae_utils.last_close()
work['s3_key'] = '{}_{}'.format(
work['ticker'],
last_close_date.strftime(ae_consts.COMMON_DATE_FORMAT))
work['redis_key'] = '{}_{}'.format(
work['ticker'],
last_close_date.strftime(ae_consts.COMMON_DATE_FORMAT))
path_to_tasks = 'analysis_engine.work_tasks'
task_name = ('{}.get_new_pricing_data.get_new_pricing_data'.format(
path_to_tasks))
task_res = None
if ae_consts.is_celery_disabled() or run_offline:
work['celery_disabled'] = True
log.debug('starting without celery work={} offline={}'.format(
ae_consts.ppj(work), run_offline))
task_res = task_pricing.get_new_pricing_data(work)
if debug:
log.info('done - result={} '
'task={} status={} '
'err={} label={}'.format(
ae_consts.ppj(task_res), task_name,
ae_consts.get_status(status=task_res['status']),
task_res['err'], work['label']))
else:
log.info('done - result '
'task={} status={} '
'err={} label={}'.format(
task_name,
ae_consts.get_status(status=task_res['status']),
task_res['err'], work['label']))
# if/else debug
else:
log.info('connecting to broker={} backend={}'.format(
broker_url, backend_url))
# Get the Celery app
app = get_celery_app.get_celery_app(
name=__name__,
auth_url=broker_url,
backend_url=backend_url,
path_to_config_module=celery_config_module,
ssl_options=ssl_options,
transport_options=transport_options,
include_tasks=include_tasks)
log.info('calling task={} - work={}'.format(
task_name, ae_consts.ppj(work)))
job_id = app.send_task(task_name, (work, ))
log.info('calling task={} - success job_id={}'.format(
task_name, job_id))
def run_aws_backup():
"""run_aws_backup
Run buy and sell analysis on a stock to send alerts to subscribed
users
"""
log.debug('start - aws-backup')
parser = argparse.ArgumentParser(description=('stock analysis tool'))
parser.add_argument('-t', help=('ticker'), required=True, dest='ticker')
parser.add_argument('-e',
help=('file path to extract an '
'algorithm-ready datasets from redis'),
required=False,
dest='algo_extract_loc')
parser.add_argument('-l',
help=('show dataset in this file'),
required=False,
dest='show_from_file')
parser.add_argument('-H',
help=('show trading history dataset in this file'),
required=False,
dest='show_history_from_file')
parser.add_argument(
'-E',
help=('show trading performance report dataset in this file'),
required=False,
dest='show_report_from_file')
parser.add_argument(
'-L',
help=('restore an algorithm-ready dataset file back into redis'),
required=False,
dest='restore_algo_file')
parser.add_argument('-f',
help=('run in mode: prepare dataset from '
'redis key or s3 key'),
required=False,
dest='prepare_mode',
action='store_true')
parser.add_argument(
'-J',
help=('plot action - after preparing you can use: '
'-J show to open the image (good for debugging)'),
required=False,
dest='plot_action')
parser.add_argument(
'-b',
help=('run a backtest using the dataset in '
'a file path/s3 key/redis key formats: '
'file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
's3://algoready/SPY-latest.json or '
'redis:SPY-latest'),
required=False,
dest='backtest_loc')
parser.add_argument('-B',
help=('optional - broker url for Celery'),
required=False,
dest='broker_url')
parser.add_argument('-C',
help=('optional - broker url for Celery'),
required=False,
dest='backend_url')
parser.add_argument(
'-w',
help=('optional - flag for publishing an algorithm job '
'using Celery to the ae workers'),
required=False,
dest='run_on_engine',
action='store_true')
parser.add_argument('-k',
help=('optional - s3 access key'),
required=False,
dest='s3_access_key')
parser.add_argument('-K',
help=('optional - s3 secret key'),
required=False,
dest='s3_secret_key')
parser.add_argument('-a',
help=('optional - s3 address format: <host:port>'),
required=False,
dest='s3_address')
parser.add_argument('-Z',
help=('optional - s3 secure: default False'),
required=False,
dest='s3_secure')
parser.add_argument('-s',
help=('optional - start date: YYYY-MM-DD'),
required=False,
dest='start_date')
parser.add_argument('-n',
help=('optional - end date: YYYY-MM-DD'),
required=False,
dest='end_date')
parser.add_argument('-u',
help=('optional - s3 bucket name'),
required=False,
dest='s3_bucket_name')
parser.add_argument('-G',
help=('optional - s3 region name'),
required=False,
dest='s3_region_name')
parser.add_argument(
'-g',
help=('Path to a custom algorithm module file '
'on disk. This module must have a single '
'class that inherits from: '
'https://github.com/AlgoTraders/stock-analysis-engine/'
'blob/master/'
'analysis_engine/algo.py Additionally you '
'can find the Example-Minute-Algorithm here: '
'https://github.com/AlgoTraders/stock-analysis-engine/'
'blob/master/analysis_engine/mocks/'
'example_algo_minute.py'),
required=False,
dest='run_algo_in_file')
parser.add_argument('-p',
help=('optional - s3 bucket/file for trading history'),
required=False,
dest='algo_history_loc')
parser.add_argument(
'-o',
help=('optional - s3 bucket/file for trading performance report'),
required=False,
dest='algo_report_loc')
parser.add_argument('-r',
help=('optional - redis_address format: <host:port>'),
required=False,
dest='redis_address')
parser.add_argument('-R',
help=('optional - redis and s3 key name'),
required=False,
dest='keyname')
parser.add_argument(
'-m',
help=('optional - redis database number (0 by default)'),
required=False,
dest='redis_db')
parser.add_argument('-x',
help=('optional - redis expiration in seconds'),
required=False,
dest='redis_expire')
parser.add_argument('-z',
help=('optional - strike price'),
required=False,
dest='strike')
parser.add_argument(
'-c',
help=('optional - algorithm config_file path for setting '
'up internal algorithm trading strategies and '
'indicators'),
required=False,
dest='config_file')
parser.add_argument(
'-P',
help=('optional - get pricing data if "1" or "0" disabled'),
required=False,
dest='get_pricing')
parser.add_argument(
'-N',
help=('optional - get news data if "1" or "0" disabled'),
required=False,
dest='get_news')
parser.add_argument(
'-O',
help=('optional - get options data if "1" or "0" disabled'),
required=False,
dest='get_options')
parser.add_argument(
'-i',
help=('optional - ignore column names (comma separated)'),
required=False,
dest='ignore_columns')
parser.add_argument('-d',
help=('debug'),
required=False,
dest='debug',
action='store_true')
args = parser.parse_args()
mode = 'prepare'
plot_action = ae_consts.PLOT_ACTION_SHOW
ticker = ae_consts.TICKER
ticker_id = ae_consts.TICKER_ID
ssl_options = ae_consts.SSL_OPTIONS
transport_options = ae_consts.TRANSPORT_OPTIONS
broker_url = ae_consts.WORKER_BROKER_URL
backend_url = ae_consts.WORKER_BACKEND_URL
path_to_config_module = ae_consts.WORKER_CELERY_CONFIG_MODULE
include_tasks = ae_consts.INCLUDE_TASKS
s3_access_key = ae_consts.S3_ACCESS_KEY
s3_secret_key = ae_consts.S3_SECRET_KEY
s3_region_name = ae_consts.S3_REGION_NAME
s3_address = ae_consts.S3_ADDRESS
s3_secure = ae_consts.S3_SECURE
s3_bucket_name = ae_consts.S3_BUCKET
s3_key = ae_consts.S3_KEY
redis_address = ae_consts.REDIS_ADDRESS
redis_key = ae_consts.REDIS_KEY
redis_password = ae_consts.REDIS_PASSWORD
redis_db = ae_consts.REDIS_DB
redis_expire = ae_consts.REDIS_EXPIRE
dataset_type = ae_consts.SA_DATASET_TYPE_ALGO_READY
serialize_datasets = ae_consts.DEFAULT_SERIALIZED_DATASETS
output_redis_key = None
output_s3_bucket = None
output_s3_key = None
ignore_columns = None
compress = False
encoding = 'utf-8'
slack_enabled = False
slack_code_block = False
slack_full_width = False
verbose = False
debug = False
redis_serializer = 'json'
redis_encoding = 'utf-8'
output_redis_key = None
output_s3_bucket = None
output_s3_key = None
s3_enabled = True
redis_enabled = True
ignore_columns = None
debug = False
run_on_engine = False
show_from_file = None
show_history_from_file = None
show_report_from_file = None
restore_algo_file = None
backtest_loc = None
use_custom_algo = False
algo_history_loc = 's3://algohistory'
algo_report_loc = 's3://algoreport'
algo_extract_loc = 's3://algoready'
use_balance = 5000.0
use_commission = 6.0
auto_fill = True
use_start_date = None
use_end_date = None
use_config_file = None
use_name = 'myalgo'
if args.ticker:
ticker = args.ticker.upper()
if args.broker_url:
broker_url = args.broker_url
if args.backend_url:
backend_url = args.backend_url
if args.s3_access_key:
s3_access_key = args.s3_access_key
if args.s3_secret_key:
s3_secret_key = args.s3_secret_key
if args.s3_region_name:
s3_region_name = args.s3_region_name
if args.s3_address:
s3_address = args.s3_address
s3_enabled = True
if args.s3_secure:
s3_secure = args.s3_secure
if args.s3_bucket_name:
s3_bucket_name = args.s3_bucket_name
if args.keyname:
s3_key = args.keyname
redis_key = args.keyname
if args.redis_address:
redis_address = args.redis_address
if args.redis_db:
redis_db = args.redis_db
if args.redis_expire:
redis_expire = args.redis_expire
if args.prepare_mode:
mode = ae_consts.SA_MODE_PREPARE
if args.ignore_columns:
ignore_columns_org = args.ignore_columns
ignore_columns = ignore_columns_org.split(",")
if args.plot_action:
if str(args.plot_action).lower() == 'show':
plot_action = ae_consts.PLOT_ACTION_SHOW
elif str(args.plot_action).lower() == 's3':
plot_action = ae_consts.PLOT_ACTION_SAVE_TO_S3
elif str(args.plot_action).lower() == 'save':
plot_action = ae_consts.PLOT_ACTION_SAVE_AS_FILE
else:
plot_action = ae_consts.PLOT_ACTION_SHOW
log.warning('unsupported plot_action: {}'.format(args.plot_action))
if args.debug:
debug = True
if args.algo_extract_loc:
mode = ae_consts.SA_MODE_EXTRACT
if args.show_from_file:
show_from_file = args.show_from_file
mode = ae_consts.SA_MODE_SHOW_DATASET
if args.show_history_from_file:
show_history_from_file = args.show_history_from_file
mode = ae_consts.SA_MODE_SHOW_HISTORY_DATASET
if args.show_report_from_file:
show_report_from_file = args.show_report_from_file
mode = ae_consts.SA_MODE_SHOW_REPORT_DATASET
if args.restore_algo_file:
restore_algo_file = args.restore_algo_file
mode = ae_consts.SA_MODE_RESTORE_REDIS_DATASET
if args.run_algo_in_file:
mode = ae_consts.SA_MODE_RUN_ALGO
if args.backtest_loc:
mode = ae_consts.SA_MODE_RUN_ALGO
if args.start_date:
try:
use_start_date = '{} 00:00:00'.format(str(args.start_date))
datetime.datetime.strptime(args.start_date,
ae_consts.COMMON_DATE_FORMAT)
except Exception as e:
msg = ('please use a start date formatted as: {}'
'\n'
'error was: {}'.format(ae_consts.COMMON_DATE_FORMAT, e))
log.error(msg)
sys.exit(1)
# end of testing for a valid date
# end of args.start_date
if args.end_date:
try:
use_end_date = '{} 00:00:00'.format(str(args.end_date))
datetime.datetime.strptime(args.end_date,
ae_consts.COMMON_DATE_FORMAT)
except Exception as e:
msg = ('please use an end date formatted as: {}'
'\n'
'error was: {}'.format(ae_consts.COMMON_DATE_FORMAT, e))
log.error(msg)
sys.exit(1)
# end of testing for a valid date
# end of args.end_date
if args.config_file:
use_config_file = args.config_file
if not os.path.exists(use_config_file):
log.error('Failed: unable to find config file: -c {}'.format(
use_config_file))
sys.exit(1)
config_dict = None
load_from_s3_bucket = None
load_from_s3_key = None
load_from_redis_key = None
load_from_file = None
load_compress = False
load_publish = True
load_config = None
report_redis_key = None
report_s3_bucket = None
report_s3_key = None
report_file = None
report_compress = False
report_publish = False
report_config = None
history_redis_key = None
history_s3_bucket = None
history_s3_key = None
history_file = None
history_compress = False
history_publish = False
history_config = None
extract_redis_key = None
extract_s3_bucket = None
extract_s3_key = None
extract_file = None
extract_save_dir = None
extract_compress = False
extract_publish = True
extract_config = None
publish_to_slack = False
publish_to_s3 = True
publish_to_redis = False
use_timeseries = 'day'
use_trade_strategy = 'count'
valid = False
required_task = False
work = None
task_name = None
work = {}
path_to_tasks = 'analysis_engine.work_tasks'
if mode == ae_consts.SA_MODE_PREPARE:
task_name = ('{}.'
'prepare_pricing_dataset.prepare_pricing_dataset'
).format(path_to_tasks)
work = api_requests.build_prepare_dataset_request()
if output_s3_key:
work['prepared_s3_key'] = output_s3_key
if output_s3_bucket:
work['prepared_s3_bucket'] = output_s3_bucket
if output_redis_key:
work['prepared_redis_key'] = output_redis_key
work['ignore_columns'] = ignore_columns
valid = True
required_task = True
elif mode == ae_consts.SA_MODE_EXTRACT:
if args.algo_extract_loc:
algo_extract_loc = args.algo_extract_loc
if ('file:/' not in algo_extract_loc
and 's3://' not in algo_extract_loc
and 'redis://' not in algo_extract_loc):
log.error(
'invalid -e <extract_to_file_or_s3_key_or_redis_key> '
'specified. please use either: '
'-e file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
'-e s3://algoready/SPY-latest.json or '
'-e redis://SPY-latest')
sys.exit(1)
if 's3://' in algo_extract_loc:
extract_s3_bucket = algo_extract_loc.split('/')[-2]
extract_s3_key = algo_extract_loc.split('/')[-1]
elif 'redis://' in algo_extract_loc:
extract_redis_key = algo_extract_loc.split('/')[-1]
elif 'file:/' in algo_extract_loc:
extract_file = algo_extract_loc.split(':')[-1]
# end of parsing supported transport for loading
use_custom_algo = True
elif mode == ae_consts.SA_MODE_SHOW_DATASET:
examine_dataset_in_file(ticker=ticker, path_to_file=show_from_file)
log.info('done showing {} dataset from file={}'.format(
ticker, show_from_file))
sys.exit(0)
elif mode == ae_consts.SA_MODE_SHOW_HISTORY_DATASET:
examine_dataset_in_file(
ticker=ticker,
dataset_type=ae_consts.SA_DATASET_TYPE_TRADING_HISTORY,
path_to_file=show_history_from_file)
log.info('done showing trading history {} dataset from '
'file={}'.format(ticker, show_from_file))
sys.exit(0)
elif mode == ae_consts.SA_MODE_SHOW_REPORT_DATASET:
examine_dataset_in_file(
ticker=ticker,
dataset_type=ae_consts.SA_DATASET_TYPE_TRADING_REPORT,
path_to_file=show_report_from_file)
log.info('done showing trading performance report {} dataset from '
'file={}'.format(ticker, show_from_file))
sys.exit(0)
elif mode == ae_consts.SA_MODE_RESTORE_REDIS_DATASET:
restore_missing_dataset_values_from_algo_ready_file(
ticker=ticker,
path_to_file=restore_algo_file,
redis_address=redis_address,
redis_password=redis_password,
redis_db=redis_db,
output_redis_db=redis_db,
dataset_type=ae_consts.SA_DATASET_TYPE_ALGO_READY,
serialize_datasets=ae_consts.DEFAULT_SERIALIZED_DATASETS)
log.info(
'done restoring {} dataset from file={} into redis_db={}'.format(
ticker, restore_algo_file, redis_db))
sys.exit(0)
elif mode == ae_consts.SA_MODE_RUN_ALGO:
if args.run_algo_in_file:
if not os.path.exists(args.run_algo_in_file):
log.error('missing algorithm module file: {}'.format(
args.run_algo_in_file))
sys.exit(1)
if args.backtest_loc:
backtest_loc = args.backtest_loc
if ('file:/' not in backtest_loc and 's3://' not in backtest_loc
and 'redis://' not in backtest_loc):
log.error(
'invalid -b <backtest dataset file> specified. '
'{} '
'please use either: '
'-b file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
'-b s3://algoready/SPY-latest.json or '
'-b redis://SPY-latest'.format(backtest_loc))
sys.exit(1)
if 's3://' in backtest_loc:
load_from_s3_bucket = backtest_loc.split('/')[-2]
load_from_s3_key = backtest_loc.split('/')[-1]
elif 'redis://' in backtest_loc:
load_from_redis_key = backtest_loc.split('/')[-1]
elif 'file:/' in backtest_loc:
load_from_file = backtest_loc.split(':')[-1]
load_publish = True
# end of parsing supported transport - loading an algo-ready
if args.algo_history_loc:
algo_history_loc = args.algo_history_loc
if ('file:/' not in algo_history_loc
and 's3://' not in algo_history_loc
and 'redis://' not in algo_history_loc):
log.error(
'invalid -p <backtest dataset file> specified. '
'{} '
'please use either: '
'-p file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
'-p s3://algoready/SPY-latest.json or '
'-p redis://SPY-latest'.format(algo_history_loc))
sys.exit(1)
if 's3://' in algo_history_loc:
history_s3_bucket = algo_history_loc.split('/')[-2]
history_s3_key = algo_history_loc.split('/')[-1]
elif 'redis://' in algo_history_loc:
history_redis_key = algo_history_loc.split('/')[-1]
elif 'file:/' in algo_history_loc:
history_file = algo_history_loc.split(':')[-1]
history_publish = True
# end of parsing supported transport - trading history
if args.algo_report_loc:
algo_report_loc = args.algo_report_loc
if ('file:/' not in algo_report_loc
and 's3://' not in algo_report_loc
and 'redis://' not in algo_report_loc):
log.error(
'invalid -o <backtest dataset file> specified. '
'{} '
'please use either: '
'-o file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
'-o s3://algoready/SPY-latest.json or '
'-o redis://SPY-latest'.format(algo_report_loc))
sys.exit(1)
if 's3://' in algo_report_loc:
report_s3_bucket = algo_report_loc.split('/')[-2]
report_s3_key = algo_report_loc.split('/')[-1]
elif 'redis://' in algo_report_loc:
report_redis_key = algo_report_loc.split('/')[-1]
elif 'file:/' in algo_report_loc:
report_file = algo_report_loc.split(':')[-1]
report_publish = True
# end of parsing supported transport - trading performance report
if args.algo_extract_loc:
algo_extract_loc = args.algo_extract_loc
if ('file:/' not in algo_extract_loc
and 's3://' not in algo_extract_loc
and 'redis://' not in algo_extract_loc):
log.error(
'invalid -e <backtest dataset file> specified. '
'{} '
'please use either: '
'-e file:/opt/sa/tests/datasets/algo/SPY-latest.json or '
'-e s3://algoready/SPY-latest.json or '
'-e redis://SPY-latest'.format(algo_extract_loc))
sys.exit(1)
if 's3://' in algo_extract_loc:
extract_s3_bucket = algo_extract_loc.split('/')[-2]
extract_s3_key = algo_extract_loc.split('/')[-1]
elif 'redis://' in algo_extract_loc:
extract_redis_key = algo_extract_loc.split('/')[-1]
elif 'file:/' in algo_extract_loc:
extract_file = algo_extract_loc.split(':')[-1]
extract_publish = True
# end of parsing supported transport - extract algorithm-ready
use_custom_algo = True
# end of set up for backtest
if use_custom_algo:
if args.run_on_engine:
run_on_engine = True
log.info('starting algo on the engine')
else:
log.info('starting algo')
algo_res = run_custom_algo.run_custom_algo(
mod_path=args.run_algo_in_file,
ticker=ticker,
balance=use_balance,
commission=use_commission,
start_date=use_start_date,
end_date=use_end_date,
config_file=use_config_file,
name=use_name,
auto_fill=auto_fill,
config_dict=config_dict,
load_from_s3_bucket=load_from_s3_bucket,
load_from_s3_key=load_from_s3_key,
load_from_redis_key=load_from_redis_key,
load_from_file=load_from_file,
load_compress=load_compress,
load_publish=load_publish,
load_config=load_config,
report_redis_key=report_redis_key,
report_s3_bucket=report_s3_bucket,
report_s3_key=report_s3_key,
report_file=report_file,
report_compress=report_compress,
report_publish=report_publish,
report_config=report_config,
history_redis_key=history_redis_key,
history_s3_bucket=history_s3_bucket,
history_s3_key=history_s3_key,
history_file=history_file,
history_compress=history_compress,
history_publish=history_publish,
history_config=history_config,
extract_redis_key=extract_redis_key,
extract_s3_bucket=extract_s3_bucket,
extract_s3_key=extract_s3_key,
extract_file=extract_file,
extract_save_dir=extract_save_dir,
extract_compress=extract_compress,
extract_publish=extract_publish,
extract_config=extract_config,
publish_to_slack=publish_to_slack,
publish_to_s3=publish_to_s3,
publish_to_redis=publish_to_redis,
dataset_type=dataset_type,
serialize_datasets=serialize_datasets,
compress=compress,
encoding=encoding,
redis_enabled=redis_enabled,
redis_key=redis_key,
redis_address=redis_address,
redis_db=redis_db,
redis_password=redis_password,
redis_expire=redis_expire,
redis_serializer=redis_serializer,
redis_encoding=redis_encoding,
s3_enabled=s3_enabled,
s3_key=s3_key,
s3_address=s3_address,
s3_bucket=s3_bucket_name,
s3_access_key=s3_access_key,
s3_secret_key=s3_secret_key,
s3_region_name=s3_region_name,
s3_secure=s3_secure,
slack_enabled=slack_enabled,
slack_code_block=slack_code_block,
slack_full_width=slack_full_width,
dataset_publish_extract=extract_publish,
dataset_publish_history=history_publish,
dataset_publish_report=report_publish,
run_on_engine=run_on_engine,
auth_url=broker_url,
backend_url=backend_url,
include_tasks=include_tasks,
ssl_options=ssl_options,
transport_options=transport_options,
path_to_config_module=path_to_config_module,
timeseries=use_timeseries,
trade_strategy=use_trade_strategy,
verbose=verbose)
show_label = 'algo.name={}'.format(use_name)
show_extract = '{}'.format(algo_extract_loc)
show_history = '{}'.format(algo_history_loc)
show_report = '{}'.format(algo_report_loc)
base_label = ('load={} '
'extract={} '
'history={} '
'report={}'.format(args.run_algo_in_file, show_extract,
show_history, show_report))
show_label = ('{} running in engine task_id={} {}'.format(
ticker, algo_res['rec'].get('task_id', 'missing-task-id'),
base_label))
if not run_on_engine:
algo_trade_history_recs = algo_res['rec'].get('history', [])
show_label = ('{} algo.name={} {} trade_history_len={}'.format(
ticker, use_name, base_label, len(algo_trade_history_recs)))
if args.debug:
log.info('algo_res={}'.format(algo_res))
if algo_res['status'] == ae_consts.SUCCESS:
log.info('{} - done running {}'.format(
ae_consts.get_status(status=algo_res['status']),
show_label))
else:
log.error('{} - done running {}'.format(
ae_consts.get_status(status=algo_res['status']),
show_label))
else:
if algo_res['status'] == ae_consts.SUCCESS:
log.info('{} - done running {}'.format(
ae_consts.get_status(status=algo_res['status']),
show_label))
else:
log.error('run_custom_algo returned error: {}'.format(
algo_res['err']))
sys.exit(1)
# end of running the custom algo handler
if mode == ae_consts.SA_MODE_EXTRACT:
log.info('done extracting dataset - {}'.format(ticker))
elif mode == ae_consts.SA_MODE_RUN_ALGO:
log.info('done running algo - {}'.format(ticker))
sys.exit(0)
# end of handling mode-specific arg assignments
# sanity checking the work and task are valid
if not valid:
log.error('usage error: missing a supported mode: '
'-f (for prepare a dataset) ')
sys.exit(1)
if required_task and not task_name:
log.error('usage error: missing a supported task_name')
sys.exit(1)
# end of sanity checks
work['ticker'] = ticker
work['ticker_id'] = ticker_id
work['s3_bucket'] = s3_bucket_name
work['s3_key'] = s3_key
work['redis_key'] = redis_key
work['s3_access_key'] = s3_access_key
work['s3_secret_key'] = s3_secret_key
work['s3_region_name'] = s3_region_name
work['s3_address'] = s3_address
work['s3_secure'] = s3_secure
work['redis_address'] = redis_address
work['redis_password'] = redis_password
work['redis_db'] = redis_db
work['redis_expire'] = redis_expire
work['s3_enabled'] = s3_enabled
work['redis_enabled'] = redis_enabled
work['debug'] = debug
work['label'] = 'ticker={}'.format(ticker)
task_res = None
if ae_consts.is_celery_disabled():
work['celery_disabled'] = True
log.debug('starting without celery work={}'.format(
ae_consts.ppj(work)))
if mode == ae_consts.SA_MODE_PREPARE:
task_res = prep_dataset.prepare_pricing_dataset(work)
if debug:
log.info('done - result={} '
'task={} status={} '
'err={} label={}'.format(
ae_consts.ppj(task_res), task_name,
ae_consts.get_status(status=task_res['status']),
task_res['err'], work['label']))
else:
log.info('done - result '
'task={} status={} '
'err={} label={}'.format(
task_name,
ae_consts.get_status(status=task_res['status']),
task_res['err'], work['label']))
if task_res['status'] == ae_consts.SUCCESS:
image_res = None
label = work['label']
ticker = work['ticker']
if plot_action == ae_consts.PLOT_ACTION_SHOW:
log.info('showing plot')
"""
minute_key = '{}_minute'.format(
redis_key)
minute_df_res = build_df.build_df_from_redis(
label=label',
address=redis_address,
db=redis_db,
key=minute_key)
minute_df = None
if (
minute_df_res['status'] == SUCCESS
and minute_df_res['rec']['valid_df']):
minute_df = minute_df_res['rec']['data']
print(minute_df.columns.values)
column_list = [
'close',
'date'
]
"""
today_str = datetime.datetime.now().strftime('%Y-%m-%d')
extract_req = work
extract_req['redis_key'] = '{}_minute'.format(
work['redis_key'])
extract_status, minute_df = \
extract_utils.extract_minute_dataset(
work_dict=work)
if extract_status == ae_consts.SUCCESS:
log.info('{} - ticker={} creating chart date={}'.format(
label, ticker, today_str))
"""
Plot Pricing with the Volume Overlay:
"""
image_res = ae_charts.plot_overlay_pricing_and_volume(
log_label=label,
ticker=ticker,
date_format=ae_consts.IEX_MINUTE_DATE_FORMAT,
df=minute_df,
show_plot=True)
"""
Plot the High-Low-Open-Close Pricing:
"""
"""
image_res = ae_charts.plot_hloc_pricing(
log_label=label,
ticker=ticker,
title='{} - Minute Pricing - {}'.format(
ticker,
today_str),
df=minute_df,
show_plot=True)
"""
"""
Plot by custom columns in the DataFrame
"""
"""
column_list = minute_df.columns.values
column_list = [
'date',
'close',
'high',
'low',
'open'
]
image_res = ae_charts.plot_df(
log_label=label,
title='Pricing Title',
column_list=column_list,
df=minute_df,
xcol='date',
xlabel='Date',
ylabel='Pricing',
show_plot=True)
"""
elif plot_action == ae_consts.PLOT_ACTION_SAVE_TO_S3:
log.info('coming soon - support to save to s3')
elif plot_action == ae_consts.PLOT_ACTION_SAVE_AS_FILE:
log.info('coming soon - support to save as file')
if image_res:
log.info('{} show plot - status={} err={}'.format(
label, ae_consts.get_status(image_res['status']),
image_res['err']))
else:
log.info('connecting to broker={} backend={}'.format(
broker_url, backend_url))
# Get the Celery app
app = get_celery_app.get_celery_app(
name=__name__,
auth_url=broker_url,
backend_url=backend_url,
path_to_config_module=path_to_config_module,
ssl_options=ssl_options,
transport_options=transport_options,
include_tasks=include_tasks)
log.info('calling task={} - work={}'.format(task_name,
ae_consts.ppj(work)))
job_id = app.send_task(task_name, (work, ))
log.info('calling task={} - success job_id={}'.format(
task_name, job_id))
def prepare_pricing_dataset(
self,
work_dict):
"""prepare_pricing_dataset
Prepare dataset for analysis. Supports loading dataset from
s3 if not found in redis. Outputs prepared artifact as a csv
to s3 and redis.
:param work_dict: dictionary for key/values
"""
label = 'prepare'
log.info(
'task - {} - start '
'work_dict={}'.format(
label,
work_dict))
initial_data = None
ticker = ae_consts.TICKER
ticker_id = ae_consts.TICKER_ID
rec = {
'ticker': None,
'ticker_id': None,
's3_enabled': True,
'redis_enabled': True,
's3_bucket': None,
's3_key': None,
'redis_key': None,
'prepared_s3_key': None,
'prepared_s3_bucket': None,
'prepared_redis_key': None,
'prepared_data': None,
'prepared_size': None,
'initial_data': None,
'initial_size': None,
'ignore_columns': None,
'updated': None
}
res = build_result.build_result(
status=ae_consts.NOT_RUN,
err=None,
rec=rec)
try:
ticker = work_dict.get(
'ticker',
ae_consts.TICKER)
ticker_id = int(work_dict.get(
'ticker_id',
ae_consts.TICKER_ID))
if not ticker:
res = build_result.build_result(
status=ae_consts.ERR,
err='missing ticker',
rec=rec)
return res
label = work_dict.get(
'label',
label)
s3_key = work_dict.get(
's3_key',
None)
s3_bucket_name = work_dict.get(
's3_bucket',
'pricing')
s3_access_key = work_dict.get(
's3_access_key',
ae_consts.S3_ACCESS_KEY)
s3_secret_key = work_dict.get(
's3_secret_key',
ae_consts.S3_SECRET_KEY)
s3_region_name = work_dict.get(
's3_region_name',
ae_consts.S3_REGION_NAME)
s3_address = work_dict.get(
's3_address',
ae_consts.S3_ADDRESS)
s3_secure = work_dict.get(
's3_secure',
ae_consts.S3_SECURE) == '1'
redis_address = work_dict.get(
'redis_address',
ae_consts.REDIS_ADDRESS)
redis_key = work_dict.get(
'redis_key',
ae_consts.REDIS_KEY)
redis_password = work_dict.get(
'redis_password',
ae_consts.REDIS_PASSWORD)
redis_db = work_dict.get(
'redis_db',
None)
if not redis_db:
redis_db = ae_consts.REDIS_DB
redis_expire = None
if 'redis_expire' in work_dict:
redis_expire = work_dict.get(
'redis_expire',
ae_consts.REDIS_EXPIRE)
updated = work_dict.get(
'updated',
datetime.datetime.utcnow().strftime(
'%Y_%m_%d_%H_%M_%S'))
prepared_s3_key = work_dict.get(
'prepared_s3_key',
'{}_{}.csv'.format(
ticker,
updated))
prepared_s3_bucket = work_dict.get(
'prepared_s3_bucket',
'prepared')
prepared_redis_key = work_dict.get(
'prepared_redis_key',
'prepared')
ignore_columns = work_dict.get(
'ignore_columns',
None)
log.info(
'{} redis enabled address={}@{} '
'key={} prepare_s3={}:{} prepare_redis={} '
'ignore_columns={}'.format(
label,
redis_address,
redis_db,
redis_key,
prepared_s3_bucket,
prepared_s3_key,
prepared_redis_key,
ignore_columns))
redis_host = redis_address.split(':')[0]
redis_port = redis_address.split(':')[1]
enable_s3 = True
enable_redis_publish = True
rec['ticker'] = ticker
rec['ticker_id'] = ticker_id
rec['s3_bucket'] = s3_bucket_name
rec['s3_key'] = s3_key
rec['redis_key'] = redis_key
rec['prepared_s3_key'] = prepared_s3_key
rec['prepared_s3_bucket'] = prepared_s3_bucket
rec['prepared_redis_key'] = prepared_redis_key
rec['updated'] = updated
rec['s3_enabled'] = enable_s3
rec['redis_enabled'] = enable_redis_publish
try:
log.info(
'{} connecting redis={}:{} '
'db={} key={} '
'updated={} expire={}'.format(
label,
redis_host,
redis_port,
redis_db,
redis_key,
updated,
redis_expire))
rc = redis.Redis(
host=redis_host,
port=redis_port,
password=redis_password,
db=redis_db)
except Exception as e:
err = (
'{} failed - redis connection to address={}@{} '
'key={} ex={}'.format(
label,
redis_address,
redis_key,
redis_db,
e))
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
# end of try/ex for connecting to redis
initial_data_res = redis_get.get_data_from_redis_key(
label=label,
client=rc,
key=redis_key)
log.info(
'{} get redis key={} status={} err={}'.format(
label,
redis_key,
ae_consts.get_status(initial_data_res['status']),
initial_data_res['err']))
initial_data = initial_data_res['rec'].get(
'data',
None)
if enable_s3 and not initial_data:
log.info(
'{} failed to find redis_key={} trying s3 '
'from s3_key={} s3_bucket={} s3_address={}'.format(
label,
redis_key,
s3_key,
s3_bucket_name,
s3_address))
get_from_s3_req = \
api_requests.build_publish_from_s3_to_redis_request()
get_from_s3_req['s3_enabled'] = enable_s3
get_from_s3_req['s3_access_key'] = s3_access_key
get_from_s3_req['s3_secret_key'] = s3_secret_key
get_from_s3_req['s3_region_name'] = s3_region_name
get_from_s3_req['s3_address'] = s3_address
get_from_s3_req['s3_secure'] = s3_secure
get_from_s3_req['s3_key'] = s3_key
get_from_s3_req['s3_bucket'] = s3_bucket_name
get_from_s3_req['redis_key'] = redis_key
get_from_s3_req['label'] = (
'{}-run_publish_from_s3_to_redis'.format(
label))
log.info(
'{} load from s3={} to '
'redis={}'.format(
label,
s3_key,
redis_key))
try:
# run in synchronous mode:
get_from_s3_req['celery_disabled'] = True
task_res = s3_to_redis.run_publish_from_s3_to_redis(
get_from_s3_req)
if task_res.get(
'status',
ae_consts.ERR) == ae_consts.SUCCESS:
log.info(
'{} loaded s3={}:{} '
'to redis={} retrying'.format(
label,
s3_bucket_name,
s3_key,
redis_key))
initial_data_res = redis_get.get_data_from_redis_key(
label=label,
client=rc,
key=redis_key)
log.info(
'{} get redis try=2 key={} status={} err={}'.format(
label,
redis_key,
ae_consts.get_status(initial_data_res['status']),
initial_data_res['err']))
initial_data = initial_data_res['rec'].get(
'data',
None)
else:
err = (
'{} ERR failed loading from bucket={} '
's3_key={} to redis_key={} with res={}'.format(
label,
s3_bucket_name,
s3_key,
redis_key,
task_res))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
except Exception as e:
err = (
'{} extract from s3 and publish to redis failed loading '
'data from bucket={} in '
's3_key={} with publish to redis_key={} '
'with ex={}'.format(
label,
s3_bucket_name,
s3_key,
redis_key,
e))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
# end of try/ex for publishing from s3->redis
# end of if enable_s3
if not initial_data:
err = (
'{} did not find any data to prepare in redis_key={} or '
's3_key={} in bucket={}'.format(
label,
redis_key,
s3_key,
s3_bucket_name))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
initial_data_num_chars = len(str(initial_data))
initial_size_value = None
initial_size_str = None
if initial_data_num_chars < ae_consts.PREPARE_DATA_MIN_SIZE:
err = (
'{} not enough data={} in redis_key={} or '
's3_key={} in bucket={}'.format(
label,
initial_data_num_chars,
redis_key,
s3_key,
s3_bucket_name))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
else:
initial_size_value = initial_data_num_chars / 1024000
initial_size_str = ae_consts.to_f(initial_size_value)
if ae_consts.ev('DEBUG_PREPARE', '0') == '1':
log.info(
'{} initial - redis_key={} data={}'.format(
label,
redis_key,
str(initial_data)))
else:
log.info(
'{} initial - redis_key={} data size={} MB'.format(
label,
redis_key,
initial_size_str))
# end of trying to get initial_data
rec['initial_data'] = initial_data
rec['initial_size'] = initial_data_num_chars
prepare_data = None
try:
if ae_consts.ev('DEBUG_PREPARE', '0') == '1':
log.info(
'{} data={} - flatten - {} MB from '
'redis_key={}'.format(
label,
ae_consts.ppj(initial_data),
initial_size_str,
redis_key))
else:
log.info(
'{} flatten - {} MB from '
'redis_key={}'.format(
label,
initial_size_str,
redis_key))
prepare_data = dict_to_csv.flatten_dict(
data=initial_data)
except Exception as e:
prepare_data = None
err = (
'{} flatten - convert to csv failed with ex={} '
'redis_key={}'.format(
label,
e,
redis_key))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
# end of try/ex
if not prepare_data:
err = (
'{} flatten - did not return any data from redis_key={} '
'or s3_key={} in bucket={}'.format(
label,
redis_key,
s3_key,
s3_bucket_name))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
# end of prepare_data
prepare_data_num_chars = len(str(prepare_data))
prepare_size_value = None
if prepare_data_num_chars < ae_consts.PREPARE_DATA_MIN_SIZE:
err = (
'{} prepare - there is not enough data={} in redis_key={}'
''.format(
label,
prepare_data_num_chars,
redis_key))
log.error(err)
res = build_result.build_result(
status=ae_consts.ERR,
err=err,
rec=rec)
return res
else:
prepare_size_value = prepare_data_num_chars / 1024000
prepare_size_str = ae_consts.to_f(prepare_size_value)
if ae_consts.ev('DEBUG_PREPARE', '0') == '1':
log.info(
'{} data={} - prepare - redis_key={}'.format(
label,
redis_key,
ae_consts.ppj(prepare_data)))
else:
log.info(
'{} prepare - redis_key={} data size={} MB'.format(
label,
redis_key,
prepare_size_str))
# end of trying to the size of the prepared data
rec['prepared_data'] = prepare_data
rec['prepared_size'] = prepare_data_num_chars
res = build_result.build_result(
status=ae_consts.SUCCESS,
err=None,
rec=rec)
rc = None
except Exception as e:
res = build_result.build_result(
status=ae_consts.ERR,
err=(
'failed - prepare_pricing_dataset '
'dict={} with ex={}').format(
work_dict,
e),
rec=rec)
log.error(
'{} - {}'.format(
label,
res['err']))
# end of try/ex
log.info(
'task - prepare_pricing_dataset done - '
'{} - status={}'.format(
label,
ae_consts.get_status(res['status'])))
return get_task_results.get_task_results(
work_dict=work_dict,
result=res)
def get_report(self, verbose=False):
"""get_report
Get the indicator's current output node
that is used for the trading performance report
generated at the end of the algorithm
.. note:: the report dict should mostly be numeric
types to enable AI predictions after removing
non-numeric columns
:param verbose: optional - boolean for toggling
to show the report
"""
cur_report_dict = {}
# allow derived indicators to build their own report prefix
report_prefix_key_name = self.get_report_prefix()
for key in self.report_dict:
is_valid = True
if key in self.report_ignore_keys:
is_valid = False
if is_valid:
report_key = self.build_report_key(
key,
prefix_key=report_prefix_key_name,
key_type='report',
cur_report_dict=cur_report_dict)
cur_report_dict[report_key] = self.report_dict[key]
# for all keys to output into the report
buy_value = None
sell_value = None
for key in self.configurables:
is_valid = True
if key in self.report_ignore_keys:
is_valid = False
elif key not in self.__dict__:
is_valid = False
if is_valid:
report_key = self.build_report_key(
key,
prefix_key=report_prefix_key_name,
key_type='conf',
cur_report_dict=cur_report_dict)
use_value = None
if key == 'is_buy':
buy_value = self.__dict__[key]
if buy_value:
use_value = \
ae_consts.INDICATOR_ACTIONS[buy_value]
else:
use_value = \
ae_consts.INT_INDICATOR_NOT_PROCESSED
elif key == 'is_sell':
sell_value = self.__dict__[key]
if sell_value:
use_value = \
ae_consts.INDICATOR_ACTIONS[sell_value]
else:
use_value = \
ae_consts.INT_INDICATOR_NOT_PROCESSED
else:
use_value = self.__dict__[key]
# end of deciding value
cur_report_dict[report_key] = use_value
# if valid
# end of all configurables for this indicator
if verbose or self.verbose:
self.lg('indicator={} '
'report={} '
'buy={} sell={}'.format(self.name,
ae_consts.ppj(cur_report_dict),
buy_value, sell_value))
return cur_report_dict
def get_configurables(self, **kwargs):
"""get_configurables
helper for setting up algorithm configs for this indicator
and programmatically set the values based off the domain
rules
.. code-block:: python
from analysis_engine.indicators.macd \
import IndicatorMACD
ind = IndicatorMACD(config_dict={
'verbose': True
}).get_configurables()
:param kwargs: keyword args dictionary
"""
self.ind_confs = []
# common:
self.build_base_configurables(ind_type='momentum',
category='technical',
uses_data=self.config.get(
'uses_data', 'minute'),
version=1)
# custom:
self.ind_confs.append(
self.build_configurable_node(name='num_points',
conf_type='int',
max_value=200,
min_value=2,
default_value=100,
inc_interval=1))
self.ind_confs.append(
self.build_configurable_node(name='buy_below_percent',
conf_type='percent',
min_value=3.0,
max_value=100.0,
default_value=5.0,
inc_interval=1))
self.ind_confs.append(
self.build_configurable_node(name='buy_above_percent',
conf_type='percent',
min_value=3.0,
max_value=100.0,
default_value=5.0,
inc_interval=1))
self.ind_confs.append(
self.build_configurable_node(name='sell_below_percent',
conf_type='percent',
min_value=3.0,
max_value=100.0,
default_value=5.0,
inc_interval=1))
self.ind_confs.append(
self.build_configurable_node(name='sell_above_percent',
conf_type='percent',
min_value=3.0,
max_value=100.0,
default_value=5.0,
inc_interval=1))
self.ind_confs.append(
self.build_configurable_node(name='fast_period',
conf_type='float',
inc_interval=1))
self.ind_confs.append(
self.build_configurable_node(name='slow_period',
conf_type='float',
inc_interval=1))
self.ind_confs.append(
self.build_configurable_node(name='signal_period',
conf_type='float',
inc_interval=1))
# output / reporting:
self.ind_confs.append(
self.build_configurable_node(name='macd_value',
conf_type='float',
is_output_only=True))
self.ind_confs.append(
self.build_configurable_node(name='macd_signal',
conf_type='float',
is_output_only=True))
self.ind_confs.append(
self.build_configurable_node(name='macd_hist',
conf_type='float',
is_output_only=True))
self.ind_confs.append(
self.build_configurable_node(name='close',
conf_type='float',
is_output_only=True))
self.ind_confs.append(
self.build_configurable_node(name='percent_value',
conf_type='percent',
is_output_only=True))
default_values_dict = {}
for node in self.ind_confs:
name = node['name']
default_value = node.get('default', None)
default_values_dict[name] = default_value
use_file = None
try:
if __file__:
use_file = __file__
except Exception:
use_file = None
self.starter_dict = {
'name': self.__class__.__name__.lower().replace('indicator', ''),
'module_path': use_file,
'category': default_values_dict.get('category', 'momentum'),
'type': default_values_dict.get('type', 'technical'),
'uses_data': default_values_dict.get('uses_data', 'minute'),
'verbose': default_values_dict.get('verbose', False)
}
self.starter_dict.update(default_values_dict)
self.lg('configurables={} for class={} in file={} '
'starter:\n {}'.format(ae_consts.ppj(self.ind_confs),
self.__class__.__name__, use_file,
ae_consts.ppj(self.starter_dict)))
return self.ind_confs
def publish_ticker_aggregate_from_s3(self, work_dict):
"""publish_ticker_aggregate_from_s3
Publish Aggregated Ticker Data from S3 to Redis
:param work_dict: dictionary for key/values
"""
label = 'pub-tic-agg-s3-to-redis'
log.info('task - {} - start ' 'work_dict={}'.format(label, work_dict))
ticker = TICKER
ticker_id = TICKER_ID
rec = {
'ticker': None,
'ticker_id': None,
's3_read_enabled': True,
's3_upload_enabled': True,
'redis_enabled': True,
's3_bucket': None,
's3_compiled_bucket': None,
's3_key': None,
'redis_key': None,
'updated': None
}
res = build_result.build_result(status=NOT_RUN, err=None, rec=rec)
try:
ticker = work_dict.get('ticker', TICKER)
ticker_id = int(work_dict.get('ticker_id', TICKER_ID))
if not ticker:
res = build_result.build_result(status=ERR,
err='missing ticker',
rec=rec)
return res
label = work_dict.get('label', label)
s3_key = work_dict.get('s3_key', None)
s3_bucket_name = work_dict.get('s3_bucket', 'pricing')
s3_compiled_bucket_name = work_dict.get('s3_compiled_bucket',
'compileddatasets')
redis_key = work_dict.get('redis_key', None)
updated = work_dict.get('updated', None)
enable_s3_upload = work_dict.get('s3_upload_enabled',
ENABLED_S3_UPLOAD)
enable_redis_publish = work_dict.get('redis_enabled',
ENABLED_REDIS_PUBLISH)
serializer = work_dict.get('serializer', 'json')
encoding = work_dict.get('encoding', 'utf-8')
enable_s3_read = True
rec['ticker'] = ticker
rec['ticker_id'] = ticker_id
rec['s3_bucket'] = s3_bucket_name
rec['s3_compiled_bucket'] = s3_compiled_bucket_name
rec['s3_key'] = s3_key
rec['redis_key'] = redis_key
rec['updated'] = updated
rec['s3_read_enabled'] = enable_s3_read
rec['s3_upload_enabled'] = enable_s3_upload
rec['redis_enabled'] = enable_redis_publish
if enable_s3_read:
log.info('{} parsing s3 values'.format(label))
access_key = work_dict.get('s3_access_key', S3_ACCESS_KEY)
secret_key = work_dict.get('s3_secret_key', S3_SECRET_KEY)
region_name = work_dict.get('s3_region_name', S3_REGION_NAME)
service_address = work_dict.get('s3_address', S3_ADDRESS)
secure = work_dict.get('s3_secure', S3_SECURE) == '1'
endpoint_url = 'http://{}'.format(service_address)
if secure:
endpoint_url = 'https://{}'.format(service_address)
log.info('{} building s3 endpoint_url={} '
'region={}'.format(label, endpoint_url, region_name))
s3 = boto3.resource(
's3',
endpoint_url=endpoint_url,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region_name,
config=boto3.session.Config(signature_version='s3v4'))
try:
log.info('{} checking bucket={} exists'.format(
label, s3_bucket_name))
if s3.Bucket(s3_bucket_name) not in s3.buckets.all():
log.info('{} creating bucket={}'.format(
label, s3_bucket_name))
s3.create_bucket(Bucket=s3_bucket_name)
except Exception as e:
log.info('{} failed creating bucket={} '
'with ex={}'.format(label, s3_bucket_name, e))
# end of try/ex for creating bucket
try:
log.info('{} checking bucket={} keys'.format(
label, s3_bucket_name))
date_keys = []
keys = []
# {TICKER}_YYYY-DD-MM regex
reg = r'^.*_\d{4}-(0?[1-9]|1[012])-(0?[1-9]|[12][0-9]|3[01])$'
for bucket in s3.buckets.all():
for key in bucket.objects.all():
if (ticker.lower() in key.key.lower()
and bool(re.compile(reg).search(key.key))):
keys.append(key.key)
date_keys.append(
key.key.split('{}_'.format(ticker))[1])
except Exception as e:
log.info('{} failed to get bucket={} '
'keys with ex={}'.format(label, s3_bucket_name, e))
# end of try/ex for getting bucket keys
if keys:
data = []
for idx, key in enumerate(keys):
try:
log.info('{} reading to s3={}/{} '
'updated={}'.format(label, s3_bucket_name,
key, updated))
loop_data = s3_read_contents_from_key.\
s3_read_contents_from_key(
s3=s3,
s3_bucket_name=s3_bucket_name,
s3_key=key,
encoding=encoding,
convert_as_json=True)
initial_size_value = \
len(str(loop_data)) / 1024000
initial_size_str = to_f(initial_size_value)
if ev('DEBUG_S3', '0') == '1':
log.info('{} read s3={}/{} data={}'.format(
label, s3_bucket_name, key, ppj(loop_data)))
else:
log.info('{} read s3={}/{} data size={} MB'.format(
label, s3_bucket_name, key, initial_size_str))
data.append({'{}'.format(date_keys[idx]): loop_data})
except Exception as e:
err = ('{} failed reading bucket={} '
'key={} ex={}').format(label, s3_bucket_name,
key, e)
log.error(err)
res = build_result.build_result(status=NOT_RUN,
err=err,
rec=rec)
# end of try/ex for creating bucket
else:
log.info('{} No keys found in '
'S3 bucket={} for ticker={}'.format(
label, s3_bucket_name, ticker))
else:
log.info('{} SKIP S3 read bucket={} '
'ticker={}'.format(label, s3_bucket_name, ticker))
# end of if enable_s3_read
if data and enable_s3_upload:
try:
log.info('{} checking bucket={} exists'.format(
label, s3_compiled_bucket_name))
if s3.Bucket(s3_compiled_bucket_name) not in s3.buckets.all():
log.info('{} creating bucket={}'.format(
label, s3_compiled_bucket_name))
s3.create_bucket(Bucket=s3_compiled_bucket_name)
except Exception as e:
log.info('{} failed creating bucket={} '
'with ex={}'.format(label, s3_compiled_bucket_name,
e))
# end of try/ex for creating bucket
try:
cmpr_data = zlib.compress(json.dumps(data).encode(encoding), 9)
if ev('DEBUG_S3', '0') == '1':
log.info(
'{} uploading to s3={}/{} data={} updated={}'.format(
label, s3_compiled_bucket_name, s3_key,
ppj(loop_data), updated))
else:
sizes = {
'MB': 1024000,
'GB': 1024000000,
'TB': 1024000000000,
'PB': 1024000000000000
}
initial_size_value = len(str(data))
org_data_size = 'MB'
for key in sizes.keys():
size = float(initial_size_value) / float(sizes[key])
if size > 1024:
continue
org_data_size = key
initial_size_value = size
break
initial_size_str = to_f(initial_size_value)
cmpr_data_size_value = len(cmpr_data)
cmpr_data_size = 'MB'
for key in sizes.keys():
size = float(cmpr_data_size_value) / float(sizes[key])
if size > 1024:
continue
cmpr_data_size = key
cmpr_data_size_value = size
break
cmpr_size_str = to_f(cmpr_data_size_value)
log.info(
'{} uploading to s3={}/{} data original_size={} {} '
'compressed_size={} {} updated={}'.format(
label, s3_compiled_bucket_name, s3_key,
initial_size_str, org_data_size, cmpr_size_str,
cmpr_data_size, updated))
s3.Bucket(s3_compiled_bucket_name).put_object(Key=s3_key,
Body=cmpr_data)
except Exception as e:
log.error('{} failed uploading bucket={} '
'key={} ex={}'.format(label, s3_compiled_bucket_name,
s3_key, e))
# end of try/ex for creating bucket
else:
log.info('{} SKIP S3 upload bucket={} '
'key={}'.format(label, s3_bucket_name, s3_key))
# end of if enable_s3_upload
if data and enable_redis_publish:
redis_address = work_dict.get('redis_address', REDIS_ADDRESS)
redis_key = work_dict.get('redis_key', REDIS_KEY)
redis_password = work_dict.get('redis_password', REDIS_PASSWORD)
redis_db = work_dict.get('redis_db', None)
if not redis_db:
redis_db = REDIS_DB
redis_expire = None
if 'redis_expire' in work_dict:
redis_expire = work_dict.get('redis_expire', REDIS_EXPIRE)
log.info('redis enabled address={}@{} '
'key={}'.format(redis_address, redis_db, redis_key))
redis_host = redis_address.split(':')[0]
redis_port = redis_address.split(':')[1]
try:
if ev('DEBUG_REDIS', '0') == '1':
log.info('{} publishing redis={}:{} '
'db={} key={} '
'updated={} expire={} '
'data={}'.format(label, redis_host, redis_port,
redis_db, redis_key, updated,
redis_expire, ppj(data)))
else:
log.info('{} publishing redis={}:{} '
'db={} key={} '
'updated={} expire={}'.format(
label, redis_host, redis_port, redis_db,
redis_key, updated, redis_expire))
# end of if/else
rc = redis.Redis(host=redis_host,
port=redis_port,
password=redis_password,
db=redis_db)
redis_set_res = redis_set.set_data_in_redis_key(
label=label,
client=rc,
key=redis_key,
data=data,
serializer=serializer,
encoding=encoding,
expire=redis_expire,
px=None,
nx=False,
xx=False)
log.info('{} redis_set status={} err={}'.format(
label, get_status(redis_set_res['status']),
redis_set_res['err']))
except Exception as e:
log.error('{} failed - redis publish to '
'key={} ex={}'.format(label, redis_key, e))
# end of try/ex for creating bucket
else:
log.info('{} SKIP REDIS publish '
'key={}'.format(label, redis_key))
# end of if enable_redis_publish
res = build_result.build_result(status=SUCCESS, err=None, rec=rec)
except Exception as e:
res = build_result.build_result(status=ERR,
err=('failed - publish_from_s3 '
'dict={} with ex={}').format(
work_dict, e),
rec=rec)
log.error('{} - {}'.format(label, res['err']))
# end of try/ex
log.info('task - publish_from_s3 done - '
'{} - status={}'.format(label, get_status(res['status'])))
return analysis_engine.get_task_results.get_task_results(
work_dict=work_dict, result=res)
def build_buy_order(ticker,
num_owned,
close,
balance,
commission,
date,
details,
use_key,
minute=None,
shares=None,
version=1,
auto_fill=True,
is_live_trading=False,
backtest_shares_default=10,
reason=None):
"""build_buy_order
Create an algorithm buy order as a dictionary
.. note:: setting the ``minute`` is required to build
a minute-by-minute ``Trading History``
:param ticker: ticker
:param num_owned: integer current owned
number of shares for this asset
:param close: float closing price of the asset
:param balance: float amount of available capital
:param commission: float for commission costs
:param date: string trade date for that row usually
``COMMON_DATE_FORMAT`` (``YYYY-MM-DD``)
:param minute: optional - string with the minute that the
order was placed. format is
``COMMON_TICK_DATE_FORMAT`` (``YYYY-MM-DD HH:MM:SS``)
:param details: dictionary for full row of values to review
all buys after the algorithm finishes.
(usually ``row.to_json()``)
:param use_key: string for redis and s3 publishing of the algorithm
result dictionary as a json-serialized dictionary
:param shares: optional - integer number of shares to buy
if None buy max number of shares at the ``close`` with the
available ``balance`` amount.
:param version: optional - version tracking integer
:param auto_fill: optional - bool for not assuming the trade
filled (default ``True``)
:param is_live_trading: optional - bool for filling trades
for live trading or for backtest tuning filled
(default ``False`` which is backtest mode)
:param backtest_shares_default: optional - integer for
simulating shares during a backtest even if there
are not enough funds
(default ``10``)
:param reason: optional - string for recording why the algo
decided to buy for review after the algorithm finishes
"""
status = ae_consts.TRADE_OPEN
s3_bucket_name = ae_consts.ALGO_BUYS_S3_BUCKET_NAME
s3_key = use_key
redis_key = use_key
s3_enabled = True
redis_enabled = True
cost_of_trade = None
new_shares = num_owned
new_balance = balance
created_date = None
tradable_funds = balance - (2.0 * commission)
if not is_live_trading:
if not shares:
shares = backtest_shares_default
tradable_funds = ((shares * close) + (2.0 * commission))
if close > 0.1 and tradable_funds > 10.0:
can_buy_num_shares = shares
if not can_buy_num_shares:
can_buy_num_shares = int(tradable_funds / close)
cost_of_trade = ae_consts.to_f(val=(can_buy_num_shares * close) +
commission)
if can_buy_num_shares > 0:
if cost_of_trade > balance:
status = ae_consts.TRADE_NOT_ENOUGH_FUNDS
else:
created_date = ae_utils.utc_now_str()
if auto_fill:
new_shares = int(num_owned + can_buy_num_shares)
new_balance = ae_consts.to_f(balance - cost_of_trade)
status = ae_consts.TRADE_FILLED
else:
new_shares = shares
new_balance = balance
else:
status = ae_consts.TRADE_NOT_ENOUGH_FUNDS
else:
status = ae_consts.TRADE_NOT_ENOUGH_FUNDS
order_dict = {
'ticker': ticker,
'status': status,
'balance': new_balance,
'shares': new_shares,
'buy_price': cost_of_trade,
'prev_balance': balance,
'prev_shares': num_owned,
'close': close,
'details': details,
'reason': reason,
'date': date,
'minute': minute,
'created': created_date,
's3_bucket': s3_bucket_name,
's3_key': s3_key,
'redis_key': redis_key,
's3_enabled': s3_enabled,
'redis_enabled': redis_enabled,
'version': version
}
use_date = minute
if not use_date:
use_date = date
log.debug('{} {} buy {} order={}'.format(
ticker, use_date, ae_consts.get_status(status=order_dict['status']),
ae_consts.ppj(order_dict)))
return order_dict
def examine_dataset_in_file(
path_to_file,
compress=False,
encoding='utf-8',
ticker=None,
dataset_type=ae_consts.SA_DATASET_TYPE_ALGO_READY,
serialize_datasets=ae_consts.DEFAULT_SERIALIZED_DATASETS,
):
"""examine_dataset_in_file
Show the internal dataset dictionary structure in dataset file
:param path_to_file: path to file
:param compress: optional - boolean flag for decompressing
the contents of the ``path_to_file`` if necessary
(default is ``False`` and algorithms
use ``zlib`` for compression)
:param encoding: optional - string for data encoding
:param ticker: optional - string ticker symbol
to verify is in the dataset
:param dataset_type: optional - dataset type
(default is ``SA_DATASET_TYPE_ALGO_READY``)
:param serialize_datasets: optional - list of dataset names to
deserialize in the dataset
"""
if dataset_type == ae_consts.SA_DATASET_TYPE_ALGO_READY:
log.info('show start - load dataset from file={}'.format(path_to_file))
show_dataset.show_dataset(path_to_file=path_to_file,
compress=compress,
encoding=encoding,
dataset_type=dataset_type,
serialize_datasets=serialize_datasets)
log.info('show done - dataset in file={}'.format(path_to_file))
elif dataset_type == ae_consts.SA_DATASET_TYPE_TRADING_HISTORY:
log.info('load trading history dataset '
'from file={}'.format(path_to_file))
trading_history_dict = load_history.load_history_dataset_from_file(
path_to_file=path_to_file, compress=compress, encoding=encoding)
history_df = trading_history_dict[ticker]
first_date = history_df['date'].iloc[0]
end_date = history_df['date'].iloc[-1]
title = ('Trading History {} for Algo {}\n'
'Backtest dates from {} to {}'.format(
ticker, trading_history_dict['algo_name'], first_date,
end_date))
xcol = 'date'
xlabel = 'Dates vs {} values'.format(trading_history_dict['algo_name'])
ylabel = 'Algo Values from columns:\n{}'.format(
trading_history_dict['algo_name'], list(history_df.columns.values))
df_filter = (history_df['close'] > 0.01)
# set default hloc columns:
red = 'close'
blue = 'low'
green = 'high'
orange = 'open'
log.info('available columns to plot in dataset: {}'.format(
ae_consts.ppj(list(history_df.columns.values))))
plot_trading_history.plot_trading_history(title=title,
df=history_df,
red=red,
blue=blue,
green=green,
orange=orange,
date_col=xcol,
xlabel=xlabel,
ylabel=ylabel,
df_filter=df_filter,
show_plot=True,
dropna_for_all=False)
elif dataset_type == ae_consts.SA_DATASET_TYPE_TRADING_REPORT:
log.info('load trading performance report dataset '
'from file={}'.format(path_to_file))
trading_report_dict = load_report.load_report_dataset_from_file(
path_to_file=path_to_file, compress=compress, encoding=encoding)
print(trading_report_dict)
else:
log.error('show unsupported dataset type={} for file={}'.format(
dataset_type, path_to_file))
return