def do_backfill(self):
#
self.logger.info("Run EXO backfill from {0}".format(
self.args.backfill))
assetindex = AssetIndexMongo(MONGO_CONNSTR, MONGO_EXO_DB)
exostorage = EXOStorage(MONGO_CONNSTR, MONGO_EXO_DB)
futures_limit = 3
options_limit = 20
# datasource = DataSourceMongo(mongo_connstr, mongo_db_name, assetindex, futures_limit, options_limit, exostorage)
datasource = DataSourceSQL(SQL_HOST, SQL_USER, SQL_PASS, assetindex,
futures_limit, options_limit, exostorage)
exos = exostorage.exo_list(exo_filter=self.args.instrument + '_',
return_names=True)
exo_start_dates = {}
exec_time, decision_time = AssetIndexMongo.get_exec_time(
self.args.backfill, self.asset_info)
current_time = decision_time
if len(exos) > 0:
for exo_name in exos:
series = exostorage.load_series(exo_name)[0]
if series is not None:
last_date = series.index[-1] + timedelta(days=1)
exec_time, decision_time = AssetIndexMongo.get_exec_time(
last_date, self.asset_info)
self.logger.info(
'Updating existing {0} series from: {1}'.format(
exo_name, decision_time))
exo_start_dates[exo_name] = decision_time
else:
self.logger.info('Updating new EXO series from: {0}'.format(
self.args.backfill))
exec_time, decision_time = AssetIndexMongo.get_exec_time(
self.args.backfill, self.asset_info)
exec_time_end, decision_time_end = AssetIndexMongo.get_exec_time(
datetime.now(), self.asset_info)
while current_time <= decision_time_end:
self.logger.info("Backfilling: {0}".format(current_time))
self.run_exo_calc(datasource,
current_time,
args.instrument,
backfill_dict=exo_start_dates)
current_time += timedelta(days=1)
exec_time += timedelta(days=1)
def process(self, exo_name, swm_callback=None):
"""
Run this method every time when new EXO quote comes
:param exo_name: exo_name in MongoDB
:param swm_callback: swarm_callback after swarm is updated
:return:
"""
# Read EXO quote data from Mongo
exo_storage = EXOStorage(self.mongo_connstr, self.mongo_dbname)
# Read swarm parameters from Mongo
StrategyClass = self.strategy_context['strategy']['class']
# Load All swarms for Strategy and EXO name
swarms_data = self.load(exo_name, StrategyClass.name)
for swm_dict in swarms_data:
# Generate context for swarm
context = self.strategy_context
context['strategy']['exo_storage'] = exo_storage
context['strategy']['exo_name'] = exo_name
# Restoring swarms last state from dict
swm = Swarm.laststate_from_dict(swm_dict, context)
# Update swarm equity dynamic and last state
swm.update()
# Saving swarm state to Mongo
self.save(swm)
if swm_callback is not None:
# Run Swarm callback (to notify framework that swarm is updated)
swm_callback(swm)
def on_new_quote(self, appclass, appname, data):
if data.mtype != MsgQuoteNotification.mtype:
return
# Check data integrity
if not self.check_quote_data(appname, appclass, data):
return
exec_time, decision_time = AssetIndexMongo.get_exec_time(
datetime.now(), self.asset_info)
start_time = time.time()
quote_date = data['date']
symbol = appname
if quote_date > decision_time:
# TODO: Check to avoid dupe launch
# Run first EXO calculation for this day
self.logger.info(
"Run EXO calculation, at decision time: {0}".format(
decision_time))
assetindex = AssetIndexMongo(MONGO_CONNSTR, MONGO_EXO_DB)
exostorage = EXOStorage(MONGO_CONNSTR, MONGO_EXO_DB)
futures_limit = 3
options_limit = 10
#datasource = DataSourceMongo(mongo_connstr, mongo_db_name, assetindex, futures_limit, options_limit, exostorage)
#datasource = DataSourceSQL(SQL_HOST, SQL_USER, SQL_PASS, assetindex, futures_limit, options_limit, exostorage)
#
# Test DB temporary credentials
#
tmp_mongo_connstr = 'mongodb://*****:*****@10.0.1.2/tmldb_test?authMechanism=SCRAM-SHA-1'
tmp_mongo_db = 'tmldb_test'
datasource = DataSourceHybrid(SQL_HOST, SQL_USER, SQL_PASS,
assetindex, tmp_mongo_connstr,
tmp_mongo_db, futures_limit,
options_limit, exostorage)
# Run EXO calculation
self.run_exo_calc(datasource,
decision_time,
symbol,
backfill_dict=None)
end_time = time.time()
self.signalapp.send(
MsgStatus('OK',
'EXO processed for {0} at {1}'.format(
symbol, quote_date),
context={
'instrument': symbol,
'date': quote_date,
'exec_time': end_time - start_time
},
notify=True))
else:
self.logger.debug("Waiting next decision time")
def __init__(self, smartexo_class, **kwargs):
self.verbosive_logging = kwargs.get('verbosive_logging', False)
self.futures_limit = kwargs.get('futures_limit', 3)
self.options_limit = kwargs.get('options_limit', 20)
self.smartexo_class = smartexo_class
if self.verbosive_logging:
logging.basicConfig(format='%(message)s', level=logging.DEBUG)
else:
logging.basicConfig(format='%(message)s', level=logging.INFO)
self.assetindex = AssetIndexMongo(MONGO_CONNSTR, MONGO_EXO_DB)
self.exostorage = EXOStorage(MONGO_CONNSTR, MONGO_EXO_DB)
self.datasource = DataSourceSQL(SQL_HOST, SQL_USER, SQL_PASS,
self.assetindex, self.futures_limit,
self.options_limit, self.exostorage)
def setUpClass(self):
mongo_connstr = 'mongodb://*****:*****@10.0.1.2/tmldb?authMechanism=SCRAM-SHA-1'
mongo_db_name = 'tmldb'
assetindex = AssetIndexMongo(mongo_connstr, mongo_db_name)
exostorage = EXOStorage(mongo_connstr, mongo_db_name)
# base_date = datetime(2011, 6, 13, 12, 45, 0)
futures_limit = 3
options_limit = 10
server = 'h9ggwlagd1.database.windows.net'
user = '******'
password = '******'
self.datasource = DataSourceSQL(server, user, password, assetindex,
futures_limit, options_limit,
exostorage)
def test_real_world_debug(self):
return
assetindex = AssetIndexMongo(MONGO_CONNSTR, MONGO_EXO_DB)
storage = EXOStorage(MONGO_CONNSTR, MONGO_EXO_DB)
# datasource = DataSourceSQL(SQL_HOST, SQL_USER, SQL_PASS, assetindex, futures_limit=10, options_limit=10)
datasource = DataSourceMongo(MONGO_CONNSTR,
MONGO_EXO_DB,
assetindex,
futures_limit=10,
options_limit=10,
exostorage=storage)
exmgr = ExecutionManager(MONGO_CONNSTR,
datasource,
dbname=MONGO_EXO_DB)
positions = exmgr.account_positions_process(write_to_db=False)
def __init__(self, args, loglevel):
self.args = args
self.loglevel = loglevel
logging.getLogger("pika").setLevel(logging.WARNING)
logger = logging.getLogger('TradingOnlineScript')
logger.setLevel(loglevel)
# create console handler with a higher log level
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(loglevel)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
self.log = logger
self.log.info('Init TradingOnlineScript')
self.signal_app = SignalApp('TradingOnlineScript', APPCLASS_SIGNALS,
RABBIT_HOST, RABBIT_USER, RABBIT_PASSW)
self.signal_app.send(
MsgStatus("INIT", 'Initiating trading online engine'))
self.alpha_app = SignalApp('*', APPCLASS_ALPHA, RABBIT_HOST,
RABBIT_USER, RABBIT_PASSW)
#
# Init EXO engine datasource
#
assetindex = AssetIndexMongo(MONGO_CONNSTR, MONGO_EXO_DB)
exostorage = EXOStorage(MONGO_CONNSTR, MONGO_EXO_DB)
futures_limit = 3
options_limit = 10
self.datasource = DataSourceBase(assetindex, futures_limit,
options_limit, exostorage)
self.exmgr = ExecutionManager(MONGO_CONNSTR, self.datasource,
MONGO_EXO_DB)
1.0,
itm_put.price,
leg_name='opt_itm_leg'),
Transaction(otm_put,
self.date,
-1.0,
otm_put.price,
leg_name='opt_otm_leg'),
]
if __name__ == "__main__":
mongo_connstr = 'mongodb://*****:*****@10.0.1.2/tmldb?authMechanism=SCRAM-SHA-1'
mongo_db_name = 'tmldb'
assetindex = AssetIndexMongo(mongo_connstr, mongo_db_name)
exostorage = EXOStorage(mongo_connstr, mongo_db_name)
base_date = datetime(2014, 1, 13, 12, 45, 0)
futures_limit = 3
options_limit = 10
DEBUG = '.'
datasource = DataSourceMongo(mongo_connstr, mongo_db_name, assetindex,
futures_limit, options_limit, exostorage)
server = 'h9ggwlagd1.database.windows.net'
user = '******'
password = '******'
datasource = DataSourceSQL(server, user, password, assetindex,
futures_limit, options_limit, exostorage)
import pandas as pd
from smartcampaign import SmartCampaignBase
import os
#
# Init V1 framework datasource
#
from exobuilder.data.exostorage import EXOStorage
from scripts.settings import *
storage = EXOStorage(MONGO_CONNSTR, MONGO_EXO_DB)
CAMPAIGN_DICT = {
'name': "BasicSmartCampaign",
# Allowed values for both ALPHA_RISK_TYPE and CAMPAIGN_RISK_TYPE
# 'atr' - ATR(RISK_PERIOD) of alpha/campaign equity risk metric
# 'atrmax' - Highest(ATR(RISK_PERIOD)) of alpha/campaign equity risk metric
# 'ddavg' - rolling mean (RISK_PERIOD) of drawdown of alpha/campaign equity
# 'ddmax' - rolling maximum (RISK_PERIOD) of drawdown of alpha/campaign equity
# 'ddq95' - rolling 95% quantile (RISK_PERIOD) of drawdown of alpha/campaign equity
'alpha_risk_type': 'atr',
'alpha_risk_period': 100,
'alpha_min_risk': 100,
'campaign_risk_type': 'atrmax',
'campaign_risk_period': 100,
'campaign_min_risk': 100,
#
class SmartEXOUtils:
def __init__(self, smartexo_class, **kwargs):
self.verbosive_logging = kwargs.get('verbosive_logging', False)
self.futures_limit = kwargs.get('futures_limit', 3)
self.options_limit = kwargs.get('options_limit', 20)
self.smartexo_class = smartexo_class
if self.verbosive_logging:
logging.basicConfig(format='%(message)s', level=logging.DEBUG)
else:
logging.basicConfig(format='%(message)s', level=logging.INFO)
self.assetindex = AssetIndexMongo(MONGO_CONNSTR, MONGO_EXO_DB)
self.exostorage = EXOStorage(MONGO_CONNSTR, MONGO_EXO_DB)
self.datasource = DataSourceSQL(SQL_HOST, SQL_USER, SQL_PASS,
self.assetindex, self.futures_limit,
self.options_limit, self.exostorage)
def plot_transactions_payoff(self, smart_exo_position_func, analysis_date,
analysis_instrument, **whatif_kwargs):
payoff = PayoffAnalyzer(self.datasource)
instr = self.datasource.get(analysis_instrument, analysis_date)
rh = RolloverHelper(instr)
fut, opt_chain = rh.get_active_chains()
strikes_on_graph = whatif_kwargs.get('strikes_on_graph', 30)
whatif_iv_change = whatif_kwargs.get('whatif_iv_change', 0)
whatif_days_to_expiration = whatif_kwargs.get(
'whatif_days_to_expiration', int(opt_chain.to_expiration_days / 2))
payoff.load_transactions(
smart_exo_position_func(analysis_date, fut, opt_chain),
analysis_date)
payoff.plot(strikes_on_graph, whatif_iv_change,
whatif_days_to_expiration)
def clear_smartexo(self):
logging.info("Deleting all SmartEXO of :" +
self.smartexo_class.EXO_NAME)
client = MongoClient(MONGO_CONNSTR)
db = client[MONGO_EXO_DB]
db['exo_data'].delete_many({
'name': {
'$regex': '.*{0}*.'.format(self.smartexo_class.EXO_NAME)
}
})
def build_smartexo(self, start_date, **smartexo_kwargs):
self.clear_smartexo()
logging.info(
"Starting EXO calculation process from: {0}".format(start_date))
if self.smartexo_class.ASSET_LIST is None:
warnings.warn(
"You must define ASSET_LIST inside SmartEXO class. Aborting..."
)
return
for ticker in self.smartexo_class.ASSET_LIST:
logging.info("Processing: " + ticker)
currdate = start_date
enddate = datetime.combine(datetime.now().date(), dttime(0, 0, 0))
while currdate <= enddate:
start_time = time.time()
date = currdate
asset_info = self.assetindex.get_instrument_info(ticker)
exec_time_end, decision_time_end = AssetIndexMongo.get_exec_time(
date, asset_info)
logging.info("\t\tRun on {0}".format(decision_time_end))
with self.smartexo_class(ticker, 0, decision_time_end,
self.datasource,
**smartexo_kwargs) as exo_engine:
try:
asset_list = exo_engine.ASSET_LIST
# Checking if current symbol is present in EXO class ASSET_LIST
if asset_list is not None:
if ticker not in asset_list:
# Skipping assets which are not in the list
continue
except AttributeError:
warnings.warn(
"EXO class {0} doesn't contain ASSET_LIST attribute filter, calculating all assets"
.format(self.smartexo_class))
# Load EXO information from mongo
exo_engine.load()
exo_engine.calculate()
end_time = time.time()
currdate += timedelta(days=1)
logging.debug("Elapsed: {0}".format(end_time - start_time))
logging.info('Done')
def plot_smartexo_price(self):
if self.smartexo_class.ASSET_LIST is None:
warnings.warn(
"You must define ASSET_LIST inside SmartEXO class. Aborting..."
)
return
for ticker in self.smartexo_class.ASSET_LIST:
exo_df, exo_info = self.exostorage.load_series('{0}_{1}'.format(
ticker, self.smartexo_class.EXO_NAME))
f, (ax1, ax2) = plt.subplots(2,
gridspec_kw={'height_ratios': [3, 1]})
exo_df['exo'].plot(ax=ax1,
title='{0}_{1}'.format(
ticker, self.smartexo_class.EXO_NAME))
ax = exo_df['regime'].plot(ax=ax1, secondary_y=True)
ax.set_ylim(-2, 2)
exo_df['delta'].plot(ax=ax2)
ax2.set_title('Delta')
plt.show()
from .settings import *
except SystemError:
from scripts.settings import *
try:
from .settings_local import *
except SystemError:
try:
from scripts.settings_local import *
except ImportError:
pass
pass
mongo_db_name = 'tmldb'
assetindex = AssetIndexMongo(MONGO_CONNSTR, mongo_db_name)
exostorage = EXOStorage(MONGO_CONNSTR, mongo_db_name)
base_date = datetime(2011, 3, 1, 10, 15, 0)
futures_limit = 3
options_limit = 10
DEBUG = '.'
datasource = DataSourceMongo(MONGO_CONNSTR, mongo_db_name, assetindex,
futures_limit, options_limit, exostorage)
server = 'h9ggwlagd1.database.windows.net'
user = '******'
password = '******'
datasource = DataSourceSQL(server, user, password, assetindex,
def main(args, loglevel):
if args.logfile == '':
logging.basicConfig(
stream=sys.stdout,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=loglevel)
else:
logging.basicConfig(
filename=args.logfile,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=loglevel)
signalapp = SignalApp("AlphaRebalancer", APPCLASS_ALPHA, RABBIT_HOST,
RABBIT_USER, RABBIT_PASSW)
signalapp.send(MsgStatus('INIT', 'Initiating alpha rebalancer script'))
#exo_names = get_exo_names_mat()
logging.getLogger("pika").setLevel(logging.WARNING)
logging.info("Starting...")
exo_storage = EXOStorage(MONGO_CONNSTR, MONGO_EXO_DB)
exo_names = exo_storage.exo_list()
for exo in exo_names:
logging.info("Processing EXO: " + exo)
# Load alpha modules to process
for module in os.listdir('alphas'):
#
# Custom EXO folder found
#
swm = None
context = None
if module.lower() == exo.lower() and os.path.isdir(
os.path.join('alphas', module)):
for custom_file in os.listdir(os.path.join('alphas', module)):
if 'alpha_' in custom_file and '.py' in custom_file:
logging.debug(
'Processing custom module: ' +
os.path.join('alphas', module, custom_file))
try:
m = importlib.import_module(
'scripts.alphas.{0}.{1}'.format(
module, custom_file.replace('.py', '')))
context = m.STRATEGY_CONTEXT
context['strategy']['exo_name'] = exo
context['strategy'][
'suffix'] = m.STRATEGY_SUFFIX + 'custom'
context['strategy']['exo_storage'] = exo_storage
logging.info(
'Running CUSTOM alpha: ' + Swarm.get_name(
m.STRATEGY_CONTEXT, m.STRATEGY_SUFFIX))
if 'exo_name' in context['strategy'] and context[
'strategy']['exo_name'] != exo:
logging.error(
"Custom strategy context exo_name != current EXO name (folder mismatch?)"
)
raise ValueError(
"Custom strategy context exo_name != current EXO name (folder mismatch?)"
)
swm = Swarm(context)
swm.run_swarm()
swm.pick()
#
# Saving last EXO state to online DB
#
swmonline = SwarmOnlineManager(
MONGO_CONNSTR, MONGO_EXO_DB,
m.STRATEGY_CONTEXT)
logging.debug('Saving: {0}'.format(swm.name))
swmonline.save(swm)
except:
logging.exception('Exception occurred:')
signalapp.send(
MsgStatus('ERROR',
'Exception in {0}'.format(
Swarm.get_name(
m.STRATEGY_CONTEXT,
m.STRATEGY_SUFFIX)),
notify=True))
elif 'alpha_' in module and '.py' in module:
logging.debug('Processing generic module: ' + module)
try:
m = importlib.import_module('scripts.alphas.{0}'.format(
module.replace('.py', '')))
for direction in [-1, 1]:
context = m.STRATEGY_CONTEXT
context['strategy']['exo_name'] = exo
context['strategy']['opt_params'][0] = OptParamArray(
'Direction', [direction])
context['strategy']['suffix'] = m.STRATEGY_SUFFIX
context['strategy']['exo_storage'] = exo_storage
logging.info('Running alpha: ' +
Swarm.get_name(m.STRATEGY_CONTEXT) +
' Direction: {0}'.format(direction))
swm = Swarm(context)
swm.run_swarm()
swm.pick()
#
# Saving last EXO state to online DB
#
swmonline = SwarmOnlineManager(MONGO_CONNSTR,
MONGO_EXO_DB,
m.STRATEGY_CONTEXT)
logging.debug('Saving: {0}'.format(swm.name))
swmonline.save(swm)
except:
logging.exception('Exception occurred:')
signalapp.send(
MsgStatus('ERROR',
'Exception in {0}'.format(
Swarm.get_name(m.STRATEGY_CONTEXT,
m.STRATEGY_SUFFIX)),
notify=True))
logging.info("Processing accounts positions")
assetindex = AssetIndexMongo(MONGO_CONNSTR, MONGO_EXO_DB)
datasource = DataSourceMongo(MONGO_CONNSTR,
MONGO_EXO_DB,
assetindex,
futures_limit=10,
options_limit=10,
exostorage=exo_storage)
exmgr = ExecutionManager(MONGO_CONNSTR, datasource, dbname=MONGO_EXO_DB)
exmgr.account_positions_process(write_to_db=True)
signalapp.send(MsgStatus('DONE', 'Alpha rebalancer script', notify=True))
logging.info("Done.")