def update_fx_prices():
"""
Update FX prices stored in Arctic (Mongo) with interactive brokers prices (usually going back about a year)
:return: Nothing
"""
mongo_db = mongoDb(
) # will use default database, host unles specified here
log = logger("Update-FX-prices", mongo_db=mongo_db)
ib_conn = connectionIB(log=log.setup(
component="IB-connection")) # will use default port, host
ibfxpricedata = ibFxPricesData(ib_conn,
log=log.setup(component="ibFxPricesData"))
arcticfxdata = arcticFxPricesData(
mongo_db=mongo_db, log=log.setup(component="arcticFxPricesData"))
list_of_codes_all = ibfxpricedata.get_list_of_fxcodes(
) # codes must be in .csv file /sysbrokers/IB/ibConfigSpotFx.csv
log.msg("FX Codes: %s" % str(list_of_codes_all))
for fx_code in list_of_codes_all:
log.label(currency_code=fx_code)
new_fx_prices = ibfxpricedata.get_fx_prices(
fx_code) # returns fxPrices object
arcticfxdata.update_fx_prices(fx_code, new_fx_prices)
ib_conn.disconnect()
def __init__(
self,
mongo_db=arg_not_supplied,
idoffset=arg_not_supplied,
log=logtoscreen("mongoIDTracker"),
):
if mongo_db is arg_not_supplied:
mongo_db = mongoDb()
if idoffset is arg_not_supplied:
_notused_ipaddress, _notused_port, idoffset = ib_defaults()
self._mongo = mongoConnection(IB_CLIENT_COLLECTION, mongo_db=mongo_db)
# this won't create the index if it already exists
self._mongo.create_index("client_id")
self.name = "Tracking IB client IDs, mongodb %s/%s @ %s -p %s " % (
self._mongo.database_name,
self._mongo.collection_name,
self._mongo.host,
self._mongo.port,
)
self.log = log
self._idoffset = idoffset
def update_fx_prices():
"""
Update FX prices stored in Arctic (Mongo) with interactive brokers prices (usually going back about a year)
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-FX-prices", mongo_db=mongo_db) as log,\
connectionIB(log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob("ibFxPricesData arcticFxPricesData",
ib_conn=ib_conn,
mongo_db=mongo_db)
list_of_codes_all = data.ib_fx_prices.get_list_of_fxcodes(
) # codes must be in .csv file /sysbrokers/IB/ibConfigSpotFx.csv
log.msg("FX Codes: %s" % str(list_of_codes_all))
for fx_code in list_of_codes_all:
try:
log.label(currency_code=fx_code)
update_fx_prices_for_code(fx_code, data)
except Exception as e:
log.warn("Something went wrong with FX update %s" % e)
return success
def update_historical_prices():
"""
Do a daily update for futures contract prices, using IB historical data
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-Historical-prices", mongo_db=mongo_db) as log,\
connectionIB(mongo_db = mongo_db, log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob(
"ibFuturesContractPriceData arcticFuturesContractPriceData \
arcticFuturesMultiplePricesData mongoFuturesContractData",
mongo_db=mongo_db,
log=log,
ib_conn=ib_conn)
list_of_codes_all = data.arctic_futures_multiple_prices.get_list_of_instruments(
)
for instrument_code in list_of_codes_all:
update_historical_prices_for_instrument(
instrument_code,
data,
log=log.setup(instrument_code=instrument_code))
return success
def update_multiple_adjusted_prices_daily():
"""
Do a daily update for multiple and adjusted prices
:return: Nothing
"""
with mongoDb() as mongo_db, \
logger("Update-multiple-adjusted-prices(daily)", mongo_db=mongo_db) as log:
data = dataBlob("arcticFuturesContractPriceData \
arcticFuturesMultiplePricesData \
arcticFuturesAdjustedPricesData",
mongo_db=mongo_db,
log=log)
list_of_codes_all = data.arctic_futures_multiple_prices.get_list_of_instruments(
)
for instrument_code in ['CRUDE_W']:
try:
update_multiple_adjusted_prices_for_instrument(
instrument_code,
data,
log=log.setup(instrument_code=instrument_code))
except Exception as e:
log.warn("ERROR: Multiple price update went wrong: %s" %
str(e))
return success
def update_manual_check_fx_prices(fx_code):
"""
Update FX prices stored in Arctic (Mongo) with interactive brokers prices (usually going back about a year)
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-FX-prices", mongo_db=mongo_db) as log,\
connectionIB(log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob("ibFxPricesData arcticFxPricesData",
ib_conn=ib_conn,
mongo_db=mongo_db)
list_of_codes_all = data.ib_fx_prices.get_list_of_fxcodes(
) # codes must be in .csv file /sysbrokers/IB/ibConfigSpotFx.csv
if fx_code not in list_of_codes_all:
print("\n\n\ %s is not an FX code (valid codes: %s) \n\n" %
(fx_code, list_of_codes_all))
raise Exception()
log.label(currency_code=fx_code)
update_manual_check_fx_prices_for_code(fx_code, data)
return success
def mongo_db(self):
mongo_db = getattr(self, "_mongo_db", arg_not_supplied)
if mongo_db is arg_not_supplied:
mongo_db = mongoDb()
self._mongo_db = mongo_db
return mongo_db
def _get_config_data_object(self):
database_name = self._database_name
host = self._host
port = self._port
data_object = mongoFuturesInstrumentData(
mongoDb(database_name=database_name, host=host))
return data_object
def _get_all_prices_data_object(self):
database_name = self._database_name
host = self._host
port = self._port
multiple_prices_data_object = arcticFuturesMultiplePricesData(
mongoDb(database_name=database_name, host=host))
multiple_prices_data_object.log = self.log
return multiple_prices_data_object
def _get_fx_data_object(self):
database_name = self._database_name
host = self._host
port = self._port
fx_prices_data_object = arcticFxPricesData(
mongoDb(database_name=database_name, host=host))
fx_prices_data_object.log = self.log
return fx_prices_data_object
def _get_adj_prices_data_object(self):
database_name = self._database_name
host = self._host
port = self._port
adj_prices_data = arcticFuturesAdjustedPricesData(
mongoDb(database_name=database_name, host=host))
adj_prices_data.log = self.log
return adj_prices_data
def __init__(self, mongo_db=arg_not_supplied, log=logger("arcticSimData")):
"""
Use a different database
"""
super().__init__(log=log)
if mongo_db is arg_not_supplied:
mongo_db = mongoDb()
self.mongo_db = mongo_db
def run_strategy_order_generator():
with mongoDb() as mongo_db, \
logger("run_strategy_order_generator", mongo_db=mongo_db) as log:
data = dataBlob(mongo_db=mongo_db, log=log)
# FIX ME CODE TO RUN MULTIPLE TIMES
# FOR NOW JUST RUN ONCE A DAY
order_handler = orderHandlerAcrossStrategies(data)
order_handler.check_for_orders_across_strategies()
return None
def update_roll_state(instrument_code: str):
"""
Update the roll state for a particular instrument
This includes the option, where possible, to switch the adjusted price series on to a new contract
:param instrument_code: str
:return: None
"""
"""
mongo_db = mongoDb()
log=logger("Update-Sampled_Contracts", mongo_db=mongo_db)
"""
with mongoDb() as mongo_db,\
logger("Update-Roll-Adjusted-Prices", mongo_db=mongo_db, instrument_code=instrument_code) as log:
data = dataBlob("mongoRollStateData", mongo_db=mongo_db, log=log)
## First get the roll info
# This will also update to console
report_results = run_report_with_data_blob(
roll_report_config, data, instrument_code=instrument_code)
if report_results is failure:
print("Can't run roll report, so can't change status")
return failure
current_roll_status, roll_state_required = get_required_roll_state(
data, instrument_code)
if roll_state_required is no_state_available:
return failure
data.mongo_roll_state.set_roll_state(instrument_code,
roll_state_required)
if roll_state_required is roll_adj_state:
## Going to roll adjusted prices
roll_result = _roll_adjusted_and_multiple_prices(
data, instrument_code)
if roll_result is success:
## Return the state back to default (no roll) state
data.log.msg(
"Successful roll! Returning roll state of %s to %s" %
(instrument_code, default_state))
data.mongo_roll_state.set_roll_state(instrument_code,
default_state)
else:
data.log.msg(
"Something has gone wrong with rolling adjusted of %s! Returning roll state to previous state of %s"
% (instrument_code, current_roll_status))
data.mongo_roll_state.set_roll_state(instrument_code,
current_roll_status)
return success
def update_fx_prices():
"""
Update FX prices stored in Arctic (Mongo) with interactive brokers prices (usually going back about a year)
:return: Nothing
"""
log = logger("Update-FX-prices")
ib_conn = connectionIB(log=log.setup(
component="IB-connection")) # will use default port, host
mongo_db = mongoDb(
) # will use default database, host unles specified here
ibfxpricedata = ibFxPricesData(ib_conn,
mongo_db=mongo_db,
log=log.setup(component="ibFxPricesData"))
arcticfxdata = arcticFxPricesData(
mongo_db=mongo_db, log=log.setup(component="arcticFxPricesData"))
list_of_codes_all = ibfxpricedata.get_list_of_fxcodes(
) # codes must be in .csv file /sysbrokers/IB/ibConfigSpotFx.csv
log.msg("FX Codes: %s" % str(list_of_codes_all))
for fx_code in list_of_codes_all:
log.label(currency_code=fx_code)
new_fx_prices = ibfxpricedata.get_fx_prices(
fx_code) # returns fxPrices object
if len(new_fx_prices) == 0:
log.error("Error trying to get data for %s" % fx_code)
continue
old_fx_prices = arcticfxdata.get_fx_prices(fx_code)
new_fx_prices = new_fx_prices[
new_fx_prices.index > old_fx_prices.index[-1]]
if len(new_fx_prices) == 0:
log.msg("No additional data for %s" % fx_code)
continue
fx_prices = pd.concat([old_fx_prices, new_fx_prices], axis=0)
fx_prices = fx_prices.sort_index()
# remove duplicates
fx_prices = fx_prices[~fx_prices.index.duplicated(keep='first')]
# write
arcticfxdata.add_fx_prices(fx_code, fx_prices, ignore_duplication=True)
def run_report(report_config, **kwargs):
"""
:param report_config:
:return:
"""
with mongoDb() as mongo_db,\
logger("Reporting %s" % report_config.title, mongo_db=mongo_db) as log:
data = dataBlob(mongo_db = mongo_db, log = log)
report_result = run_report_with_data_blob(report_config, data, **kwargs)
return report_result
def run_system():
with mongoDb() as mongo_db, \
logger("runSystem", mongo_db=mongo_db, strategy = strategy_name) as log:
data = dataBlob(mongo_db=mongo_db, log=log)
capital_value = get_capital(data, strategy_name)
system = production_futures_system(backtest_config_filename,
log=log, notional_trading_capital=capital_value,
base_currency=account_currency)
updated_buffered_positions(data, strategy_name, system)
store_backtest_state(data, system, strategy_name=strategy_name,
backtest_config_filename=backtest_config_filename)
return success
def __init__(self, collection_name, mongo_db=None):
if mongo_db is None:
mongo_db = mongoDb()
database_name = mongo_db.database_name
host = mongo_db.host
# Arctic doesn't accept a port
store = Arctic(host)
self.database_name = database_name
self.collection_name = collection_name
self.host = host
self.store = store
self.library = self._setup_lib(store, database_name, collection_name)
def __init__(self,
arg_string=arg_not_supplied,
mongo_db=arg_not_supplied,
ib_conn=arg_not_supplied,
csv_data_paths=arg_not_supplied,
log=logtoscreen("")):
"""
Set up of a data pipeline with standard attribute names, logging, links to DB etc
Class names we know how to handle are:
'ib*', 'mongo*', 'arctic*', 'csv*'
So: "arcticFuturesContractPriceData arcticFuturesContractPriceData mongoFuturesContractData'
.... is equivalent of this sort of thing:
ib_pricedata = ibFuturesContractPriceData(ib_conn, log=log.setup(component="IB-price-data"))
arctic_pricedata = arcticFuturesContractPriceData(mongo_db=mongo_db,
log=log.setup(component="arcticFuturesContractPriceData"))
mongo_contractsdata = mongoFuturesContractData(mongo_db=mongo_db,
log = log.setup(component="mongoFuturesContractData"))
:param arg_string: str like a named tuple in the form 'classNameOfData1 classNameOfData2' and so on
:param mongo_db: mongo DB object
:param ib_conn: ib connection
:param csv_data_paths: dict, keynames are function names eg csvFuturesContractPriceData, values are paths
:param log: logger
"""
if mongo_db is arg_not_supplied:
mongo_db = mongoDb()
self.mongo_db = mongo_db
self.ib_conn = ib_conn
self.log = log
self.csv_data_paths = csv_data_paths
self.attr_list = []
self.class_list = []
if arg_string is arg_not_supplied:
# can set up dynamically later
return None
self.add_class_list(arg_string)
def update_account_values():
"""
Do a daily update of accounting information
Get the total account value from IB, and calculate the p&l since we last ran
This calculation is done using a user specified handler, which can deal with eg multiple accounts if required
Needs to know about any withdrawals.
Does spike checking: large changes in account value are checked before writing
If your strategy has very high risk you may wish to do this more frequently than daily
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-Account-Values", mongo_db=mongo_db) as log,\
connectionIB(mongo_db = mongo_db, log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob(mongo_db=mongo_db, log=log, ib_conn=ib_conn)
capital_data = dataCapital(data)
## This assumes that each account only reports eithier in one currency or for each currency, i.e. no double counting
total_account_value_in_base_currency = capital_data.get_ib_total_capital_value(
)
log.msg("Broker account value is %f" %
total_account_value_in_base_currency)
# Update total capital
try:
new_capital = capital_data.total_capital_calculator.\
get_total_capital_with_new_broker_account_value(total_account_value_in_base_currency)
except Exception as e:
## Problem, most likely spike
log.critical(
"Error %s whilst updating total capital; you may have to use update_capital_manual script or function"
% e)
return failure
log.msg("New capital is %f" % new_capital)
return success
def update_run_systems():
with mongoDb() as mongo_db, \
logger("update_run_systems", mongo_db=mongo_db) as log:
data = dataBlob(mongo_db=mongo_db, log=log)
strategy_dict = get_private_then_default_key_value('strategy_list')
for strategy_name in strategy_dict:
data.log.label(strategy = strategy_name)
try:
launch_function, launch_args = _get_launch_config(strategy_dict[strategy_name])
except Exception as e:
log.critical("Error %s with config in defaults or private yaml files for strategy_list:%s:overnight_launcher" % (e,strategy_name))
# By convention, arg is strategy_name, data, kwargs are the rest of config
try:
launch_function(strategy_name, data, **launch_args)
except Exception as e:
log.critical("Error %s running system for %s" % (e,strategy_name))
def update_strategy_capital():
"""
Allocate capital to different strategies
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-Strategy-Capital", mongo_db=mongo_db) as log,\
connectionIB(mongo_db = mongo_db, log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob(mongo_db = mongo_db, log = log, ib_conn = ib_conn)
try:
strategy_allocation(data)
except Exception as e:
## Problem, will send email
log.critical("Error %s whilst allocating strategy capital" % e)
return success
def update_capital_manual():
"""
Interactive session that allows you to manipulate capital manually
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-Capital-Manual", mongo_db=mongo_db) as log,\
connectionIB(mongo_db = mongo_db, log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob(mongo_db = mongo_db, log = log, ib_conn = ib_conn)
data_capital = dataCapital(data)
still_running = True
while still_running:
# display capital and get input
user_option_int = print_capital_and_get_user_input(data_capital)
if user_option_int==0:
setup_initial_capital(data_capital)
elif user_option_int==1:
update_capital_from_ib(data_capital)
elif user_option_int==2:
adjust_capital_for_delta(data_capital)
elif user_option_int==3:
modify_any_value(data_capital)
elif user_option_int==4:
delete_capital_since_time(data_capital)
elif user_option_int==919:
delete_all_capital(data_capital)
elif user_option_int==5:
still_running=False
break
else:
print("%d is not a valid option but was in list of possible options: check code" % str(user_option_int))
## Back to top of while loop
return success
def update_manual_check_historical_prices(instrument_code:str):
"""
Do a daily update for futures contract prices, using IB historical data
If any 'spikes' are found, run manual checks
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-Historical-prices-manually", mongo_db=mongo_db) as log,\
connectionIB(mongo_db = mongo_db, log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob("ibFuturesContractPriceData arcticFuturesContractPriceData \
arcticFuturesMultiplePricesData mongoFuturesContractData",
mongo_db = mongo_db, log = log, ib_conn = ib_conn)
list_of_codes_all = data.arctic_futures_contract_price.get_instruments_with_price_data()
if instrument_code not in list_of_codes_all:
print("\n\n\ %s is not an instrument with price data \n\n" % instrument_code)
raise Exception()
update_historical_prices_with_checks_for_instrument(instrument_code, data, log=log.setup(instrument_code = instrument_code))
return success
def __init__(self, collection_name, mongo_db=None):
if mongo_db is None:
mongo_db = mongoDb()
database_name = mongo_db.database_name
host = mongo_db.host
# Arctic doesn't accept a port
store = Arctic(host)
library_name = database_name + "." + collection_name
store.initialize_library(
library_name) # will this fail if already exists??
library = store[library_name]
self.database_name = database_name
self.collection_name = collection_name
self.host = host
self.store = store
self.library_name = library_name
self.library = library
def update_sampled_contracts():
"""
Update the active contracts, according to what is available in IB for a given instrument
These are stored in mongoDB
The active contracts list is used to see what contracts have historical data sampled for
It does not tell you what the current priced, forward, or carry contract are - that is in multiple prices (DRY)
However we base the list of theoretical active contracts on the current priced, forward, and carry contracts
We will end up adding to this list when we roll; this will change the current priced, forward, and carry contract
When we add a new contract (because one has become available), we get the exact expiry date from IB and save this with the
contract data.
We do not sample contracts on the list when they have passed their expiry date
Contracts are never deleted from the database
We don't check IB for contracts; since we can't reverse engineer a YYYYMM identifier from a YYYYMMDD
:returns: None
"""
with mongoDb() as mongo_db,\
logger("Update-Sampled_Contracts", mongo_db=mongo_db) as log,\
connectionIB(log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob("arcticFuturesMultiplePricesData ibFuturesContractPriceData\
mongoFuturesContractData mongoRollParametersData",
mongo_db, ib_conn, log=log)
list_of_codes_all = data.arctic_futures_multiple_prices.get_list_of_instruments()
for instrument_code in ['CRUDE_W']:
new_log = log.setup(instrument_code = instrument_code)
update_active_contracts_for_instrument(instrument_code, data, log=new_log)
def _get_new_mongo_db(self) -> mongoDb:
mongo_db = mongoDb()
return mongo_db
def update_account_values():
"""
Do a daily update of accounting information
Get the total account value from IB, and calculate the p&l since we last ran
This calculation is done using a user specified handler, which can deal with eg multiple accounts if required
Needs to know about any withdrawals.
Does spike checking: large changes in account value are checked before writing
Also get various other capital information
accountValues
accountSummary
portfolio
positions
reqPnL
pnl
cancelPnL
reqPnLSingle
pnlSingle
cancelPnLSingle
All of this is passed to a single capital allocator function that calculates how much capital each strategy should be allocated
and works out the p&l for each strategy
This can be done in various ways, eg by instrument, by asset class, as a % of total account value
... allow each strategy to keep it's p&l, or redistribute
What about margin? Might need to know 'in real time' what the margin is per strategy, if when
using whatif trades need to check. However better doing this on a global basis for account
If your strategy has very high risk you may wish to do this more frequently than daily
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-Historical-prices", mongo_db=mongo_db) as log,\
connectionIB(mongo_db = mongo_db, log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob(
"ibFuturesContractPriceData arcticFuturesContractPriceData \
arcticFuturesMultiplePricesData mongoFuturesContractData",
mongo_db=mongo_db,
log=log,
ib_conn=ib_conn)
currency_data = currencyData(data)
values_across_accounts = data.ib_conn.broker_get_account_value_across_currency_across_accounts(
)
## This assumes that each account only reports eithier in one currency or for each currency, i.e. no double counting
total_account_value_in_base_currency = currency_data.total_of_list_of_currency_values_in_base(
values_across_accounts)
return success
"""
Update spot FX prices using interactive brokers data, dump into mongodb
"""
from sysbrokers.IB.ibConnection import connectionIB
from sysdata.mongodb.mongo_connection import mongoDb
from sysbrokers.IB.ibSpotFXData import ibFxPricesData
from sysdata.arctic.arctic_spotfx_prices import arcticFxPricesData
from syslogdiag.log import logToMongod as logger
mongo_db = mongoDb() # will use default database, host unles specified here
log = logger("Update-FX-prices", mongo_db=mongo_db)
def update_fx_prices():
"""
Update FX prices stored in Arctic (Mongo) with interactive brokers prices (usually going back about a year)
:return: Nothing
"""
mongo_db = mongoDb(
) # will use default database, host unles specified here
log = logger("Update-FX-prices", mongo_db=mongo_db)
ib_conn = connectionIB(log=log.setup(
component="IB-connection")) # will use default port, host
ibfxpricedata = ibFxPricesData(ib_conn,
log=log.setup(component="ibFxPricesData"))
arcticfxdata = arcticFxPricesData(
