def update_historical_prices_for_instrument(instrument_code,
data,
log=logger("")):
"""
Do a daily update for futures contract prices, using IB historical data
:param instrument_code: str
:param data: dataBlob
:param log: logger
:return: None
"""
all_contracts_list = data.mongo_futures_contract.get_all_contract_objects_for_instrument_code(
instrument_code)
contract_list = all_contracts_list.currently_sampling()
if len(contract_list) == 0:
log.warn("No contracts marked for sampling for %s" % instrument_code)
return failure
for contract_object in contract_list:
update_historical_prices_for_instrument_and_contract(
contract_object,
data,
log=log.setup(contract_date=contract_object.date))
return success
def update_historical_prices_for_instrument_and_contract(
contract_object, data, log=logger("")):
"""
Do a daily update for futures contract prices, using IB historical data
:param contract_object: futuresContract
:param data: data blob
:param log: logger
:return: None
"""
ib_prices = data.ib_futures_contract_price.get_prices_for_contract_object(
contract_object)
if len(ib_prices) == 0:
log.warn("No IB prices found for %s" % str(contract_object))
return failure
rows_added = data.arctic_futures_contract_price.update_prices_for_contract(
contract_object, ib_prices, check_for_spike=True)
if rows_added is data_error:
## SPIKE
## Need to email user about this as will need manually checking
msg = "Spike found in prices for %s: need to manually check by running update_manual_check_historical_prices" % str(
contract_object)
log.warn(msg)
try:
send_mail_msg(msg, "Price Spike")
except:
log.warn("Couldn't send email about price spike")
return success
def update_historical_prices():
"""
Do a daily update for futures contract prices, using IB historical data
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-Historical-prices", mongo_db=mongo_db) as log,\
connectionIB(mongo_db = mongo_db, log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob(
"ibFuturesContractPriceData arcticFuturesContractPriceData \
arcticFuturesMultiplePricesData mongoFuturesContractData",
mongo_db=mongo_db,
log=log,
ib_conn=ib_conn)
list_of_codes_all = data.arctic_futures_multiple_prices.get_list_of_instruments(
)
for instrument_code in list_of_codes_all:
update_historical_prices_for_instrument(
instrument_code,
data,
log=log.setup(instrument_code=instrument_code))
return success
def update_historical_prices_for_instrument_and_contract(
contract_object, data, log=logger("")):
"""
Do a daily update for futures contract prices, using IB historical data
:param contract_object: futuresContract
:param data: data blob
:param log: logger
:return: None
"""
diag_prices = diagPrices(data)
intraday_frequency = diag_prices.get_intraday_frequency_for_historical_download(
)
result = get_and_add_prices_for_frequency(data,
log,
contract_object,
frequency=intraday_frequency)
if result is failure:
# Skip daily data if intraday not working
return failure
result = get_and_add_prices_for_frequency(data,
log,
contract_object,
frequency="D")
return result
def production_futures_system(config_filename, log=logger("futures_system"),
notional_trading_capital=1000000, base_currency="USD"):
"""
:param data: data object (defaults to reading from csv files)
:type data: sysdata.data.simData, or anything that inherits from it
:param config: Configuration object (defaults to futuresconfig.yaml in this directory)
:type config: sysdata.configdata.Config
:param log_level: How much logging to do
:type log_level: str
"""
log_level = "on"
data = arcticFuturesSimData()
config = Config(
config_filename)
# Overwrite capital
config.notional_trading_capital = notional_trading_capital
config.base_currency = base_currency
system = futures_system(data=data, config=config)
system.log = log
system.set_logging_level(log_level)
return system
def read_fx_prices(currency_code: str, tail_size: int = 20):
"""
Retrieve FX prices from a database, print last tail_size rows
Will only work for codes in database, so won't do cross rate conversions to find a non existent rate
:param currency_code: The currency code, for example 'GBPUSD'
:param tail_size: The length of the tail to print
:return: None, but print results
"""
log = logger()
arcticfxdata = arcticFxPricesData(log=log.setup(
component="arcticFxPricesData"))
list_of_codes_all = arcticfxdata.get_list_of_fxcodes()
try:
assert currency_code in list_of_codes_all
except:
raise Exception("Currency code %s not in possible codes %s" %
(currency_code, list_of_codes_all))
fx_prices = arcticfxdata.get_fx_prices(currency_code)
print("/n Last %d FX rates for %s \n\n" % (tail_size, currency_code))
print(fx_prices.tail(tail_size))
def update_fx_prices():
"""
Update FX prices stored in Arctic (Mongo) with interactive brokers prices (usually going back about a year)
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-FX-prices", mongo_db=mongo_db) as log,\
connectionIB(log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob("ibFxPricesData arcticFxPricesData",
ib_conn=ib_conn,
mongo_db=mongo_db)
list_of_codes_all = data.ib_fx_prices.get_list_of_fxcodes(
) # codes must be in .csv file /sysbrokers/IB/ibConfigSpotFx.csv
log.msg("FX Codes: %s" % str(list_of_codes_all))
for fx_code in list_of_codes_all:
try:
log.label(currency_code=fx_code)
update_fx_prices_for_code(fx_code, data)
except Exception as e:
log.warn("Something went wrong with FX update %s" % e)
return success
def update_multiple_adjusted_prices_daily():
"""
Do a daily update for multiple and adjusted prices
:return: Nothing
"""
with mongoDb() as mongo_db, \
logger("Update-multiple-adjusted-prices(daily)", mongo_db=mongo_db) as log:
data = dataBlob("arcticFuturesContractPriceData \
arcticFuturesMultiplePricesData \
arcticFuturesAdjustedPricesData",
mongo_db=mongo_db,
log=log)
list_of_codes_all = data.arctic_futures_multiple_prices.get_list_of_instruments(
)
for instrument_code in ['CRUDE_W']:
try:
update_multiple_adjusted_prices_for_instrument(
instrument_code,
data,
log=log.setup(instrument_code=instrument_code))
except Exception as e:
log.warn("ERROR: Multiple price update went wrong: %s" %
str(e))
return success
def update_manual_check_fx_prices(fx_code):
"""
Update FX prices stored in Arctic (Mongo) with interactive brokers prices (usually going back about a year)
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-FX-prices", mongo_db=mongo_db) as log,\
connectionIB(log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob("ibFxPricesData arcticFxPricesData",
ib_conn=ib_conn,
mongo_db=mongo_db)
list_of_codes_all = data.ib_fx_prices.get_list_of_fxcodes(
) # codes must be in .csv file /sysbrokers/IB/ibConfigSpotFx.csv
if fx_code not in list_of_codes_all:
print("\n\n\ %s is not an FX code (valid codes: %s) \n\n" %
(fx_code, list_of_codes_all))
raise Exception()
log.label(currency_code=fx_code)
update_manual_check_fx_prices_for_code(fx_code, data)
return success
def update_historical_prices_with_checks_for_instrument(
instrument_code, data, log=logger("")):
"""
Do a daily update for futures contract prices, using IB historical data
Any 'spikes' are manually checked
:param instrument_code: str
:param data: dataBlob
:param log: logger
:return: None
"""
diag_contracts = diagContracts(data)
all_contracts_list = diag_contracts.get_all_contract_objects_for_instrument_code(
instrument_code)
contract_list = all_contracts_list.currently_sampling()
if len(contract_list) == 0:
log.warn("No contracts marked for sampling for %s" % instrument_code)
return failure
for contract_object in contract_list:
update_historical_prices_with_checks_for_instrument_and_contract(
contract_object,
data,
log=log.setup(contract_date=contract_object.date))
return success
def update_historical_prices_for_instrument_and_contract(
contract_object, data, log=logger("")):
"""
Do a daily update for futures contract prices, using IB historical data
:param contract_object: futuresContract
:param data: data blob
:param log: logger
:return: None
"""
intraday_frequency = get_private_then_default_key_value(
"intraday_frequency")
result = get_and_add_prices_for_frequency(data,
log,
contract_object,
frequency=intraday_frequency)
if result is failure:
# Skip daily data if intraday not working
return failure
result = get_and_add_prices_for_frequency(data,
log,
contract_object,
frequency="D")
return result
def update_fx_prices():
"""
Update FX prices stored in Arctic (Mongo) with interactive brokers prices (usually going back about a year)
:return: Nothing
"""
mongo_db = mongoDb(
) # will use default database, host unles specified here
log = logger("Update-FX-prices", mongo_db=mongo_db)
ib_conn = connectionIB(log=log.setup(
component="IB-connection")) # will use default port, host
ibfxpricedata = ibFxPricesData(ib_conn,
log=log.setup(component="ibFxPricesData"))
arcticfxdata = arcticFxPricesData(
mongo_db=mongo_db, log=log.setup(component="arcticFxPricesData"))
list_of_codes_all = ibfxpricedata.get_list_of_fxcodes(
) # codes must be in .csv file /sysbrokers/IB/ibConfigSpotFx.csv
log.msg("FX Codes: %s" % str(list_of_codes_all))
for fx_code in list_of_codes_all:
log.label(currency_code=fx_code)
new_fx_prices = ibfxpricedata.get_fx_prices(
fx_code) # returns fxPrices object
arcticfxdata.update_fx_prices(fx_code, new_fx_prices)
ib_conn.disconnect()
def update_multiple_adjusted_prices_for_instrument(instrument_code, data, log=logger("")):
"""
Update for multiple and adjusted prices for a given instrument
:param instrument_code:
:param data: dataBlob
:param log: logger
:return: None
"""
# update multiple prices with new prices
# (method in multiple prices object and possible in data socket)
existing_adjusted_prices = data.arctic_futures_adjusted_prices.get_adjusted_prices(instrument_code)
existing_multiple_prices = data.arctic_futures_multiple_prices.get_multiple_prices(instrument_code)
relevant_contracts = existing_multiple_prices.current_contract_dict()
new_prices_dict = get_dict_of_new_prices_and_contractid(instrument_code, relevant_contracts, data)
updated_multiple_prices = existing_multiple_prices.update_multiple_prices_with_dict(new_prices_dict)
updated_adjusted_prices = existing_adjusted_prices.update_with_multiple_prices_no_roll(updated_multiple_prices)
if updated_multiple_prices is no_update_roll_has_occured:
raise Exception("Can't update adjusted prices as roll has occured but not registered properly")
data.arctic_futures_multiple_prices.add_multiple_prices(instrument_code, updated_multiple_prices, ignore_duplication=True)
data.arctic_futures_adjusted_prices.add_adjusted_prices(instrument_code, updated_adjusted_prices, ignore_duplication=True)
return success
def log(self):
log = getattr(self, "_log", arg_not_supplied)
if log is arg_not_supplied:
log = logger(self._log_name, mongo_db=self.mongo_db)
log.set_logging_level("on")
self._log = log
return log
def __init__(self, mongo_db=arg_not_supplied, log=logger("arcticSimData")):
"""
Use a different database
"""
super().__init__(log=log)
if mongo_db is arg_not_supplied:
mongo_db = mongoDb()
self.mongo_db = mongo_db
def run_strategy_order_generator():
with mongoDb() as mongo_db, \
logger("run_strategy_order_generator", mongo_db=mongo_db) as log:
data = dataBlob(mongo_db=mongo_db, log=log)
# FIX ME CODE TO RUN MULTIPLE TIMES
# FOR NOW JUST RUN ONCE A DAY
order_handler = orderHandlerAcrossStrategies(data)
order_handler.check_for_orders_across_strategies()
return None
def update_roll_state(instrument_code: str):
"""
Update the roll state for a particular instrument
This includes the option, where possible, to switch the adjusted price series on to a new contract
:param instrument_code: str
:return: None
"""
"""
mongo_db = mongoDb()
log=logger("Update-Sampled_Contracts", mongo_db=mongo_db)
"""
with mongoDb() as mongo_db,\
logger("Update-Roll-Adjusted-Prices", mongo_db=mongo_db, instrument_code=instrument_code) as log:
data = dataBlob("mongoRollStateData", mongo_db=mongo_db, log=log)
## First get the roll info
# This will also update to console
report_results = run_report_with_data_blob(
roll_report_config, data, instrument_code=instrument_code)
if report_results is failure:
print("Can't run roll report, so can't change status")
return failure
current_roll_status, roll_state_required = get_required_roll_state(
data, instrument_code)
if roll_state_required is no_state_available:
return failure
data.mongo_roll_state.set_roll_state(instrument_code,
roll_state_required)
if roll_state_required is roll_adj_state:
## Going to roll adjusted prices
roll_result = _roll_adjusted_and_multiple_prices(
data, instrument_code)
if roll_result is success:
## Return the state back to default (no roll) state
data.log.msg(
"Successful roll! Returning roll state of %s to %s" %
(instrument_code, default_state))
data.mongo_roll_state.set_roll_state(instrument_code,
default_state)
else:
data.log.msg(
"Something has gone wrong with rolling adjusted of %s! Returning roll state to previous state of %s"
% (instrument_code, current_roll_status))
data.mongo_roll_state.set_roll_state(instrument_code,
current_roll_status)
return success
def update_fx_prices():
"""
Update FX prices stored in Arctic (Mongo) with interactive brokers prices (usually going back about a year)
:return: Nothing
"""
log = logger("Update-FX-prices")
ib_conn = connectionIB(log=log.setup(
component="IB-connection")) # will use default port, host
mongo_db = mongoDb(
) # will use default database, host unles specified here
ibfxpricedata = ibFxPricesData(ib_conn,
mongo_db=mongo_db,
log=log.setup(component="ibFxPricesData"))
arcticfxdata = arcticFxPricesData(
mongo_db=mongo_db, log=log.setup(component="arcticFxPricesData"))
list_of_codes_all = ibfxpricedata.get_list_of_fxcodes(
) # codes must be in .csv file /sysbrokers/IB/ibConfigSpotFx.csv
log.msg("FX Codes: %s" % str(list_of_codes_all))
for fx_code in list_of_codes_all:
log.label(currency_code=fx_code)
new_fx_prices = ibfxpricedata.get_fx_prices(
fx_code) # returns fxPrices object
if len(new_fx_prices) == 0:
log.error("Error trying to get data for %s" % fx_code)
continue
old_fx_prices = arcticfxdata.get_fx_prices(fx_code)
new_fx_prices = new_fx_prices[
new_fx_prices.index > old_fx_prices.index[-1]]
if len(new_fx_prices) == 0:
log.msg("No additional data for %s" % fx_code)
continue
fx_prices = pd.concat([old_fx_prices, new_fx_prices], axis=0)
fx_prices = fx_prices.sort_index()
# remove duplicates
fx_prices = fx_prices[~fx_prices.index.duplicated(keep='first')]
# write
arcticfxdata.add_fx_prices(fx_code, fx_prices, ignore_duplication=True)
def run_report(report_config, **kwargs):
"""
:param report_config:
:return:
"""
with mongoDb() as mongo_db,\
logger("Reporting %s" % report_config.title, mongo_db=mongo_db) as log:
data = dataBlob(mongo_db = mongo_db, log = log)
report_result = run_report_with_data_blob(report_config, data, **kwargs)
return report_result
def run_system():
with mongoDb() as mongo_db, \
logger("runSystem", mongo_db=mongo_db, strategy = strategy_name) as log:
data = dataBlob(mongo_db=mongo_db, log=log)
capital_value = get_capital(data, strategy_name)
system = production_futures_system(backtest_config_filename,
log=log, notional_trading_capital=capital_value,
base_currency=account_currency)
updated_buffered_positions(data, strategy_name, system)
store_backtest_state(data, system, strategy_name=strategy_name,
backtest_config_filename=backtest_config_filename)
return success
def update_account_values():
"""
Do a daily update of accounting information
Get the total account value from IB, and calculate the p&l since we last ran
This calculation is done using a user specified handler, which can deal with eg multiple accounts if required
Needs to know about any withdrawals.
Does spike checking: large changes in account value are checked before writing
If your strategy has very high risk you may wish to do this more frequently than daily
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-Account-Values", mongo_db=mongo_db) as log,\
connectionIB(mongo_db = mongo_db, log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob(mongo_db=mongo_db, log=log, ib_conn=ib_conn)
capital_data = dataCapital(data)
## This assumes that each account only reports eithier in one currency or for each currency, i.e. no double counting
total_account_value_in_base_currency = capital_data.get_ib_total_capital_value(
)
log.msg("Broker account value is %f" %
total_account_value_in_base_currency)
# Update total capital
try:
new_capital = capital_data.total_capital_calculator.\
get_total_capital_with_new_broker_account_value(total_account_value_in_base_currency)
except Exception as e:
## Problem, most likely spike
log.critical(
"Error %s whilst updating total capital; you may have to use update_capital_manual script or function"
% e)
return failure
log.msg("New capital is %f" % new_capital)
return success
def update_run_systems():
with mongoDb() as mongo_db, \
logger("update_run_systems", mongo_db=mongo_db) as log:
data = dataBlob(mongo_db=mongo_db, log=log)
strategy_dict = get_private_then_default_key_value('strategy_list')
for strategy_name in strategy_dict:
data.log.label(strategy = strategy_name)
try:
launch_function, launch_args = _get_launch_config(strategy_dict[strategy_name])
except Exception as e:
log.critical("Error %s with config in defaults or private yaml files for strategy_list:%s:overnight_launcher" % (e,strategy_name))
# By convention, arg is strategy_name, data, kwargs are the rest of config
try:
launch_function(strategy_name, data, **launch_args)
except Exception as e:
log.critical("Error %s running system for %s" % (e,strategy_name))
def update_strategy_capital():
"""
Allocate capital to different strategies
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-Strategy-Capital", mongo_db=mongo_db) as log,\
connectionIB(mongo_db = mongo_db, log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob(mongo_db = mongo_db, log = log, ib_conn = ib_conn)
try:
strategy_allocation(data)
except Exception as e:
## Problem, will send email
log.critical("Error %s whilst allocating strategy capital" % e)
return success
def update_historical_prices_for_instrument_and_contract(
contract_object, data, log=logger("")):
"""
Do a daily update for futures contract prices, using IB historical data
:param contract_object: futuresContract
:param data: data blob
:param log: logger
:return: None
"""
ib_prices = data.ib_futures_contract_price.get_prices_for_contract_object(
contract_object)
if len(ib_prices) == 0:
log.warn("No IB prices found for %s" % str(contract_object))
return failure
data.arctic_futures_contract_price.update_prices_for_contract(
contract_object, ib_prices)
return success
def update_historical_prices_with_checks_for_instrument_and_contract(
contract_object, data, log=logger("")):
"""
Do a daily update for futures contract prices, using IB historical data, with checking
:param contract_object: futuresContract
:param data: data blob
:param log: logger
:return: None
"""
intraday_frequency = get_private_then_default_key_value(
"intraday_frequency")
get_and_check_prices_for_frequency(data,
log,
contract_object,
frequency=intraday_frequency)
get_and_check_prices_for_frequency(data,
log,
contract_object,
frequency="D")
return success
def update_capital_manual():
"""
Interactive session that allows you to manipulate capital manually
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-Capital-Manual", mongo_db=mongo_db) as log,\
connectionIB(mongo_db = mongo_db, log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob(mongo_db = mongo_db, log = log, ib_conn = ib_conn)
data_capital = dataCapital(data)
still_running = True
while still_running:
# display capital and get input
user_option_int = print_capital_and_get_user_input(data_capital)
if user_option_int==0:
setup_initial_capital(data_capital)
elif user_option_int==1:
update_capital_from_ib(data_capital)
elif user_option_int==2:
adjust_capital_for_delta(data_capital)
elif user_option_int==3:
modify_any_value(data_capital)
elif user_option_int==4:
delete_capital_since_time(data_capital)
elif user_option_int==919:
delete_all_capital(data_capital)
elif user_option_int==5:
still_running=False
break
else:
print("%d is not a valid option but was in list of possible options: check code" % str(user_option_int))
## Back to top of while loop
return success
def update_manual_check_historical_prices(instrument_code:str):
"""
Do a daily update for futures contract prices, using IB historical data
If any 'spikes' are found, run manual checks
:return: Nothing
"""
with mongoDb() as mongo_db,\
logger("Update-Historical-prices-manually", mongo_db=mongo_db) as log,\
connectionIB(mongo_db = mongo_db, log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob("ibFuturesContractPriceData arcticFuturesContractPriceData \
arcticFuturesMultiplePricesData mongoFuturesContractData",
mongo_db = mongo_db, log = log, ib_conn = ib_conn)
list_of_codes_all = data.arctic_futures_contract_price.get_instruments_with_price_data()
if instrument_code not in list_of_codes_all:
print("\n\n\ %s is not an instrument with price data \n\n" % instrument_code)
raise Exception()
update_historical_prices_with_checks_for_instrument(instrument_code, data, log=log.setup(instrument_code = instrument_code))
return success
def update_sampled_contracts():
"""
Update the active contracts, according to what is available in IB for a given instrument
These are stored in mongoDB
The active contracts list is used to see what contracts have historical data sampled for
It does not tell you what the current priced, forward, or carry contract are - that is in multiple prices (DRY)
However we base the list of theoretical active contracts on the current priced, forward, and carry contracts
We will end up adding to this list when we roll; this will change the current priced, forward, and carry contract
When we add a new contract (because one has become available), we get the exact expiry date from IB and save this with the
contract data.
We do not sample contracts on the list when they have passed their expiry date
Contracts are never deleted from the database
We don't check IB for contracts; since we can't reverse engineer a YYYYMM identifier from a YYYYMMDD
:returns: None
"""
with mongoDb() as mongo_db,\
logger("Update-Sampled_Contracts", mongo_db=mongo_db) as log,\
connectionIB(log=log.setup(component="IB-connection")) as ib_conn:
data = dataBlob("arcticFuturesMultiplePricesData ibFuturesContractPriceData\
mongoFuturesContractData mongoRollParametersData",
mongo_db, ib_conn, log=log)
list_of_codes_all = data.arctic_futures_multiple_prices.get_list_of_instruments()
for instrument_code in ['CRUDE_W']:
new_log = log.setup(instrument_code = instrument_code)
update_active_contracts_for_instrument(instrument_code, data, log=new_log)
def __init__(self, log=logger("logData")):
super().__init__(log=log)
def update_historical_prices_with_checks_for_instrument_and_contract(contract_object, data, log=logger("")):
"""
Do a daily update for futures contract prices, using IB historical data
:param contract_object: futuresContract
:param data: data blob
:param log: logger
:return: None
"""
ib_prices = data.ib_futures_contract_price.get_prices_for_contract_object(contract_object)
if len(ib_prices)==0:
log.warn("No IB prices found for %s" % str(contract_object))
return failure
old_prices = data.arctic_futures_contract_price.get_prices_for_contract_object(contract_object)
print("\n\n Manually checking prices for %s \n\n" % str(contract_object))
new_prices_checked = manual_price_checker(old_prices, ib_prices,
column_to_check = 'FINAL',
delta_columns = ['OPEN','HIGH','LOW'],
type_new_data = futuresContractPrices
)
rows_added = data.arctic_futures_contract_price.update_prices_for_contract(contract_object, new_prices_checked,
check_for_spike=False)
return success
