def add_missing_contract_to_database(data: dataBlob,
contract_to_add: futuresContract):
diag_contracts = diagContracts(data)
update_contracts = updateContracts(data)
instrument_code = contract_to_add.instrument_code
contract_date_str = contract_to_add.date_str
if diag_contracts.is_contract_in_data(instrument_code, contract_date_str):
contract_to_add = diag_contracts.get_contract_object(
instrument_code, contract_date_str)
# Mark it as sampling
contract_to_add.sampling_on()
# Add it to the database
# We are happy to overwrite
update_contracts.add_contract_data(contract_to_add,
ignore_duplication=True)
log = data.log.setup(instrument_code=instrument_code)
log.msg("Contract %s now added to database and sampling" %
str(contract_to_add))
def _get_contract_object_with_IB_metadata(self, contract_object: futuresContract) -> futuresContract:
futures_instrument_with_ib_data = self.ib_futures_instrument_data.get_futures_instrument_object_with_IB_data(
contract_object.instrument_code
)
if futures_instrument_with_ib_data is missing_instrument:
return missing_contract
contract_object_with_ib_data = (
contract_object.new_contract_with_replaced_instrument_object(
futures_instrument_with_ib_data
)
)
return contract_object_with_ib_data
def update_expiry_for_single_contract(self, original_contract: futuresContract) -> futuresContract:
actual_expiry = self.get_actual_expiry_date_for_single_contract(
original_contract
)
if actual_expiry is missing_contract:
log = original_contract.specific_log(self.data.log)
log.warn("Contract %s is missing from IB probably expired - need to manually close on DB" % str(
original_contract))
new_contract = copy(original_contract)
else:
expiry_date_as_str = actual_expiry.as_str()
instrument_code = original_contract.instrument_code
new_contract = futuresContract(instrument_code, expiry_date_as_str)
return new_contract
def _delete_prices_for_contract_object_with_no_checks_be_careful(
self, futures_contract_object: futuresContract):
"""
Delete prices for a given contract object without performing any checks
WILL THIS WORK IF DOESN'T EXIST?
:param futures_contract_object:
:return: None
"""
log = futures_contract_object.log(self.log)
ident = from_contract_to_key(futures_contract_object)
self.arctic_connection.delete(ident)
log.msg("Deleted all prices for %s from %s" %
(futures_contract_object.key, str(self)))
def get_min_tick_size_for_contract(self, contract_object: futuresContract) -> float:
new_log = contract_object.log(self.log)
contract_object_with_ib_data = self.get_contract_object_with_IB_data(contract_object)
if contract_object_with_ib_data is missing_contract:
new_log.msg("Can't resolve contract so can't find tick size")
return missing_contract
min_tick_size = self.ibconnection.ib_get_min_tick_size(
contract_object_with_ib_data
)
if min_tick_size is missing_contract:
new_log.msg("No tick size found")
return missing_contract
return min_tick_size
def check_and_stop_expired_contract_sampling(contract: futuresContract, data: dataBlob):
data_contracts = dataContracts(data)
db_contract = data_contracts.get_contract_from_db(contract)
contract_expired = db_contract.expired()
contract_sampling = db_contract.currently_sampling
if contract_expired and contract_sampling:
# Mark it as stop sampling in the database
data_contracts.mark_contract_as_not_sampling(contract)
log = contract.specific_log(data.log)
log.msg(
"Contract %s has expired so now stopped sampling" % str(contract),
contract_date=contract.date_str,
)
def _get_prices_for_contract_object_no_checking(
self, futures_contract_object: futuresContract
) -> futuresContractPrices:
"""
Read back the prices for a given contract object
:param contract_object: futuresContract
:return: data
"""
keyname = self._keyname_given_contract_object(futures_contract_object)
filename = self._filename_given_key_name(keyname)
config = self.config
date_format = config.input_date_format
date_time_column = config.input_date_index_name
input_column_mapping = config.input_column_mapping
skiprows = config.input_skiprows
skipfooter = config.input_skipfooter
multiplier = config.apply_multiplier
inverse = config.apply_inverse
try:
instrpricedata = pd_readcsv(
filename,
date_index_name=date_time_column,
date_format=date_format,
input_column_mapping=input_column_mapping,
skiprows=skiprows,
skipfooter=skipfooter,
)
except OSError:
log = futures_contract_object.log(self.log)
log.warn("Can't find adjusted price file %s" % filename)
return futuresContractPrices.create_empty()
instrpricedata = instrpricedata.groupby(level=0).last()
for col_name in ["OPEN", "HIGH", "LOW", "FINAL"]:
column_series = instrpricedata[col_name]
if inverse:
column_series = 1 / column_series
column_series *= multiplier
instrpricedata[col_name] = column_series.round(2)
instrpricedata = futuresContractPrices(instrpricedata)
return instrpricedata
def _get_contract_cache_key(
self,
contract_object_to_use: futuresContract,
trade_list_for_multiple_legs: tradeQuantity = None) -> str:
if not contract_object_to_use.is_spread_contract():
trade_list_suffix = ""
else:
# WANT TO TREAT EG -2,2 AND -4,4 AS THE SAME BUT DIFFERENT FROM
# -2,1 OR -1,2,-1...
trade_list_suffix = str(
list_of_ints_with_highest_common_factor_positive_first(
trade_list_for_multiple_legs))
key = contract_object_to_use.key + trade_list_suffix
return key
def delete_prices_for_contract_object(
self, futures_contract_object: futuresContract, areyousure=False):
"""
:param futures_contract_object:
:return:
"""
if not areyousure:
raise Exception(
"You have to be sure to delete prices_for_contract_object!")
if self.has_data_for_contract(futures_contract_object):
self._delete_prices_for_contract_object_with_no_checks_be_careful(
futures_contract_object)
else:
log = futures_contract_object.log(self.log)
log.warn("Tried to delete non existent contract")
def cancel_market_data_for_contract_object(
self,
contract_object_with_ib_data: futuresContract,
trade_list_for_multiple_legs: tradeQuantity = None):
specific_log = contract_object_with_ib_data.specific_log(self.log)
ibcontract = self.ib_futures_contract(
contract_object_with_ib_data,
trade_list_for_multiple_legs=trade_list_for_multiple_legs,
)
if ibcontract is missing_contract:
specific_log.warn("Can't find matching IB contract for %s" %
str(contract_object_with_ib_data))
return missing_contract
self.ib.cancelMktData(ibcontract)
def ib_get_min_tick_size(self, contract_object_with_ib_data: futuresContract) -> float:
specific_log = contract_object_with_ib_data.specific_log(self.log)
ib_contract = self.ib_futures_contract(
contract_object_with_ib_data, always_return_single_leg=True
)
if ib_contract is missing_contract:
specific_log.warn("Can't get tick size as contract missing")
return missing_contract
ib_contract_details = self.ib.reqContractDetails(ib_contract)[0]
try:
min_tick = ib_contract_details.minTick
except Exception as e:
specific_log.warn("%s when getting min tick size from %s!" % (str(e), str(ib_contract_details)))
return missing_contract
return min_tick
def _get_prices_at_frequency_for_contract_object_no_checking(
self, contract_object: futuresContract,
freq: Frequency) -> futuresContractPrices:
"""
Get historical prices at a particular frequency
We override this method, rather than _get_prices_at_frequency_for_contract_object_no_checking
Because the list of dates returned by contracts_with_price_data is likely to not match (expiries)
:param contract_object: futuresContract
:param freq: str; one of D, H, 15M, 5M, M, 10S, S
:return: data
"""
new_log = contract_object.log(self.log)
contract_object_with_ib_broker_config = (
self.futures_contract_data.get_contract_object_with_IB_data(
contract_object))
if contract_object_with_ib_broker_config is missing_contract:
new_log.warn("Can't get data for %s" % str(contract_object))
return futuresContractPrices.create_empty()
price_data = self.ib_client.broker_get_historical_futures_data_for_contract(
contract_object_with_ib_broker_config, bar_freq=freq)
if price_data is missing_data:
new_log.warn("Something went wrong getting IB price data for %s" %
str(contract_object))
price_data = futuresContractPrices.create_empty()
elif len(price_data) == 0:
new_log.warn("No IB price data found for %s" %
str(contract_object))
price_data = futuresContractPrices.create_empty()
else:
price_data = futuresContractPrices(price_data)
## It's important that the data is in local time zone so that this works
price_data = price_data.remove_future_data()
## Some contract data is marked to model, don't want this
price_data = price_data.remove_zero_volumes()
return price_data
def update_prices_for_contract(
self,
contract_object: futuresContract,
new_futures_per_contract_prices: futuresContractPrices,
check_for_spike: bool = True,
) -> int:
"""
Reads existing data, merges with new_futures_prices, writes merged data
:param new_futures_prices:
:return: int, number of rows
"""
new_log = contract_object.log(self.log)
old_prices = self.get_prices_for_contract_object(contract_object)
merged_prices = old_prices.add_rows_to_existing_data(
new_futures_per_contract_prices, check_for_spike=check_for_spike)
if merged_prices is data_error:
new_log.msg(
"Price has moved too much - will need to manually check - no price updated done"
)
return data_error
rows_added = len(merged_prices) - len(old_prices)
if rows_added == 0:
if len(old_prices) == 0:
new_log.msg("No existing or additional data")
return 0
else:
new_log.msg("No additional data since %s " %
str(old_prices.index[-1]))
return 0
# We have guaranteed no duplication
self.write_prices_for_contract_object(contract_object,
merged_prices,
ignore_duplication=True)
new_log.msg("Added %d additional rows of data" % rows_added)
return rows_added
def ib_get_trading_hours(self, contract_object_with_ib_data: futuresContract) -> list:
specific_log = contract_object_with_ib_data.specific_log(self.log)
ib_contract = self.ib_futures_contract(
contract_object_with_ib_data, always_return_single_leg=True
)
if ib_contract is missing_contract:
specific_log.warn("Can't get trading hours as contract is missing")
return missing_contract
# returns a list but should only have one element
ib_contract_details_list = self.ib.reqContractDetails(ib_contract)
ib_contract_details = ib_contract_details_list[0]
try:
trading_hours = get_trading_hours(ib_contract_details)
except Exception as e:
specific_log.warn("%s when getting trading hours from %s!" % (str(e), str(ib_contract_details)))
return missing_contract
return trading_hours
def get_actual_expiry_date_for_contract(
self, contract_object: futuresContract) -> expiryDate:
"""
FIXME CONSIDER USE OF get_contract_object_with_IB_data INSTEAD
Get the actual expiry date of a contract from IB
:param contract_object: type futuresContract
:return: YYYYMMDD or None
"""
log = contract_object.log(self.log)
contract_object_with_ib_data = self._get_contract_object_with_IB_metadata(
contract_object)
if contract_object_with_ib_data is missing_contract:
log.msg("Can't resolve contract so can't find expiry date")
return missing_contract
expiry_date = self._get_actual_expiry_date_given_contract_with_ib_metadata(
contract_object_with_ib_data)
return expiry_date
def add_contract_data(self,
contract_object: futuresContract,
ignore_duplication: bool = False):
instrument_code = contract_object.instrument_code
contract_date = contract_object.date_str
log = contract_object.log(self.log)
if self.is_contract_in_data(instrument_code, contract_date):
if ignore_duplication:
pass
else:
log.warn(
"There is already %s in the data, you have to delete it first"
% (contract_object.key))
return None
self._add_contract_object_without_checking_for_existing_entry(
contract_object)
log.terse("Added contract %s %s" % (instrument_code, contract_date))
def _write_prices_for_contract_object_no_checking(
self, futures_contract_object: futuresContract,
futures_price_data: futuresContractPrices):
"""
Write prices
CHECK prices are overriden on second write
:param futures_contract_object: futuresContract
:param futures_price_data: futuresContractPriceData
:return: None
"""
log = futures_contract_object.log(self.log)
ident = from_contract_to_key(futures_contract_object)
futures_price_data_as_pd = pd.DataFrame(futures_price_data)
self.arctic_connection.write(ident, futures_price_data_as_pd)
log.msg("Wrote %s lines of prices for %s to %s" %
(len(futures_price_data), str(
futures_contract_object.key), str(self)))
def _get_prices_at_frequency_for_contract_object_no_checking(self, contract_object: futuresContract,
freq: str
) -> futuresContractPrices:
"""
Get historical prices at a particular frequency
We override this method, rather than _get_prices_at_frequency_for_contract_object_no_checking
Because the list of dates returned by contracts_with_price_data is likely to not match (expiries)
:param contract_object: futuresContract
:param freq: str; one of D, H, 15M, 5M, M, 10S, S
:return: data
"""
new_log = contract_object.log(self.log)
contract_object_with_ib_broker_config = (
self.futures_contract_data.get_contract_object_with_IB_data(
contract_object
)
)
if contract_object_with_ib_broker_config is missing_contract:
new_log.warn("Can't get data for %s" % str(contract_object))
return futuresContractPrices.create_empty()
price_data = self.ibconnection.broker_get_historical_futures_data_for_contract(
contract_object_with_ib_broker_config, bar_freq=freq)
if len(price_data) == 0:
new_log.warn(
"No IB price data found for %s" %
str(contract_object))
price_data = futuresContractPrices.create_empty()
else:
price_data = futuresContractPrices(price_data)
price_data = price_data.remove_future_data()
price_data = price_data.remove_zero_volumes()
return price_data
def get_trading_hours_for_contract(self, futures_contract: futuresContract) :
"""
:param futures_contract:
:return: list of paired date times
"""
new_log = futures_contract.log(self.log)
contract_object_with_ib_data = self.get_contract_object_with_IB_data(futures_contract)
if contract_object_with_ib_data is missing_contract:
new_log.msg("Can't resolve contract")
return missing_contract
trading_hours = self.ibconnection.ib_get_trading_hours(
contract_object_with_ib_data
)
if trading_hours is missing_contract:
new_log.msg("No IB expiry date found")
trading_hours = []
return trading_hours
def _get_prices_for_contract_object_no_checking(
self,
futures_contract_object: futuresContract) -> futuresContractPrices:
"""
Read back the prices for a given contract object
:param contract_object: futuresContract
:return: data
"""
keyname = self._keyname_given_contract_object(futures_contract_object)
filename = self._filename_given_key_name(keyname)
config = self.config
date_format = config.input_date_format
date_time_column = config.input_date_index_name
input_column_mapping = config.input_column_mapping
skiprows = config.input_skiprows
skipfooter = config.input_skipfooter
try:
instrpricedata = pd_readcsv(
filename,
date_index_name=date_time_column,
date_format=date_format,
input_column_mapping=input_column_mapping,
skiprows=skiprows,
skipfooter=skipfooter,
)
except OSError:
log = futures_contract_object.log(self.log)
log.warning("Can't find adjusted price file %s" % filename)
return futuresContractPrices.create_empty()
instrpricedata = instrpricedata.groupby(level=0).last()
instrpricedata = futuresContractPrices(instrpricedata)
return instrpricedata
def check_and_update_sampling_status(contract: futuresContract, data: dataBlob,
contract_chain: listOfFuturesContracts):
unsample = False
reason = ""
data_contracts = dataContracts(data)
db_contract = data_contracts.get_contract_from_db(contract)
if db_contract.expired():
unsample = True
reason = "has expired"
elif db_contract not in contract_chain:
unsample = True
reason = "not in chain"
if unsample:
# Mark it as stop sampling in the database
data_contracts.mark_contract_as_not_sampling(contract)
log = contract.specific_log(data.log)
log.msg(
"Contract %s %s so now stopped sampling" % (str(contract), reason),
contract_date=contract.date_str,
)
def ib_get_recent_bid_ask_tick_data(
self,
contract_object_with_ib_data: futuresContract,
tick_count=200,
) -> list:
"""
:param contract_object_with_ib_data:
:return:
"""
specific_log = self.log.setup(
instrument_code=contract_object_with_ib_data.instrument_code,
contract_date=contract_object_with_ib_data.date_str,
)
if contract_object_with_ib_data.is_spread_contract():
error_msg = "Can't get historical data for combo"
specific_log.critical(error_msg)
raise Exception(error_msg)
ibcontract = self.ib_futures_contract(contract_object_with_ib_data)
if ibcontract is missing_contract:
specific_log.warn("Can't find matching IB contract for %s" %
str(contract_object_with_ib_data))
return missing_contract
recent_ib_time = self.ib.reqCurrentTime() - datetime.timedelta(
seconds=60)
tick_data = self.ib.reqHistoricalTicks(ibcontract,
recent_ib_time,
"",
tick_count,
"BID_ASK",
useRth=False)
return tick_data
def write_prices_for_contract_object(
self,
futures_contract_object: futuresContract,
futures_price_data: futuresContractPrices,
ignore_duplication=False,
):
"""
Write some prices
:param futures_contract_object:
:param futures_price_data:
:param ignore_duplication: bool, to stop us overwriting existing prices
:return: None
"""
not_ignoring_duplication = not ignore_duplication
if not_ignoring_duplication:
if self.has_data_for_contract(futures_contract_object):
log = futures_contract_object.log(self.log)
log.warn("There is already existing data for %s" %
futures_contract_object.key)
return None
self._write_prices_for_contract_object_no_checking(
futures_contract_object, futures_price_data)
def _update_positions_for_individual_contract_leg(
self, contract: futuresContract, trade_done: int,
time_date: datetime.datetime):
current_position = self.diag_positions.get_position_for_contract(
contract)
new_position = current_position + trade_done
self.db_contract_position_data.update_position_for_contract_object(
contract, new_position, date=time_date)
# check
new_position_db = self.diag_positions.get_position_for_contract(
contract)
log = contract.specific_log(self.log)
log.msg(
"Updated position of %s from %d to %d; new position in db is %d" %
(
str(contract),
current_position,
new_position,
new_position_db,
))
def _add_contract_object_without_checking_for_existing_entry(
self, contract_object: futuresContract):
contract_object_as_dict = contract_object.as_dict()
key = contract_object.key
self.mongo_data.add_data(key, contract_object_as_dict, allow_overwrite=True)
