def add_missing_contract_to_database(data: dataBlob, contract_to_add: futuresContract): diag_contracts = diagContracts(data) update_contracts = updateContracts(data) instrument_code = contract_to_add.instrument_code contract_date_str = contract_to_add.date_str if diag_contracts.is_contract_in_data(instrument_code, contract_date_str): contract_to_add = diag_contracts.get_contract_object( instrument_code, contract_date_str) # Mark it as sampling contract_to_add.sampling_on() # Add it to the database # We are happy to overwrite update_contracts.add_contract_data(contract_to_add, ignore_duplication=True) log = data.log.setup(instrument_code=instrument_code) log.msg("Contract %s now added to database and sampling" % str(contract_to_add))
def _get_contract_object_with_IB_metadata(self, contract_object: futuresContract) -> futuresContract: futures_instrument_with_ib_data = self.ib_futures_instrument_data.get_futures_instrument_object_with_IB_data( contract_object.instrument_code ) if futures_instrument_with_ib_data is missing_instrument: return missing_contract contract_object_with_ib_data = ( contract_object.new_contract_with_replaced_instrument_object( futures_instrument_with_ib_data ) ) return contract_object_with_ib_data
def update_expiry_for_single_contract(self, original_contract: futuresContract) -> futuresContract: actual_expiry = self.get_actual_expiry_date_for_single_contract( original_contract ) if actual_expiry is missing_contract: log = original_contract.specific_log(self.data.log) log.warn("Contract %s is missing from IB probably expired - need to manually close on DB" % str( original_contract)) new_contract = copy(original_contract) else: expiry_date_as_str = actual_expiry.as_str() instrument_code = original_contract.instrument_code new_contract = futuresContract(instrument_code, expiry_date_as_str) return new_contract
def _delete_prices_for_contract_object_with_no_checks_be_careful( self, futures_contract_object: futuresContract): """ Delete prices for a given contract object without performing any checks WILL THIS WORK IF DOESN'T EXIST? :param futures_contract_object: :return: None """ log = futures_contract_object.log(self.log) ident = from_contract_to_key(futures_contract_object) self.arctic_connection.delete(ident) log.msg("Deleted all prices for %s from %s" % (futures_contract_object.key, str(self)))
def get_min_tick_size_for_contract(self, contract_object: futuresContract) -> float: new_log = contract_object.log(self.log) contract_object_with_ib_data = self.get_contract_object_with_IB_data(contract_object) if contract_object_with_ib_data is missing_contract: new_log.msg("Can't resolve contract so can't find tick size") return missing_contract min_tick_size = self.ibconnection.ib_get_min_tick_size( contract_object_with_ib_data ) if min_tick_size is missing_contract: new_log.msg("No tick size found") return missing_contract return min_tick_size
def check_and_stop_expired_contract_sampling(contract: futuresContract, data: dataBlob): data_contracts = dataContracts(data) db_contract = data_contracts.get_contract_from_db(contract) contract_expired = db_contract.expired() contract_sampling = db_contract.currently_sampling if contract_expired and contract_sampling: # Mark it as stop sampling in the database data_contracts.mark_contract_as_not_sampling(contract) log = contract.specific_log(data.log) log.msg( "Contract %s has expired so now stopped sampling" % str(contract), contract_date=contract.date_str, )
def _get_prices_for_contract_object_no_checking( self, futures_contract_object: futuresContract ) -> futuresContractPrices: """ Read back the prices for a given contract object :param contract_object: futuresContract :return: data """ keyname = self._keyname_given_contract_object(futures_contract_object) filename = self._filename_given_key_name(keyname) config = self.config date_format = config.input_date_format date_time_column = config.input_date_index_name input_column_mapping = config.input_column_mapping skiprows = config.input_skiprows skipfooter = config.input_skipfooter multiplier = config.apply_multiplier inverse = config.apply_inverse try: instrpricedata = pd_readcsv( filename, date_index_name=date_time_column, date_format=date_format, input_column_mapping=input_column_mapping, skiprows=skiprows, skipfooter=skipfooter, ) except OSError: log = futures_contract_object.log(self.log) log.warn("Can't find adjusted price file %s" % filename) return futuresContractPrices.create_empty() instrpricedata = instrpricedata.groupby(level=0).last() for col_name in ["OPEN", "HIGH", "LOW", "FINAL"]: column_series = instrpricedata[col_name] if inverse: column_series = 1 / column_series column_series *= multiplier instrpricedata[col_name] = column_series.round(2) instrpricedata = futuresContractPrices(instrpricedata) return instrpricedata
def _get_contract_cache_key( self, contract_object_to_use: futuresContract, trade_list_for_multiple_legs: tradeQuantity = None) -> str: if not contract_object_to_use.is_spread_contract(): trade_list_suffix = "" else: # WANT TO TREAT EG -2,2 AND -4,4 AS THE SAME BUT DIFFERENT FROM # -2,1 OR -1,2,-1... trade_list_suffix = str( list_of_ints_with_highest_common_factor_positive_first( trade_list_for_multiple_legs)) key = contract_object_to_use.key + trade_list_suffix return key
def delete_prices_for_contract_object( self, futures_contract_object: futuresContract, areyousure=False): """ :param futures_contract_object: :return: """ if not areyousure: raise Exception( "You have to be sure to delete prices_for_contract_object!") if self.has_data_for_contract(futures_contract_object): self._delete_prices_for_contract_object_with_no_checks_be_careful( futures_contract_object) else: log = futures_contract_object.log(self.log) log.warn("Tried to delete non existent contract")
def cancel_market_data_for_contract_object( self, contract_object_with_ib_data: futuresContract, trade_list_for_multiple_legs: tradeQuantity = None): specific_log = contract_object_with_ib_data.specific_log(self.log) ibcontract = self.ib_futures_contract( contract_object_with_ib_data, trade_list_for_multiple_legs=trade_list_for_multiple_legs, ) if ibcontract is missing_contract: specific_log.warn("Can't find matching IB contract for %s" % str(contract_object_with_ib_data)) return missing_contract self.ib.cancelMktData(ibcontract)
def ib_get_min_tick_size(self, contract_object_with_ib_data: futuresContract) -> float: specific_log = contract_object_with_ib_data.specific_log(self.log) ib_contract = self.ib_futures_contract( contract_object_with_ib_data, always_return_single_leg=True ) if ib_contract is missing_contract: specific_log.warn("Can't get tick size as contract missing") return missing_contract ib_contract_details = self.ib.reqContractDetails(ib_contract)[0] try: min_tick = ib_contract_details.minTick except Exception as e: specific_log.warn("%s when getting min tick size from %s!" % (str(e), str(ib_contract_details))) return missing_contract return min_tick
def _get_prices_at_frequency_for_contract_object_no_checking( self, contract_object: futuresContract, freq: Frequency) -> futuresContractPrices: """ Get historical prices at a particular frequency We override this method, rather than _get_prices_at_frequency_for_contract_object_no_checking Because the list of dates returned by contracts_with_price_data is likely to not match (expiries) :param contract_object: futuresContract :param freq: str; one of D, H, 15M, 5M, M, 10S, S :return: data """ new_log = contract_object.log(self.log) contract_object_with_ib_broker_config = ( self.futures_contract_data.get_contract_object_with_IB_data( contract_object)) if contract_object_with_ib_broker_config is missing_contract: new_log.warn("Can't get data for %s" % str(contract_object)) return futuresContractPrices.create_empty() price_data = self.ib_client.broker_get_historical_futures_data_for_contract( contract_object_with_ib_broker_config, bar_freq=freq) if price_data is missing_data: new_log.warn("Something went wrong getting IB price data for %s" % str(contract_object)) price_data = futuresContractPrices.create_empty() elif len(price_data) == 0: new_log.warn("No IB price data found for %s" % str(contract_object)) price_data = futuresContractPrices.create_empty() else: price_data = futuresContractPrices(price_data) ## It's important that the data is in local time zone so that this works price_data = price_data.remove_future_data() ## Some contract data is marked to model, don't want this price_data = price_data.remove_zero_volumes() return price_data
def update_prices_for_contract( self, contract_object: futuresContract, new_futures_per_contract_prices: futuresContractPrices, check_for_spike: bool = True, ) -> int: """ Reads existing data, merges with new_futures_prices, writes merged data :param new_futures_prices: :return: int, number of rows """ new_log = contract_object.log(self.log) old_prices = self.get_prices_for_contract_object(contract_object) merged_prices = old_prices.add_rows_to_existing_data( new_futures_per_contract_prices, check_for_spike=check_for_spike) if merged_prices is data_error: new_log.msg( "Price has moved too much - will need to manually check - no price updated done" ) return data_error rows_added = len(merged_prices) - len(old_prices) if rows_added == 0: if len(old_prices) == 0: new_log.msg("No existing or additional data") return 0 else: new_log.msg("No additional data since %s " % str(old_prices.index[-1])) return 0 # We have guaranteed no duplication self.write_prices_for_contract_object(contract_object, merged_prices, ignore_duplication=True) new_log.msg("Added %d additional rows of data" % rows_added) return rows_added
def ib_get_trading_hours(self, contract_object_with_ib_data: futuresContract) -> list: specific_log = contract_object_with_ib_data.specific_log(self.log) ib_contract = self.ib_futures_contract( contract_object_with_ib_data, always_return_single_leg=True ) if ib_contract is missing_contract: specific_log.warn("Can't get trading hours as contract is missing") return missing_contract # returns a list but should only have one element ib_contract_details_list = self.ib.reqContractDetails(ib_contract) ib_contract_details = ib_contract_details_list[0] try: trading_hours = get_trading_hours(ib_contract_details) except Exception as e: specific_log.warn("%s when getting trading hours from %s!" % (str(e), str(ib_contract_details))) return missing_contract return trading_hours
def get_actual_expiry_date_for_contract( self, contract_object: futuresContract) -> expiryDate: """ FIXME CONSIDER USE OF get_contract_object_with_IB_data INSTEAD Get the actual expiry date of a contract from IB :param contract_object: type futuresContract :return: YYYYMMDD or None """ log = contract_object.log(self.log) contract_object_with_ib_data = self._get_contract_object_with_IB_metadata( contract_object) if contract_object_with_ib_data is missing_contract: log.msg("Can't resolve contract so can't find expiry date") return missing_contract expiry_date = self._get_actual_expiry_date_given_contract_with_ib_metadata( contract_object_with_ib_data) return expiry_date
def add_contract_data(self, contract_object: futuresContract, ignore_duplication: bool = False): instrument_code = contract_object.instrument_code contract_date = contract_object.date_str log = contract_object.log(self.log) if self.is_contract_in_data(instrument_code, contract_date): if ignore_duplication: pass else: log.warn( "There is already %s in the data, you have to delete it first" % (contract_object.key)) return None self._add_contract_object_without_checking_for_existing_entry( contract_object) log.terse("Added contract %s %s" % (instrument_code, contract_date))
def _write_prices_for_contract_object_no_checking( self, futures_contract_object: futuresContract, futures_price_data: futuresContractPrices): """ Write prices CHECK prices are overriden on second write :param futures_contract_object: futuresContract :param futures_price_data: futuresContractPriceData :return: None """ log = futures_contract_object.log(self.log) ident = from_contract_to_key(futures_contract_object) futures_price_data_as_pd = pd.DataFrame(futures_price_data) self.arctic_connection.write(ident, futures_price_data_as_pd) log.msg("Wrote %s lines of prices for %s to %s" % (len(futures_price_data), str( futures_contract_object.key), str(self)))
def _get_prices_at_frequency_for_contract_object_no_checking(self, contract_object: futuresContract, freq: str ) -> futuresContractPrices: """ Get historical prices at a particular frequency We override this method, rather than _get_prices_at_frequency_for_contract_object_no_checking Because the list of dates returned by contracts_with_price_data is likely to not match (expiries) :param contract_object: futuresContract :param freq: str; one of D, H, 15M, 5M, M, 10S, S :return: data """ new_log = contract_object.log(self.log) contract_object_with_ib_broker_config = ( self.futures_contract_data.get_contract_object_with_IB_data( contract_object ) ) if contract_object_with_ib_broker_config is missing_contract: new_log.warn("Can't get data for %s" % str(contract_object)) return futuresContractPrices.create_empty() price_data = self.ibconnection.broker_get_historical_futures_data_for_contract( contract_object_with_ib_broker_config, bar_freq=freq) if len(price_data) == 0: new_log.warn( "No IB price data found for %s" % str(contract_object)) price_data = futuresContractPrices.create_empty() else: price_data = futuresContractPrices(price_data) price_data = price_data.remove_future_data() price_data = price_data.remove_zero_volumes() return price_data
def get_trading_hours_for_contract(self, futures_contract: futuresContract) : """ :param futures_contract: :return: list of paired date times """ new_log = futures_contract.log(self.log) contract_object_with_ib_data = self.get_contract_object_with_IB_data(futures_contract) if contract_object_with_ib_data is missing_contract: new_log.msg("Can't resolve contract") return missing_contract trading_hours = self.ibconnection.ib_get_trading_hours( contract_object_with_ib_data ) if trading_hours is missing_contract: new_log.msg("No IB expiry date found") trading_hours = [] return trading_hours
def _get_prices_for_contract_object_no_checking( self, futures_contract_object: futuresContract) -> futuresContractPrices: """ Read back the prices for a given contract object :param contract_object: futuresContract :return: data """ keyname = self._keyname_given_contract_object(futures_contract_object) filename = self._filename_given_key_name(keyname) config = self.config date_format = config.input_date_format date_time_column = config.input_date_index_name input_column_mapping = config.input_column_mapping skiprows = config.input_skiprows skipfooter = config.input_skipfooter try: instrpricedata = pd_readcsv( filename, date_index_name=date_time_column, date_format=date_format, input_column_mapping=input_column_mapping, skiprows=skiprows, skipfooter=skipfooter, ) except OSError: log = futures_contract_object.log(self.log) log.warning("Can't find adjusted price file %s" % filename) return futuresContractPrices.create_empty() instrpricedata = instrpricedata.groupby(level=0).last() instrpricedata = futuresContractPrices(instrpricedata) return instrpricedata
def check_and_update_sampling_status(contract: futuresContract, data: dataBlob, contract_chain: listOfFuturesContracts): unsample = False reason = "" data_contracts = dataContracts(data) db_contract = data_contracts.get_contract_from_db(contract) if db_contract.expired(): unsample = True reason = "has expired" elif db_contract not in contract_chain: unsample = True reason = "not in chain" if unsample: # Mark it as stop sampling in the database data_contracts.mark_contract_as_not_sampling(contract) log = contract.specific_log(data.log) log.msg( "Contract %s %s so now stopped sampling" % (str(contract), reason), contract_date=contract.date_str, )
def ib_get_recent_bid_ask_tick_data( self, contract_object_with_ib_data: futuresContract, tick_count=200, ) -> list: """ :param contract_object_with_ib_data: :return: """ specific_log = self.log.setup( instrument_code=contract_object_with_ib_data.instrument_code, contract_date=contract_object_with_ib_data.date_str, ) if contract_object_with_ib_data.is_spread_contract(): error_msg = "Can't get historical data for combo" specific_log.critical(error_msg) raise Exception(error_msg) ibcontract = self.ib_futures_contract(contract_object_with_ib_data) if ibcontract is missing_contract: specific_log.warn("Can't find matching IB contract for %s" % str(contract_object_with_ib_data)) return missing_contract recent_ib_time = self.ib.reqCurrentTime() - datetime.timedelta( seconds=60) tick_data = self.ib.reqHistoricalTicks(ibcontract, recent_ib_time, "", tick_count, "BID_ASK", useRth=False) return tick_data
def write_prices_for_contract_object( self, futures_contract_object: futuresContract, futures_price_data: futuresContractPrices, ignore_duplication=False, ): """ Write some prices :param futures_contract_object: :param futures_price_data: :param ignore_duplication: bool, to stop us overwriting existing prices :return: None """ not_ignoring_duplication = not ignore_duplication if not_ignoring_duplication: if self.has_data_for_contract(futures_contract_object): log = futures_contract_object.log(self.log) log.warn("There is already existing data for %s" % futures_contract_object.key) return None self._write_prices_for_contract_object_no_checking( futures_contract_object, futures_price_data)
def _update_positions_for_individual_contract_leg( self, contract: futuresContract, trade_done: int, time_date: datetime.datetime): current_position = self.diag_positions.get_position_for_contract( contract) new_position = current_position + trade_done self.db_contract_position_data.update_position_for_contract_object( contract, new_position, date=time_date) # check new_position_db = self.diag_positions.get_position_for_contract( contract) log = contract.specific_log(self.log) log.msg( "Updated position of %s from %d to %d; new position in db is %d" % ( str(contract), current_position, new_position, new_position_db, ))
def _add_contract_object_without_checking_for_existing_entry( self, contract_object: futuresContract): contract_object_as_dict = contract_object.as_dict() key = contract_object.key self.mongo_data.add_data(key, contract_object_as_dict, allow_overwrite=True)