def update_historical_prices_for_instrument_and_contract(
contract_object: futuresContract, data: dataBlob):
"""
Do a daily update for futures contract prices, using IB historical data
There are two different calls, the first using intraday frequency, the second daily.
So in a typical session we'd get the hourly intraday prices for a given instrument,
and then if that instrument has closed we'd also get a new end of day price.
The end of day prices are given the artifical datetime stamp of 23:00:00
(because I never collect data round that time).
If the intraday call fails, we don't want to get daily data.
Otherwise we wouldn't be able to subsequently backfill hourly prices that occured
before the daily close being added (the code doesn't allow you to add prices that have
a timestamp which is before the last timestamp in the existing data).
That's why the result of the first call matters, and is used to abort the function prematurely before we get to daily data.
We don't care about the second call succeeding or failing, and so the result of that is ignored.
:param contract_object: futuresContract
:param data: data blob
:return: None
"""
diag_prices = diagPrices(data)
intraday_frequency = diag_prices.get_intraday_frequency_for_historical_download(
)
daily_frequency = DAILY_PRICE_FREQ
# Get *intraday* data (defaults to hourly)
result = get_and_add_prices_for_frequency(data,
contract_object,
frequency=intraday_frequency)
if result is failure:
# Skip daily data if intraday not working
return None
# Get daily data
# we don't care about the result flag for this
get_and_add_prices_for_frequency(data,
contract_object,
frequency=daily_frequency)
def update_multiple_adjusted_prices_for_instrument(instrument_code, data):
"""
Update for multiple and adjusted prices for a given instrument
:param instrument_code:
:param data: dataBlob
:param log: logger
:return: None
"""
log = data.log.setup(instrument_code=instrument_code)
diag_prices = diagPrices(data)
update_prices = updatePrices(data)
# update multiple prices with new prices
# (method in multiple prices object and possible in data socket)
existing_adjusted_prices = diag_prices.get_adjusted_prices(instrument_code)
existing_multiple_prices = diag_prices.get_multiple_prices(instrument_code)
relevant_contracts = existing_multiple_prices.current_contract_dict()
new_prices_dict = get_dict_of_new_prices_and_contractid(
instrument_code, relevant_contracts, data)
updated_multiple_prices = existing_multiple_prices.update_multiple_prices_with_dict(
new_prices_dict)
updated_adjusted_prices = existing_adjusted_prices.update_with_multiple_prices_no_roll(
updated_multiple_prices)
if updated_adjusted_prices is no_update_roll_has_occured:
log.critical(
"Can't update adjusted prices for %s as roll has occured but not registered properly"
% instrument_code)
raise Exception()
update_prices.add_multiple_prices(instrument_code,
updated_multiple_prices,
ignore_duplication=True)
update_prices.add_adjusted_prices(instrument_code,
updated_adjusted_prices,
ignore_duplication=True)
return success
def calculate_adjusted_price_for_a_direct_child_order(
data: dataBlob,
child_order: contractOrder,
original_contract_date: str,
original_price: float
) -> float:
"""
:param data:
:param child_order:
:param original_contract_date:
:param original_price:
:return: float or missing data
"""
instrument_code = child_order.instrument_code
try:
assert not child_order.calendar_spread_order
except BaseException:
raise Exception(
"You have tried to adjust the price for a spread contract order assuming it is a single leg order"
)
child_contract_date = child_order.contract_date_key
if original_contract_date == child_contract_date:
return original_price
diag_prices = diagPrices(data)
contract_list = [original_contract_date, child_contract_date]
_last_matched_date, list_of_matching_prices = diag_prices.get_last_matched_date_and_prices_for_contract_list(
instrument_code, contract_list
)
differential = list_of_matching_prices[1] - list_of_matching_prices[0]
if np.isnan(differential):
# can't adjust
# note need to test code there may be other ways in which this fails
return missing_data
adjusted_price = original_price + differential
return adjusted_price
def get_dates_to_choose_from(
data: dataBlob,
instrument_code: str,
only_priced_contracts: bool = False) -> listOfContractDateStr:
diag_contracts = dataContracts(data)
diag_prices = diagPrices(data)
if only_priced_contracts:
dates_to_choose_from = (
diag_prices.contract_dates_with_price_data_for_instrument_code(
instrument_code))
else:
contract_list = diag_contracts.get_all_contract_objects_for_instrument_code(
instrument_code)
dates_to_choose_from = contract_list.list_of_dates()
dates_to_choose_from = listOfContractDateStr(dates_to_choose_from)
dates_to_choose_from = dates_to_choose_from.sorted_date_str()
return dates_to_choose_from
def get_liquidity_data_df(data: dataBlob):
diag_prices = diagPrices(data)
instrument_list = diag_prices.get_list_of_instruments_with_contract_prices(
)
print("Getting data... patience")
p = progressBar(len(instrument_list))
all_liquidity = []
for instrument_code in instrument_list:
p.iterate()
liquidity_this_instrument = get_liquidity_dict_for_instrument_code(
data, instrument_code)
all_liquidity.append(liquidity_this_instrument)
all_liquidity_df = pd.DataFrame(all_liquidity)
all_liquidity_df.index = instrument_list
all_liquidity_df["contracts"] = all_liquidity_df["contracts"].round(0)
return all_liquidity_df
def update_historical_prices_with_checks_for_instrument_and_contract(
contract_object, data, log
):
"""
Do a daily update for futures contract prices, using IB historical data, with checking
:param contract_object: futuresContract
:param data: data blob
:param log: logger
:return: None
"""
diag_prices = diagPrices(data)
intraday_frequency = diag_prices.get_intraday_frequency_for_historical_download()
get_and_check_prices_for_frequency(
data, log, contract_object, frequency=intraday_frequency
)
get_and_check_prices_for_frequency(
data, log, contract_object, frequency="D")
return success
def calc_update_adjusted_prices(
data: dataBlob, instrument_code: str,
updated_multiple_prices: futuresMultiplePrices
) -> futuresAdjustedPrices:
diag_prices = diagPrices(data)
existing_adjusted_prices = diag_prices.get_adjusted_prices(instrument_code)
updated_adjusted_prices = (
existing_adjusted_prices.update_with_multiple_prices_no_roll(
updated_multiple_prices))
if updated_adjusted_prices is no_update_roll_has_occured:
msg = (
"Can't update adjusted prices for %s as roll has occured but not registered properly"
% instrument_code)
data.log.critical(msg)
raise Exception(msg)
return updated_adjusted_prices
def get_list_of_instruments_to_auto_cycle(data: dataBlob, days_ahead: int = 10) -> list:
diag_prices = diagPrices()
list_of_potential_instruments = (
diag_prices.get_list_of_instruments_in_multiple_prices()
)
instrument_list = [
instrument_code
for instrument_code in list_of_potential_instruments
if include_instrument_in_auto_cycle(
data=data, instrument_code=instrument_code, days_ahead=days_ahead
)
]
print_with_landing_strips_around(
"Identified following instruments that are near expiry %s"
% str(instrument_list)
)
return instrument_list
def get_valid_instrument_code_and_contractid_from_user(
data, instrument_code=None, include_priced_contracts=False):
diag_contracts = diagContracts(data)
diag_prices = diagPrices(data)
invalid_input = True
while invalid_input:
if instrument_code is None:
instrument_code = get_valid_instrument_code_from_user(
data, source='single')
if include_priced_contracts:
dates_to_choose_from = diag_prices.contract_dates_with_price_data_for_instrument_code(
instrument_code)
else:
contract_list = diag_contracts.get_all_contract_objects_for_instrument_code(
instrument_code)
dates_to_choose_from = contract_list.list_of_dates()
dates_to_display = diag_contracts.get_labelled_list_of_contracts_from_contract_date_list(
instrument_code, dates_to_choose_from)
if len(dates_to_choose_from) == 0:
print("%s is not an instrument with contract data" %
instrument_code)
instrument_code = None
continue
print("Available contract dates %s" % str(dates_to_display))
print("p = currently priced, c=current carry, f= current forward")
contract_date = input(
"Contract date? [yyyymm or yyyymmdd] (ignore suffixes)")
if len(contract_date) == 6:
contract_date = contract_date + "00"
if contract_date in dates_to_choose_from:
break
else:
print("%s is not in list %s" %
(contract_date, dates_to_choose_from))
continue # not required
return instrument_code, contract_date
def update_historical_prices_for_instrument_and_contract(
contract_object: futuresContract,
data: dataBlob,
cleaning_config: priceFilterConfig = arg_not_supplied,
interactive_mode: bool = False):
diag_prices = diagPrices(data)
intraday_frequency = diag_prices.get_intraday_frequency_for_historical_download(
)
daily_frequency = DAILY_PRICE_FREQ
# Get *intraday* data (defaults to hourly)
result = get_and_add_prices_for_frequency(
data,
contract_object,
frequency=intraday_frequency,
cleaning_config=cleaning_config,
interactive_mode=interactive_mode)
if result is failure:
# Skip daily data if intraday not working
if cleaning_config.dont_sample_daily_if_intraday_fails:
data.log.msg(
"Had a problem samping intraday, skipping daily to avoid price gaps"
)
return failure
else:
data.log.warn(
"Had a problem samping intraday, but **NOT** skipping daily - may be price gaps"
)
# Get daily data
# we don't care about the result flag for this
get_and_add_prices_for_frequency(
data,
contract_object,
frequency=daily_frequency,
cleaning_config=cleaning_config,
interactive_mode=interactive_mode,
)
return success
def vol_calculations_for_slippage_row(slippage_row, data):
## What's a tick worth in base currency?
diag_prices = diagPrices(data)
rolling_daily_vol = diag_prices.get_quick_std_of_adjusted_prices(
slippage_row.instrument_code)
if len(rolling_daily_vol) == 0:
last_daily_vol = np.nan
else:
last_daily_vol = rolling_daily_vol.ffill().values[-1]
last_annual_vol = last_daily_vol * 16
input_items = [
'delay', 'bid_ask', 'execution', 'versus_limit', 'versus_parent_limit',
'total_trading'
]
output = [
10000 * slippage_row[input_name] / last_annual_vol
for input_name in input_items
]
return tuple(output + [last_annual_vol])
def interactive_manual_check_historical_prices(instrument_code: str):
"""
Do a daily update for futures contract prices, using IB historical data
If any 'spikes' are found, run manual checks
:return: Nothing
"""
with dataBlob(log_name="Update-Historical-prices-manually") as data:
diag_prices = diagPrices(data)
list_of_codes_all = diag_prices.get_list_of_instruments_with_contract_prices()
if instrument_code not in list_of_codes_all:
print(
"\n\n\ %s is not an instrument with price data \n\n" %
instrument_code)
raise Exception()
update_historical_prices_with_checks_for_instrument(
instrument_code, data, log=data.log.setup(
instrument_code=instrument_code))
return success
def get_dict_of_new_prices_and_contractid(instrument_code, contract_date_dict,
data):
"""
:param instrument_code: str
:param contract_list: dict of 'yyyymmdd' str, keynames 'CARRY, PRICE, FORWARD'
:param data:
:return: dict of futures contract prices for each contract, plus contract id column
"""
diag_prices = diagPrices(data)
# get prices for relevant contracts, return as dict labelled with column for contractids
relevant_contract_prices = dict()
for key, contract_date in contract_date_dict.items():
price_series = diag_prices.get_prices_for_instrument_code_and_contract_date(
instrument_code, contract_date)
relevant_contract_prices[key] = price_series.return_final_prices()
new_prices_dict = dictFuturesContractFinalPricesWithContractID.\
create_from_two_dicts(relevant_contract_prices, contract_date_dict)
return new_prices_dict
def create_contract_orders_spread(data, instrument_code, priced_contract_id,
forward_contract_id, position_in_priced):
diag_prices = diagPrices(data)
reference_price_priced_contract, reference_price_forward_contract = \
tuple(diag_prices.get_last_matched_prices_for_contract_list(
instrument_code, [priced_contract_id, forward_contract_id]))
strategy = ROLL_PSEUDO_STRATEGY
contract_id_list = [priced_contract_id, forward_contract_id]
trade_list = [-position_in_priced, position_in_priced]
spread_reference_price = reference_price_priced_contract - reference_price_forward_contract
spread_order = contractOrder(strategy,
instrument_code,
contract_id_list,
trade_list,
reference_price=spread_reference_price,
roll_order=True)
return [spread_order]
def calculate_adjusted_price_for_a_direct_child_order(data, instrument_code,
child_order,
original_contract_date,
original_price):
"""
:param data:
:param instrument_code:
:param child_order:
:param original_contract_date:
:param original_price:
:return: float or missing data
"""
try:
assert len(child_order.contract_id) == 1
except:
raise Exception(
"You have tried to adjust the price for a spread contract order assuming it is a direct order"
)
child_contract_date = child_order.contract_id[0]
if original_contract_date == child_contract_date:
return original_price
diag_prices = diagPrices(data)
contract_list = [original_contract_date, child_contract_date]
list_of_prices = diag_prices.get_last_matched_prices_for_contract_list(
instrument_code, contract_list)
differential = list_of_prices[1] - list_of_prices[0]
if np.isnan(differential):
# can't adjust
# note need to test code there may be other ways in which this fails
return missing_data
adjusted_price = original_price + differential
return adjusted_price
def update_multiple_adjusted_prices_daily():
"""
Do a daily update for multiple and adjusted prices
:return: Nothing
"""
with dataBlob(log_name="Update-multiple-adjusted-prices(daily)") as data:
diag_prices = diagPrices(data)
list_of_codes_all = diag_prices.get_list_of_instruments_in_multiple_prices(
)
for instrument_code in list_of_codes_all:
try:
update_multiple_adjusted_prices_for_instrument(
instrument_code, data)
except Exception as e:
data.log.warn("ERROR: Multiple price update went wrong: %s" %
str(e))
return success
def update_historical_prices_for_instrument_and_contract(
contract_object: futuresContract, data: dataBlob):
"""
Do a daily update for futures contract prices, using IB historical data
:param contract_object: futuresContract
:param data: data blob
:return: None
"""
diag_prices = diagPrices(data)
intraday_frequency = diag_prices.get_intraday_frequency_for_historical_download()
daily_frequency = DAILY_PRICE_FREQ
result = get_and_add_prices_for_frequency(
data, contract_object, frequency=intraday_frequency
)
if result is failure:
# Skip daily data if intraday not working
return None
get_and_add_prices_for_frequency(
data, contract_object, frequency=daily_frequency)
def get_and_check_prices_for_frequency(
data, log, contract_object, frequency="D"):
broker_data = dataBroker(data)
price_data = diagPrices(data)
price_updater = updatePrices(data)
try:
old_prices = price_data.get_prices_for_contract_object(contract_object)
ib_prices = broker_data.get_prices_at_frequency_for_contract_object(
contract_object, frequency
)
if len(ib_prices) == 0:
raise Exception(
"No IB prices found for %s nothing to check" %
str(contract_object))
print(
"\n\n Manually checking prices for %s \n\n" %
str(contract_object))
new_prices_checked = manual_price_checker(
old_prices,
ib_prices,
column_to_check="FINAL",
delta_columns=["OPEN", "HIGH", "LOW"],
type_new_data=futuresContractPrices,
)
result = price_updater.update_prices_for_contract(
contract_object, new_prices_checked, check_for_spike=False
)
return result
except Exception as e:
log.warn(
"Exception %s when getting or checking data at frequency %s for %s"
% (e, frequency, str(contract_object))
)
return failure
def get_trading_hours_for_all_instruments(data=arg_not_supplied):
if data is arg_not_supplied:
data = dataBlob()
diag_prices = diagPrices(data)
list_of_instruments = diag_prices.get_list_of_instruments_with_contract_prices()
p = progressBar(len(list_of_instruments))
all_trading_hours = {}
for instrument_code in list_of_instruments:
p.iterate()
trading_hours = get_trading_hours_for_instrument(data, instrument_code)
if trading_hours is missing_contract:
print("*** NO EXPIRY FOR %s ***" % instrument_code)
continue
## will have several days use first one
check_trading_hours(trading_hours, instrument_code)
all_trading_hours[instrument_code] = trading_hours
p.finished()
return all_trading_hours
def update_historical_prices_for_instrument_and_contract(
contract_object, data, log):
"""
Do a daily update for futures contract prices, using IB historical data
:param contract_object: futuresContract
:param data: data blob
:param log: logger
:return: None
"""
diag_prices = diagPrices(data)
intraday_frequency = diag_prices.get_intraday_frequency_for_historical_download()
result = get_and_add_prices_for_frequency(
data, log, contract_object, frequency=intraday_frequency
)
if result is failure:
# Skip daily data if intraday not working
return failure
result = get_and_add_prices_for_frequency(
data, log, contract_object, frequency="D")
return result
def get_contract_chain(instrument_code, data):
diag_contracts = diagContracts(data)
diag_prices = diagPrices(data)
roll_parameters = diag_contracts.get_roll_parameters(instrument_code)
# Get the last contract currently being used
multiple_prices = diag_prices.get_multiple_prices(instrument_code)
current_contract_dict = multiple_prices.current_contract_dict()
current_contract_list = list(current_contract_dict.values())
furthest_out_contract_date = max(current_contract_list)
furthest_out_contract = contractDateWithRollParameters(
roll_parameters, furthest_out_contract_date
)
# To give us wiggle room, and ensure we start collecting the new forward a
# little in advance
final_contract = furthest_out_contract.next_priced_contract()
contract_date_chain = (
final_contract.get_unexpired_contracts_from_now_to_contract_date()
)
# We have a list of contract_date objects, need futureContracts
# create a 'bare' instrument object
instrument_object = futuresInstrument(instrument_code)
contract_object_chain_as_list = [
futuresContract(instrument_object, contract_date_object)
for contract_date_object in contract_date_chain
]
contract_object_chain = listOfFuturesContracts(
contract_object_chain_as_list)
return contract_object_chain
def get_price_series_for_contract(data, instrument_code, contract_id):
diag_prices = diagPrices(data)
all_prices = diag_prices.get_prices_for_instrument_code_and_contract_date(instrument_code, contract_id)
price_series = all_prices.return_final_prices()
return price_series
def update_historical_prices_with_data(data: dataBlob):
price_data = diagPrices(data)
list_of_codes_all = price_data.get_list_of_instruments_in_multiple_prices()
for instrument_code in list_of_codes_all:
data.log.label(instrument_code=instrument_code)
update_historical_prices_for_instrument(instrument_code, data)
def update_active_contracts_with_data(data: dataBlob):
diag_prices = diagPrices(data)
list_of_codes_all = diag_prices.get_list_of_instruments_in_multiple_prices()
for instrument_code in list_of_codes_all:
update_active_contracts_for_instrument(
instrument_code, data)
def get_price_series(data, instrument_code):
diag_prices = diagPrices(data)
price_series = diag_prices.get_adjusted_prices(instrument_code)
return price_series
def get_last_price(data: dataBlob, instrument_code: str):
diag_prices = diagPrices(data)
prices = diag_prices.get_adjusted_prices(instrument_code)
return prices.ffill().values[-1]
def _list_of_all_instruments(self):
diag_prices = diagPrices(self.data)
list_of_instruments = diag_prices.get_list_of_instruments_in_multiple_prices(
)
return list_of_instruments
def get_current_contract_price_series_for_instrument(data, instrument_code):
diag_prices = diagPrices(data)
price_series = diag_prices.get_current_priced_contract_prices_for_instrument(
instrument_code)
return price_series
def _roll_adjusted_and_multiple_prices(data, instrument_code):
"""
Roll multiple and adjusted prices
THE POSITION MUST BE ZERO IN THE PRICED CONTRACT! WE DON'T CHECK THIS HERE
:param data: dataBlob
:param instrument_code: str
:return:
"""
print(landing_strip(80))
print("")
print("Rolling adjusted prices!")
print("")
diag_prices = diagPrices(data)
current_multiple_prices = diag_prices.get_multiple_prices(instrument_code)
# Only required for potential rollback
current_adjusted_prices = diag_prices.get_adjusted_prices(instrument_code)
try:
updated_multiple_prices = update_multiple_prices_on_roll(
data, current_multiple_prices, instrument_code)
new_adj_prices = futuresAdjustedPrices.stich_multiple_prices(
updated_multiple_prices)
except Exception as e:
data.log.warn("%s : went wrong when rolling: No roll has happened" % e)
return failure
# We want user input before we do anything
compare_old_and_new_prices([
current_multiple_prices, updated_multiple_prices,
current_adjusted_prices, new_adj_prices
], [
"Current multiple prices", "New multiple prices",
"Current adjusted prices", "New adjusted prices"
])
print("")
confirm_roll = input(
"Confirm roll adjusted prices for %s are you sure y/n:" %
instrument_code)
if confirm_roll != "y":
print(
"\nUSER DID NOT WANT TO ROLL: Setting roll status back to previous state"
)
return failure
try:
# Apparently good let's try and write rolled data
price_updater = updatePrices(data)
price_updater.add_adjusted_prices(instrument_code,
new_adj_prices,
ignore_duplication=True)
price_updater.add_multiple_prices(instrument_code,
updated_multiple_prices,
ignore_duplication=True)
except Exception as e:
data.log.warn(
"%s went wrong when rolling: Going to roll-back to original multiple/adjusted prices"
% e)
rollback_adjustment(data, instrument_code, current_adjusted_prices,
current_multiple_prices)
return failure
return success
