def add_interest_rate(curve_name):
interest_rate_factor = utils.Factor(
'InterestRate', utils.check_rate_name(curve_name))
dependent_factors.update(get_rates(interest_rate_factor, {}))
for sub_factor in utils.traverse_dependents(
interest_rate_factor, dependent_factors):
dependent_factor_tenors[sub_factor] = reset_dates
def fetch_all_calibration_factors(self, override={}):
"""
Assumes valid marketdata and calibration config files have been loaded (via parse_json)
and returns the list of price factors that have mapped price models.
The return value of this method is suitable to pass to the calibrate_factors method.
"""
model_factor = {}
for factor in self.params.get('Price Factors', {}):
price_factor = utils.check_rate_name(factor)
model_config = self.params['Model Configuration'].search(
utils.Factor(price_factor[0], price_factor[1:]),
self.params['Price Factors'][factor])
model = override.get(model_config, model_config)
if model:
subtype = self.params['Price Factors'][factor].get('Sub_Type')
model_name = utils.Factor(model, price_factor[1:])
archive_name = utils.Factor(
price_factor[0] +
(subtype if subtype and subtype != "None" else ''),
price_factor[1:])
model_factor[factor] = utils.RateInfo(
utils.check_tuple_name(model_name),
utils.check_tuple_name(archive_name),
self.calibration_process_map.get(model))
remaining_factor = {}
remaining_rates = set([
col.split(',')[0] for col in self.archive.columns
]).difference(model_factor.keys())
for factor in remaining_rates:
price_factor = utils.check_rate_name(factor)
model = self.params['Model Configuration'].search(
utils.Factor(price_factor[0], price_factor[1:]), {})
if model:
model_name = utils.Factor(model, price_factor[1:])
archive_name = utils.Factor(price_factor[0], price_factor[1:])
remaining_factor[factor] = utils.RateInfo(
utils.check_tuple_name(model_name),
utils.check_tuple_name(archive_name),
self.calibration_process_map.get(model))
return {'present': model_factor, 'absent': remaining_factor}
def get_rates(factor, instrument):
rates_to_add = {factor: []}
factor_name = utils.check_tuple_name(factor)
if factor.type in dependant_fields:
linked_factors = [
utils.Factor(
dep_field[1],
utils.check_rate_name(self.params['Price Factors']
[factor_name][dep_field[0]]))
for dep_field in dependant_fields[factor.type]
if self.params['Price Factors'][factor_name][dep_field[0]]
]
for linked_factor in linked_factors:
# add it assuming no dependencies
rates_to_add.setdefault(linked_factor, [])
# update and dependencies
if linked_factor.type in dependant_fields:
rates_to_add.update(
get_rates(linked_factor, instrument))
# link it to the original factor
rates_to_add[factor].append(linked_factor)
# check that we include any nested factors
if linked_factor.type in nested_fields:
for i in range(1, len(linked_factor.name) + 1):
rates_to_add.update({
utils.Factor(linked_factor.type, linked_factor.name[:i]):
[
utils.Factor(linked_factor.type,
linked_factor.name[:i - 1])
] if i > 1 else []
})
if factor.type in nested_fields:
for i in range(1, len(factor.name) + 1):
rates_to_add.update({
utils.Factor(factor.type, factor.name[:i]):
[utils.Factor(factor.type, factor.name[:i - 1])]
if i > 1 else []
})
if factor.type in conditional_fields:
for conditional_factor in conditional_fields[factor.type](
instrument, self.params['Price Factors'][factor_name],
self.params['Price Factors']):
rates_to_add[factor].append(conditional_factor)
rates_to_add.update({conditional_factor: []})
return rates_to_add
def get_price_factors(rates_to_add, rate_tenors, instrument):
reval_dates = instrument.get_reval_dates()
for field_name, factor_types in instrument.factor_fields.items(
):
for field_value in utils.get_fieldname(
field_name, instrument.field):
for factor in [
utils.Factor(
factor_type,
utils.check_rate_name(field_value))
for factor_type in factor_types
]:
# store the max tenor for just the factor alone
if reval_dates:
rate_tenors.setdefault(factor, set()).update(
{max(reval_dates)})
# now look at dependent factors
if factor not in rates_to_add:
try:
rates_to_add.update(
get_rates(factor, instrument))
except KeyError as e:
logging.warning(
'Price Factor {0} not found in market data'
.format(e))
if factor.type == 'DiscountRate':
logging.info(
'Creating default Discount {0}'.
format(factor))
self.params['Price Factors'][
utils.check_tuple_name(
factor)] = OrderedDict([
('Interest_Rate',
'.'.join(factor.name))
])
# try again
rates_to_add.update(
get_rates(factor, instrument))
else:
logging.error('Skipping Price Factor')
def get_forwardprice(self):
return utils.check_rate_name(self.param['ForwardPrice'])
def get_currency(self):
return utils.check_rate_name(self.param['Currency'])
def get_interest_rate(self):
return utils.check_rate_name(self.param['Interest_Rate'])
def get_repo_curve_name(self):
return utils.check_rate_name(
self.param['Interest_Rate'] if self.
param['Interest_Rate'] else self.param['Currency'])
def get_domestic_currency(self, default):
return utils.check_rate_name(self.param['Domestic_Currency'] if self.
param['Domestic_Currency'] else default)
def get_repo_curve_name(self, default):
return utils.check_rate_name(self.param['Interest_Rate'] if self.
param['Interest_Rate'] else default)
def calculate_dependencies(self,
options,
base_date,
base_MTM_dates,
calc_dates=True):
"""
Works out the risk factors (and risk models) in the given set of deals.
These factors are cross referenced in the marketdata file and matched by
name. This can be extended as needed.
Returns the dependant factors, the stochastic models, the full list of
reset dates and optionally the potential currency settlements
"""
def get_rates(factor, instrument):
rates_to_add = {factor: []}
factor_name = utils.check_tuple_name(factor)
if factor.type in dependant_fields:
linked_factors = [
utils.Factor(
dep_field[1],
utils.check_rate_name(self.params['Price Factors']
[factor_name][dep_field[0]]))
for dep_field in dependant_fields[factor.type]
if self.params['Price Factors'][factor_name][dep_field[0]]
]
for linked_factor in linked_factors:
# add it assuming no dependencies
rates_to_add.setdefault(linked_factor, [])
# update and dependencies
if linked_factor.type in dependant_fields:
rates_to_add.update(
get_rates(linked_factor, instrument))
# link it to the original factor
rates_to_add[factor].append(linked_factor)
# check that we include any nested factors
if linked_factor.type in nested_fields:
for i in range(1, len(linked_factor.name) + 1):
rates_to_add.update({
utils.Factor(linked_factor.type, linked_factor.name[:i]):
[
utils.Factor(linked_factor.type,
linked_factor.name[:i - 1])
] if i > 1 else []
})
if factor.type in nested_fields:
for i in range(1, len(factor.name) + 1):
rates_to_add.update({
utils.Factor(factor.type, factor.name[:i]):
[utils.Factor(factor.type, factor.name[:i - 1])]
if i > 1 else []
})
if factor.type in conditional_fields:
for conditional_factor in conditional_fields[factor.type](
instrument, self.params['Price Factors'][factor_name],
self.params['Price Factors']):
rates_to_add[factor].append(conditional_factor)
rates_to_add.update({conditional_factor: []})
return rates_to_add
def walk_groups(deals, price_factors, factor_tenors):
def get_price_factors(rates_to_add, rate_tenors, instrument):
reval_dates = instrument.get_reval_dates()
for field_name, factor_types in instrument.factor_fields.items(
):
for field_value in utils.get_fieldname(
field_name, instrument.field):
for factor in [
utils.Factor(
factor_type,
utils.check_rate_name(field_value))
for factor_type in factor_types
]:
# store the max tenor for just the factor alone
if reval_dates:
rate_tenors.setdefault(factor, set()).update(
{max(reval_dates)})
# now look at dependent factors
if factor not in rates_to_add:
try:
rates_to_add.update(
get_rates(factor, instrument))
except KeyError as e:
logging.warning(
'Price Factor {0} not found in market data'
.format(e))
if factor.type == 'DiscountRate':
logging.info(
'Creating default Discount {0}'.
format(factor))
self.params['Price Factors'][
utils.check_tuple_name(
factor)] = OrderedDict([
('Interest_Rate',
'.'.join(factor.name))
])
# try again
rates_to_add.update(
get_rates(factor, instrument))
else:
logging.error('Skipping Price Factor')
resets = {base_date}
children = []
settlement_currencies = {}
for node in deals:
# get the instrument
instrument = node['instrument']
if node.get('Ignore') == 'True':
continue
# get a list of children ready to pass back to the parent
children.append(instrument)
if node.get('Children'):
node_children, node_resets, node_settlements = walk_groups(
node['Children'], price_factors, factor_tenors)
# sort out dates and calendars
instrument.reset(self.holidays)
if calc_dates:
instrument.finalize_dates(self.parse_grid, base_date,
base_MTM_dates,
node_children, node_resets,
node_settlements)
# get it's price factors
get_price_factors(price_factors, factor_tenors, instrument)
# merge dates
resets.update(node_resets)
for key, value in node_settlements.items():
settlement_currencies.setdefault(key,
set()).update(value)
else:
# sort out dates and calendars
instrument.reset(self.holidays)
if calc_dates:
instrument.finalize_dates(self.parse_grid, base_date,
base_MTM_dates, None, resets,
settlement_currencies)
# get it's price factors
get_price_factors(price_factors, factor_tenors, instrument)
return children, resets, settlement_currencies
def add_interest_rate(curve_name):
interest_rate_factor = utils.Factor(
'InterestRate', utils.check_rate_name(curve_name))
dependent_factors.update(get_rates(interest_rate_factor, {}))
for sub_factor in utils.traverse_dependents(
interest_rate_factor, dependent_factors):
dependent_factor_tenors[sub_factor] = reset_dates
# derived fields are fields that embed other risk factors
dependant_fields = {
'FxRate': [('Interest_Rate', 'InterestRate')],
'DiscountRate': [('Interest_Rate', 'InterestRate')],
'ForwardPrice': [('Currency', 'FxRate')],
'ReferencePrice': [('ForwardPrice', 'ForwardPrice')],
'ReferenceVol': [('ForwardPriceVol', 'ForwardPriceVol'),
('ReferencePrice', 'ReferencePrice')],
'InflationRate': [('Price_Index', 'PriceIndex')],
'EquityPrice': [('Interest_Rate', 'InterestRate'),
('Currency', 'FxRate')]
}
# nested fields need to include all their children
nested_fields = {'InterestRate'}
# conditional fields need to potentially include correlation and fx vol surfaces (e.g. reference prices)
conditional_fields = {
'ReferenceVol':
lambda instrument, factor_fields, params: [
utils.Factor(
'Correlation',
tuple('FxRate.{0}.{1}/ReferencePrice.{2}.{0}'.format(
params[utils.check_tuple_name(
utils.Factor('ForwardPrice', (instrument.field[
'Reference_Type'], )))]['Currency'], instrument
.field['Currency'], instrument.field['Reference_Type'])
.split('.')))
] if instrument.field['Currency'] != params[utils.check_tuple_name(
utils.Factor('ForwardPrice', (instrument.field[
'Reference_Type'], )))]['Currency'] else [],
'ForwardPrice':
lambda instrument, factor_fields, params: [
utils.Factor(
'FXVol',
tuple(
sorted([
instrument.field['Currency'], factor_fields[
'Currency']
])))
]
if instrument.field['Currency'] != factor_fields['Currency'] else [
]
}
# the list of returned factors
dependent_factors = set()
stochastic_factors = OrderedDict()
additional_factors = OrderedDict()
# complete list of reset dates referenced
reset_dates = set()
# complete list of currency settlement dates
currency_settlement_dates = {}
# only if we have a portfolio of trades can we calculate its dependencies
if self.deals:
# get the reporting currency
report_currency = options['Currency']
# get the base currency factor
base_factor = utils.Factor(
'FxRate',
utils.check_rate_name(
self.params['System Parameters']['Base_Currency']))
# add the base Fx rate
dependent_factors = get_rates(base_factor, {})
# store the max date the factor is needed
dependent_factor_tenors = {}
# grab all the factor fields in the portfolio
dependant_deals, reset_dates, currency_settlement_dates = walk_groups(
self.deals['Deals']['Children'], dependent_factors,
dependent_factor_tenors)
# additional factors from the options passed in
report_factor = utils.Factor(
'FxRate', utils.check_rate_name(report_currency))
report_currency_dependencies = get_rates(report_factor, {})
# add the base dependencies to the reporting currency
if report_factor != base_factor:
report_currency_dependencies[report_factor].append(base_factor)
dependent_factors.update(report_currency_dependencies)
# make sure the reporting currency is around till the end
dependent_factor_tenors[report_factor] = reset_dates
for reporting_factor in utils.traverse_dependents(
report_factor, dependent_factors):
dependent_factor_tenors[reporting_factor] = reset_dates
# check if we need to fetch survival data for CVA
if options.get('CVA'):
dependent_factors.update(
get_rates(
utils.Factor(
'SurvivalProb',
utils.check_rate_name(
options['CVA']['Counterparty'])), {}))
# check if we need to fetch curve data for FVA
if options.get('FVA'):
# add curves
add_interest_rate(options['FVA']['Funding_Interest_Curve'])
add_interest_rate(options['FVA']['Risk_Free_Curve'])
# need to weight the FVA by the survival prob of the counterparty (if defined)
if 'Counterparty' in options['FVA']:
dependent_factors.update(
get_rates(
utils.Factor(
'SurvivalProb',
utils.check_rate_name(
options['FVA']['Counterparty'])), {}))
# Check deflation
if options.get('Deflation_Interest_Rate'):
add_interest_rate(options['Deflation_Interest_Rate'])
# update the linked factor max tenors
missing_tenors = {}
for k, v in dependent_factor_tenors.items():
for linked_factor in utils.traverse_dependents(
k, dependent_factors):
missing_tenors.setdefault(linked_factor, set()).update(v)
# make sure the base currency is always first
for k, v in dependent_factors.items():
if k.type == 'FxRate' and k != base_factor and base_factor not in v:
v.append(base_factor)
# now sort the factors taking any factor dependencies into account
sorted_factors = utils.topological_sort(dependent_factors)
# merge missing tenors
for k, v in missing_tenors.items():
dependent_factor_tenors.setdefault(k, set()).update(v)
# now get the last tenor for each factor
dependent_factors = {
k: max(dependent_factor_tenors.get(k, reset_dates))
for k in sorted_factors
}
# now lookup the processes
for factor in sorted_factors:
stoch_proc = self.params['Model Configuration'].search(
factor, self.params['Price Factors'].get(
utils.check_tuple_name(factor), {}))
# might need implied parameters
additional_factor = self.params[
'Model Configuration'].additional_factors(
stoch_proc, factor)
if stoch_proc and factor.name[0] != self.params[
'System Parameters']['Base_Currency']:
factor_model = utils.Factor(stoch_proc, factor.name)
if utils.check_tuple_name(
factor_model) in self.params['Price Models']:
stochastic_factors.setdefault(factor_model, factor)
if additional_factor:
additional_factors.setdefault(
factor_model, additional_factor)
else:
logging.error(
'Risk Factor {0} using stochastic process {1} missing in Price Models section'
.format(utils.check_tuple_name(factor),
stoch_proc))
return dependent_factors, stochastic_factors, additional_factors, reset_dates, currency_settlement_dates
def calibrate_factors(self,
from_date,
to_date,
factors,
smooth=0.0,
correlation_cuttoff=0.2,
overwrite_correlations=True):
"""
Assumes a valid calibration JSON configuration file is loaded first, then proceeds to strip out only data
between from_date and to_date. The calibration rules as specified by the calibration configuration file is then
applied to the factors given. Note that factors needs to be a list of utils.RateInfo (obtained via a call to
fetch_all_calibration_factors). Also note that this method overwrites the Price Model section of the config
file in memory. To save the changes, an explicit call to write_marketdata_json must be made.
"""
correlation_names = []
consolidated_df = None
ak = []
num_indexes = 0
num_factors = 0
total_rates = reduce(operator.concat, [
self.archive_columns[rate.archive_name]
for rate in factors.values()
], [])
factor_data = utils.filter_data_frame(self.archive, from_date,
to_date)[total_rates]
for rate_name, rate_value in sorted(factors.items()):
df = factor_data[[
col for col in factor_data.columns
if col.split(',')[0] == rate_value.archive_name
]]
# now remove spikes
data_frame = df[np.abs(df - df.median()) <= (smooth * df.std())].interpolate(method='index') \
if smooth else df
# log it
logging.info(
'Calibrating {0} (archive name {1}) with raw shape {2} and cleaned non-null shape {3}'
.format(rate_name, rate_value.archive_name, str(df.shape),
str(data_frame.dropna().shape)))
# calibrate
try:
result = rate_value.calibration.calibrate(
data_frame, num_business_days=252.0)
except:
logging.error(
'Data errors in factor {0} resulting in flawed calibration - skipping'
.format(rate_value.archive_name))
continue
# check that it makes sense . . .
if (np.array(result.correlation).max() > 1) or (np.array(
result.correlation).min() < -1) or (result.delta.std()
== 0).any():
logging.error(
'Data errors in factor {0} resulting in incorrect correlations - skipping'
.format(rate_value.archive_name))
continue
model_tuple = utils.check_rate_name(rate_value.model_name)
model_factor = utils.Factor(model_tuple[0], model_tuple[1:])
# get the correlation name
process_name, addons = construct_process(
model_factor.type, None, result.param).correlation_name
for sub_factors in addons:
correlation_names.append(
utils.check_tuple_name(
utils.Factor(process_name,
model_factor.name + sub_factors)))
consolidated_df = result.delta if consolidated_df is None else pd.concat(
[consolidated_df, result.delta], axis=1)
# store the correlation coefficients
ak.append(result.correlation)
# store result back in market data file
self.params['Price Models'][rate_value.model_name] = result.param
num_indexes += result.delta.shape[1]
num_factors += rate_value.calibration.num_factors
a = np.zeros((num_factors, num_indexes))
rho = consolidated_df.corr()
offset = 0
row_num = 0
for coeff in ak:
for factor_index, factor in enumerate(coeff):
a[row_num + factor_index, offset:offset + len(factor)] = factor
row_num += len(coeff)
offset += len(factor)
# cheating here
factor_correlations = a.dot(rho).dot(a.T).clip(-1.0, 1.0)
# see if we need to delete the old correlations
if overwrite_correlations:
self.params['Correlations'] = {}
for index1 in range(len(correlation_names) - 1):
for index2 in range(index1 + 1, len(correlation_names)):
if np.fabs(factor_correlations[index1,
index2]) > correlation_cuttoff:
self.params['Correlations'][(correlation_names[index1], correlation_names[index2])] = \
factor_correlations[index1, index2]
