def add_interest_rate(curve_name): interest_rate_factor = utils.Factor( 'InterestRate', utils.check_rate_name(curve_name)) dependent_factors.update(get_rates(interest_rate_factor, {})) for sub_factor in utils.traverse_dependents( interest_rate_factor, dependent_factors): dependent_factor_tenors[sub_factor] = reset_dates
def fetch_all_calibration_factors(self, override={}): """ Assumes valid marketdata and calibration config files have been loaded (via parse_json) and returns the list of price factors that have mapped price models. The return value of this method is suitable to pass to the calibrate_factors method. """ model_factor = {} for factor in self.params.get('Price Factors', {}): price_factor = utils.check_rate_name(factor) model_config = self.params['Model Configuration'].search( utils.Factor(price_factor[0], price_factor[1:]), self.params['Price Factors'][factor]) model = override.get(model_config, model_config) if model: subtype = self.params['Price Factors'][factor].get('Sub_Type') model_name = utils.Factor(model, price_factor[1:]) archive_name = utils.Factor( price_factor[0] + (subtype if subtype and subtype != "None" else ''), price_factor[1:]) model_factor[factor] = utils.RateInfo( utils.check_tuple_name(model_name), utils.check_tuple_name(archive_name), self.calibration_process_map.get(model)) remaining_factor = {} remaining_rates = set([ col.split(',')[0] for col in self.archive.columns ]).difference(model_factor.keys()) for factor in remaining_rates: price_factor = utils.check_rate_name(factor) model = self.params['Model Configuration'].search( utils.Factor(price_factor[0], price_factor[1:]), {}) if model: model_name = utils.Factor(model, price_factor[1:]) archive_name = utils.Factor(price_factor[0], price_factor[1:]) remaining_factor[factor] = utils.RateInfo( utils.check_tuple_name(model_name), utils.check_tuple_name(archive_name), self.calibration_process_map.get(model)) return {'present': model_factor, 'absent': remaining_factor}
def get_rates(factor, instrument): rates_to_add = {factor: []} factor_name = utils.check_tuple_name(factor) if factor.type in dependant_fields: linked_factors = [ utils.Factor( dep_field[1], utils.check_rate_name(self.params['Price Factors'] [factor_name][dep_field[0]])) for dep_field in dependant_fields[factor.type] if self.params['Price Factors'][factor_name][dep_field[0]] ] for linked_factor in linked_factors: # add it assuming no dependencies rates_to_add.setdefault(linked_factor, []) # update and dependencies if linked_factor.type in dependant_fields: rates_to_add.update( get_rates(linked_factor, instrument)) # link it to the original factor rates_to_add[factor].append(linked_factor) # check that we include any nested factors if linked_factor.type in nested_fields: for i in range(1, len(linked_factor.name) + 1): rates_to_add.update({ utils.Factor(linked_factor.type, linked_factor.name[:i]): [ utils.Factor(linked_factor.type, linked_factor.name[:i - 1]) ] if i > 1 else [] }) if factor.type in nested_fields: for i in range(1, len(factor.name) + 1): rates_to_add.update({ utils.Factor(factor.type, factor.name[:i]): [utils.Factor(factor.type, factor.name[:i - 1])] if i > 1 else [] }) if factor.type in conditional_fields: for conditional_factor in conditional_fields[factor.type]( instrument, self.params['Price Factors'][factor_name], self.params['Price Factors']): rates_to_add[factor].append(conditional_factor) rates_to_add.update({conditional_factor: []}) return rates_to_add
def get_price_factors(rates_to_add, rate_tenors, instrument): reval_dates = instrument.get_reval_dates() for field_name, factor_types in instrument.factor_fields.items( ): for field_value in utils.get_fieldname( field_name, instrument.field): for factor in [ utils.Factor( factor_type, utils.check_rate_name(field_value)) for factor_type in factor_types ]: # store the max tenor for just the factor alone if reval_dates: rate_tenors.setdefault(factor, set()).update( {max(reval_dates)}) # now look at dependent factors if factor not in rates_to_add: try: rates_to_add.update( get_rates(factor, instrument)) except KeyError as e: logging.warning( 'Price Factor {0} not found in market data' .format(e)) if factor.type == 'DiscountRate': logging.info( 'Creating default Discount {0}'. format(factor)) self.params['Price Factors'][ utils.check_tuple_name( factor)] = OrderedDict([ ('Interest_Rate', '.'.join(factor.name)) ]) # try again rates_to_add.update( get_rates(factor, instrument)) else: logging.error('Skipping Price Factor')
def get_forwardprice(self): return utils.check_rate_name(self.param['ForwardPrice'])
def get_currency(self): return utils.check_rate_name(self.param['Currency'])
def get_interest_rate(self): return utils.check_rate_name(self.param['Interest_Rate'])
def get_repo_curve_name(self): return utils.check_rate_name( self.param['Interest_Rate'] if self. param['Interest_Rate'] else self.param['Currency'])
def get_domestic_currency(self, default): return utils.check_rate_name(self.param['Domestic_Currency'] if self. param['Domestic_Currency'] else default)
def get_repo_curve_name(self, default): return utils.check_rate_name(self.param['Interest_Rate'] if self. param['Interest_Rate'] else default)
def calculate_dependencies(self, options, base_date, base_MTM_dates, calc_dates=True): """ Works out the risk factors (and risk models) in the given set of deals. These factors are cross referenced in the marketdata file and matched by name. This can be extended as needed. Returns the dependant factors, the stochastic models, the full list of reset dates and optionally the potential currency settlements """ def get_rates(factor, instrument): rates_to_add = {factor: []} factor_name = utils.check_tuple_name(factor) if factor.type in dependant_fields: linked_factors = [ utils.Factor( dep_field[1], utils.check_rate_name(self.params['Price Factors'] [factor_name][dep_field[0]])) for dep_field in dependant_fields[factor.type] if self.params['Price Factors'][factor_name][dep_field[0]] ] for linked_factor in linked_factors: # add it assuming no dependencies rates_to_add.setdefault(linked_factor, []) # update and dependencies if linked_factor.type in dependant_fields: rates_to_add.update( get_rates(linked_factor, instrument)) # link it to the original factor rates_to_add[factor].append(linked_factor) # check that we include any nested factors if linked_factor.type in nested_fields: for i in range(1, len(linked_factor.name) + 1): rates_to_add.update({ utils.Factor(linked_factor.type, linked_factor.name[:i]): [ utils.Factor(linked_factor.type, linked_factor.name[:i - 1]) ] if i > 1 else [] }) if factor.type in nested_fields: for i in range(1, len(factor.name) + 1): rates_to_add.update({ utils.Factor(factor.type, factor.name[:i]): [utils.Factor(factor.type, factor.name[:i - 1])] if i > 1 else [] }) if factor.type in conditional_fields: for conditional_factor in conditional_fields[factor.type]( instrument, self.params['Price Factors'][factor_name], self.params['Price Factors']): rates_to_add[factor].append(conditional_factor) rates_to_add.update({conditional_factor: []}) return rates_to_add def walk_groups(deals, price_factors, factor_tenors): def get_price_factors(rates_to_add, rate_tenors, instrument): reval_dates = instrument.get_reval_dates() for field_name, factor_types in instrument.factor_fields.items( ): for field_value in utils.get_fieldname( field_name, instrument.field): for factor in [ utils.Factor( factor_type, utils.check_rate_name(field_value)) for factor_type in factor_types ]: # store the max tenor for just the factor alone if reval_dates: rate_tenors.setdefault(factor, set()).update( {max(reval_dates)}) # now look at dependent factors if factor not in rates_to_add: try: rates_to_add.update( get_rates(factor, instrument)) except KeyError as e: logging.warning( 'Price Factor {0} not found in market data' .format(e)) if factor.type == 'DiscountRate': logging.info( 'Creating default Discount {0}'. format(factor)) self.params['Price Factors'][ utils.check_tuple_name( factor)] = OrderedDict([ ('Interest_Rate', '.'.join(factor.name)) ]) # try again rates_to_add.update( get_rates(factor, instrument)) else: logging.error('Skipping Price Factor') resets = {base_date} children = [] settlement_currencies = {} for node in deals: # get the instrument instrument = node['instrument'] if node.get('Ignore') == 'True': continue # get a list of children ready to pass back to the parent children.append(instrument) if node.get('Children'): node_children, node_resets, node_settlements = walk_groups( node['Children'], price_factors, factor_tenors) # sort out dates and calendars instrument.reset(self.holidays) if calc_dates: instrument.finalize_dates(self.parse_grid, base_date, base_MTM_dates, node_children, node_resets, node_settlements) # get it's price factors get_price_factors(price_factors, factor_tenors, instrument) # merge dates resets.update(node_resets) for key, value in node_settlements.items(): settlement_currencies.setdefault(key, set()).update(value) else: # sort out dates and calendars instrument.reset(self.holidays) if calc_dates: instrument.finalize_dates(self.parse_grid, base_date, base_MTM_dates, None, resets, settlement_currencies) # get it's price factors get_price_factors(price_factors, factor_tenors, instrument) return children, resets, settlement_currencies def add_interest_rate(curve_name): interest_rate_factor = utils.Factor( 'InterestRate', utils.check_rate_name(curve_name)) dependent_factors.update(get_rates(interest_rate_factor, {})) for sub_factor in utils.traverse_dependents( interest_rate_factor, dependent_factors): dependent_factor_tenors[sub_factor] = reset_dates # derived fields are fields that embed other risk factors dependant_fields = { 'FxRate': [('Interest_Rate', 'InterestRate')], 'DiscountRate': [('Interest_Rate', 'InterestRate')], 'ForwardPrice': [('Currency', 'FxRate')], 'ReferencePrice': [('ForwardPrice', 'ForwardPrice')], 'ReferenceVol': [('ForwardPriceVol', 'ForwardPriceVol'), ('ReferencePrice', 'ReferencePrice')], 'InflationRate': [('Price_Index', 'PriceIndex')], 'EquityPrice': [('Interest_Rate', 'InterestRate'), ('Currency', 'FxRate')] } # nested fields need to include all their children nested_fields = {'InterestRate'} # conditional fields need to potentially include correlation and fx vol surfaces (e.g. reference prices) conditional_fields = { 'ReferenceVol': lambda instrument, factor_fields, params: [ utils.Factor( 'Correlation', tuple('FxRate.{0}.{1}/ReferencePrice.{2}.{0}'.format( params[utils.check_tuple_name( utils.Factor('ForwardPrice', (instrument.field[ 'Reference_Type'], )))]['Currency'], instrument .field['Currency'], instrument.field['Reference_Type']) .split('.'))) ] if instrument.field['Currency'] != params[utils.check_tuple_name( utils.Factor('ForwardPrice', (instrument.field[ 'Reference_Type'], )))]['Currency'] else [], 'ForwardPrice': lambda instrument, factor_fields, params: [ utils.Factor( 'FXVol', tuple( sorted([ instrument.field['Currency'], factor_fields[ 'Currency'] ]))) ] if instrument.field['Currency'] != factor_fields['Currency'] else [ ] } # the list of returned factors dependent_factors = set() stochastic_factors = OrderedDict() additional_factors = OrderedDict() # complete list of reset dates referenced reset_dates = set() # complete list of currency settlement dates currency_settlement_dates = {} # only if we have a portfolio of trades can we calculate its dependencies if self.deals: # get the reporting currency report_currency = options['Currency'] # get the base currency factor base_factor = utils.Factor( 'FxRate', utils.check_rate_name( self.params['System Parameters']['Base_Currency'])) # add the base Fx rate dependent_factors = get_rates(base_factor, {}) # store the max date the factor is needed dependent_factor_tenors = {} # grab all the factor fields in the portfolio dependant_deals, reset_dates, currency_settlement_dates = walk_groups( self.deals['Deals']['Children'], dependent_factors, dependent_factor_tenors) # additional factors from the options passed in report_factor = utils.Factor( 'FxRate', utils.check_rate_name(report_currency)) report_currency_dependencies = get_rates(report_factor, {}) # add the base dependencies to the reporting currency if report_factor != base_factor: report_currency_dependencies[report_factor].append(base_factor) dependent_factors.update(report_currency_dependencies) # make sure the reporting currency is around till the end dependent_factor_tenors[report_factor] = reset_dates for reporting_factor in utils.traverse_dependents( report_factor, dependent_factors): dependent_factor_tenors[reporting_factor] = reset_dates # check if we need to fetch survival data for CVA if options.get('CVA'): dependent_factors.update( get_rates( utils.Factor( 'SurvivalProb', utils.check_rate_name( options['CVA']['Counterparty'])), {})) # check if we need to fetch curve data for FVA if options.get('FVA'): # add curves add_interest_rate(options['FVA']['Funding_Interest_Curve']) add_interest_rate(options['FVA']['Risk_Free_Curve']) # need to weight the FVA by the survival prob of the counterparty (if defined) if 'Counterparty' in options['FVA']: dependent_factors.update( get_rates( utils.Factor( 'SurvivalProb', utils.check_rate_name( options['FVA']['Counterparty'])), {})) # Check deflation if options.get('Deflation_Interest_Rate'): add_interest_rate(options['Deflation_Interest_Rate']) # update the linked factor max tenors missing_tenors = {} for k, v in dependent_factor_tenors.items(): for linked_factor in utils.traverse_dependents( k, dependent_factors): missing_tenors.setdefault(linked_factor, set()).update(v) # make sure the base currency is always first for k, v in dependent_factors.items(): if k.type == 'FxRate' and k != base_factor and base_factor not in v: v.append(base_factor) # now sort the factors taking any factor dependencies into account sorted_factors = utils.topological_sort(dependent_factors) # merge missing tenors for k, v in missing_tenors.items(): dependent_factor_tenors.setdefault(k, set()).update(v) # now get the last tenor for each factor dependent_factors = { k: max(dependent_factor_tenors.get(k, reset_dates)) for k in sorted_factors } # now lookup the processes for factor in sorted_factors: stoch_proc = self.params['Model Configuration'].search( factor, self.params['Price Factors'].get( utils.check_tuple_name(factor), {})) # might need implied parameters additional_factor = self.params[ 'Model Configuration'].additional_factors( stoch_proc, factor) if stoch_proc and factor.name[0] != self.params[ 'System Parameters']['Base_Currency']: factor_model = utils.Factor(stoch_proc, factor.name) if utils.check_tuple_name( factor_model) in self.params['Price Models']: stochastic_factors.setdefault(factor_model, factor) if additional_factor: additional_factors.setdefault( factor_model, additional_factor) else: logging.error( 'Risk Factor {0} using stochastic process {1} missing in Price Models section' .format(utils.check_tuple_name(factor), stoch_proc)) return dependent_factors, stochastic_factors, additional_factors, reset_dates, currency_settlement_dates
def calibrate_factors(self, from_date, to_date, factors, smooth=0.0, correlation_cuttoff=0.2, overwrite_correlations=True): """ Assumes a valid calibration JSON configuration file is loaded first, then proceeds to strip out only data between from_date and to_date. The calibration rules as specified by the calibration configuration file is then applied to the factors given. Note that factors needs to be a list of utils.RateInfo (obtained via a call to fetch_all_calibration_factors). Also note that this method overwrites the Price Model section of the config file in memory. To save the changes, an explicit call to write_marketdata_json must be made. """ correlation_names = [] consolidated_df = None ak = [] num_indexes = 0 num_factors = 0 total_rates = reduce(operator.concat, [ self.archive_columns[rate.archive_name] for rate in factors.values() ], []) factor_data = utils.filter_data_frame(self.archive, from_date, to_date)[total_rates] for rate_name, rate_value in sorted(factors.items()): df = factor_data[[ col for col in factor_data.columns if col.split(',')[0] == rate_value.archive_name ]] # now remove spikes data_frame = df[np.abs(df - df.median()) <= (smooth * df.std())].interpolate(method='index') \ if smooth else df # log it logging.info( 'Calibrating {0} (archive name {1}) with raw shape {2} and cleaned non-null shape {3}' .format(rate_name, rate_value.archive_name, str(df.shape), str(data_frame.dropna().shape))) # calibrate try: result = rate_value.calibration.calibrate( data_frame, num_business_days=252.0) except: logging.error( 'Data errors in factor {0} resulting in flawed calibration - skipping' .format(rate_value.archive_name)) continue # check that it makes sense . . . if (np.array(result.correlation).max() > 1) or (np.array( result.correlation).min() < -1) or (result.delta.std() == 0).any(): logging.error( 'Data errors in factor {0} resulting in incorrect correlations - skipping' .format(rate_value.archive_name)) continue model_tuple = utils.check_rate_name(rate_value.model_name) model_factor = utils.Factor(model_tuple[0], model_tuple[1:]) # get the correlation name process_name, addons = construct_process( model_factor.type, None, result.param).correlation_name for sub_factors in addons: correlation_names.append( utils.check_tuple_name( utils.Factor(process_name, model_factor.name + sub_factors))) consolidated_df = result.delta if consolidated_df is None else pd.concat( [consolidated_df, result.delta], axis=1) # store the correlation coefficients ak.append(result.correlation) # store result back in market data file self.params['Price Models'][rate_value.model_name] = result.param num_indexes += result.delta.shape[1] num_factors += rate_value.calibration.num_factors a = np.zeros((num_factors, num_indexes)) rho = consolidated_df.corr() offset = 0 row_num = 0 for coeff in ak: for factor_index, factor in enumerate(coeff): a[row_num + factor_index, offset:offset + len(factor)] = factor row_num += len(coeff) offset += len(factor) # cheating here factor_correlations = a.dot(rho).dot(a.T).clip(-1.0, 1.0) # see if we need to delete the old correlations if overwrite_correlations: self.params['Correlations'] = {} for index1 in range(len(correlation_names) - 1): for index2 in range(index1 + 1, len(correlation_names)): if np.fabs(factor_correlations[index1, index2]) > correlation_cuttoff: self.params['Correlations'][(correlation_names[index1], correlation_names[index2])] = \ factor_correlations[index1, index2]