def __history(self, history_start, history_end, history_part, with_error):
if history_start is not None and history_end is not None:
assert(history_start >= 0)
assert(history_start < len(self._dates))
assert(history_end > history_start)
assert(history_end <= len(self._dates))
dates = self._dates[history_start:history_end]
else:
assert(history_part > 0)
assert(history_part <= 1.0)
dates = self._dates[:len(self._dates)*history_part]
#Get history of parameters
nb_swo_params = len(self._default_params)
columns_orig = ['OrigParam%d' % x for x in range(nb_swo_params)]
columns_hist = ['HistParam%d' % x for x in range(nb_swo_params)]
df_model = du.from_hdf5(self.key_model, du.h5file)
#Pick the best of the two
swo_param_history_orig = df_model.loc[dates][columns_orig]
swo_param_history_hist = df_model.loc[dates][columns_hist]
orig_vs_hist = df_model.loc[dates]['OrigObjective'].values < df_model.loc[dates]['HistObjective'].values
swo_param_history = swo_param_history_orig.copy(deep=True)
swo_param_history[~orig_vs_hist] = swo_param_history_hist[~orig_vs_hist]
#Get history of errors
if with_error:
df_error = du.from_hdf5(self.key_error, du.h5file)
swo_error_history = df_error.loc[dates]
else:
swo_error_history = None
return (dates, swo_param_history, swo_error_history)
def objective_shape(self, predictive_model, date, nb_samples_x=100, nb_samples_y=100):
df = du.from_hdf5(self.key_model, du.h5file)
self.set_date(date)
params_predict = predictive_model.predict((self.values, self._ircurve.values))
params_predict = params_predict.reshape((params_predict.shape[1],))
self.model.setParams(ql.Array(params_predict.tolist()))
print("Predict value = %f" % self.model.value(self.model.params(), self.helpers))
orig_objective = df.ix[date, 'OrigObjective']
hist_objective = df.ix[date, 'HistObjective']
if orig_objective < hist_objective:
name = 'OrigParam'
else:
name = 'HistParam'
params_calib = np.array([df.ix[date, name + '0'],
df.ix[date, name + '1'],
df.ix[date, name + '2'],
df.ix[date, name + '3'],
df.ix[date, name + '4']])
self.model.setParams(ql.Array(params_calib.tolist()))
print("Calib value = %f" % self.model.value(self.model.params(), self.helpers))
params_optim = np.array(self._default_params)
self.model.setParams(self._default_params)
print("Optim value = %f" % self.model.value(self.model.params(), self.helpers))
#The intention is to sample the plane that joins the three points:
#params_predict, params_calib, and params_optim
#A point on that plane can be described by Q(alpha, beta)
#Q(alpha, beta) = params_predict+(alpha-beta(A*B)/(A*A))A+beta B
#with
A = params_calib - params_predict
# and
B = params_optim - params_predict
Aa = np.sqrt(np.dot(A, A))
Ba = np.dot(A, B)/Aa
lim_alpha = np.array([np.min((Ba, 0)), np.max((Ba/Aa, 1))])
da = lim_alpha[1] - lim_alpha[0]
lim_alpha += np.array([-1.0, 1.0])*da/10
lim_beta = np.array([-0.1, 1.1])
ls_alpha = np.linspace(lim_alpha[0], lim_alpha[1], nb_samples_x)
ls_beta = np.linspace(lim_beta[0], lim_beta[1], nb_samples_y)
xv, xy = np.meshgrid(ls_alpha, ls_beta)
sh = xv.shape
samples = [params_predict + (alpha-beta*Ba/Aa)*A+beta*B
for alpha, beta in zip(xv.reshape((-1,)), xy.reshape((-1,)))]
objectives = np.empty((len(samples), ))
for i, params in enumerate(samples):
try:
self.model.setParams(ql.Array(params.tolist()))
objectives[i] = self.model.value(self.model.params(), self.helpers)
except RuntimeError:
objectives[i] = np.nan
return (objectives.reshape(sh), lim_alpha, lim_beta)
def errors(self, predictive_model, date):
df_error = du.from_hdf5(self.key_error, du.h5file)
orig_errors = df_error.loc[date]
self.refdate = ql.Date(1, 1, 1901)
self.set_date(date)
params = predictive_model.predict((self.values, self._ircurve.values))
self.model.setParams(ql.Array(params.tolist()[0]))
_, errors = self.__errors()
return (orig_errors, errors)
def compare_history(self, predictive_model, dates=None, plot_results=False):
df = du.from_hdf5(self.key_model, du.h5file)
self.refdate = ql.Date(1, 1, 1901)
vals = np.zeros((len(df.index),4))
values = np.zeros((len(df.index),13))
if dates is None:
dates = self._dates
method = ql.LevenbergMarquardt()
end_criteria = ql.EndCriteria(250, 200, 1e-7, 1e-7, 1e-7)
if 'constraint' in self._model_dict:
constraint = self._model_dict['constraint']
else:
constraint = ql.NoConstraint()
for i, date in enumerate(dates):
self.set_date(date)
params = predictive_model.predict((self.values, self._ircurve.values))
self.model.setParams(ql.Array(params.tolist()[0]))
meanErrorPrior, _ = self.__errors()
try:
objectivePrior = self.model.value(self.model.params(), self.helpers)
except RuntimeError:
objectivePrior = np.nan
self.model.calibrate(self.helpers, method,
end_criteria, constraint)
meanErrorAfter, _ = self.__errors()
paramsC = self.model.params()
try:
objectiveAfter = self.model.value(self.model.params(), self.helpers)
except RuntimeError:
objectiveAfter = np.nan
orig_mean_error = df.ix[date, 'OrigMeanError']
hist_mean_error = df.ix[date, 'HistMeanError']
orig_objective = df.ix[date, 'OrigObjective']
hist_objective = df.ix[date, 'HistObjective']
values[i, 0] = orig_mean_error
values[i, 1] = hist_mean_error
values[i, 2] = meanErrorPrior
values[i, 3] = orig_objective
values[i, 4] = hist_objective
values[i, 5] = objectivePrior
values[i, 6] = meanErrorAfter
values[i, 7] = objectiveAfter
if orig_objective < hist_objective:
values[i, 8] = df.ix[date, 'OrigParam0']
values[i, 9] = df.ix[date, 'OrigParam1']
values[i, 10] = df.ix[date, 'OrigParam2']
values[i, 11] = df.ix[date, 'OrigParam3']
values[i, 12] = df.ix[date, 'OrigParam4']
else:
values[i, 8] = df.ix[date, 'HistParam0']
values[i, 9] = df.ix[date, 'HistParam1']
values[i, 10] = df.ix[date, 'HistParam2']
values[i, 11] = df.ix[date, 'HistParam3']
values[i, 12] = df.ix[date, 'HistParam4']
print('Date=%s' % date)
print('Vola: Orig=%s Hist=%s ModelPrior=%s ModelAfter=%s' % (orig_mean_error, hist_mean_error, meanErrorPrior, meanErrorAfter))
print('NPV: Orig=%s Hist=%s Model=%s ModelAfter=%s' % (orig_objective, hist_objective, objectivePrior, objectiveAfter))
print('Param0: Cal:%s , Model:%s, Cal-Mod:%s' % (values[i, 8], params[0][0], paramsC[0]))
print('Param1: Cal:%s , Model:%s, Cal-Mod:%s' % (values[i, 9], params[0][1], paramsC[1]))
print('Param2: Cal:%s , Model:%s, Cal-Mod:%s' % (values[i, 10], params[0][2], paramsC[2]))
print('Param3: Cal:%s , Model:%s, Cal-Mod:%s' % (values[i, 11], params[0][3], paramsC[3]))
print('Param4: Cal:%s , Model:%s, Cal-Mod:%s' % (values[i, 12], params[0][4], paramsC[4]))
vals[i, 0] = (meanErrorPrior - orig_mean_error)/orig_mean_error*100.0
vals[i, 1] = (meanErrorPrior - hist_mean_error)/hist_mean_error*100.0
vals[i, 2] = (meanErrorAfter - orig_mean_error)/orig_mean_error*100.0
vals[i, 3] = (meanErrorAfter - hist_mean_error)/hist_mean_error*100.0
print(' impO=%s impH=%s impAfterO=%s impAfterH=%s' % (vals[i, 0], vals[i, 1], vals[i,2], vals[i, 3]))
if plot_results:
r = range(vals.shape[0])
fig = plt.figure(figsize=(16, 16))
f1 = fig.add_subplot(211)
f1.plot(r, vals[:, 0])
f2 = fig.add_subplot(212)
f2.plot(r, vals[:, 1])
return (dates, values)
def calibrate_history(self, *args):
clean = True
start = 0
end = len(self._dates)
if len(args) > 0:
if len(args) != 2:
raise RuntimeError("Invalid input")
clean = False
start = args[0]
if args[1] > 0:
end = args[1]
prev_params = self._default_params
nb_params = len(prev_params)
nb_instruments = len(self.helpers)
columns = ['OrigEvals', 'OptimEvals', 'OrigObjective', 'OrigMeanError',
'HistEvals', 'HistObjective', 'HistMeanError',
'HistObjectivePrior', 'HistMeanErrorPrior']
size_cols = len(columns)
columns = columns + ['OrigParam' + str(x) for x in range(nb_params)]
columns = columns + ['HistParam' + str(x) for x in range(nb_params)]
if clean:
rows_model = np.empty((len(self._dates), len(columns)))
rows_model.fill(np.nan)
rows_error = np.zeros((len(self._dates), nb_instruments))
else:
df_model = du.from_hdf5(self.key_model, du.h5file)
if len(df_model.columns) != len(columns) or \
len(df_model.index) != len(self._dates):
raise RuntimeError("Incompatible file")
rows_model = df_model.values
df_error = du.from_hdf5(self.key_error, du.h5file)
if len(df_error.columns) != nb_instruments or \
len(df_error.index) != len(self._dates):
raise RuntimeError("Incompatible file")
rows_error = df_error.values
header = self.ok_format % ('maturity','length','volatility','implied','error')
dblrule = '=' * len(header)
for iDate in range(start, end):
self.model.setParams(self._default_params)
#Return tuple (date, orig_evals, optim_evals, hist_evals,
#orig_objective, orig_mean_error, hist_objective, hist_mean_error,
#original params, hist parameters, errors)
try:
res = self.calibrate(self._dates[iDate], prev_params)
except RuntimeError as e:
print('')
print(dblrule)
print(self.name + " " + str(self._dates[iDate]))
print("Error: %s" % e)
print(dblrule)
res = (self._dates[iDate], -1, -1, -1, -1, -1, -1, -1, -1, -1,
self._default_params, self._default_params, np.zeros((1, nb_instruments)))
rows_model[iDate, 0:size_cols] = res[1:size_cols+1]
rows_model[iDate, size_cols:size_cols+nb_params] = res[size_cols+1]
rows_model[iDate, size_cols+nb_params:size_cols+nb_params*2] = res[size_cols+2]
rows_error[iDate, :] = res[-1]
prev_params = self.model.params()
df_model = pd.DataFrame(rows_model, index=self._dates, columns = columns)
du.store_hdf5(du.h5file, self.key_model, df_model)
df_error = pd.DataFrame(rows_error, index=self._dates)
du.store_hdf5(du.h5file, self.key_error, df_error)