def testWgtCopula(self):
"""!
@brief Test ability to construct copula
given samples with unequal weights.
Compose two bivariate gauss dists, one with
positive and one with negative depencence.
Sample from dists.
Assign large sample weights to positive gauss
and low sample weights to neg gauss.
Combine weighted samples into a single "X" shaped distribution.
Refit weighted samples and ensure positive depencence
"""
# construct gaussian margins; mu={0, 0}, sd={1.0, 2}
marg1 = Uvm("gauss")(1e-3, 1.)
marg2 = Uvm("gauss")(1e-3, 2.)
# construct gaussian copula positive dep
cop1 = Copula("gauss")
cop1.fittedParams = [0.7]
# construct gaussian copula neg dep
cop2 = Copula("gauss")
cop2.fittedParams = [-0.7]
# draw 1000 samples from each model
n = 1000
rvs1 = marg1.rvs(size=n)
rvs2 = marg2.rvs(size=n)
x1, y1 = cop1.sampleScale(rvs1, rvs2, marg1.cdf, marg2.cdf)
x2, y2 = cop2.sampleScale(rvs1, rvs2, marg1.cdf, marg2.cdf)
# assign weights to each gauss sample group
cop1_wgts = np.ones(n) * 0.95
cop2_wgts = np.ones(n) * 0.05
# combine both gauss models into dbl gauss model
x = np.append(x1, x2)
y = np.append(y1, y2)
wgts = np.append(cop1_wgts, cop2_wgts)
# plot
data = pd.DataFrame([x, y]).T
matrixPairPlot(data, weights=wgts, savefig='x_gauss_original.png')
# fit copula to weighted data
copModel = PairCopula(x, y, wgts)
copModel.copulaTournament()
# verify that a positive dep copula was produced with a
# dep parameter of slightly less than 0.7
x_wt, y_wt = copModel.copulaModel.sampleScale(rvs1, rvs2, marg1.cdf,
marg2.cdf)
self.assertTrue(copModel.copulaModel.kTau() > 0.)
self.assertTrue((copModel.copulaModel.fittedParams[0] > 0.)
& (copModel.copulaModel.fittedParams[0] < 0.7))
# plot
data = pd.DataFrame([x_wt, y_wt]).T
matrixPairPlot(data, savefig='x_gauss_weighted_fit.png')
def testWgtResampledCopula(self):
"""!
@brief Test ability to construct copula
given samples with unequal weights using a resampling strat
"""
np.random.seed(123)
# construct gaussian margins; mu={0, 0}, sd={1.0, 2}
# marg1 = Uvm("gauss")(1e-3, 1.)
marg1 = norm(loc=1e-3, scale=1.0)
# marg2 = Uvm("gauss")(1e-3, 2.)
marg2 = norm(loc=1e-3, scale=2.0)
# construct gaussian copula positive dep
cop1 = Copula("gauss")
cop1.fittedParams = [0.7]
# construct gaussian copula neg dep
cop2 = Copula("gauss")
cop2.fittedParams = [-0.7]
# draw 1000 samples from each model
n = 1000
x1, y1 = cop1.sampleScale(marg1, marg2, n)
x2, y2 = cop2.sampleScale(marg1, marg2, n)
# assign weights to each gauss sample group
cop1_wgts = np.ones(n) * 0.95
cop2_wgts = np.ones(n) * 0.05
# combine both gauss models into dbl gauss model
x = np.append(x1, x2)
y = np.append(y1, y2)
wgts = np.append(cop1_wgts, cop2_wgts)
# fit copula to weighted data
copModel = PairCopula(x, y, wgts, resample=10)
copModel.copulaTournament()
resampled_data = pd.DataFrame([copModel.x, copModel.y]).T
matrixPairPlot(resampled_data, savefig='x_gauss_resampled.png')
# verify that a positive dep copula was produced with a
# dep parameter of slightly less than 0.7
x_wt, y_wt = copModel.copulaModel.sampleScale(marg1, marg2, n)
self.assertTrue(copModel.copulaModel.kTau() > 0.)
self.assertTrue((copModel.copulaModel.fittedParams[0] > 0.)
& (copModel.copulaModel.fittedParams[0] < 0.7))
# plot
data = pd.DataFrame([x_wt, y_wt]).T
matrixPairPlot(data, savefig='x_gauss_resampled_fit.png')
def testGumbelRotate(self):
expectedKtaus = {0: 0.87499, 1: -0.87499, 2: 0.87499, 3: -0.87499}
shapeParam = 8.0
family = {'gumbel': 0,
'gumbel-90': 1,
'gumbel-180': 2,
'gumbel-270': 3}
for rotation, expectedKtau in iteritems(expectedKtaus):
gumbel = Copula("gumbel", rotation)
u, v = gumbel.sample(40000, *(shapeParam,))
g = sns.jointplot(u, v, stat_func=kendalltau)
g.savefig("gumbel_sample_pdf_" + str(rotation) + ".png")
gumbel.fittedParams = (shapeParam,)
c_kTau = gumbel.kTau()
# check kTau
self.assertAlmostEqual(c_kTau, expectedKtau, delta=0.001)
# compute rank corr coeff from resampled data
gumbel_model = PairCopula(u, v, family=family)
gumbel_model.copulaTournament()
print(gumbel_model.copulaParams)
self.assertTrue("gumbel" in gumbel_model.copulaModel.name)
# Ensure fitted shape parameter is same as original
self.assertAlmostEqual(shapeParam, gumbel_model.copulaParams[1][0], delta=0.2)
self.assertEqual(rotation, gumbel_model.copulaParams[3])
# Ensure kTau is nearly the same from resampled data
self.assertAlmostEqual(c_kTau, gumbel_model.copulaModel.kTau(), delta=0.02)
# fit to resampled data
u_model, v_model = gumbel_model.copulaModel.sample(40000)
gumbel_refit = PairCopula(u_model, v_model, family=family)
gumbel_refit.copulaTournament()
u_resample, v_resample = gumbel_refit.copulaModel.sample(4000)
self.assertAlmostEqual(c_kTau, gumbel_refit.copulaModel.kTau(), delta=0.05)
# plot resampled data
g_resample = sns.jointplot(u_resample, v_resample, stat_func=kendalltau)
g_resample.savefig("gumbel_resample_pdf_" + str(rotation) + ".png")
def testKtauFitGauss(self):
# Load matlab data set
stocks = np.loadtxt(dataDir + 'stocks.csv', delimiter=',')
x = stocks[:, 0]
y = stocks[:, 1]
stockModel = PairCopula(x, y)
# Try to fit all copula
stockModel.copulaTournament()
# Check gaussian copula parameters for correctness
self.assertAlmostEqual(stockModel.copulaParams[1][0], 0.73874003, 4)
# Obtain Ktau
kTauFitted = stockModel.copulaModel.kTau()
# Use fitted kTau to specify a new copula
stockModelRefit = Copula("gauss")
# Fit gauss copula param given kTau
fittedParam = stockModelRefit.fitKtau(kTauFitted)[0]
print("Fitted Ktau = %f" % kTauFitted)
print("Expected Theta = %f" % 0.738740)
print("Computed Theta = %f" % fittedParam)
# check result
self.assertAlmostEqual(0.73874003, fittedParam, delta=0.01)
def setTrialCopula(self, family={}):
if not family:
family = self.defaultFamily
self.trialFamily = family
self.copulaBank = {}
for name, rotation in iteritems(self.trialFamily):
self.copulaBank[name] = Copula(name, rotation)
def testGaussRotate(self):
np.random.seed(123)
shapes = {0: 0.7777777, 1: -0.7777777, 2: 0.7777777, 3: -0.7777777}
family = {'gauss': 0}
for rotation, shapeParam in iteritems(shapes):
gauss = Copula("gauss", 0)
u, v = gauss.sample(10000, *(shapeParam,))
g = sns.jointplot(u, v, stat_func=kendalltau)
g.savefig("gauss_sample_pdf_" + str(rotation) + ".png")
gauss.fittedParams = (shapeParam,)
c_kTau = gauss.kTau()
# compute rank corr coeff from resampled data
gauss_model = PairCopula(u, v, family=family)
gauss_model.copulaTournament()
print(gauss_model.copulaParams)
self.assertTrue("gauss" in gauss_model.copulaModel.name)
# Ensure fitted shape parameter is same as original
self.assertAlmostEqual(shapeParam, gauss_model.copulaParams[1][0], delta=0.2)
# Ensure kTau is nearly the same from resampled data
self.assertAlmostEqual(c_kTau, gauss_model.copulaModel.kTau(), delta=0.02)
# fit to resampled data
u_model, v_model = gauss_model.copulaModel.sample(10000)
gauss_refit = PairCopula(u_model, v_model, family=family)
gauss_refit.copulaTournament()
u_resample, v_resample = gauss_refit.copulaModel.sample(2000)
self.assertAlmostEqual(c_kTau, gauss_refit.copulaModel.kTau(), delta=0.05)
self.assertAlmostEqual(shapeParam, gauss_refit.copulaParams[1][0], delta=0.2)
# plot resampled data
g_resample = sns.jointplot(u_resample, v_resample, stat_func=kendalltau)
g_resample.savefig("gauss_resample_pdf_" + str(rotation) + ".png")
def testClaytonSample(self):
# 0 deg
clayton00 = Copula("clayton", 0)
u00, v00 = clayton00.sample(10000, *(0.5, ))
clayton00.fittedParams = (0.5, )
c00_kTau = clayton00.kTau()
# check kTau
self.assertAlmostEqual(c00_kTau, 0.2)
# compute rank corr coeff from resampled data
clayton00_model = PairCopula(u00, v00)
clayton00_model.copulaTournament()
# check that clayton copula won since
# we seeded with samples from a clayton df
self.assertTrue("clayton" in clayton00_model.copulaModel.name)
# Ensure kTau is nearly the same from resampled data
self.assertAlmostEqual(c00_kTau,
clayton00_model.copulaModel.kTau(),
delta=0.02)
# 90 deg
clayton90 = Copula("clayton", 1)
u90, v90 = clayton90.sample(10000, *(0.5, ))
clayton90.fittedParams = (0.5, )
c90_kTau = clayton90.kTau()
self.assertAlmostEqual(c90_kTau, -0.2)
# compute rank corr coeff from resampled data
clayton90_model = PairCopula(u90, v90, family={"clayton": 1})
clayton90_model.copulaTournament()
# check that clayton copula won since
# we seeded with samples from a clayton df
self.assertTrue("clayton" in clayton90_model.copulaModel.name)
self.assertTrue(1 == clayton90_model.copulaModel.rotation)
# Ensure kTau is nearly the same from resampled data
self.assertAlmostEqual(c90_kTau,
clayton90_model.copulaModel.kTau(),
delta=0.02)
# 180 deg
clayton180 = Copula("clayton", 2)
u180, v180 = clayton180.sample(1000, *(0.5, ))
# 270 deg
clayton270 = Copula("clayton", 3)
u270, v270 = clayton270.sample(1000, *(0.5, ))
def testClaytonRotate(self):
expectedKtaus = {
0: 0.7777777,
1: -0.7777777,
2: 0.7777777,
3: -0.7777777
}
shapeParam = 7.0
family = {
'gauss': 0,
'clayton': 0,
'clayton-90': 1,
'clayton-180': 2,
'clayton-270': 3,
'gumbel': 0,
'gumbel-90': 1,
'gumbel-180': 2,
'gumbel-270': 3,
}
for rotation, expectedKtau in iteritems(expectedKtaus):
clayton = Copula("clayton", rotation)
u, v = clayton.sample(40000, *(shapeParam, ))
g = sns.jointplot(u, v)
g.savefig("clayton_sample_pdf_" + str(rotation) + ".png")
clayton.fittedParams = (shapeParam, )
c_kTau = clayton.kTau()
# check kTau
self.assertAlmostEqual(c_kTau, expectedKtau, delta=0.001)
# compute rank corr coeff from resampled data
clayton_model = PairCopula(u, v, family=family)
clayton_model.copulaTournament()
print(clayton_model.copulaParams)
self.assertTrue("clayton" in clayton_model.copulaModel.name)
# Ensure fitted shape parameter is same as original
self.assertAlmostEqual(shapeParam,
clayton_model.copulaParams[1][0],
delta=0.2)
self.assertEqual(rotation, clayton_model.copulaParams[3])
# Ensure kTau is nearly the same from resampled data
self.assertAlmostEqual(c_kTau,
clayton_model.copulaModel.kTau(),
delta=0.02)
# fit to resampled data
u_model, v_model = clayton_model.copulaModel.sample(10000)
clayton_refit = PairCopula(u_model, v_model, family=family)
clayton_refit.copulaTournament()
u_resample, v_resample = clayton_refit.copulaModel.sample(1000)
self.assertAlmostEqual(c_kTau,
clayton_refit.copulaModel.kTau(),
delta=0.05)
# plot resampled data
g_resample = sns.jointplot(u_resample, v_resample)
g_resample.savefig("clayton_resample_pdf_" + str(rotation) +
".png")
def testFrankSample(self):
# 0 deg
frank00 = Copula("frank", 0)
u00, v00 = frank00.sample(30000, *(8.0, ))
frank00.fittedParams = (8.0, )
c00_kTau = frank00.kTau()
# check kTau
print(c00_kTau)
self.assertAlmostEqual(c00_kTau, 0.602619667, 6)
# compute rank corr coeff from resampled data
frank00_model = PairCopula(u00, v00, family={"frank": 0})
frank00_model.copulaTournament()
print(frank00_model.copulaParams)
# check that frank copula won since
# we seeded with samples from a frank df
self.assertTrue("frank" in frank00_model.copulaModel.name)
# Ensure kTau is nearly the same from resampled data
self.assertAlmostEqual(c00_kTau,
frank00_model.copulaModel.kTau(),
delta=0.02)
# 90 deg
frank90 = Copula("frank", 1)
u90, v90 = frank90.sample(30000, *(8.0, ))
frank90.fittedParams = (8.0, )
c90_kTau = frank90.kTau()
self.assertAlmostEqual(c90_kTau, -0.602619667, 6)
# compute rank corr coeff from resampled data
frank90_model = PairCopula(u90, v90, family={"frank": 1})
frank90_model.copulaTournament()
print(frank90_model.copulaParams)
# check that frank copula won since
# we seeded with samples from a frank df
self.assertTrue("frank" in frank90_model.copulaModel.name)
# Ensure kTau is nearly the same from resampled data
self.assertAlmostEqual(c90_kTau,
frank90_model.copulaModel.kTau(),
delta=0.02)
def testGumbelSample(self):
# TODO: TEST FAILS FOR SMALL GUMBEL PARAMETER VALUES
# 0 deg
gumbel00 = Copula("gumbel", 0)
u00, v00 = gumbel00.sample(10000, *(8.0, ))
gumbel00.fittedParams = (8.0, )
c00_kTau = gumbel00.kTau()
# check kTau
self.assertAlmostEqual(c00_kTau, 0.8749999999999, 6)
# compute rank corr coeff from resampled data
gumbel00_model = PairCopula(u00, v00, family={"gumbel": 0})
gumbel00_model.copulaTournament()
print(gumbel00_model.copulaParams)
# check that gumbel copula won since
# we seeded with samples from a gumbel df
self.assertTrue("gumbel" in gumbel00_model.copulaModel.name)
# Ensure kTau is nearly the same from resampled data
self.assertAlmostEqual(c00_kTau,
gumbel00_model.copulaModel.kTau(),
delta=0.04)
# 90 deg
gumbel90 = Copula("gumbel", 1)
u90, v90 = gumbel90.sample(10000, *(8.0, ))
gumbel90.fittedParams = (8.0, )
c90_kTau = gumbel90.kTau()
self.assertAlmostEqual(c90_kTau, -0.87499999999, 6)
# compute rank corr coeff from resampled data
gumbel90_model = PairCopula(u90, v90, family={"gumbel": 1})
gumbel90_model.copulaTournament()
print(gumbel90_model.copulaParams)
# check that gumbel copula won since
# we seeded with samples from a gumbel df
self.assertTrue("gumbel" in gumbel90_model.copulaModel.name)
# Ensure kTau is nearly the same from resampled data
self.assertAlmostEqual(c90_kTau,
gumbel90_model.copulaModel.kTau(),
delta=0.04)
def testFrankRotate(self):
expectedKtaus = {
0: 0.602619667,
1: -0.602619667,
2: 0.602619667,
3: -0.602619667
}
shapeParam = 8.0
family = {
'gauss': 0,
'frank': 0,
'frank-90': 1,
'frank-180': 2,
'frank-270': 3,
}
for rotation, expectedKtau in iteritems(expectedKtaus):
frank = Copula("frank", rotation)
u, v = frank.sample(10000, *(shapeParam, ))
g = sns.jointplot(u, v, stat_func=kendalltau)
g.savefig("frank_sample_pdf_" + str(rotation) + ".png")
frank.fittedParams = (shapeParam, )
c_kTau = frank.kTau()
# check kTau
self.assertAlmostEqual(c_kTau, expectedKtau, delta=0.001)
# compute rank corr coeff from resampled data
frank_model = PairCopula(u, v, family=family)
frank_model.copulaTournament()
print(frank_model.copulaParams)
self.assertTrue("frank" in frank_model.copulaModel.name)
# Ensure refitted shape parameter is same as original
self.assertAlmostEqual(shapeParam,
frank_model.copulaParams[1][0],
delta=0.2)
# Ensure kTau is nearly the same from resampled data
self.assertAlmostEqual(c_kTau,
frank_model.copulaModel.kTau(),
delta=0.02)
# fit to resampled data
u_model, v_model = frank_model.copulaModel.sample(10000)
frank_refit = PairCopula(u_model, v_model, family=family)
frank_refit.copulaTournament()
u_resample, v_resample = frank_refit.copulaModel.sample(1000)
self.assertAlmostEqual(c_kTau,
frank_refit.copulaModel.kTau(),
delta=0.05)
# plot resampled data
g_resample = sns.jointplot(u_resample,
v_resample,
stat_func=kendalltau)
g_resample.savefig("frank_resample_pdf_" + str(rotation) + ".png")
def compute_kc_metric(self, copula, log=True, log_dir='Kc_logs'):
"""!
@brief Compute l2 norm of differences between the empirical Kc function
and the fitted copula's Kc function.
@param copula starvine.bvcopula.copula.copula_base.CopulaBase instance
"""
# rotate current data into the proposed copula orientation
if copula.name in ["gauss", "t"
] and copula.rotation == 0 and self.empKTau_ < 0:
# force positive correlation
rt_UU, rt_VV = self.rotate_data(self.UU, self.VV, -1)
else:
rt_UU, rt_VV = self.rotate_data(self.UU, self.VV, copula.rotation)
# compute emperical Kc on the rotated data
rt_t_emp, rt_kc_emp = jit_empKc(rt_UU, rt_VV)
# fit the un-rotated copula
base_copula = Copula(copula.name, 0)
base_copula.fitMLE(rt_UU, rt_VV, *(
None,
None,
), weights=self.weights)
mask = ((rt_t_emp > 0.005) & (rt_t_emp < 1.0))
kc_metric = []
for i in range(4):
fitted_kc = base_copula.kC(rt_t_emp[mask])
kc_metric.append(np.linalg.norm(fitted_kc - rt_kc_emp[mask]))
if log:
if not os.path.exists(log_dir):
os.makedirs(log_dir)
np.savetxt(log_dir + '/kc_log_' + str(base_copula.name) + "_" +
str(copula.rotation) + '.txt',
np.array([rt_t_emp[mask], fitted_kc,
rt_kc_emp[mask]]).T,
header="Kendalls fn log for Copula: " +
str(base_copula.name) + "_" + str(copula.rotation))
return np.average(kc_metric)
def setTrialCopula(self, family):
self.trialFamily = family
self.copulaBank = {}
for name, rotation in iteritems(self.trialFamily):
self.copulaBank[name] = Copula(name, rotation)
def testKtauFitGauss(self):
# Load matlab data set
stocks = np.loadtxt(dataDir + 'stocks.csv', delimiter=',')
x = stocks[:, 0]
y = stocks[:, 1]
stockModel = PairCopula(x, y)
# Try to fit all copula
stockModel.copulaTournament()
# Check gaussian copula parameters for correctness
self.assertAlmostEqual(stockModel.copulaParams[1], 0.73874003, 4)
# Obtain Ktau
kTauFitted = stockModel.copulaModel.kTau()
# Use fitted kTau to specify a new copula
stockModelRefit = Copula("gauss")
# Fit gauss copula param given kTau
fittedParam = stockModelRefit.fitKtau(kTauFitted)[0]
print("Fitted Ktau = %f" % kTauFitted)
print("Expected Theta = %f" % 0.738740)
print("Computed Theta = %f" % fittedParam)
# Compute kendall's function
t_in = np.linspace(0.01, 0.99, 20)
k_c = stockModelRefit.kC(t_in)
print("Gauss Kendall's function")
print(t_in)
print(k_c)
# And specify gumbel model
stockModelRefit = Copula("gumbel")
fittedParam = stockModelRefit.fitKtau(kTauFitted)[0]
k_c_gumbel = stockModelRefit.kC(t_in)
print("Gumbel Kendall's function")
print(t_in)
print(k_c_gumbel)
print("Fitted Gumbel Param= %f" % fittedParam)
# expected gubel param for ktau=0.9, 10.0
# expected gubel param for ktau=0.8, 5.0
# expected gubel param for ktau=0.7, 3.33
# check that k_c is monotonic
self.assertTrue(k_c[0] < k_c[1])
self.assertTrue(k_c[-2] < k_c[-1])
# Compute emperical kendall's function
t_emp, kc_emp = stockModel.empKc()
# plot
try:
import pylab as pl
pl.figure()
pl.plot(t_in, t_in - k_c, label="gauss")
pl.plot(t_in, t_in - k_c_gumbel, label="gumbel")
pl.plot(t_emp, t_emp - kc_emp, label="emperical")
pl.xlabel("t")
pl.ylabel("t - Kc(t)")
pl.legend()
pl.savefig("ktau_function_plot.png")
pl.close()
except:
pass