def kurtosis_score(x, dim=0):
'''Test whether a dataset has normal kurtosis.
This function tests the null hypothesis that the kurtosis
of the population from which the sample was drawn is that
of the normal distribution: ``kurtosis = 3(n-1)/(n+1)``.
ripoff from: `scipy.stats.kurtosistest`.
Args:
a: Array of the sample data
axis: Axis along which to compute test. Default is 0. If None,
compute over the whole array `a`.
Returns:
statistic: The computed z-score for this test.
p-value: A 2-sided chi squared probability for the hypothesis test.
'''
x, n, dim = _x_n_dim(x, dim)
if n < 20:
raise ValueError(
"Number of elements has to be >= 20 to compute kurtosis")
b2 = (x**4).mean(dim) / (x**2).mean(dim)**2
E = 3.0 * (n - 1) / (n + 1)
varb2 = 24.0 * n * (n - 2) * (n - 3) / ((n + 1)**2 * (n + 3) * (n + 5))
x = (b2 - E) / math.sqrt(varb2)
sqrtbeta1 = 6.0 * (n * n - 5 * n + 2) / ((n + 7) * (n + 9)) *\
math.sqrt((6.0 * (n + 3) * (n + 5)) / (n * (n - 2) * (n - 3)))
A = 6.0 + 8.0 / sqrtbeta1 * \
(2.0 / sqrtbeta1 + math.sqrt(1 + 4.0 / (sqrtbeta1**2)))
term1 = 1 - 2 / (9.0 * A)
denom = 1 + x * math.sqrt(2 / (A - 4.0))
term2 = torch.sign(denom) * torch.pow((1 - 2.0 / A) /
torch.abs(denom), 1 / 3.0)
Z = (term1 - term2) / math.sqrt(2 / (9.0 * A))
return Z, 1 + torch.erf(-math.sqrt(0.5) * torch.abs(Z))
def skewness_score(x, dim=0):
'''Test whether the skew is different from the normal distribution.
This function tests the null hypothesis that the skewness of
the population that the sample was drawn from is the same
as that of a corresponding normal distribution.
ripoff from: `scipy.stats.skewtest`.
Args:
a: Array of the sample data
axis: Axis along which to compute test. Default is 0. If None,
compute over the whole array `a`.
Returns:
statistic: The computed z-score for this test.
p-value: A 2-sided chi squared probability for the hypothesis test.
'''
x, n, dim = _x_n_dim(x, dim)
b2 = (x**3).mean(dim) / (x**2).mean(dim)**1.5
y = b2 * math.sqrt(((n + 1) * (n + 3)) / (6.0 * (n - 2)))
beta2 = 3.0 * (n**2 + 27 * n - 70) * (n + 1) * (n + 3) /\
((n - 2.0) * (n + 5) * (n + 7) * (n + 9))
W2 = -1.0 + math.sqrt(2 * (beta2 - 1))
delta = 1.0 / math.sqrt(0.5 * math.log(W2))
alpha = math.sqrt(2.0 / (W2 - 1))
y[y == 0] = 1
yalpha = y / alpha
Z = delta * torch.log(yalpha + torch.sqrt(yalpha**2 + 1))
return Z, 1 + torch.erf(-math.sqrt(0.5) * torch.abs(Z))
def gelu(x):
"""Implementation of the gelu activation function.
For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
"""
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
def gelu(x):
if hasattr(torch.nn.functional, 'gelu'):
return torch.nn.functional.gelu(x.float()).type_as(x)
else:
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
def gelu(tensor):
return 0.5 * tensor * (1.0 + torch.erf(tensor / math.sqrt(2.0)))
def gelu(x):
'''Gaussian Error Linear Unitという活性化関数です。
LeLUが0でカクっと不連続なので、そこを連続になるように滑らかにした形のLeLUです。
'''
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
def f_gelu(x):
pdtype = x.dtype
x = x.float()
y = x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
return y.to(pdtype)
def test_erf(x, y):
c = torch.erf(torch.add(x, y))
return c
def _gelu_python(self, input: Tensor) -> Tensor:
return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0)))
def cdf(self, x, mu=0., sig=1.):
return 0.5 * (1 + torch.erf((x - mu) / (sig * math.sqrt(2))))
def probit(x):
return 0.5 * (1.0 + torch.erf(x / (2.0**0.5))) * (1 - 2e-3) + 1e-3
def GPupdate(self,
x,
y,
kfunc,
kparams,
alpha=None,
C=None,
bvs=None,
update_method='c',
rawx=None):
kstar = kfunc(x, x, kparams)
#noisevar = kparams['noisevar']
noise = torch.exp(self.noisevar) + 0.01 # for numerical stability
mx = self.getPriorMean(x)
if bvs is None:
# first ever update
alpha = (y - mx) / kstar
# print('alpha', alpha)
C = Variable(dtype(np.zeros((1, 1))))
if usecuda:
C = C.cuda()
C[0] = -1 / (kstar + noise)
bvs = x
else:
# subsequent updates (projected process approximation)
nbvs = bvs.shape[0]
k = self.getKernelMatrix(bvs, kfunc, kparams, X2=x)
m = torch.dot(k.view(-1), alpha.view(-1)) - mx
# print("\nalpha", alpha)
# print("k", torch.t(k))
# print("m", m)
Ck = torch.matmul(C, k)
if self.verbose:
print('Ck', Ck)
s2 = kstar + torch.matmul(torch.t(k), Ck) + noise
if (s2 < self.minvar).all():
print("==== WARNING! =====")
print('m', m, 's2', s2, 'k', k, 'Ck', Ck, 'alpha', alpha)
s2[0] = self.minvar[0]
sx = torch.sqrt(s2)
z0 = m / sx
z = y * z0
Erfz = (torch.erf(z / self.sqrt2) + 1) / 2
# print('Erfz', Erfz)
regl = self.reg_const
constl = regl * 1.0 / np.sqrt(2 * np.pi)
dErfz = torch.exp(-torch.pow(z, 2.0) / 2.0) * constl
dErfz2 = dErfz * (-z) * regl
if update_method == 'c':
rclp = 1.0 # clamp value for numerical stability
q = (y / sx) * (dErfz / Erfz) # EQUATION 11 of the paper
q = torch.clamp(q, -rclp, rclp)
r = (1.0 / s2) * ((dErfz2 / Erfz) - torch.pow(
(dErfz / Erfz), 2.0))
r = torch.clamp(r, -rclp, rclp)
# print('r', r)
else:
# regression updates
r = -1.0 / (s2)
q = -r * (y - m)
if (r != r).any() or (q != q).any():
return (alpha, C, bvs)
# grow and update alpha and C
s = torch.cat((Ck.view(-1), self.one.view(-1))).view(1, -1)
alpha = torch.cat((alpha.view(-1), self.zero.view(-1)))
nbvs += 1
bvs = torch.cat((bvs, x))
zerocol = Variable(dtype(np.zeros((nbvs - 1, 1))),
requires_grad=False)
zerorow = Variable(dtype(np.zeros((1, nbvs))), requires_grad=False)
if usecuda:
zerocol = zerocol.cuda()
zerorow = zerorow.cuda()
C = torch.cat((C, zerocol), 1)
C = torch.cat((C, zerorow))
C = C + r * torch.matmul(s.t(), s)
alpha = alpha + s * q
# print("q", q)
# print("alpha 2", alpha)
# print("C", C)
# print("k", k)
# print("Ck", Ck)
return (alpha, C, bvs)
def fastncdf(x, mu, sigma2):
return 0.5 * (1 + torch.erf((x - mu) / torch.sqrt(sigma2 * 2)))
def gelu(x):
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
def regularizer(self, x):
''' Gaussian CDF. '''
return 0.5 * (1 + torch.erf(x / math.sqrt(2)))
def cdf(self, value):
self._validate_log_prob_arg(value)
return 0.5 * (1 + torch.erf((value - self.loc) * self.scale.reciprocal() / math.sqrt(2)))
def forward(self, x): # pylint: disable=no-self-use
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
def normal_logcdf(x, mean, log_scale):
z = (x - mean) * torch.exp(-log_scale)
return torch.log(0.5 * torch.erf(z / np.sqrt(2)) + 0.5 + 1e-10)
def normal_standard_cdf(val):
return 1 / 2 * (1 + torch.erf(val / np.sqrt(2)))
def erf(x):
return torch.erf(x)
def normcdf(a, b):
c = 0.7071067811865476 # = math.sqrt(0.5)
return 0.5 * (torch.erf(c * b) - torch.erf(c * a))
def bias_gelu(bias, y):
x = bias + y
return x * 0.5 * (1.0 + torch.erf(x / 1.41421))
def gelu(x):
""""Implementation of the gelu activation function by Hugging Face."""
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
def norm_cdf(x, mean=0, std=1):
# Computes standard normal cumulative distribution function
x = (x - mean) / std
return (1. + torch.erf(x / math.sqrt(2.))) / 2.
def _compute_cdf(self, eta):
e = torch.erf(eta / self.c2)
cdf = 0.5 * (1.0 + e)
return cdf
def normcdf(value, mu=0.0, stddev=1.0):
sinv = (1.0 / stddev) if isinstance(stddev, Number) else stddev.reciprocal()
return 0.5 * (1.0 + torch.erf((value - mu) * sinv / np.sqrt(2.0)))
def gelu(features: torch.Tensor, approximate: bool = False):
if approximate:
return 0.5 * features * (1.0 + nn.tanh(0.7978845608028654 * (features + 0.044715 * (features ** 3))))
else:
return 0.5 * features * (1.0 + torch.erf(features / 1.4142135623730951))
def fn_test_erf(x):
return F.relu(torch.erf(x) - torch.erfc(x))
def cdf(self, value):
if self._validate_args:
self._validate_sample(value)
return 0.5 * (1 + torch.erf((value - self.loc) * self.scale.reciprocal() / math.sqrt(2)))
def gelu(x: torch.Tensor) -> torch.Tensor:
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
def forward(self, x):
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
def gelu(x: torch.Tensor) -> torch.Tensor:
if hasattr(torch.nn.functional, "gelu"):
return torch.nn.functional.gelu(x.float()).type_as(x)
else:
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
def n_dist(x):
"""
Cumulative distribution function of the standard normal distribution.
"""
return 0.5 * (1 + torch.erf(x / math.sqrt(2)))
def gelu(x):
"""Implementation of the gelu activation function.
For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
"""
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
def cdf(self, value):
if self._validate_args:
self._validate_sample(value)
return 0.5 * (1 + torch.erf((value - self.loc) * self.scale.reciprocal() / math.sqrt(2)))
