def _loss_fn(self, matrix, rels_reversed): """Given a numpy array with vectors for u, v and negative samples, computes loss value. Parameters ---------- matrix : numpy.array Array containing vectors for u, v and negative samples, of shape (2 + negative_size, dim). rels_reversed : bool Returns ------- float Computed loss value. Warnings -------- Only used for autograd gradients, since autograd requires a specific function signature. """ vector_u = matrix[0] vectors_v = matrix[1:] if not rels_reversed: entailment_penalty = grad_np.maximum(0, vector_u - vectors_v) # (1 + negative_size, dim). else: entailment_penalty = grad_np.maximum(0, - vector_u + vectors_v) # (1 + negative_size, dim). energy_vec = grad_np.linalg.norm(entailment_penalty, axis=1) ** 2 positive_term = energy_vec[0] negative_terms = energy_vec[1:] return positive_term + grad_np.maximum(0, self.margin - negative_terms).sum()
def get_default_theta(self): if self.k: # kernel2 MF # sn2 + (output_scale + lengthscale) + (output_scale + lengthscales) * 2 theta = np.random.randn(3 + 2 * self.dim) theta[2] = np.maximum( -100, np.log(0.5 * (self.train_x[self.dim - 1].max() - self.train_x[self.dim - 1].min()))) for i in range(self.dim - 1): tmp = np.maximum( -100, np.log(0.5 * (self.train_x[i].max() - self.train_x[i].min()))) theta[4 + i] = tmp theta[4 + self.dim + i] = tmp else: # kernel1 RBF # sn2 + output_scale + lengthscales theta = np.random.randn(2 + self.dim) for i in range(self.dim): theta[2 + i] = np.maximum( -100, np.log(0.5 * (self.train_x[i].max() - self.train_x[i].min()))) theta[0] = np.log(np.std(self.train_y) + 0.000001) # sn2 return theta
def forward(self, X, WEIGHTS, BIASES): # Forward passs through the network... input = activation(previous_layer_output)*Weights + Biases self.X = X inp_next = X for i in range(self.N): output = anp.dot(anp.array(inp_next), WEIGHTS[i]) + anp.array( [BIASES[i]] * inp_next.shape[0]) activation = self.a_hidden_layers[i] if (activation == "relu"): output = anp.maximum(output, 0.0) elif (activation == "sigmoid"): output = (1.0) / (1 + e**(-anp.array(output))) else: pass inp_next = output output = anp.dot(anp.array(inp_next), WEIGHTS[-1]) + anp.array( [BIASES[-1]] * inp_next.shape[0]) if (self.fit_type == "classification" ): # LAst layer is softmax if classifiaction output = self.softmax(output) else: output = anp.maximum(output, 0.0) return output
def equations_in_RealPigments( K, S, r, h, eps=1e-8, model='normal' ): ## r is substrate reflectance, h is layer thicness, all of parameters can either be array or scalar values K = np.maximum(K, eps) S = np.maximum(S, eps) a = 1 + K / S # b=sqrt(a**2-1.0) b = (a**2 - 1.0)**(1 / 2.0) if model == 'normal': d = mycoth(b * S * h) numerator = 1 - r * (a - b * d) denumerator = a - r + b * d R = numerator / denumerator elif model == 'infinite': R = a - b else: print 'wrong option!' return R
def logphi(x): if x**2 < 0.0492: lp0 = -x / np.sqrt(2 * np.pi) c = np.array([ 0.00048204, -0.00142906, 0.0013200243174, 0.0009461589032, -0.0045563339802, 0.00556964649138, 0.00125993961762116, -0.01621575378835404, 0.02629651521057465, -0.001829764677455021, 2 * (1 - np.pi / 3), (4 - np.pi) / 3, 1, 1 ]) f = 0 for i in range(14): f = lp0 * (c[i] + f) return -2 * f - np.log(2) elif x < -11.3137: r = np.array([ 1.2753666447299659525, 5.019049726784267463450, 6.1602098531096305441, 7.409740605964741794425, 2.9788656263939928886 ]) q = np.array([ 2.260528520767326969592, 9.3960340162350541504, 12.048951927855129036034, 17.081440747466004316, 9.608965327192787870698, 3.3690752069827527677 ]) num = 0.5641895835477550741 for i in range(5): num = -x * num / np.sqrt(2) + r[i] den = 1.0 for i in range(6): den = -x * den / np.sqrt(2) + q[i] return np.log(0.5 * np.maximum(0.000001, num / den)) - 0.5 * (x**2) else: return np.log(0.5 * np.maximum(0.000001, (1.0 - erf(-x / np.sqrt(2)))))
def __init__(self, layer_sizes, **kwargs): # set default values for layer sizes, activation, and scale activation = 'relu' # decide on these parameters via user input if 'activation' in kwargs: activation = kwargs['activation'] # switches if activation == 'linear': self.activation = lambda data: data elif activation == 'tanh': self.activation = lambda data: np.tanh(data) elif activation == 'relu': self.activation = lambda data: np.maximum(0, data) elif activation == 'sinc': self.activation = lambda data: np.sinc(data) elif activation == 'sin': self.activation = lambda data: np.sin(data) elif activation == 'maxout': self.activation = lambda data1, data2: np.maximum(data1, data2) # select layer sizes and scale self.layer_sizes = layer_sizes self.scale = 0.1 if 'scale' in kwargs: self.scale = kwargs['scale'] # assign initializer / feature transforms function if activation == 'linear' or activation == 'tanh' or activation == 'relu' or activation == 'sinc' or activation == 'sin': self.initializer = self.standard_initializer self.feature_transforms = self.standard_feature_transforms elif activation == 'maxout': self.initializer = self.maxout_initializer self.feature_transforms = self.maxout_feature_transforms
def __init__(self, **kwargs): if 'layer_sizes' in kwargs: self.layer_sizes = kwargs['layer_sizes'] # Set layer sizes else: # Else create default setup N = 1 # Input dimensions M = 1 # Output dimensions U = 10 # 10-unit hidden layer self.layer_sizes = [ N, U, M ] # Build layer sizes to generate weight matrix if 'scale' in kwargs: self.scale = kwargs['scale'] # Set scale else: self.scale = 0.1 a = 'relu' # Set default activation to ReLU if 'activation' in kwargs: a = kwargs['activation'] # Manually set activation if present self.activation_name = a if a == 'relu': self.activation = lambda data: np.maximum(0, data) elif a == 'tanh': self.activation = lambda data: np.tanh(data) elif a == 'maxout': self.activation = lambda data1, data2: np.maximum(data1, data2) self.weight_matrix = self.maxout_init_weights self.transforms = self.maxout_feature_transforms else: # Manual activation self.activation = kwargs['activation'] if a in ['relu', 'tanh']: self.weight_matrix = self.init_weights self.transforms = self.feature_transforms
def forward_pass(W1, W2, W3, b1, b2, b3, x): """ forward-pass for an fully connected neural network with 2 hidden layers of M neurons Inputs: W1 : (M, 784) weights of first (hidden) layer W2 : (M, M) weights of second (hidden) layer W3 : (10, M) weights of third (output) layer b1 : (M, 1) biases of first (hidden) layer b2 : (M, 1) biases of second (hidden) layer b3 : (10, 1) biases of third (output) layer x : (N, 784) training inputs Outputs: Fhat : (N, 10) output of the neural network at training inputs """ H1 = np.maximum(0, np.dot(x, W1.T) + b1.T) # layer 1 neurons with ReLU activation, shape (N, M) H2 = np.maximum(0, np.dot(H1, W2.T) + b2.T) # layer 2 neurons with ReLU activation, shape (N, M) Fhat = np.dot(H2, W3.T) + b3.T # layer 3 (output) neurons with linear activation, shape (N, 10) a = np.max(Fhat, axis=1) # expand the value of a into a matrix to compute log-exp trick easy A = -1*np.ones(np.shape(Fhat))*a[:, np.newaxis] log_sum = np.log(np.sum(np.exp(np.add(Fhat, A)), axis=1)) # turn sums into a matrix as well Log_Sum = -1*np.ones(np.shape(Fhat))*log_sum[:, np.newaxis] # log-exp trick on each element of Fhat Fhat = np.add(np.add(Fhat, A), Log_Sum) return Fhat
def rand_x(self, n=1): tmp = np.random.uniform(0, 1, (n)) idx = (tmp < 0.4) x = np.random.uniform(-0.5, 0.5, (self.dim, n)) x[:, idx] = (0.05 * np.random.uniform(-0.5, 0.5, (self.dim, idx.sum())).T + self.best_x[1]).T x[:, idx] = np.maximum(-0.5, np.minimum(0.5, x[:, idx])) idx = (tmp < 0.5) * (tmp > 0.4) x[:, idx] = (0.05 * np.random.uniform(-0.5, 0.5, (self.dim, idx.sum())).T + self.best_x[0]).T x[:, idx] = np.maximum(-0.5, np.minimum(0.5, x[:, idx])) idx = (tmp > 0.5) * (tmp < 0.6) num = idx.sum() num_seed = np.minimum(self.dataset['high_y'].shape[1], 5) idx = np.argsort(idx)[-num:] idx1 = np.random.randint(0, self.dim, num) idx2 = np.random.randint(0, num_seed, num) idx3 = np.random.randint(0, num_seed, num) for i in range(num): while idx2[i] == idx3[i]: idx3[i] = random.randint(0, num_seed) x[:idx1[i], i] = self.dataset['high_x'][:idx1[i], -idx2[i]] x[idx1[i]:, i] = self.dataset['high_x'][idx1[i]:, -idx3[i]] return x
def rand_theta(self, scale): if self.k: # kernel2 # sn2 + (output_scale + lengthscale) + (output_scale + lengthscales) * 2 # 1 + 2 + (1 + self.dim - 1)*2 = 3 + 2*self.dim theta = scale * np.random.randn(3 + 2 * self.dim) theta[2] = np.maximum( -100, np.log(0.5 * (self.train_x[self.dim - 1].max() - self.train_x[self.dim - 1].min()))) for i in range(self.dim - 1): tmp = np.maximum( -100, np.log(0.5 * (self.train_x[i].max() - self.train_x[i].min()))) theta[4 + i] = tmp theta[4 + self.dim + i] = tmp else: # kernel1 RBF # sn2 + output_scale + lengthscales, 1 + 1 + self.dim theta = scale * np.random.randn(2 + self.dim) for i in range(self.dim): theta[2 + i] = np.maximum( -100, np.log(0.5 * (self.train_x[i].max() - self.train_x[i].min()))) theta[0] = np.log(np.std(self.train_y)) # sn2 # theta[1] = np.log(np.std(self.train_y)) return theta
def forward_pass(W1, W2, W3, b1, b2, b3, x): """ forward-pass for an fully connected neural network with 2 hidden layers of M neurons Inputs: W1 : (M, 784) weights of first (hidden) layer W2 : (M, M) weights of second (hidden) layer W3 : (10, M) weights of third (output) layer b1 : (M, 1) biases of first (hidden) layer b2 : (M, 1) biases of second (hidden) layer b3 : (10, 1) biases of third (output) layer x : (N, 784) training inputs Outputs: Fhat : (N, 10) output of the neural network at training inputs """ H1 = np.maximum(0, np.dot(x, W1.T) + b1.T) # layer 1 neurons with ReLU activation, shape (N, M) H2 = np.maximum(0, np.dot(H1, W2.T) + b2.T) # layer 2 neurons with ReLU activation, shape (N, M) Fhat = np.dot( H2, W3.T ) + b3.T # layer 3 (output) neurons with linear activation, shape (N, 10) # ####### # Note that the activation function at the output layer is linear! # You must impliment a stable log-softmax activation function at the ouput layer # ####### return Fhat
def rand_theta(self, scale): theta = scale * np.random.randn(self.num_param) theta[0] = 1.0 theta[1] = np.log(np.std(self.low_y)) theta[2] = np.log(np.std(self.high_y)) for i in range(self.dim): theta[4+i] = np.maximum(-100, np.log(0.5*(self.low_x[i].max() - self.low_x[i].min()))) theta[5+self.dim+i] = np.maximum(-100, np.log(0.5*(self.high_x[i].max() - self.high_x[i].min()))) return theta
def odeInt(f, x0, y0, xT): # 1993 Solving Ordinary Differential Equations I, page 169 # initial step size f0 = f(x0, y0) d0 = rmsNorm(y0) d1 = rmsNorm(f0) if d0 < 1e-5 or d1 < 1e-5: h0 = 1e-6 else: h0 = 1e-2 * d0 / d1 y1 = y0 + f0 * h0 f1 = f(x0 + h0, y1) d2 = rmsNorm(f1 - f0) / h0 maxD = np.maximum(d1, d2) if maxD <= 1e-15: h1 = np.maximum(1e-6, h0 * 1e-3) else: h1 = np.power(1e-2 / maxD, 1 / (P + 1)) step = np.minimum(1e2 * h0, h1) # integrate x = x0 y = y0 k1 = f0 while x < xT: rejected = False accepted = False while not accepted: ks, y1, y1h = odeIntStep(step, f, x, y, k1) xNew = x + step scale = ABS_TOL + np.maximum(np.abs(y1), np.abs(y1h)) * REL_TOL errNorm = rmsNorm((y1 - y1h) / scale) if errNorm < 1: accepted = True if errNorm == 0: updateFactor = MAX_UPDATE_FACTOR else: updateFactor = np.minimum( MAX_UPDATE_FACTOR, SAFETY_FACTOR * np.power(errNorm, ERROR_EXP)) if rejected: updateFactor = np.minimum(1, updateFactor) step *= updateFactor else: rejected = True updateFactor = np.maximum( MIN_UPDATE_FACTOR, SAFETY_FACTOR * np.power(errNorm, ERROR_EXP)) step *= updateFactor # interpolate # update x = xNew y = y1 k1 = ks[6] step = xT - x ks, y1, y1h = odeIntStep(step, f, x, y, k1) return y1
def test_minmax(): grad_test(lambda x: ti.min(x, 0), lambda x: np.minimum(x, 0)) grad_test(lambda x: ti.min(x, 1), lambda x: np.minimum(x, 1)) grad_test(lambda x: ti.min(0, x), lambda x: np.minimum(0, x)) grad_test(lambda x: ti.min(1, x), lambda x: np.minimum(1, x)) grad_test(lambda x: ti.max(x, 0), lambda x: np.maximum(x, 0)) grad_test(lambda x: ti.max(x, 1), lambda x: np.maximum(x, 1)) grad_test(lambda x: ti.max(0, x), lambda x: np.maximum(0, x)) grad_test(lambda x: ti.max(1, x), lambda x: np.maximum(1, x))
def compute_xnew(inputs, lambda_): x, dc, dv = unpack(inputs) # avoid dividing by zero outside the design region dv = np.where(np.ravel(args['mask']) > 0, dv, 1) # square root is not defined for negative numbers, which can happen due to # small numerical errors in the computed gradients. xnew = x * np.maximum(-dc / (lambda_ * dv), 0) ** eta lower = np.maximum(0.0, x - max_move) upper = np.minimum(1.0, x + max_move) # note: autograd does not define gradients for np.clip return np.minimum(np.maximum(xnew, lower), upper)
def invert(self, data, input=None, mask=None, tag=None): yhat = smooth(data,20) xhat = self.link(np.clip(yhat, 0.01, np.inf)) xhat = self._invert(xhat, input=input, mask=mask, tag=tag) for t in range(xhat.shape[0]): if np.all(xhat[np.max([0,t-2]):t+3]>0.99) and t>2: xhat[np.minimum(0,t-2):] = 1.01*np.ones(np.shape(xhat[np.minimum(0,t-2):])) xhat[:np.maximum(0,t-2)] = np.clip(xhat[:np.maximum(0,t-2)], -0.5,0.95) if np.abs(xhat[0])>1.0: xhat[0] = 0.5 + 0.01*npr.randn(1,np.shape(xhat)[1]) return xhat
def _loss_fn(self, matrix, rels_reversed): """Given a numpy array with vectors for u, v and negative samples, computes loss value. Parameters ---------- matrix : numpy.array Array containing vectors for u, v and negative samples, of shape (2 + negative_size, dim). rels_reversed : bool Returns ------- float Computed loss value. Warnings -------- Only used for autograd gradients, since autograd requires a specific function signature. """ vector_u = matrix[0] vectors_v = matrix[1:] norm_u = grad_np.linalg.norm(vector_u) norms_v = grad_np.linalg.norm(vectors_v, axis=1) euclidean_dists = grad_np.linalg.norm(vector_u - vectors_v, axis=1) dot_prod = (vector_u * vectors_v).sum(axis=1) if not rels_reversed: # u is x , v is y cos_angle_child = (dot_prod * (1 + norm_u ** 2) - norm_u ** 2 * (1 + norms_v ** 2)) /\ (norm_u * euclidean_dists * grad_np.sqrt(1 + norms_v ** 2 * norm_u ** 2 - 2 * dot_prod)) angles_psi_parent = grad_np.arcsin(self.K * (1 - norm_u**2) / norm_u) # scalar else: # v is x , u is y cos_angle_child = (dot_prod * (1 + norms_v ** 2) - norms_v **2 * (1 + norm_u ** 2) ) /\ (norms_v * euclidean_dists * grad_np.sqrt(1 + norms_v**2 * norm_u**2 - 2 * dot_prod)) angles_psi_parent = grad_np.arcsin(self.K * (1 - norms_v**2) / norms_v) # 1 + neg_size # To avoid numerical errors clipped_cos_angle_child = grad_np.maximum(cos_angle_child, -1 + EPS) clipped_cos_angle_child = grad_np.minimum(clipped_cos_angle_child, 1 - EPS) angles_child = grad_np.arccos(clipped_cos_angle_child) # 1 + neg_size energy_vec = grad_np.maximum(0, angles_child - angles_psi_parent) positive_term = energy_vec[0] negative_terms = energy_vec[1:] return positive_term + grad_np.maximum( 0, self.margin - negative_terms).sum()
def IoG(self, box): inter_xmin = np.maximum(box[:, 0], self.IoG_gt[:, 0]) inter_ymin = np.maximum(box[:, 1], self.IoG_gt[:, 1]) inter_xmax = np.minimum(box[:, 2], self.IoG_gt[:, 2]) inter_ymax = np.minimum(box[:, 3], self.IoG_gt[:, 3]) Iw = np.clip(inter_xmax - inter_xmin, 0, 5) Ih = np.clip(inter_ymax - inter_ymin, 0, 5) I = Iw * Ih G = (self.IoG_gt[:, 2] - self.IoG_gt[:, 0]) * (self.IoG_gt[:, 3] - self.IoG_gt[:, 1]) iog = I / G smln = smoothln(iog) n_p = float(I.shape[0]) return smln.sum() / n_p
def feed_forward(features, w1, b1, w2, b2, w3, b3): HL1 = np.matmul(w1, features) HL1_with_bias = np.add(HL1, b1) HL1_with_bias_and_activation = np.maximum(HL1_with_bias, np.zeros((4, 1))) HL2 = np.matmul(w2, HL1_with_bias_and_activation) HL2_with_bias = np.add(HL2, b2) HL2_with_bias_and_activation = np.maximum(HL2_with_bias, np.zeros((3, 1))) targets_predicted = np.matmul(w3, HL2_with_bias_and_activation) targets_predicted = np.add(targets_predicted, b3) # Use sigmoid for the output activation targets_predicted = sigmoid(targets_predicted) return targets_predicted
def to_safe_common_arr(topics_KV, min_eps=MIN_EPS): ''' Force provided topics_KV array to be numerically safe. Returns ------- topics_KV : 2D array, size K x V minimum value of each row is min_eps each row will sum to 1.0 (+/- min_eps) ''' K, V = topics_KV.shape topics_KV = topics_KV.copy() for rep in range(2): topics_KV /= topics_KV.sum(axis=1)[:, np.newaxis] np.maximum(topics_KV, min_eps, out=topics_KV) return topics_KV
def integrate(self, t, *args, **kwargs): """ This integral is a simple linear interpolant, which I would like to do as a spline. However, I need to do it manually, since it needs to be autograd differentiable, which splines are not. The method here is not especially efficent. """ tau = self.get_param('tau', **kwargs) kappa = self.get_param('kappa', **kwargs) mu = self.get_param('mu', 0.0, **kwargs) f_kappa = self.f_kappa(kappa=kappa, mu=mu) t = np.reshape(t, (-1, 1)) delta = np.diff(tau) each = np.maximum( 0, (t - tau[:-1].reshape(1, -1)) ) each = np.minimum( each, delta.reshape(1, -1) ) return np.sum( each * np.reshape(f_kappa, (1, -1)), 1 ) + (mu * t.ravel())
def IoU(self, box): inter_xmin = np.maximum(box[:, 0], self.IoU_const[:, 0]) inter_ymin = np.maximum(box[:, 1], self.IoU_const[:, 1]) inter_xmax = np.minimum(box[:, 2], self.IoU_const[:, 2]) inter_ymax = np.minimum(box[:, 3], self.IoU_const[:, 3]) Iw = np.clip(inter_xmax - inter_xmin, 0, 5) Ih = np.clip(inter_ymax - inter_ymin, 0, 5) I = Iw * Ih A1 = (self.box[:, 2] - self.box[:, 0]) * (self.box[:, 3] - self.box[:, 1]) A2 = (self.IoU_const[:, 2] - self.IoU_const[:, 0]) * ( self.IoU_const[:, 3] - self.IoU_const[:, 1]) iou = I / (A1 + A2 - I) smln = smoothln(iog) n_p = float(I.shape[0]) return smln.sum()
def _composite_log_likelihood(data, demo, mut_rate=None, truncate_probs=0.0, vector=False, p_missing=None, use_pairwise_diffs=False, **kwargs): try: sfs = data.sfs except AttributeError: sfs = data sfs_probs = np.maximum( expected_sfs(demo, sfs.configs, normalized=True, **kwargs), truncate_probs) log_lik = sfs._integrate_sfs(np.log(sfs_probs), vector=vector) # add on log likelihood of poisson distribution for total number of SNPs if mut_rate is not None: log_lik = log_lik + \ _mut_factor(sfs, demo, mut_rate, vector, p_missing, use_pairwise_diffs) if not vector: log_lik = np.squeeze(log_lik) return log_lik
def __init__(self, **kwargs): # set default values for layer sizes, activation, and scale activation = 'relu' # decide on these parameters via user input if 'activation' in kwargs: activation = kwargs['activation'] # switches if activation == 'linear': self.activation = lambda data: data elif activation == 'tanh': self.activation = lambda data: np.tanh(data) elif activation == 'relu': self.activation = lambda data: np.maximum(0, data) elif activation == 'sinc': self.activation = lambda data: np.sinc(data) elif activation == 'sin': self.activation = lambda data: np.sin(data) else: # user-defined activation self.activation = kwargs['activation'] # select layer sizes and scale N = 1 M = 1 U = 10 self.layer_sizes = [N, U, M] self.scale = 0.1 if 'layer_sizes' in kwargs: self.layer_sizes = kwargs['layer_sizes'] if 'scale' in kwargs: self.scale = kwargs['scale']
def log_logist(theta, Z, nu, mu, Sig): dots = -np.dot(Z, theta.T) log_lik = -np.sum(np.maximum(dots, 0) + np.log1p(np.exp(-np.abs(dots))), axis=0) log_pri = log_multivariate_t(theta[:, :-1], mu, Sig, nu) + log_cauchy( theta[:, -1]) return log_pri + log_lik
def make_positive_definite(m, tol=None): ''' Computes a matrix close to the original matrix m that is positive definite. This function is just a transcript of R' make.positive.definite function. m (2d array): A matrix that is not necessary psd. tol (int): A tolerence level controlling how "different" the psd matrice can be from the original matrix --------------------------------------------------------------- returns (2d array): A psd matrix ''' d = m.shape[0] if (m.shape[1] != d): raise RuntimeError("Input matrix is not square!") eigvalues, eigvect = eigh(m) # Sort the eigen values idx = eigvalues.argsort()[::-1] eigvalues = eigvalues[idx] eigvect = eigvect[:, idx] if (tol == None): tol = d * np.max(np.abs(eigvalues)) * sys.float_info.epsilon delta = 2 * tol tau = np.maximum(0, delta - eigvalues) dm = multi_dot([eigvect, np.diag(tau), eigvect.T]) return (m + dm)
def log_likelihood(self, theta): sn2, sp2, log_lscale, w = self.split_theta(theta) scaled_x = scale_x(log_lscale, self.train_x) Phi = self.calc_Phi(w, scaled_x) Phi_y = np.dot(Phi, self.train_y.T) sp2 = np.maximum(sp2, 0.000001) A = np.dot(Phi, Phi.T) + self.m * sn2 / sp2 * np.eye( self.m) # A.shape: self.m, self.m LA = np.linalg.cholesky(A) logDetA = 0 for i in range(self.m): logDetA = 2 * np.log(LA[i][i]) datafit = (np.dot(self.train_y, self.train_y.T) - np.dot(Phi_y.T, chol_inv(LA, Phi_y))) / sn2 neg_likelihood = 0.5 * (datafit + self.num_train * np.log(2 * np.pi * sn2) + logDetA - self.m * np.log(self.m * sn2 / sp2)) if (np.isnan(neg_likelihood)): neg_likelihood = np.inf w_nobias = self.nn.w_nobias(w, self.dim) l1_reg = self.l1 * np.abs(w_nobias).sum() l2_reg = self.l2 * np.dot(w_nobias, w_nobias.T) neg_likelihood += l1_reg + l2_reg if neg_likelihood < self.loss: self.loss = neg_likelihood self.theta = theta.copy() self.A = A.copy() self.LA = LA.copy() return neg_likelihood
def conditional_probability_alive(self, frequency, recency, T): """ Compute conditional probability alive. Compute the probability that a customer with history (frequency, recency, T) is currently alive. From http://www.brucehardie.com/notes/021/palive_for_BGNBD.pdf Parameters ---------- frequency: array or scalar historical frequency of customer. recency: array or scalar historical recency of customer. T: array or scalar age of the customer. Returns ------- array value representing a probability """ r, alpha, a, b = self._unload_params("r", "alpha", "a", "b") log_div = (r + frequency) * np.log( (alpha + T) / (alpha + recency)) + np.log(a / (b + np.maximum(frequency, 1) - 1)) return np.atleast_1d(np.where(frequency == 0, 1.0, expit(-log_div)))
def _negative_log_likelihood(log_params, freq, rec, T, weights, penalizer_coef): """ The following method for calculatating the *log-likelihood* uses the method specified in section 7 of [2]_. More information can also be found in [3]_. References ---------- .. [2] Fader, Peter S., Bruce G.S. Hardie, and Ka Lok Lee (2005a), "Counting Your Customers the Easy Way: An Alternative to the Pareto/NBD Model," Marketing Science, 24 (2), 275-84. .. [3] http://brucehardie.com/notes/004/ """ warnings.simplefilter(action="ignore", category=FutureWarning) params = np.exp(log_params) r, alpha, a, b = params A_1 = gammaln(r + freq) - gammaln(r) + r * np.log(alpha) A_2 = gammaln(a + b) + gammaln(b + freq) - gammaln(b) - gammaln(a + b + freq) A_3 = -(r + freq) * np.log(alpha + T) A_4 = np.log(a) - np.log(b + np.maximum(freq, 1) - 1) - (r + freq) * np.log(rec + alpha) penalizer_term = penalizer_coef * sum(params**2) ll = weights * (A_1 + A_2 + np.log(np.exp(A_3) + np.exp(A_4) * (freq > 0))) return -ll.sum() / weights.sum() + penalizer_term
def predict(samples_q, X): # First layer K = samples_q.shape[0] # print('shapes', list(shapes)) (m, n) = shapes[0] # print('m={},n={},k={}'.format(m,n,K)) # print('samples_q shape', samples_q.shape) # print('-------------------after shape------------') # samples_q[ : , : m * n ] # print('samples_q shape', samples_q[ : , : m * n ].shape) # print('-------------------after samples_q------------') # samples_q[ : , : m * n ].reshape((n * K, m)) # print('-------------------after samples_q reshape------------') W = samples_q[:, :m * n].reshape((n * K, m)).T # print('-------------------after W------------') b = samples_q[:, m * n:m * n + n].reshape((1, n * K)) # print('-------------------after b------------') a = np.dot(X, W) + b h = np.maximum(a, 0) # Second layer samples_q = samples_q[:, m * n + n:] (m, n) = shapes[1] b = samples_q[:, m * n:m * n + n].T a = np.sum((samples_q[:, :m * n].reshape((1, -1)) * h).reshape( (K * X.shape[0], m)), 1).reshape((X.shape[0], K)) + b return a
def ye_limit(x, trackwidth): k = TrackCurvature(x) N = len(k) lowlimit = -trackwidth/2 * np.ones(N) highlimit = trackwidth/2 * np.ones(N) # use a 5% margin so we can't actually hit the center of curvature lowlimit[k < 0] = np.maximum(0.95/k[k < 0], -trackwidth/2) highlimit[k > 0] = np.minimum(0.95/k[k > 0], trackwidth/2) return lowlimit, highlimit
def projectLB(v, lb): """ project vector v onto constraint v >= lb, used for nonnegative constraint Parameter --------- v: shape(nVars,) input vector lb: float lower bound Return ------ w: shape(nVars,) projection of v to constraint v >= lb """ return np.maximum(v, lb)
def projectSimplex(mat): """ project each row vector to the simplex """ nPoints, nVars = mat.shape mu = np.fliplr(np.sort(mat, axis=1)) sum_hist = np.cumsum(mu, axis=1) flag = (mu - 1./np.tile(np.arange(1,nVars+1),(nPoints,1))*(sum_hist-1) > 0) f_flag = lambda flagPoint: len(flagPoint) - 1 - \ flagPoint[::-1].argmax() lastTrue = map(f_flag, flag) sm_row = sum_hist[np.arange(nPoints), lastTrue] theta = (sm_row - 1)*1./(np.array(lastTrue)+1.) w = np.maximum(mat - np.tile(theta, (nVars,1)).T, 0.) return w
def predict(samples_q, X): # First layer K = samples_q.shape[ 0 ] (m, n) = shapes[ 0 ] W = samples_q[ : , : m * n ].reshape(n * K, m).T b = samples_q[ : , m * n : m * n + n ].reshape(1, n * K) a = np.dot(X, W) + b h = np.maximum(a, 0) # Second layer samples_q = samples_q[ : , m * n + n : ] (m, n) = shapes[ 1 ] b = samples_q[ : , m * n : m * n + n ].T a = np.sum((samples_q[ : , : m * n ].reshape(1, -1) * h).reshape((K * X.shape[ 0 ], m)), 1).reshape((X.shape[ 0 ], K)) + b return a
def fit_gaussian_draw(X, J, seed=28, reg=1e-7, eig_pow=1.0): """ Fit a multivariate normal to the data X (n x d) and draw J points from the fit. - reg: regularizer to use with the covariance matrix - eig_pow: raise eigenvalues of the covariance matrix to this power to construct a new covariance matrix before drawing samples. Useful to shrink the spread of the variance. """ with NumpySeedContext(seed=seed): d = X.shape[1] mean_x = np.mean(X, 0) cov_x = np.cov(X.T) if d==1: cov_x = np.array([[cov_x]]) [evals, evecs] = np.linalg.eig(cov_x) evals = np.maximum(0, np.real(evals)) assert np.all(np.isfinite(evals)) evecs = np.real(evecs) shrunk_cov = evecs.dot(np.diag(evals**eig_pow)).dot(evecs.T) + reg*np.eye(d) V = np.random.multivariate_normal(mean_x, shrunk_cov, J) return V
def projectSimplex_vec(v): """ project vector v onto the probability simplex Parameter --------- v: shape(nVars,) input vector Returns ------- w: shape(nVars,) projection of v onto the probability simplex """ nVars = v.shape[0] mu = np.sort(v,kind='quicksort')[::-1] sm_hist = np.cumsum(mu) flag = (mu - 1./np.arange(1,nVars+1)*(sm_hist-1) > 0) lastTrue = len(flag) - 1 - flag[::-1].argmax() sm_row = sm_hist[lastTrue] theta = 1./(lastTrue+1) * (sm_row - 1) w = np.maximum(v-theta, 0.) return w
def relu(x): return np.maximum(0, x) def init_net_params(scale, layer_sizes, rs=npr.RandomState(0)):
def relu(z): return np.maximum(0, z)
def relu(x): return np.maximum(x, 0.0)
def FFNN(X, P): H = np.dot(X,P[0]) + P[1] H = np.maximum(H, H*0.01) return np.dot(H, P[2])
def NN_Obj(W): # shallow net of the form : max( X^T W[:-1,] )^T W[-1,] return 0.001*np.linalg.norm(np.dot(np.maximum( np.dot( X, W[:-1,] ), 0 ) , W[-1,]) - Y, 2) ** 2; # MSE
def relu(x): return _np.maximum(0, x)
def fun(x): return to_scalar(np.maximum( np.array( [x, x, 0.5]), np.array([[x, 0.5, x], [x, x, 0.5]]))) d_fun = lambda x : to_scalar(grad(fun)(x))
def fun(x): return to_scalar(np.maximum(x, x)) d_fun = lambda x : to_scalar(grad(fun)(x))
def relu(x): np.maximum(0, x)
D = 1 rs = npr.RandomState(0) inputs = np.concatenate([np.linspace(0, 2, num=n_data/2), np.linspace(6, 8, num=n_data/2)]) targets = np.cos(inputs) + rs.randn(n_data) * noise_std inputs = (inputs - 4.0) / 4.0 inputs = inputs.reshape((len(inputs), D)) targets = targets.reshape((len(targets), D)) return inputs, targets if __name__ == '__main__': # Specify inference problem by its unnormalized log-posterior. rbf = lambda x: np.exp(-x**2) relu = lambda x: np.maximum(x, 0.) num_weights, predictions, logprob = \ make_nn_funs(layer_sizes=[1, 20, 20, 1], L2_reg=0.1, noise_variance=0.01, nonlinearity=rbf) inputs, targets = build_toy_dataset() log_posterior = lambda weights, t: logprob(weights, inputs, targets) # Build variational objective. objective, gradient, unpack_params = \ black_box_variational_inference(log_posterior, num_weights, num_samples=20) # Set up figure. fig = plt.figure(figsize=(12, 8), facecolor='white') ax = fig.add_subplot(111, frameon=False)
def relu(v): return np.maximum(v, 0)
def relu(x): return np.maximum(0, x) def sigmoid(x): return 0.5 * (np.tanh(x) + 1.0)
def forward_pass(self, data, params): return np.maximum(data, np.zeros(data.shape))
def nn_predict(inputs, t, params): for W, b in params: outputs = np.dot(inputs, W) + b inputs = np.maximum(0, outputs) return outputs