def predict(Theta1, Theta2, X, y): m = X.shape[0] h1 = sigmoid(X.dot(Theta1.T)) h1 = hstack((ones((m,1)),h1)) h2 = sigmoid(h1.dot(Theta2.T)) p = argmax(h2,axis=1) return mean(p == y.T.tolist())*100\
def predict_value(Theta1, Theta2, frame): m = X.shape[0] h1 = sigmoid(X.dot(Theta1.T)) h1 = hstack((ones((m,1)),h1)) h2 = sigmoid(h1.dot(Theta2.T)) p = argmax(h2,axis=1) return p
def eval_mixed_guidance(data, v): assert False, "TODO: Normalize by amount of guidance" x = data.x_neighbor x_low = data.x_low x_high = data.x_high scale = data.scale w, b = unpack_linear(v) y = apply_linear(x, w, b) y_low = apply_linear(x_low, w, b) y_high = apply_linear(x_high, w, b) """ if (y_low + eps >= y_high).any() or (y + eps >= y_high).any(): return np.inf """ sig1 = sigmoid((y_high - y_low) / scale) sig2 = sigmoid((2 * y - y_high - y_low) / scale) diff = sig1 - sig2 # assert (np.sign(diff) > 0).all() small_constant = getattr(data, "eps", eps) # assert False, 'Should this be infinity instead?' # diff[diff < 0] = 0 vals2 = -np.log(diff + small_constant) I = np.isnan(vals2) if I.any(): # print 'eval_linear_neighbor_logistic: inf = ' + str(I.mean()) return np.inf val2 = vals2.sum() # assert norm(val - val2)/norm(val) < 1e-6 return val2
def eval_mixed_guidance(data, v): assert False, "TODO: Normalize by amount of guidance" w, b = unpack_linear(v) x = data.x_bound bounds = data.bounds scale = data.scale y = apply_linear(x, w, b) assert y.size == bounds.shape[0] c1 = bounds[:, 0] c2 = bounds[:, 1] """ y_c1 = (y-c1)/scale y_c2 = (y-c2)/scale loss_num = np.log(np.exp(y_c1) - np.exp(y_c2)) loss_denom = np.log(1 + np.exp(y_c1) + np.exp(y_c2) + np.exp(y_c1+y_c2)) vals = (-loss_num + loss_denom) val = vals.sum() """ sig1 = sigmoid((c2 - y) / scale) sig2 = sigmoid((c1 - y) / scale) small_constant = getattr(data, "eps", eps) diff = sig1 - sig2 + small_constant vals2 = -np.log(sig1 - sig2 + small_constant) val2 = vals2.sum() I = np.isnan(vals2) if I.any(): assert False # assert norm(val - val2)/norm(val) < 1e-6 return val2
def cost(Theta, X, y, lmda): Theta1 = Theta[:401*25].reshape((25,401)) Theta2 = Theta[401*25:].reshape((10,26)) m = X.shape[0] a1 = X z2 = a1.dot(Theta1.T) a2 =hstack((ones((m,1)),sigmoid(z2))) z3 =a2.dot(Theta2.T) h = sigmoid(z3) k = h.shape[1] Y = zeros((m,k)) Y[arange(m),y.T] = 1 d3 = h-Y d2 = d3.dot(Theta2[:,1:])*(sigmoid(z2)*(1-sigmoid(z2))) D2 = d3.T.dot(a2) D1 = d2.T.dot(a1) reg = lmda/float(m) Theta1_grad = (D1/m)+(reg*hstack(((zeros((Theta1.shape[0],1))), Theta1[:,1:]))) Theta2_grad = (D2/m)+reg*hstack(((zeros((Theta2.shape[0],1))), Theta2[:,1:])) reg = sum(sum(Theta1[:,1:]**2,axis=1))+sum(sum(Theta2[:,1:]**2,axis=1)) J = (1.0/m)*sum(sum(-Y*log(h)-((1-Y)*log(1-h)),axis=1))+(lmda/(2.0*m))*reg grad = hstack((Theta1_grad.ravel(),Theta2_grad.ravel())) return J,grad
def elbo_grad(z_sample, mu, sigma_sq, y, X, P, prior_sigma): score_mu = (z_sample - mu)/(sigma_sq) score_logsigma_sq = (-1/(2*sigma_sq) + np.power((z_sample - mu),2)/(2*np.power(sigma_sq,2))) * sigma_sq log_p = np.sum(y * np.log(sigmoid(np.dot(X,z_sample))) + (1-y) * np.log(1-sigmoid(np.dot(X,z_sample))))\ + np.sum(norm.logpdf(z_sample, np.zeros(P), prior_sigma*np.ones(P))) log_q = np.sum(norm.logpdf(z_sample, mu, np.sqrt(sigma_sq))) return np.concatenate([score_mu,score_logsigma_sq])*(log_p - log_q)
def feed_forward(self, input_value): self._input_layer = np.insert(input_value, 0, 1, axis=1) # print self._input_layer self._hidden_layer = np.insert(sigmoid(np.dot(self._input_layer, self._hidden_layer_weights)), 0, 1, axis=1) self._output_layer = sigmoid(np.dot(self._hidden_layer, self._output_layer_weights)) # print self._output_layer return self._output_layer
def pushdown(self, hid_pats): """ push hidden pats into visible, BUT JUST CALC PROBS """ if self.DROPOUT: dropout = rng.randint(2, size=(hid_pats.shape[0], self.num_hid)) vis_prob1 = sigmoid(np.dot(hid_pats*dropout, self.W) + self.vis_bias) else: FACTOR = 0.5 vis_prob1 = sigmoid(np.dot(hid_pats, FACTOR*self.W) + self.vis_bias) return vis_prob1
def pushup(self, vis_pats, noise=True): """ push visible pats into hidden layer""" psi = np.dot(vis_pats, self.W.T) + self.hid_bias if self.hid_type == 'logistic': # BERNOULLI units hid_prob1 = sigmoid(psi) if noise == False: return hid_prob1 elif noise == True: return 1*(hid_prob1 > rng.random(size=hid_prob1.shape)) elif self.hid_type == 'relu': # ReLU units if noise == True: psi = psi + rng.normal(0.,0.001+np.sqrt(sigmoid(psi)), size=psi.shape) return np.maximum(0.0, psi)
def _train(start, stop): X = alias.train_x y = alias.train_y N = alias.N weights = alias.weights weights0 = alias.weights0 weights1 = alias.weights1 weights10 = alias.weights10 alpha = alias.alpha w1 = np.zeros_like(weights1) w10 = np.zeros_like(weights10) w = np.zeros_like(weights) w0 = np.zeros_like(weights0) for i in range(start, stop): x = X[i] a1 = x z2 = x @ weights + weights0 a2 = sigmoid(z2) z3 = a2 @ weights1 + weights10 a3 = sigmoid(z3) delta3 = -(y[i] - a3) delta2 = weights1 @ delta3 * a2 * (1 - a2) w1 += alpha * (a2[:, None] @ delta3[None, :]) w10 += alpha * delta3 w += alpha * (a1[:, None] @ delta2[None, :]) w0 += alpha * delta2 w0 /= N w /= N w10 /= N w1 /= N # wc = WeightCache() # wc.weights = w # wc.weights0 = w0 # wc.weights1 = w1 # wc.weights10= w10 # update_cache_weight.delay(wc.key) # with one_lock: alias.weights -= w alias.weights0 -= w0 alias.weights1 -= w1 alias.weights10 -= w10 cache.incr('n_s', stop-start)
def transform(in_vector, conn_matrix): """ Transform in_vector into out_vector using conn_matrix Last column of conn_matrix is bias """ raw_out = conn_matrix.dot(np.append(in_vector, 1.0)) return sigmoid(raw_out)
def predictions(dbhandler, theta, candidates, expected_prob): for c in candidates: # [Intercept item, Degree, Python, Octave, C++, Java] test = np.array([1, c[2], c[3], c[4], c[5], c[6]]) p = sigmoid(test.dot(theta)) # If the algorithm predicts a value > expected_prob, HR will contact the candidate if p > expected_prob: query = "INSERT INTO good_candidates VALUES(%s,%s,%s,%s,%s,%s,%s,%s)" data = ( c[0], c[1], c[2], c[3], c[4], c[5], c[6], float(p), ) # Must cast to float because mysqlconnector doesn't recognize numpy formats dbhandler.execute(query, data) # Add a new entry to the training set with this candidate info query = "INSERT INTO training_set VALUES(%s,%s,%s,%s,%s,%s)" data = (c[2], c[3], c[4], c[5], c[6], 1) dbhandler.execute(query, data) # Otherwise, save the candidates just in case HR need to contact them else: query = "INSERT INTO bad_candidates VALUES(%s,%s,%s,%s,%s,%s,%s,%s)" data = (c[0], c[1], c[2], c[3], c[4], c[5], c[6], float(p)) dbhandler.execute(query, data) query = "INSERT INTO training_set VALUES(%s,%s,%s,%s,%s,%s)" data = (c[2], c[3], c[4], c[5], c[6], 0) dbhandler.execute(query, data) dbhandler.commit()
def data_simple_sigmoid(N): X = np.random.uniform(-10., 10., N) Y = sigmoid(X) Y += np.random.normal(0., .1, N) Y += np.random.binomial(1, .5, N) Y, X = Y.reshape((N, 1)), X.reshape((N, 1)) return X, Y
def prob_function_sigmoid_rd(candidates, cand_available, inst, inst_rank, school_info, weights, **kwargs): cand_p = np.zeros(len(candidates), dtype=float) for i, (candidate, candidate_rank) in enumerate(candidates): if cand_available[i]: cand_p[i] = sigmoid(weights[0] + weights[1]*(inst_rank-candidate_rank)) #cand_p /= cand_p.sum() return cand_p
def grad_mixed_guidance(data, v): x_low = data.x_low x_high = data.x_high if x_low is None: return 0 scale = data.scale n = x_low.shape[0] d = (apply_linear(x_high, v) - apply_linear(x_low, v)) / scale sig = sigmoid(d) g = np.zeros(v.size) """ for i in range(n): dx = (x_high[i,:] - x_low[i,:])/scale t = 1-sig[i] g[0:-1] += t*dx g[-1] += t """ dx = x_high - x_low / scale g_fast = (dx.T * (1 - sig)).sum(1) g_fast = np.append(g_fast, (1 - sig).sum()) g = g_fast # rel_err = array_functions.relative_error(g,g_fast) g *= -1 g[-1] *= 0 g_m = g / n return g_m
def lr_gradient(self, w, X, y, lambda_): # y must be in {-1, +1} num_obs, num_features = X.shape grad = np.zeros((1, num_features)) grad += mat((1 - sigmoid(y * np.dot(X, w))) * y) * mat(X) # do not regularize intercep grad -= lambda_ * np.concatenate(([0], w[1:])) return -grad[0]
def fit(self, X, y, n_epochs=4000, lr=0.01, n_units=10): self.w = np.random.rand(X.shape[1], n_units) self.v = np.random.rand(n_units, y.shape[1]) for _ in range(n_epochs): h_out = sigmoid(X.dot(self.w)) out = softmax(h_out.dot(self.v)) self.v -= lr * h_out.T.dot(out - y) self.w -= lr * X.T.dot((out - y).dot(self.v.T) * (h_out * (1 - h_out)))
def prob_function_sigmoid_pd(candidates, cand_available, inst, inst_rank, school_info, weights, **kwargs): cand_p = np.zeros(len(candidates), dtype=float) for i, (candidate, candidate_rank) in enumerate(candidates): if cand_available[i]: cand_p[i] = sigmoid(np.dot(weights, [1, int(candidate.has_postdoc)])) #cand_p /= cand_p.sum() return cand_p
def grad_mixed_guidance(data, v): assert False, "TODO: Normalize by amount of guidance" x = data.x_neighbor x_low = data.x_low x_high = data.x_high scale = data.scale w, b = unpack_linear(v) y = apply_linear(x, w, b) y_low = apply_linear(x_low, w, b) y_high = apply_linear(x_high, w, b) sig1 = sigmoid((y_high - y_low) / scale) sig2 = sigmoid((2 * y - y_high - y_low) / scale) small_constant = getattr(data, "eps", eps) diff = sig1 - sig2 # diff[diff < 0] = 0 denom = diff + small_constant """ val = np.zeros(v.shape) for i in range(x.shape[0]): x1 = (x_high[i,:] - x_low[i,:])/scale x2 = (2*x[i,:] - x_low[i,:] - x_high[i,:])/scale num1 = sig1[i]*(1-sig1[i])*x1 num2 = sig2[i]*(1-sig2[i])*x2 val[0:-1] += (num1-num2)*(1/denom[i]) val[-1] = 0 """ x1 = (x_high - x_low) / scale x2 = (2 * x - x_low - x_high) / scale num1 = sig1 * (1 - sig1) num2 = sig2 * (1 - sig2) d = x1.T * num1 - x2.T * num2 g_fast = (d / denom).sum(1) g_fast = np.append(g_fast, 0) # err = array_functions.relative_error(val, g_fast) val = g_fast val *= -1 I = np.isnan(val) | np.isinf(val) if I.any(): # print 'grad_linear_neighbor_logistic: nan!' val[I] = 0 return val
def predict_adoption(self, context, max_len=DEFAULT_MAX_LEN): # type: (List[str], int) -> List[str] max_len = max(max_len, len(context)) context = [c for c in context if c in self.weights.index] logits = self.weights.loc[context].dot(self.weights.T) probabilities = sigmoid(logits).sum(axis=0).drop(context) top_idx = np.argpartition(probabilities.values, -max_len)[-max_len:] top = probabilities.iloc[top_idx].sort_values(ascending=False) return top.index.to_list()
def predict(self, X): """ Run prediction based on trained model """ if self.w is None: print("Weight w is not yet optimized. Call fit() first.") pred_proba = sigmoid(np.dot(X, self.w)) y_pred = pred_proba > 0.5 return y_pred
def getDataExpectations(self, batch): hidden_probs = sigmoid( np.matmul(batch, self.weights) + self.hidden_bias) w_expectation = np.matmul(batch.T, hidden_probs) v_expectation = batch h_expectation = hidden_probs return w_expectation, v_expectation, h_expectation
def predict(test, W): Oh = np.tanh(np.dot(W[0], test.transpose())) Oh = np.expand_dims(Oh, axis=1) Ino_ones = np.ones((1, 1)) Ino = np.vstack((Ino_ones, Oh)) Oo = sigmoid(np.dot(W[1], Ino)) return Oo
def forward_backward_propagation(main_word, context_words, negsamples, W_input, W_output, learning_rate, vocab_size): # print(W_input) # print('********************8') targets = list() for context_word in context_words: targets.append(context_word) for negsample in negsamples: targets.append(negsample) targets = np.array(targets) #forward_propagation h = W_input[targets,:] prediction = h.dot(W_output[main_word].T) prediction = sigmoid(prediction) #backward_propagation tj = np.zeros(len(targets)) for i in range(len(context_words)): tj[i] = 1 pred_error = prediction - tj # print(h.shape) # print(prediction.shape) # print(pred_error.shape) # print(W_output[:,targets].T.shape) # print('*****************************') del_W_input = np.outer(pred_error, W_output[main_word]) # print(del_W_input.shape) # print(h.shape) del_W_output = np.dot(pred_error.T,h) # print(del_W_output.shape) # print('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@') for index in range(len(targets)): W_input[targets[index]] -= learning_rate*del_W_input[index] W_output[main_word] -= learning_rate*del_W_output costr = tj * np.log(prediction + 1e-10) + (1 - tj) * np.log(1 - prediction + 1e-10) # cost = pred_error.sum() cost = -1*costr.sum() # print(W_input.shape()) return W_input, W_output, cost
def generate_data(dataset_size): x, y = [], [] for _ in range(dataset_size): x_i = 20 * np.random.random() - 10 p_i = sigmoid(np.sin(x_i)) y_i = np.random.binomial(1, p_i) x.append(x_i) y.append(y_i) return np.array(x), np.array(y).reshape(-1, 1)
def binary_crossentropy_pseudo_residuals(y_true, logits): """Pseudo residuals between true label values and predicted logits. Arguments need equal shape :param y_true: True values (np.array) :param logits: predicted logit values (np.array) """ return y_true - sigmoid(logits)
def score_vecs(vecs): vec_list = [vecs[0].T] for v in vecs[1:-1]: vec_list.extend([v, v.T]) vec_list.append(vecs[-1]) score = 1.0 for i in range(len(vec_list) - 1): score = score * sigmoid(np.dot(vec_list[i], vec_list[i + 1])) return score
def inspect_and_evaluate_model(model, games_tourney_train, games_tourney_test): # Print the model weights print(model.get_weights()) # Print the training data means print(games_tourney_train.mean()) weight = 0.14 # Print the approximate win probability predicted close game print(sigmoid(1 * weight)) # Print the approximate win probability predicted blowout game print(sigmoid(10 * weight)) # Evaluate the model on new data print( model.evaluate( games_tourney_test[['seed_diff', 'pred']], [games_tourney_test[['score_diff']], games_tourney_test[['won']]], verbose=False))
def test_hazard_and_hidden(simple_feature_matrix): X = simple_feature_matrix Id = np.identity(3) # Exp( first column of X ) assert np.allclose(hazard(X, Id[0]), np.exp([0, 1, 2, 0])) # Exp ( second column of X ) assert np.allclose(hazard(X, Id[1]), np.exp([1, 0, 5, 2])) # Exp ( third column of X ) assert np.allclose(hazard(X, Id[2]), np.exp([3, 1, -2, -2])) # sigmoid ( first column of X ) assert np.allclose(conv_prob(X, Id[0]), sigmoid(np.asarray([0, 1, 2, 0]))) # sigmoid ( second column of X ) assert np.allclose(conv_prob(X, Id[1]), sigmoid(np.asarray([1, 0, 5, 2]))) # sigmoid ( third column of X ) assert np.allclose(conv_prob(X, Id[2]), sigmoid(np.asarray([3, 1, -2, -2])))
def prob_function_sigmoid_gg(candidates, cand_available, inst, inst_rank, school_info, weights, **kwargs): job_region = school_info[inst]['Region'] cand_p = np.zeros(len(candidates), dtype=float) for i, (candidate, candidate_rank) in enumerate(candidates): if cand_available[i]: cand_p[i] = sigmoid(np.dot(weights, [1, int(job_region == candidate.phd_region)])) #cand_p /= cand_p.sum() return cand_p
def fit(self, X, y, n_epochs=4000, lr=0.01, n_units=10): self.w = np.random.rand(X.shape[1], n_units) self.v = np.random.rand(n_units, y.shape[1]) for _ in range(n_epochs): h_out = sigmoid(X.dot(self.w)) out = softmax(h_out.dot(self.v)) self.v -= lr * h_out.T.dot(out - y) self.w -= lr * X.T.dot( (out - y).dot(self.v.T) * (h_out * (1 - h_out)))
def prob_function_sigmoid_rd_rh(candidates, cand_available, inst, inst_rank, school_info, weights, **kwargs): cand_p = np.zeros(len(candidates), dtype=float) for i, (candidate, candidate_rank) in enumerate(candidates): if cand_available[i]: cand_p[i] = sigmoid( np.dot(weights, [1, inst_rank - candidate_rank, inst_rank])) #cand_p /= cand_p.sum() return cand_p
def initialize(input, hidden, output, examples): print("examples", examples, "input", input, "hidden", hidden, "output", output) x = np.random.uniform(0, 1, (examples, input)) y = np.random.uniform(0, 1, (examples, output)) weightsin = np.random.uniform(-1, 1, size=(input, hidden)) weights2 = np.random.uniform(-1,1, size = (hidden+1, hidden)) weightsout = np.random.uniform(-1, 1, size=(hidden+1, output)) error = 1 while(error !=0): #forward propogation layer1 = sigmoid(np.dot(x, weightsin)) layer1 = np.append(layer1, np.ones((examples, 1)), axis = 1) layer2 = sigmoid(np.dot(layer1,weights2)) layer2 = np.append(layer2, np.ones((examples, 1)), axis = 1) predict = sigmoid(np.dot(layer2, weightsout)) alpha = 0.006 error = 1/2 * (y - predict) ** 2 error = error.sum() #backward propogation change3 = -1 * (y - predict) * dsigmoid(predict) DJWo = np.dot(layer2.transpose(), change3) weightsout = weightsout + -alpha * DJWo change2 = np.dot(change3, weightsout[0:np.shape(weightsout)[0]-1].transpose()) * dsigmoid(layer2[:, 0:np.shape(layer2)[1]-1]) DJW2 = np.dot(layer1.transpose(), change2) weights2 = weights2 + -alpha*DJW2 change1 = np.dot(change2, weights2[0:np.shape(weights2)[0]-1, :].transpose())*dsigmoid(layer1[:, 0:np.shape(layer1)[1]-1]) DjWi = np.dot(x.transpose(), change1) weightsin += -alpha*DjWi print(error) print("Final Error", error)
def getDataExpectations(self, batch): from scipy.special import expit as sigmoid hidden_probs = sigmoid( np.matmul(batch, self.weights) + self.hidden_bias) w_expectation = np.matmul(batch.T, hidden_probs) v_expectation = batch h_expectation = hidden_probs return w_expectation, v_expectation, h_expectation
def _process_pred(self, pred): """Standarizes pred to be an integer array.""" assert pred.ndim == 3, f"expected 3-dim array, got {pred.ndim}-dim" if pred.shape[-1] == 1: # binary case if self.from_logits: pred = sigmoid(pred) pred = pred[..., 0] * 255 else: # multiclass pred = np.argmax(pred, axis=-1) return pred
def cost(self,theta, X, y, l): m = len(y) J = 0 grad = np.zeros(shape=(len(theta),1)) H = sigmoid(X.dot(theta)) y_T = y.transpose() J = 1./m * np.sum(-y_T.dot(np.log(H)) - (1-y_T).dot(np.log(1-H))) + l/(2.*m) * np.sum(theta[1:]**2) return J
def propagateForward(X_row, thetas): features = X_row for i in range(len(thetas)): theta = thetas[i] z = np.dot(theta, features) a = sigmoid(z) if i == len(thetas)-1: return a a = np.insert(a,0,1) #add bias unit features = a
def forward(self, sample): # forward propogation through the network in_data = sample self.nodes = [] for i in range(len(self.layers[:-1])): layer_data = sigmoid(np.dot(in_data, self.weights[i])) self.nodes.append(copy.deepcopy(layer_data)) in_data = copy.deepcopy(layer_data) return self.nodes[-1][ 0] #prediction after softmax -- do percentage on it somewhere?
def predict(self, x): z = 0 for (field, index, value) in x: w = self.get_weight(field, index) z += w * value p = sigmoid(z) return p
def Execute_SGD_TT(X_norm, alpha, num_iters): X_train, X_test = splitTT(0.6, X_norm) theta = np.zeros((X_norm.shape[1] - 1, 1)) theta = stochasticGD(X_train, theta, alpha, num_iters) prediction_set = sigmoid(np.dot(X_test[:, 0:5], theta)) acc = getAccuracy(prediction_set, X_test[:, 5:6]) error = errCompute(X_test, theta) return acc, error
def create_data(N, P, z_real, seed = 0): # N = 200 # P = 4 N = int(N) P = int(P) rs = np.random.RandomState(seed) X = rs.randn(N,P) # z_real = rs.randn(P) y = rs.binomial(1,sigmoid(np.dot(X,z_real))) return X, y
def activate(self, inputs): """ inputs = numpy array of input values, one per input neuron. outputs = numpy array of outputs, one per neuron in this layer. Outputs calculated using a sigmodal activation function. """ internal_activations = np.dot(self.weights, inputs) if self._apply_bias: internal_activations += self.biases return sigmoid(internal_activations)
def forward_jax(X, t, U, b, W, c): N = X.shape[0] # Perform forwards computation. G = jnp.dot(X, U.T) + b H = jnp.tanh(G) z = jnp.dot(H, W.T) + c y = sigmoid(z) return (1./N) * jnp.sum(-t * jnp.log(y) - (1 - t) * jnp.log(1 - y))
def nnCostFunc(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, lmbda): Theta1 = nn_params[0:(input_layer_size+1)*hidden_layer_size] Theta1 = Theta1.reshape((hidden_layer_size,input_layer_size+1),order='F') Theta2 = nn_params[(input_layer_size+1)*hidden_layer_size:] Theta2 = Theta2.reshape((num_labels,hidden_layer_size+1),order='F') m = size(X, 0); J = 0; #Theta1_grad = np.zeros(size(Theta1)); #Theta2_grad = np.zeros(size(Theta2)); eye_matrix = np.eye(num_labels) if num_labels == 1: y_matrix = y.astype(int) else: y_matrix = eye_matrix[y-1,:] y_matrix.shape = (size(y,0),num_labels) #-------------------------------------------------------------------------- #forward propagation #-------------------------------------------------------------------------- X = np.concatenate((np.ones(((size(X,0)),1)), X),1) hidden = sigmoid([email protected]) hidden = np.concatenate((np.ones(((size(hidden,0)),1)), hidden),1) prediction = sigmoid([email protected]) for i in range(num_labels): J = J + (1/m)*(((np.log(prediction[:,i]).T)@(-y_matrix[:,i])) - (np.log(1-prediction[:,i]).T)@(1-y_matrix[:,i])) for i in range(1,size(Theta1,1)): J= J + lmbda/(2*m)*np.sum(np.square(Theta1[:,i])) for i in range(1,size(Theta2,1)): J= J + lmbda/(2*m)*np.sum(np.square(Theta2[:,i])) J = np.sum(np.sum(J)) print (J) return J
def prob_function_sigmoid_gg(candidates, cand_available, inst, inst_rank, school_info, weights, **kwargs): job_region = school_info[inst]['Region'] cand_p = np.zeros(len(candidates), dtype=float) for i, (candidate, candidate_rank) in enumerate(candidates): if cand_available[i]: cand_p[i] = sigmoid( np.dot(weights, [1, int(job_region == candidate.phd_region)])) #cand_p /= cand_p.sum() return cand_p
def forward(self, W_in, X): # construct network tmp = X for i in xrange(len(W_in)): if i == 0: W, W_b = W_in[i][:self.n_in], W_in[i][self.n_in:] else: W, W_b = W_in[i][:self.n_hidden[i-1]], W_in[i][self.n_hidden[i-1]:] tmp = sigmoid(np.dot(tmp, W) + W_b) return tmp
def compare(i, j): if np.random.choice([0, 1], p=[0.9, 0.1]): # draw return 0 z_i = np.random.normal(mu[i], sigma[i]) z_j = np.random.normal(mu[j], sigma[j]) p = sigmoid(z_i - z_j) if np.random.choice([0, 1], p=[1 - p, p]): return 1 else: return 2
def errCompute(X_norm, theta): actual_response = X_norm[:, 5:6] predictor_set = X_norm[:, 0:5] prediction_set = sigmoid(np.dot(predictor_set, theta)) cost = (-1) * np.sum(actual_response * np.log(prediction_set) + (1 - actual_response) * np.log(1 - prediction_set)) / len(X_norm) return cost
def fit(self, y, x, n=1, epsilon=.01, regularization=None, _lambda=1.0): """ Learn the weights for a Logistic Regression Classifier from the data Params --------- y : Numpy array - The binary class labels for the data points x : Numpy array - A list of features for each data point n : int - Default: 1 - The number of iterations for the training algorithm epsilon : float - Default: 0.01 - Hyperparamter controlling overfitting of the model to the data regularization : {None, 'L1', 'L2'} - Default: None - Type of regularization to employ on the model _lambda : float - Hyper parameter for regularization """ # Initialize the weight vector w_0 = np.zeros(x.shape[1]) # Variables used for learning weights self._epsilon = epsilon self._num_training = x.shape[0] self._lambda = _lambda print 'Epsilon: {}'.format(self._epsilon) # Pick the correct update method if regularization == 'l1': print 'L1 regularization' print 'Lambda: {}'.format(self._lambda) update_func = self._l1 elif regularization == 'l2': print 'L2 regularization' print 'Lambda: {}'.format(self._lambda) update_func = self._l2 else: print 'No regularization' update_func = self._no_reg # Number of iterations for _ in range(n): # Loop over all the data points for i in range(x.shape[0]): y_minus_g = y[i] - sigmoid( np.dot(w_0, x[i]) ) w_0 = update_func(y[i], x[i], y_minus_g, w_0) # Save the learned weights self.weights = w_0 return None
def predict(images, V, W): predicted_labels = [] for img in images: z1 = np.dot(V, img.T) activation = np.tanh(z1) z1 = np.append(activation, 1) z1 = np.reshape(z1, (len(z1), 1)) z1 = np.dot(W, z1) z = sigmoid(z1) predicted_labels.append(np.argmax(z) + 1) return predicted_labels
def plot_bond_encoding(theta=None, start=1.0, end=6.0, slope=20., segments=5, show=False): if theta is None: theta = numpy.linspace(start, end, segments) distance = numpy.linspace(0, 8, 250) value = sigmoid(slope*numpy.subtract.outer(theta, distance)).T plt.plot(distance, value) plt.yticks(numpy.linspace(0, 1.1, 4)) plt.xlabel("Bond Length ($\AA$)") plt.ylabel("Weight") if show: plt.show()
def decision_function(self, X): if not self.fitted_: raise ValueError('This ELMClassifier instance is not fitted yet.') X = self._validate_X(X) if self.random_projection: H = sigmoid(dot(X, self.W) + self.b) else: H = X if hasattr(self, 'fit_intercept') and self.fit_intercept: H = np.hstack((H, np.ones((X.shape[0], 1)))) return dot(H, self.beta)
def data_sigm_multi(N, p): if not isinstance(p, (list, tuple)): assert isinstance(p, int) p = [1. / p] * p X = np.random.uniform(-10., 10., N) Y = sigmoid(X) Y += np.random.normal(0., .1, N) y = np.random.multinomial(1, p, size=N) Y += np.nonzero(y)[1] Y, X = Y.reshape((N, 1)), X.reshape((N, 1)) return X, Y
def _simulate_single_equaiton_daring(X, scale, hidden=100): pa_size = X.shape[1] z = _simulate_noise(scale) W1 = np.random.uniform(low=0.5, high=2.0, size=[pa_size, hidden]) W1[np.random.rand(*W1.shape) < 0.5] *= -1 W2 = np.random.uniform(low=0.5, high=2.0, size=hidden) W2[np.random.rand(hidden) < 0.5] *= -1 x = sigmoid(X @ W1) @ W2 + z return x
def feedforward(X, W): X_input = X[:, 0:len(X[0]) - 1] Oh = np.tanh(np.dot(W[0], X_input.transpose())) n, m = Oh.shape Ino_ones = np.ones((1, m)) Ino = np.vstack((Ino_ones, Oh)) Oo = sigmoid(np.dot(W[1], Ino)) return [Oh, Ino, Oo]
def ML_crp(counts_list): params = np.zeros(2) print("ML optimization starting at obj = ", logprob_obj(params, counts_list)) res = minimize(logprob_obj, params, args=(counts_list)) params = res.x print("ML optimization ended at obj = ", logprob_obj(params, counts_list)) spi_alpha, logit_beta = params alpha = softplus(spi_alpha) beta = sigmoid(logit_beta) return CRP(alpha, beta)
def predict(theta, X, y): # number of training examples m = shape(X)[0] # find hypothesis h = sigmoid( X.dot(theta.T) ) # make prediction using # index of max value in # each training example (row) prediction = argmax(h,axis=1) # find accuracy accuracy = (float(sum(prediction == y))/m)*100 return prediction,accuracy
def grad_mixed_guidance(data, v): bounds = data.bounds x = data.x_bound scale = data.scale w, b = unpack_linear(v) y = apply_linear(x, w, b) assert y.size == bounds.shape[0] c1 = bounds[:, 0] c2 = bounds[:, 1] sig1 = sigmoid((c2 - y) / scale) sig2 = sigmoid((c1 - y) / scale) small_constant = getattr(data, "eps", eps) denom = sig1 - sig2 + small_constant """ val = np.zeros(v.shape) assert (denom > -1).all() for i in range(x.shape[0]): num1 = sig1[i]*(1-sig1[i]) num2 = sig2[i]*(1-sig2[i]) val[0:-1] += (num1-num2)*(1/denom[i])*x[i,:]/scale val[-1] += (num1-num2)*(1/denom[i]) """ # assert not np.isnan(val).any() # Why isn't this necessary? # val *= -1 num = sig1 * (1 - sig1) - sig2 * (1 - sig2) num /= scale num /= denom g_fast = (x.T * num).sum(1) g_fast = np.append(g_fast, num.sum()) # rel_error = array_functions.relative_error(val, g_fast) val = g_fast if np.isnan(val).any(): print "grad_linear_bound_logistic: nan!" val[np.isnan(val)] = 0 if np.isinf(val).any(): # print 'grad_linear_bound_logistic: inf!' val[np.isinf(val)] = 0 return val
def _sigmoidLikelihood(self, x, label): """ Returns the sigmoid likelihood p(y=label|features; weights) x : Numpy array Feature set for a single data point """ logit = sigmoid(np.dot(x, self.weights)) if label == 0: return (1-logit) elif label == 1: return logit