def predict(Theta1, Theta2, X, y): m = X.shape[0] h1 = sigmoid(X.dot(Theta1.T)) h1 = hstack((ones((m,1)),h1)) h2 = sigmoid(h1.dot(Theta2.T)) p = argmax(h2,axis=1) return mean(p == y.T.tolist())*100\
def predict_value(Theta1, Theta2, frame): m = X.shape[0] h1 = sigmoid(X.dot(Theta1.T)) h1 = hstack((ones((m,1)),h1)) h2 = sigmoid(h1.dot(Theta2.T)) p = argmax(h2,axis=1) return p
def eval_mixed_guidance(data, v):
assert False, "TODO: Normalize by amount of guidance"
x = data.x_neighbor
x_low = data.x_low
x_high = data.x_high
scale = data.scale
w, b = unpack_linear(v)
y = apply_linear(x, w, b)
y_low = apply_linear(x_low, w, b)
y_high = apply_linear(x_high, w, b)
"""
if (y_low + eps >= y_high).any() or (y + eps >= y_high).any():
return np.inf
"""
sig1 = sigmoid((y_high - y_low) / scale)
sig2 = sigmoid((2 * y - y_high - y_low) / scale)
diff = sig1 - sig2
# assert (np.sign(diff) > 0).all()
small_constant = getattr(data, "eps", eps)
# assert False, 'Should this be infinity instead?'
# diff[diff < 0] = 0
vals2 = -np.log(diff + small_constant)
I = np.isnan(vals2)
if I.any():
# print 'eval_linear_neighbor_logistic: inf = ' + str(I.mean())
return np.inf
val2 = vals2.sum()
# assert norm(val - val2)/norm(val) < 1e-6
return val2
def eval_mixed_guidance(data, v):
assert False, "TODO: Normalize by amount of guidance"
w, b = unpack_linear(v)
x = data.x_bound
bounds = data.bounds
scale = data.scale
y = apply_linear(x, w, b)
assert y.size == bounds.shape[0]
c1 = bounds[:, 0]
c2 = bounds[:, 1]
"""
y_c1 = (y-c1)/scale
y_c2 = (y-c2)/scale
loss_num = np.log(np.exp(y_c1) - np.exp(y_c2))
loss_denom = np.log(1 + np.exp(y_c1) + np.exp(y_c2) + np.exp(y_c1+y_c2))
vals = (-loss_num + loss_denom)
val = vals.sum()
"""
sig1 = sigmoid((c2 - y) / scale)
sig2 = sigmoid((c1 - y) / scale)
small_constant = getattr(data, "eps", eps)
diff = sig1 - sig2 + small_constant
vals2 = -np.log(sig1 - sig2 + small_constant)
val2 = vals2.sum()
I = np.isnan(vals2)
if I.any():
assert False
# assert norm(val - val2)/norm(val) < 1e-6
return val2
def cost(Theta, X, y, lmda): Theta1 = Theta[:401*25].reshape((25,401)) Theta2 = Theta[401*25:].reshape((10,26)) m = X.shape[0] a1 = X z2 = a1.dot(Theta1.T) a2 =hstack((ones((m,1)),sigmoid(z2))) z3 =a2.dot(Theta2.T) h = sigmoid(z3) k = h.shape[1] Y = zeros((m,k)) Y[arange(m),y.T] = 1 d3 = h-Y d2 = d3.dot(Theta2[:,1:])*(sigmoid(z2)*(1-sigmoid(z2))) D2 = d3.T.dot(a2) D1 = d2.T.dot(a1) reg = lmda/float(m) Theta1_grad = (D1/m)+(reg*hstack(((zeros((Theta1.shape[0],1))), Theta1[:,1:]))) Theta2_grad = (D2/m)+reg*hstack(((zeros((Theta2.shape[0],1))), Theta2[:,1:])) reg = sum(sum(Theta1[:,1:]**2,axis=1))+sum(sum(Theta2[:,1:]**2,axis=1)) J = (1.0/m)*sum(sum(-Y*log(h)-((1-Y)*log(1-h)),axis=1))+(lmda/(2.0*m))*reg grad = hstack((Theta1_grad.ravel(),Theta2_grad.ravel())) return J,grad
def elbo_grad(z_sample, mu, sigma_sq, y, X, P, prior_sigma):
score_mu = (z_sample - mu)/(sigma_sq)
score_logsigma_sq = (-1/(2*sigma_sq) + np.power((z_sample - mu),2)/(2*np.power(sigma_sq,2))) * sigma_sq
log_p = np.sum(y * np.log(sigmoid(np.dot(X,z_sample))) + (1-y) * np.log(1-sigmoid(np.dot(X,z_sample))))\
+ np.sum(norm.logpdf(z_sample, np.zeros(P), prior_sigma*np.ones(P)))
log_q = np.sum(norm.logpdf(z_sample, mu, np.sqrt(sigma_sq)))
return np.concatenate([score_mu,score_logsigma_sq])*(log_p - log_q)
def feed_forward(self, input_value):
self._input_layer = np.insert(input_value, 0, 1, axis=1)
# print self._input_layer
self._hidden_layer = np.insert(sigmoid(np.dot(self._input_layer, self._hidden_layer_weights)), 0, 1, axis=1)
self._output_layer = sigmoid(np.dot(self._hidden_layer, self._output_layer_weights))
# print self._output_layer
return self._output_layer
def pushdown(self, hid_pats):
""" push hidden pats into visible, BUT JUST CALC PROBS """
if self.DROPOUT:
dropout = rng.randint(2, size=(hid_pats.shape[0], self.num_hid))
vis_prob1 = sigmoid(np.dot(hid_pats*dropout, self.W) + self.vis_bias)
else:
FACTOR = 0.5
vis_prob1 = sigmoid(np.dot(hid_pats, FACTOR*self.W) + self.vis_bias)
return vis_prob1
def pushup(self, vis_pats, noise=True):
""" push visible pats into hidden layer"""
psi = np.dot(vis_pats, self.W.T) + self.hid_bias
if self.hid_type == 'logistic': # BERNOULLI units
hid_prob1 = sigmoid(psi)
if noise == False:
return hid_prob1
elif noise == True:
return 1*(hid_prob1 > rng.random(size=hid_prob1.shape))
elif self.hid_type == 'relu': # ReLU units
if noise == True:
psi = psi + rng.normal(0.,0.001+np.sqrt(sigmoid(psi)), size=psi.shape)
return np.maximum(0.0, psi)
def _train(start, stop):
X = alias.train_x
y = alias.train_y
N = alias.N
weights = alias.weights
weights0 = alias.weights0
weights1 = alias.weights1
weights10 = alias.weights10
alpha = alias.alpha
w1 = np.zeros_like(weights1)
w10 = np.zeros_like(weights10)
w = np.zeros_like(weights)
w0 = np.zeros_like(weights0)
for i in range(start, stop):
x = X[i]
a1 = x
z2 = x @ weights + weights0
a2 = sigmoid(z2)
z3 = a2 @ weights1 + weights10
a3 = sigmoid(z3)
delta3 = -(y[i] - a3)
delta2 = weights1 @ delta3 * a2 * (1 - a2)
w1 += alpha * (a2[:, None] @ delta3[None, :])
w10 += alpha * delta3
w += alpha * (a1[:, None] @ delta2[None, :])
w0 += alpha * delta2
w0 /= N
w /= N
w10 /= N
w1 /= N
# wc = WeightCache()
# wc.weights = w
# wc.weights0 = w0
# wc.weights1 = w1
# wc.weights10= w10
# update_cache_weight.delay(wc.key)
# with one_lock:
alias.weights -= w
alias.weights0 -= w0
alias.weights1 -= w1
alias.weights10 -= w10
cache.incr('n_s', stop-start)
def transform(in_vector, conn_matrix):
"""
Transform in_vector into out_vector using conn_matrix
Last column of conn_matrix is bias
"""
raw_out = conn_matrix.dot(np.append(in_vector, 1.0))
return sigmoid(raw_out)
def predictions(dbhandler, theta, candidates, expected_prob):
for c in candidates:
# [Intercept item, Degree, Python, Octave, C++, Java]
test = np.array([1, c[2], c[3], c[4], c[5], c[6]])
p = sigmoid(test.dot(theta))
# If the algorithm predicts a value > expected_prob, HR will contact the candidate
if p > expected_prob:
query = "INSERT INTO good_candidates VALUES(%s,%s,%s,%s,%s,%s,%s,%s)"
data = (
c[0],
c[1],
c[2],
c[3],
c[4],
c[5],
c[6],
float(p),
) # Must cast to float because mysqlconnector doesn't recognize numpy formats
dbhandler.execute(query, data)
# Add a new entry to the training set with this candidate info
query = "INSERT INTO training_set VALUES(%s,%s,%s,%s,%s,%s)"
data = (c[2], c[3], c[4], c[5], c[6], 1)
dbhandler.execute(query, data)
# Otherwise, save the candidates just in case HR need to contact them
else:
query = "INSERT INTO bad_candidates VALUES(%s,%s,%s,%s,%s,%s,%s,%s)"
data = (c[0], c[1], c[2], c[3], c[4], c[5], c[6], float(p))
dbhandler.execute(query, data)
query = "INSERT INTO training_set VALUES(%s,%s,%s,%s,%s,%s)"
data = (c[2], c[3], c[4], c[5], c[6], 0)
dbhandler.execute(query, data)
dbhandler.commit()
def data_simple_sigmoid(N):
X = np.random.uniform(-10., 10., N)
Y = sigmoid(X)
Y += np.random.normal(0., .1, N)
Y += np.random.binomial(1, .5, N)
Y, X = Y.reshape((N, 1)), X.reshape((N, 1))
return X, Y
def prob_function_sigmoid_rd(candidates, cand_available, inst, inst_rank, school_info, weights, **kwargs):
cand_p = np.zeros(len(candidates), dtype=float)
for i, (candidate, candidate_rank) in enumerate(candidates):
if cand_available[i]:
cand_p[i] = sigmoid(weights[0] + weights[1]*(inst_rank-candidate_rank))
#cand_p /= cand_p.sum()
return cand_p
def grad_mixed_guidance(data, v):
x_low = data.x_low
x_high = data.x_high
if x_low is None:
return 0
scale = data.scale
n = x_low.shape[0]
d = (apply_linear(x_high, v) - apply_linear(x_low, v)) / scale
sig = sigmoid(d)
g = np.zeros(v.size)
"""
for i in range(n):
dx = (x_high[i,:] - x_low[i,:])/scale
t = 1-sig[i]
g[0:-1] += t*dx
g[-1] += t
"""
dx = x_high - x_low / scale
g_fast = (dx.T * (1 - sig)).sum(1)
g_fast = np.append(g_fast, (1 - sig).sum())
g = g_fast
# rel_err = array_functions.relative_error(g,g_fast)
g *= -1
g[-1] *= 0
g_m = g / n
return g_m
def lr_gradient(self, w, X, y, lambda_):
# y must be in {-1, +1}
num_obs, num_features = X.shape
grad = np.zeros((1, num_features))
grad += mat((1 - sigmoid(y * np.dot(X, w))) * y) * mat(X)
# do not regularize intercep
grad -= lambda_ * np.concatenate(([0], w[1:]))
return -grad[0]
def fit(self, X, y, n_epochs=4000, lr=0.01, n_units=10):
self.w = np.random.rand(X.shape[1], n_units)
self.v = np.random.rand(n_units, y.shape[1])
for _ in range(n_epochs):
h_out = sigmoid(X.dot(self.w))
out = softmax(h_out.dot(self.v))
self.v -= lr * h_out.T.dot(out - y)
self.w -= lr * X.T.dot((out - y).dot(self.v.T) * (h_out * (1 - h_out)))
def prob_function_sigmoid_pd(candidates, cand_available, inst, inst_rank, school_info, weights, **kwargs):
cand_p = np.zeros(len(candidates), dtype=float)
for i, (candidate, candidate_rank) in enumerate(candidates):
if cand_available[i]:
cand_p[i] = sigmoid(np.dot(weights, [1,
int(candidate.has_postdoc)]))
#cand_p /= cand_p.sum()
return cand_p
def grad_mixed_guidance(data, v):
assert False, "TODO: Normalize by amount of guidance"
x = data.x_neighbor
x_low = data.x_low
x_high = data.x_high
scale = data.scale
w, b = unpack_linear(v)
y = apply_linear(x, w, b)
y_low = apply_linear(x_low, w, b)
y_high = apply_linear(x_high, w, b)
sig1 = sigmoid((y_high - y_low) / scale)
sig2 = sigmoid((2 * y - y_high - y_low) / scale)
small_constant = getattr(data, "eps", eps)
diff = sig1 - sig2
# diff[diff < 0] = 0
denom = diff + small_constant
"""
val = np.zeros(v.shape)
for i in range(x.shape[0]):
x1 = (x_high[i,:] - x_low[i,:])/scale
x2 = (2*x[i,:] - x_low[i,:] - x_high[i,:])/scale
num1 = sig1[i]*(1-sig1[i])*x1
num2 = sig2[i]*(1-sig2[i])*x2
val[0:-1] += (num1-num2)*(1/denom[i])
val[-1] = 0
"""
x1 = (x_high - x_low) / scale
x2 = (2 * x - x_low - x_high) / scale
num1 = sig1 * (1 - sig1)
num2 = sig2 * (1 - sig2)
d = x1.T * num1 - x2.T * num2
g_fast = (d / denom).sum(1)
g_fast = np.append(g_fast, 0)
# err = array_functions.relative_error(val, g_fast)
val = g_fast
val *= -1
I = np.isnan(val) | np.isinf(val)
if I.any():
# print 'grad_linear_neighbor_logistic: nan!'
val[I] = 0
return val
def predict_adoption(self, context, max_len=DEFAULT_MAX_LEN):
# type: (List[str], int) -> List[str]
max_len = max(max_len, len(context))
context = [c for c in context if c in self.weights.index]
logits = self.weights.loc[context].dot(self.weights.T)
probabilities = sigmoid(logits).sum(axis=0).drop(context)
top_idx = np.argpartition(probabilities.values, -max_len)[-max_len:]
top = probabilities.iloc[top_idx].sort_values(ascending=False)
return top.index.to_list()
def predict(self, X):
"""
Run prediction based on trained model
"""
if self.w is None:
print("Weight w is not yet optimized. Call fit() first.")
pred_proba = sigmoid(np.dot(X, self.w))
y_pred = pred_proba > 0.5
return y_pred
def getDataExpectations(self, batch):
hidden_probs = sigmoid(
np.matmul(batch, self.weights) + self.hidden_bias)
w_expectation = np.matmul(batch.T, hidden_probs)
v_expectation = batch
h_expectation = hidden_probs
return w_expectation, v_expectation, h_expectation
def predict(test, W):
Oh = np.tanh(np.dot(W[0], test.transpose()))
Oh = np.expand_dims(Oh, axis=1)
Ino_ones = np.ones((1, 1))
Ino = np.vstack((Ino_ones, Oh))
Oo = sigmoid(np.dot(W[1], Ino))
return Oo
def forward_backward_propagation(main_word, context_words, negsamples, W_input, W_output, learning_rate, vocab_size):
# print(W_input)
# print('********************8')
targets = list()
for context_word in context_words:
targets.append(context_word)
for negsample in negsamples:
targets.append(negsample)
targets = np.array(targets)
#forward_propagation
h = W_input[targets,:]
prediction = h.dot(W_output[main_word].T)
prediction = sigmoid(prediction)
#backward_propagation
tj = np.zeros(len(targets))
for i in range(len(context_words)):
tj[i] = 1
pred_error = prediction - tj
# print(h.shape)
# print(prediction.shape)
# print(pred_error.shape)
# print(W_output[:,targets].T.shape)
# print('*****************************')
del_W_input = np.outer(pred_error, W_output[main_word])
# print(del_W_input.shape)
# print(h.shape)
del_W_output = np.dot(pred_error.T,h)
# print(del_W_output.shape)
# print('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@')
for index in range(len(targets)):
W_input[targets[index]] -= learning_rate*del_W_input[index]
W_output[main_word] -= learning_rate*del_W_output
costr = tj * np.log(prediction + 1e-10) + (1 - tj) * np.log(1 - prediction + 1e-10)
# cost = pred_error.sum()
cost = -1*costr.sum()
# print(W_input.shape())
return W_input, W_output, cost
def generate_data(dataset_size):
x, y = [], []
for _ in range(dataset_size):
x_i = 20 * np.random.random() - 10
p_i = sigmoid(np.sin(x_i))
y_i = np.random.binomial(1, p_i)
x.append(x_i)
y.append(y_i)
return np.array(x), np.array(y).reshape(-1, 1)
def binary_crossentropy_pseudo_residuals(y_true, logits):
"""Pseudo residuals between true label values and predicted logits.
Arguments need equal shape
:param y_true: True values (np.array)
:param logits: predicted logit values (np.array)
"""
return y_true - sigmoid(logits)
def score_vecs(vecs):
vec_list = [vecs[0].T]
for v in vecs[1:-1]:
vec_list.extend([v, v.T])
vec_list.append(vecs[-1])
score = 1.0
for i in range(len(vec_list) - 1):
score = score * sigmoid(np.dot(vec_list[i], vec_list[i + 1]))
return score
def inspect_and_evaluate_model(model, games_tourney_train, games_tourney_test):
# Print the model weights
print(model.get_weights())
# Print the training data means
print(games_tourney_train.mean())
weight = 0.14
# Print the approximate win probability predicted close game
print(sigmoid(1 * weight))
# Print the approximate win probability predicted blowout game
print(sigmoid(10 * weight))
# Evaluate the model on new data
print(
model.evaluate(
games_tourney_test[['seed_diff', 'pred']],
[games_tourney_test[['score_diff']], games_tourney_test[['won']]],
verbose=False))
def test_hazard_and_hidden(simple_feature_matrix):
X = simple_feature_matrix
Id = np.identity(3)
# Exp( first column of X )
assert np.allclose(hazard(X, Id[0]), np.exp([0, 1, 2, 0]))
# Exp ( second column of X )
assert np.allclose(hazard(X, Id[1]), np.exp([1, 0, 5, 2]))
# Exp ( third column of X )
assert np.allclose(hazard(X, Id[2]), np.exp([3, 1, -2, -2]))
# sigmoid ( first column of X )
assert np.allclose(conv_prob(X, Id[0]), sigmoid(np.asarray([0, 1, 2, 0])))
# sigmoid ( second column of X )
assert np.allclose(conv_prob(X, Id[1]), sigmoid(np.asarray([1, 0, 5, 2])))
# sigmoid ( third column of X )
assert np.allclose(conv_prob(X, Id[2]), sigmoid(np.asarray([3, 1, -2,
-2])))
def prob_function_sigmoid_gg(candidates, cand_available, inst, inst_rank, school_info, weights, **kwargs):
job_region = school_info[inst]['Region']
cand_p = np.zeros(len(candidates), dtype=float)
for i, (candidate, candidate_rank) in enumerate(candidates):
if cand_available[i]:
cand_p[i] = sigmoid(np.dot(weights, [1,
int(job_region == candidate.phd_region)]))
#cand_p /= cand_p.sum()
return cand_p
def fit(self, X, y, n_epochs=4000, lr=0.01, n_units=10):
self.w = np.random.rand(X.shape[1], n_units)
self.v = np.random.rand(n_units, y.shape[1])
for _ in range(n_epochs):
h_out = sigmoid(X.dot(self.w))
out = softmax(h_out.dot(self.v))
self.v -= lr * h_out.T.dot(out - y)
self.w -= lr * X.T.dot(
(out - y).dot(self.v.T) * (h_out * (1 - h_out)))
def prob_function_sigmoid_rd_rh(candidates, cand_available, inst, inst_rank,
school_info, weights, **kwargs):
cand_p = np.zeros(len(candidates), dtype=float)
for i, (candidate, candidate_rank) in enumerate(candidates):
if cand_available[i]:
cand_p[i] = sigmoid(
np.dot(weights, [1, inst_rank - candidate_rank, inst_rank]))
#cand_p /= cand_p.sum()
return cand_p
def initialize(input, hidden, output, examples):
print("examples", examples, "input", input, "hidden", hidden, "output", output)
x = np.random.uniform(0, 1, (examples, input))
y = np.random.uniform(0, 1, (examples, output))
weightsin = np.random.uniform(-1, 1, size=(input, hidden))
weights2 = np.random.uniform(-1,1, size = (hidden+1, hidden))
weightsout = np.random.uniform(-1, 1, size=(hidden+1, output))
error = 1
while(error !=0):
#forward propogation
layer1 = sigmoid(np.dot(x, weightsin))
layer1 = np.append(layer1, np.ones((examples, 1)), axis = 1)
layer2 = sigmoid(np.dot(layer1,weights2))
layer2 = np.append(layer2, np.ones((examples, 1)), axis = 1)
predict = sigmoid(np.dot(layer2, weightsout))
alpha = 0.006
error = 1/2 * (y - predict) ** 2
error = error.sum()
#backward propogation
change3 = -1 * (y - predict) * dsigmoid(predict)
DJWo = np.dot(layer2.transpose(), change3)
weightsout = weightsout + -alpha * DJWo
change2 = np.dot(change3, weightsout[0:np.shape(weightsout)[0]-1].transpose()) * dsigmoid(layer2[:, 0:np.shape(layer2)[1]-1])
DJW2 = np.dot(layer1.transpose(), change2)
weights2 = weights2 + -alpha*DJW2
change1 = np.dot(change2, weights2[0:np.shape(weights2)[0]-1, :].transpose())*dsigmoid(layer1[:, 0:np.shape(layer1)[1]-1])
DjWi = np.dot(x.transpose(), change1)
weightsin += -alpha*DjWi
print(error)
print("Final Error", error)
def getDataExpectations(self, batch):
from scipy.special import expit as sigmoid
hidden_probs = sigmoid(
np.matmul(batch, self.weights) + self.hidden_bias)
w_expectation = np.matmul(batch.T, hidden_probs)
v_expectation = batch
h_expectation = hidden_probs
return w_expectation, v_expectation, h_expectation
def _process_pred(self, pred):
"""Standarizes pred to be an integer array."""
assert pred.ndim == 3, f"expected 3-dim array, got {pred.ndim}-dim"
if pred.shape[-1] == 1: # binary case
if self.from_logits:
pred = sigmoid(pred)
pred = pred[..., 0] * 255
else: # multiclass
pred = np.argmax(pred, axis=-1)
return pred
def cost(self,theta, X, y, l):
m = len(y)
J = 0
grad = np.zeros(shape=(len(theta),1))
H = sigmoid(X.dot(theta))
y_T = y.transpose()
J = 1./m * np.sum(-y_T.dot(np.log(H)) - (1-y_T).dot(np.log(1-H))) + l/(2.*m) * np.sum(theta[1:]**2)
return J
def propagateForward(X_row, thetas):
features = X_row
for i in range(len(thetas)):
theta = thetas[i]
z = np.dot(theta, features)
a = sigmoid(z)
if i == len(thetas)-1:
return a
a = np.insert(a,0,1) #add bias unit
features = a
def forward(self, sample):
# forward propogation through the network
in_data = sample
self.nodes = []
for i in range(len(self.layers[:-1])):
layer_data = sigmoid(np.dot(in_data, self.weights[i]))
self.nodes.append(copy.deepcopy(layer_data))
in_data = copy.deepcopy(layer_data)
return self.nodes[-1][
0] #prediction after softmax -- do percentage on it somewhere?
def predict(self, x):
z = 0
for (field, index, value) in x:
w = self.get_weight(field, index)
z += w * value
p = sigmoid(z)
return p
def Execute_SGD_TT(X_norm, alpha, num_iters):
X_train, X_test = splitTT(0.6, X_norm)
theta = np.zeros((X_norm.shape[1] - 1, 1))
theta = stochasticGD(X_train, theta, alpha, num_iters)
prediction_set = sigmoid(np.dot(X_test[:, 0:5], theta))
acc = getAccuracy(prediction_set, X_test[:, 5:6])
error = errCompute(X_test, theta)
return acc, error
def create_data(N, P, z_real, seed = 0): # N = 200 # P = 4 N = int(N) P = int(P) rs = np.random.RandomState(seed) X = rs.randn(N,P) # z_real = rs.randn(P) y = rs.binomial(1,sigmoid(np.dot(X,z_real))) return X, y
def activate(self, inputs):
"""
inputs = numpy array of input values, one per input neuron.
outputs = numpy array of outputs, one per neuron in this layer.
Outputs calculated using a sigmodal activation function.
"""
internal_activations = np.dot(self.weights, inputs)
if self._apply_bias:
internal_activations += self.biases
return sigmoid(internal_activations)
def forward_jax(X, t, U, b, W, c):
N = X.shape[0]
# Perform forwards computation.
G = jnp.dot(X, U.T) + b
H = jnp.tanh(G)
z = jnp.dot(H, W.T) + c
y = sigmoid(z)
return (1./N) * jnp.sum(-t * jnp.log(y) - (1 - t) * jnp.log(1 - y))
def nnCostFunc(nn_params, input_layer_size, hidden_layer_size, num_labels, X, y, lmbda):
Theta1 = nn_params[0:(input_layer_size+1)*hidden_layer_size]
Theta1 = Theta1.reshape((hidden_layer_size,input_layer_size+1),order='F')
Theta2 = nn_params[(input_layer_size+1)*hidden_layer_size:]
Theta2 = Theta2.reshape((num_labels,hidden_layer_size+1),order='F')
m = size(X, 0);
J = 0;
#Theta1_grad = np.zeros(size(Theta1));
#Theta2_grad = np.zeros(size(Theta2));
eye_matrix = np.eye(num_labels)
if num_labels == 1:
y_matrix = y.astype(int)
else:
y_matrix = eye_matrix[y-1,:]
y_matrix.shape = (size(y,0),num_labels)
#--------------------------------------------------------------------------
#forward propagation
#--------------------------------------------------------------------------
X = np.concatenate((np.ones(((size(X,0)),1)), X),1)
hidden = sigmoid([email protected])
hidden = np.concatenate((np.ones(((size(hidden,0)),1)), hidden),1)
prediction = sigmoid([email protected])
for i in range(num_labels):
J = J + (1/m)*(((np.log(prediction[:,i]).T)@(-y_matrix[:,i])) - (np.log(1-prediction[:,i]).T)@(1-y_matrix[:,i]))
for i in range(1,size(Theta1,1)):
J= J + lmbda/(2*m)*np.sum(np.square(Theta1[:,i]))
for i in range(1,size(Theta2,1)):
J= J + lmbda/(2*m)*np.sum(np.square(Theta2[:,i]))
J = np.sum(np.sum(J))
print (J)
return J
def prob_function_sigmoid_gg(candidates, cand_available, inst, inst_rank,
school_info, weights, **kwargs):
job_region = school_info[inst]['Region']
cand_p = np.zeros(len(candidates), dtype=float)
for i, (candidate, candidate_rank) in enumerate(candidates):
if cand_available[i]:
cand_p[i] = sigmoid(
np.dot(weights, [1, int(job_region == candidate.phd_region)]))
#cand_p /= cand_p.sum()
return cand_p
def forward(self, W_in, X):
# construct network
tmp = X
for i in xrange(len(W_in)):
if i == 0:
W, W_b = W_in[i][:self.n_in], W_in[i][self.n_in:]
else:
W, W_b = W_in[i][:self.n_hidden[i-1]], W_in[i][self.n_hidden[i-1]:]
tmp = sigmoid(np.dot(tmp, W) + W_b)
return tmp
def compare(i, j):
if np.random.choice([0, 1], p=[0.9, 0.1]):
# draw
return 0
z_i = np.random.normal(mu[i], sigma[i])
z_j = np.random.normal(mu[j], sigma[j])
p = sigmoid(z_i - z_j)
if np.random.choice([0, 1], p=[1 - p, p]):
return 1
else:
return 2
def errCompute(X_norm, theta):
actual_response = X_norm[:, 5:6]
predictor_set = X_norm[:, 0:5]
prediction_set = sigmoid(np.dot(predictor_set, theta))
cost = (-1) * np.sum(actual_response * np.log(prediction_set) +
(1 - actual_response) *
np.log(1 - prediction_set)) / len(X_norm)
return cost
def fit(self, y, x, n=1, epsilon=.01, regularization=None, _lambda=1.0):
"""
Learn the weights for a Logistic Regression Classifier from the data
Params
---------
y : Numpy array
- The binary class labels for the data points
x : Numpy array
- A list of features for each data point
n : int - Default: 1
- The number of iterations for the training algorithm
epsilon : float - Default: 0.01
- Hyperparamter controlling overfitting of the model to the data
regularization : {None, 'L1', 'L2'} - Default: None
- Type of regularization to employ on the model
_lambda : float
- Hyper parameter for regularization
"""
# Initialize the weight vector
w_0 = np.zeros(x.shape[1])
# Variables used for learning weights
self._epsilon = epsilon
self._num_training = x.shape[0]
self._lambda = _lambda
print 'Epsilon: {}'.format(self._epsilon)
# Pick the correct update method
if regularization == 'l1':
print 'L1 regularization'
print 'Lambda: {}'.format(self._lambda)
update_func = self._l1
elif regularization == 'l2':
print 'L2 regularization'
print 'Lambda: {}'.format(self._lambda)
update_func = self._l2
else:
print 'No regularization'
update_func = self._no_reg
# Number of iterations
for _ in range(n):
# Loop over all the data points
for i in range(x.shape[0]):
y_minus_g = y[i] - sigmoid( np.dot(w_0, x[i]) )
w_0 = update_func(y[i], x[i], y_minus_g, w_0)
# Save the learned weights
self.weights = w_0
return None
def predict(images, V, W):
predicted_labels = []
for img in images:
z1 = np.dot(V, img.T)
activation = np.tanh(z1)
z1 = np.append(activation, 1)
z1 = np.reshape(z1, (len(z1), 1))
z1 = np.dot(W, z1)
z = sigmoid(z1)
predicted_labels.append(np.argmax(z) + 1)
return predicted_labels
def plot_bond_encoding(theta=None, start=1.0, end=6.0, slope=20., segments=5, show=False):
if theta is None:
theta = numpy.linspace(start, end, segments)
distance = numpy.linspace(0, 8, 250)
value = sigmoid(slope*numpy.subtract.outer(theta, distance)).T
plt.plot(distance, value)
plt.yticks(numpy.linspace(0, 1.1, 4))
plt.xlabel("Bond Length ($\AA$)")
plt.ylabel("Weight")
if show:
plt.show()
def decision_function(self, X):
if not self.fitted_:
raise ValueError('This ELMClassifier instance is not fitted yet.')
X = self._validate_X(X)
if self.random_projection:
H = sigmoid(dot(X, self.W) + self.b)
else:
H = X
if hasattr(self, 'fit_intercept') and self.fit_intercept:
H = np.hstack((H, np.ones((X.shape[0], 1))))
return dot(H, self.beta)
def data_sigm_multi(N, p):
if not isinstance(p, (list, tuple)):
assert isinstance(p, int)
p = [1. / p] * p
X = np.random.uniform(-10., 10., N)
Y = sigmoid(X)
Y += np.random.normal(0., .1, N)
y = np.random.multinomial(1, p, size=N)
Y += np.nonzero(y)[1]
Y, X = Y.reshape((N, 1)), X.reshape((N, 1))
return X, Y
def _simulate_single_equaiton_daring(X, scale, hidden=100):
pa_size = X.shape[1]
z = _simulate_noise(scale)
W1 = np.random.uniform(low=0.5, high=2.0, size=[pa_size, hidden])
W1[np.random.rand(*W1.shape) < 0.5] *= -1
W2 = np.random.uniform(low=0.5, high=2.0, size=hidden)
W2[np.random.rand(hidden) < 0.5] *= -1
x = sigmoid(X @ W1) @ W2 + z
return x
def data_sigm_multi(N, p):
if not isinstance(p, (list, tuple)):
assert isinstance(p, int)
p = [1. / p] * p
X = np.random.uniform(-10., 10., N)
Y = sigmoid(X)
Y += np.random.normal(0., .1, N)
y = np.random.multinomial(1, p, size=N)
Y += np.nonzero(y)[1]
Y, X = Y.reshape((N, 1)), X.reshape((N, 1))
return X, Y
def feedforward(X, W):
X_input = X[:, 0:len(X[0]) - 1]
Oh = np.tanh(np.dot(W[0], X_input.transpose()))
n, m = Oh.shape
Ino_ones = np.ones((1, m))
Ino = np.vstack((Ino_ones, Oh))
Oo = sigmoid(np.dot(W[1], Ino))
return [Oh, Ino, Oo]
def ML_crp(counts_list):
params = np.zeros(2)
print("ML optimization starting at obj = ",
logprob_obj(params, counts_list))
res = minimize(logprob_obj, params, args=(counts_list))
params = res.x
print("ML optimization ended at obj = ", logprob_obj(params, counts_list))
spi_alpha, logit_beta = params
alpha = softplus(spi_alpha)
beta = sigmoid(logit_beta)
return CRP(alpha, beta)
def predict(theta, X, y): # number of training examples m = shape(X)[0] # find hypothesis h = sigmoid( X.dot(theta.T) ) # make prediction using # index of max value in # each training example (row) prediction = argmax(h,axis=1) # find accuracy accuracy = (float(sum(prediction == y))/m)*100 return prediction,accuracy
def grad_mixed_guidance(data, v):
bounds = data.bounds
x = data.x_bound
scale = data.scale
w, b = unpack_linear(v)
y = apply_linear(x, w, b)
assert y.size == bounds.shape[0]
c1 = bounds[:, 0]
c2 = bounds[:, 1]
sig1 = sigmoid((c2 - y) / scale)
sig2 = sigmoid((c1 - y) / scale)
small_constant = getattr(data, "eps", eps)
denom = sig1 - sig2 + small_constant
"""
val = np.zeros(v.shape)
assert (denom > -1).all()
for i in range(x.shape[0]):
num1 = sig1[i]*(1-sig1[i])
num2 = sig2[i]*(1-sig2[i])
val[0:-1] += (num1-num2)*(1/denom[i])*x[i,:]/scale
val[-1] += (num1-num2)*(1/denom[i])
"""
# assert not np.isnan(val).any()
# Why isn't this necessary?
# val *= -1
num = sig1 * (1 - sig1) - sig2 * (1 - sig2)
num /= scale
num /= denom
g_fast = (x.T * num).sum(1)
g_fast = np.append(g_fast, num.sum())
# rel_error = array_functions.relative_error(val, g_fast)
val = g_fast
if np.isnan(val).any():
print "grad_linear_bound_logistic: nan!"
val[np.isnan(val)] = 0
if np.isinf(val).any():
# print 'grad_linear_bound_logistic: inf!'
val[np.isinf(val)] = 0
return val
def _sigmoidLikelihood(self, x, label):
"""
Returns the sigmoid likelihood p(y=label|features; weights)
x : Numpy array
Feature set for a single data point
"""
logit = sigmoid(np.dot(x, self.weights))
if label == 0:
return (1-logit)
elif label == 1:
return logit
