def expitPrime(colVec): return expit(colVec) * expit(1-colVec)
# the derivative of ("expit" = the sigmoid function)
# In[4]:
def _sigmoid(x): return (x / (1 + abs(x)) + 1) / 2
def worker(data,\
weights, hidden_bias, visible_bias,\
weight_rate, vbias_rate, hbias_rate, weightcost, isLinear, batch_num):
pos_hidden_activations = dot(data, weights) + hidden_bias
if isLinear:
pos_hidden_probs = pos_hidden_activations
pos_hidden_states = pos_hidden_probs + random.randn(len(data), len(hidden_bias)).astype(REAL)
else:
pos_hidden_probs = expit(pos_hidden_activations)
pos_hidden_states = pos_hidden_probs > random.randn(len(data), len(hidden_bias)).astype(REAL)
posprods = dot(data.T, pos_hidden_probs)
pos_hidden_act = sum(pos_hidden_probs, axis = 0)
pos_visible_act = sum(data, axis = 0)
neg_visible_activations = dot(pos_hidden_states, weights.T) + visible_bias
neg_visible_probs = expit(neg_visible_activations)
neg_hidden_activations = dot(neg_visible_probs, weights) + hidden_bias
if isLinear:
neg_hidden_probs = neg_hidden_activations
else:
neg_hidden_probs = expit(neg_hidden_activations)
negprods = dot(neg_visible_probs.T, neg_hidden_probs)
neg_hidden_act = sum(neg_hidden_probs, axis = 0)
neg_visible_act = sum(neg_visible_probs, axis = 0)
add_grad_weight = weight_rate * ((posprods - negprods) / len(data) - weightcost * weights)
add_grad_vbias = vbias_rate * (pos_visible_act - neg_visible_act) / len(data)
add_grad_hbias = hbias_rate * (pos_hidden_act - neg_hidden_act) / len(data)
error = sum((data - neg_visible_probs) ** 2)
if batch_num % 10 == 0:
print 'finish batch compute', batch_num, time.asctime(time.localtime(time.time()))
return (error, add_grad_weight, add_grad_vbias, add_grad_hbias, neg_hidden_probs)
def trainingKBComparison(kbValues,input_sentences,filename):
# This both sorts out the features and the training labels
trainingValues = []
for dict in input_sentences:
trainingValues.append(expit(dict['location-value-pair'].values()))
trainingValues = np.array(trainingValues).flatten()
# Load the KB
plt.figure()
fig, ax = plt.subplots()
for i,value in enumerate(kbValues):
kbValues[i] = expit(value)
ax.hist(kbValues,alpha=0.5,color='g',normed=True,bins=25,label='KB Values') # plt.hist passes it's arguments to np.histogram
ax.hist(trainingValues,alpha=0.5,color='r',normed=True,bins=25,label='Training Values')
plt.title("Histogram of Values")
ax.legend(loc='upper right')
ax.tick_params(axis='both', which='major', labelsize=8)
ax.tick_params(axis='both', which='minor', labelsize=8)
vals = ax.get_yticks()
ax.set_yticklabels(['{:1.1f}%'.format(x) for x in vals])
plt.savefig(filename)
plt.clf()
def loss(labels,q,M,a,b):
"""#Loss cross-entropy function with regularization
inputs: labels-row array of {0.1};q-column vector;M-matrix;a-row of columns;b-scalar
"""
l,alpha=trainConsts()
x=-(labels*np.log(s.expit(z(q,M,a,b)))+(1-labels)*np.log(1-s.expit(z(q,M,a,b))))
return np.sum(x)+l/2*(np.sum(M**2)+b**2)
def myTrain(model, docIndxPos, docIndxNeg, ngramEmbeddings, alpha = 0.1):
a_2 = expit(reshape(ngramEmbeddings, (1, ngramEmbeddings.size))*matrix(model.w_lt))
## sum_a_2 = exp(a_2).sum()
## a_2 = divide(exp(a_2), sum_a_2)
a_3Pos = expit(matrix(model.w_ld[docIndxPos])*(a_2.transpose()))
a_3Neg = expit(matrix(model.w_ld[docIndxNeg])*(a_2.transpose()))
## a_3Pos = matrix(model.w_ld[docIndxPos])*(a_2.transpose())
## a_3Neg = matrix(model.w_ld[docIndxNeg])*(a_2.transpose())
## cost = max(0, 0.5 - a_3Pos + a_3Neg);
## if(cost > 0):
err3Pos = (1 - a_3Pos)
err3Neg = (0 - a_3Neg)
err2Pos = multiply((err3Pos*model.w_ld[docIndxPos]), multiply(a_2, (1-a_2)))
err2Neg = multiply((err3Neg*model.w_ld[docIndxNeg]), multiply(a_2, (1-a_2)))
model.w_ld[docIndxPos] = model.w_ld[docIndxPos] + alpha*err3Pos*a_2
model.w_ld[docIndxNeg] = model.w_ld[docIndxNeg] + alpha*err3Neg*a_2
model.w_lt = model.w_lt + alpha*(matrix(ngramEmbeddings).transpose())*err2Pos
model.w_lt = model.w_lt + alpha*(matrix(ngramEmbeddings).transpose())*err2Neg
return
def normalize(score):
if score==0:
score=score-2
normScore= expit(score)
else:
normScore= expit(score)
return normScore
def get_ordered_logistic_probs(ability, item_difficulties):
# NB the '+' operators here represent list concatenation not addition
return array([1 - expit(ability - item_difficulties[0])] +
[expit(ability - item_difficulties[grade - 1]) -
expit(ability - item_difficulties[grade])
for grade in range(1, len(item_difficulties))] +
[expit(ability - item_difficulties[len(item_difficulties) - 1])])
def RMSE(self):
W, V, b, mu = self.AE.get_params()
print("testing process starts")
test = self.AE.test_ind
rat = []
pred = []
rmse = 0
for i,j in test:
Rt = self.AE.T[i, :].todense()
Rt1 = np.zeros(Rt.shape[1] +1)
Rt1[:6541] = Rt
#pdb.set_trace()
p = expit(np.dot(W, expit(np.dot(V, Rt1) + mu)) + b)
#p = np.tanh(np.dot(W, np.tanh(np.dot(V, Rt1) + mu)) + b)
p = p[j]
pred.append(p)
rat.append(self.AE.t[i, j])
try:
rat = np.array(rat)
pred = np.array(pred)
rmse = np.sqrt(np.mean((pred-rat)**2))
except:
print "exception"
pdb.set_trace()
np.save('test', test)
np.save('W', W)
np.save('V', V)
np.save('mu', mu)
np.save('b', b)
return rmse
def rhs(self, V, h, n, S):
s = self
tauhV = s.tauBarh / (np.cosh((V - s.thetah) / 2 / s.sigmah))
taunV = s.tauBarn / (np.cosh((V - s.thetan) / 2 / s.sigman))
hinfV = expit((s.thetah - V) / s.sigmah)
ninfV = expit((s.thetan - V) / s.sigman)
minfNaPV = expit((s.thetamNaP - V) / s.sigmamNaP)
minfNaV = expit((s.thetamNa - V) / s.sigmamNa)
SinfV = expit((s.thetaS - V) / s.sigmaS)
gsyns = np.dot(s.gsyn, S)
# m n h need expit functions
# V h n are the function parameters
# dVdt = [- Inap-h - INa - IK - IL - Itonice==0 - Isyn-e + Iapp==0 ] / C
dVdt = (
- s.gNaPBar * minfNaPV * h * (V - s.ENa) # -I_{NaP} = I_{NaP-h} for model 1. (\neq I_{Na})
- s.gNaBar * minfNaV**3 * (1 - n) * (V - s.ENa) # -I_{Na}
- s.gkBar * n**4 * (V - s.Ek) # -I_K
- s.gLBar * (V - s.EL) # -I_L
- s.gt * (V - s.Etonic) # -I_tonic
- gsyns * (V - s.Esyn) # -I-syn-e
+ s.Iapp
) / s.C * 1000 # Conversion to mV. Naturally in terms of V/s without this.
dhdt = (hinfV - h) / tauhV
dndt = (ninfV - n) / taunV
dSdt = ((1 - S) * SinfV - (s.kr * S)) / s.tauS
return dVdt, dhdt, dndt, dSdt
def activate(rnn, in_0, rec_0, bias=False, return_out=False):
W = rnn.W
# We're throwing away the b_rec.
# That can be added on a trial by trial basis.
W_in, b_in = rnn.get_layer(W, 0)
W_rec, b_rec = rnn.get_layer(W, 1)
W_out, b_out = rnn.get_layer(W, 2)
# recurrent activations
ff_input = np.dot(in_0, W_in) + b_in
if bias:
rec_input = np.dot(rec_0, W_rec) + b_rec
else:
rec_input = np.dot(rec_0, W_rec) + b_rec
rec_1 = expit(ff_input + rec_input)
# # output activations
# # we don't care about these when finding slow point
if return_out:
out_1 = expit(np.dot(rec_1, W_out) + b_out)
return out_1
# assume input doesn't change from previous step
return rec_1
def grad_L2loss(self, beta0, beta, alpha, reg_lambda, x, y):
z = beta0 + np.dot(x, beta)
if(self.distr=='poisson'):
q = self.qu(z)
s = expit(z)
grad_beta0 = np.sum(s) - np.sum(y*s/q)
grad_beta = np.transpose(np.dot(np.transpose(s), x) - np.dot(np.transpose(y*s/q), x)) \
+ reg_lambda*(1-alpha)*beta# + reg_lambda*alpha*np.sign(beta)
elif(self.distr=='normal'):
grad_beta0 = -np.sum(y-z)
grad_beta = -np.transpose(np.dot(np.transpose(y-z), x)) \
+ reg_lambda*(1-alpha)*beta# + reg_lambda*alpha*np.sign(beta)
elif(self.distr=='binomial'):
s = expit(z)
grad_beta0 = np.sum(s-y)
grad_beta = np.transpose(np.dot(np.transpose(s-y), x)) \
+ reg_lambda*(1-alpha)*beta# + reg_lambda*alpha*np.sign(beta)
elif(self.distr=='multinomial'):
# this assumes that y is already as a one-hot encoding
pred = self.qu(z)
grad_beta0 = -np.sum(y - pred)
grad_beta = -np.transpose(np.dot(np.transpose(y - pred), x)) \
+ reg_lambda*(1-alpha)*beta
return grad_beta0, grad_beta
def make_emissions(self, diff, prob_num):
if self.emission_mask[prob_num]:
#print str(time.time()) + "\tusing saved emissions"
return self.emission_mats[prob_num]
else:
#print str(time.time()) + "\tcalculating new emissions"
self.emission_mask[prob_num] = True
table = self.emission_mats[prob_num]
#guesses...
for row in range(self.total_states - 1):
table[row, 1] = expit(self.params['G_' + str(row)].get() - diff)
table[row, 0] = 1 - table[row, 1]
#and slip
table[row + 1, 0] = expit(self.params['S'].get() + diff)
table[row + 1, 1] = 1 - table[row + 1, 0]
#g = self.params['G_0'].get()
#s = self.params['S'].get()
#table = np.array([[1-g, g], [s, 1-s]])
"""
print
print np.array(table)
print self.params['G_0'].get()
print self.params['G_1'].get()
print self.params['S'].get()
print
"""
return table
def trainingTestComparison(training,oldtest,newtest,filename):
# This both sorts out the features and the training labels
trainingValues = []
oldTestValues = []
newTestValues = []
for dict in training:
trainingValues.append(expit(dict['location-value-pair'].values()))
for dict in oldtest:
oldTestValues.append(expit(dict['location-value-pair'].values()))
for dict in newtest:
newTestValues.append(expit(dict['location-value-pair'].values()))
trainingValues = np.array(trainingValues).flatten()
oldTestValues = np.array(oldTestValues).flatten()
newTestValues = np.array(newTestValues).flatten()
plt.figure()
fig, ax = plt.subplots()
ax.hist(oldTestValues,alpha=0.5,color='r',bins=25,normed=True,label='Old Test Values') # plt.hist passes it's arguments to np.histogram
ax.hist(trainingValues,alpha=0.5,color='y',bins=25,normed=True,label='Training Values')
ax.hist(newTestValues,alpha=0.5,color='b',bins=25,normed=True,label='New Test Values') # plt.hist passes it's arguments to np.histogram
plt.title("Histogram of Values")
ax.legend(loc='upper right')
ax.tick_params(axis='both', which='major', labelsize=8)
ax.tick_params(axis='both', which='minor', labelsize=8)
vals = ax.get_yticks()
ax.set_yticklabels(['{:1.1f}%'.format(x) for x in vals])
plt.savefig(filename)
plt.clf()
def get_loss(self, data):
loss = 0
for u, items in enumerate(data):
fij = np.sum(self.U[u][np.newaxis, :] * self.V[items], axis=1)
loss += np.sum(np.log(expit(fij)))
loss += np.sum(np.log(1 -expit(fij[:, np.newaxis] - fij[np.newaxis, :])))
loss -= self.reg / 2 * (np.sum(self.U * self.U) + np.sum(self.V + self.V))
return loss
def updateRBM2(lattice, W, n, method="naive"):
damp = 0.5
if method == "naive":
lattice.hid = damp * expit(np.dot(W, lattice.vis)) + (1.0 - damp) * lattice.hid
lattice.vis = damp * expit(np.dot(W.T, lattice.hid)) + (1.0 - damp) * lattice.vis
else:
lattice.hid = damp * expit(np.dot(W, lattice.vis) - (lattice.hid - .5) * np.dot((W ** 2), (lattice.vis - lattice.vis ** 2))) + (1.0 - damp) * lattice.hid
lattice.vis = damp * expit(np.dot(W.T, lattice.hid) - (lattice.vis - .5) * np.dot((W.T ** 2), (lattice.hid - lattice.hid ** 2))) + (1.0 - damp) * lattice.vis
def sigmoid_function(x, derivative=False):
# Clip x to prevent overflow.
x = np.clip(x, -500, 500)
if derivative:
sigmoid_x = expit(x)
return sigmoid_x * (1-sigmoid_x)
else:
return expit(x)
def test_large(self):
for dtype in (np.float32, np.float64, np.longdouble):
for n in (88, 89, 709, 710, 11356, 11357):
n = np.array(n, dtype=dtype)
assert_allclose(expit(n), 1.0, atol=1e-20)
assert_allclose(expit(-n), 0.0, atol=1e-20)
assert_equal(expit(n).dtype, dtype)
assert_equal(expit(-n).dtype, dtype)
def logser_solver(ab):
"""Given abundance data, solve for MLE of logseries parameter p."""
ab = check_for_support(ab, lower=1)
BOUNDS = [0, 1]
DIST_FROM_BOUND = 10 ** -15
y = lambda x: 1 / log(1 / (1 - expit(x))) * expit(x) / (1 - expit(x)) - sum(ab) / len(ab)
x = bisect(y, logit(BOUNDS[0] + DIST_FROM_BOUND), logit(BOUNDS[1] - DIST_FROM_BOUND), xtol=1.490116e-08)
return expit(x)
def likelihoodmachine(list_of_positive_x, list_of_negative_x): """ input list of items for plus and minus outputs respectively. """ poshoods=listmultiplier([special.expit(i) for i in list_of_positive_x]) neghoods=listmultiplier([1-special.expit(i) for i in list_of_negative_x]) print listmultiplier([poshoods,neghoods]) return listmultiplier([poshoods,neghoods])
def predict(Theta1, Theta2, X):
# h1 = expit()
# X = np.transpose(X)
X = np.insert(X, 0, 1)
h1 = expit(np.dot(X, np.transpose(Theta1)))
h1 = np.insert(h1, 0, 1)
h2 = expit(np.dot(h1, np.transpose(Theta2)))
return h2.argmax() + 1
def predictProbability(self, x):
numberOfSamples = x.shape[0]
h1 = expit(
np.transpose(
np.vstack((np.ones((1,1)), x.reshape((numberOfSamples,1))))) @ self.theta[0])
h2 = expit(np.hstack((np.ones((1, 1)), h1)) @ self.theta[1])
return h2.reshape((h2.size,))
def forwardProp(X, thetaO, thetaT):
X = np.matrix(X)
secLayer = expit(X * thetaO.T)
onesArray = np.ones((len(secLayer),1))
secLayer = np.hstack((onesArray, secLayer))
thrLayer = expit(secLayer * thetaT.T)
return thrLayer, secLayer
def propagate(x):
# propagate an input x to the output layer
# [4.B] FILL YOUR CODE HERE
layers[0] = x.copy()
layers[1] = expit(np.dot(x, weights[0]))
layers[2] = expit(np.dot(layers[1], weights[1]))
return layers[2]
def train(self, visible_data, \ max_epochs = 50, batch = 10, \ initialmomentum = 0.5, finalmomentum = 0.9, \ weight_rate = 0.001, vbias_rate = 0.001, hbias_rate = 0.001, \ weightcost = 0.0002): grad_weight = zeros((len(self.visible_bias), len(self.hidden_bias))) grad_hbias = zeros((1, len(self.hidden_bias))) grad_vbias = zeros((1, len(self.visible_bias))) for epoch in xrange(max_epochs): error = 0. for start in xrange(batch): if start % 10 == 0: print "epoch %d, batch %d" % (epoch, start + 1), time.asctime( time.localtime(time.time()) ) data = visible_data[start::batch] pos_hidden_activations = dot(data, self.weights) + self.hidden_bias if self.isLinear: pos_hidden_probs = pos_hidden_activations pos_hidden_states = pos_hidden_probs + random.randn(len(data), len(self.hidden_bias)) else: pos_hidden_probs = expit(pos_hidden_activations) pos_hidden_states = pos_hidden_probs > random.randn(len(data), len(self.hidden_bias)) posprods = dot(data.T, pos_hidden_probs) pos_hidden_act = sum(pos_hidden_probs, axis = 0) pos_visible_act = sum(data, axis = 0) neg_visible_activations = dot(pos_hidden_states, self.weights.T) + self.visible_bias neg_visible_probs = expit(neg_visible_activations) neg_hidden_activations = dot(neg_visible_probs, self.weights) + self.hidden_bias if self.isLinear: neg_hidden_probs = neg_hidden_activations else: neg_hidden_probs = expit(neg_hidden_activations) negprods = dot(neg_visible_probs.T, neg_hidden_probs) neg_hidden_act = sum(neg_hidden_probs, axis = 0) neg_visible_act = sum(neg_visible_probs, axis = 0) error += sum((data - neg_visible_probs) ** 2) if epoch > 5: momentum = finalmomentum else: momentum = initialmomentum if epoch == max_epochs - 1: if self.hidden_probs == None: self.hidden_probs = neg_hidden_probs else: self.hidden_probs = concatenate((self.hidden_probs, neg_hidden_probs), axis=0) grad_weight = momentum * grad_weight + weight_rate * ((posprods - negprods) / len(data) - weightcost * self.weights) grad_vbias = momentum * grad_vbias + vbias_rate * (pos_visible_act - neg_visible_act) / len(data) grad_hbias = momentum * grad_hbias + hbias_rate * (pos_hidden_act - neg_hidden_act) / len(data) self.weights += grad_weight self.visible_bias += grad_vbias self.hidden_bias += grad_hbias print "epoch %d, error %d\n" % (epoch, error)
def count_loss(self, data):
loss = 0
for u, items in enumerate(data):
fi = np.dot(self.V[items], self.U[u])
diff_fi = fi[:, np.newaxis] - fi[np.newaxis, :]
gi = expit(fi)
gij = expit(diff_fi)
loss += np.sum(gi * np.sum(gij, axis=0))/len(items)
loss -= self.reg/2 * (np.sum(self.U ** 2) + np.sum(self.V ** 2))
return loss
def updateRBM1(lattice, W, n, method="naive"):
damp = 0.5
if method == "naive":
for ii in range(n):
lattice.m[ii] = damp * expit(np.dot(W[ii], lattice.m[n:])) + (1.0 - damp) * lattice.m[ii]
lattice.m[n + ii] = damp * expit(np.dot(W.T[ii], lattice.m[0:n])) + (1.0 - damp) * lattice.m[n + ii]
else:
for ii in range(n): # TODO sequential
lattice.m[ii] = damp * expit(np.dot(W[ii], lattice.m[n:]) - (lattice.m[ii] - 0.5) * np.dot(W[ii] ** 2, lattice.m[n:] - lattice.m[n:] ** 2)) + (1.0 - damp) * lattice.m[ii]
lattice.m[n + ii] = damp * expit(np.dot(W.T[ii], lattice.m[0:n]) - (lattice.m[n + ii] - 0.5) * np.dot(W.T[ii] ** 2, lattice.m[0:n] - lattice.m[0:n] ** 2)) + (1.0 - damp) * lattice.m[n + ii]
def _forward_propagate(self, features):
"""Propagate forward"""
self.values[0][:-1] = features
for layer in range(len(self.weights)):
if layer < len(self.weights)-1:
self.values[layer+1][:-1] = expit(
np.dot(self.values[layer], self.weights[layer]))
else:
self.values[layer+1] = expit(
np.dot(self.values[layer], self.weights[layer]))
def main():
"""Prints hyperparameter estimates."""
samples = get_samples()
kprobs = get_ks_hparams(samples)
print('Mean, credible interval, HPDI')
for k in range(KMAX + 1):
print(
'Pr(k = {}) ='.format(k),
f2s(
kprobs[k].mean(),
kprobs[k].quantile(0.025), kprobs[k].quantile(0.975),
hpd(kprobs[k])))
q = kprobs[1] + kprobs[2]
print(
'Pr(k = 1 or k = 2) =',
f2s(q.mean(), q.quantile(0.025), q.quantile(0.975), hpd(q)))
q = sum(kprobs[1:])
print(
'Pr(k >= 1) =',
f2s(q.mean(), q.quantile(0.025), q.quantile(0.975), hpd(q)))
q = sum(kprobs[3:])
print(
'Pr(k >= 3) =',
f2s(q.mean(), q.quantile(0.025), q.quantile(0.975), hpd(q)))
# Medians, not means!
A, rho, theta = expit(samples['mu.1']), expit(samples['mu.2']),\
exp(samples['mu.3'])
for param, medians in zip(('A', 'rho', 'theta'), (A, rho, theta)):
medians = pd.Series(medians)
print(
param,
f2s(
medians.mean(),
medians.quantile(0.025), medians.quantile(0.975),
hpd(medians)))
# Correlations between parameters
# Correlation is obtained by dividing the covariance by the product of stdev.
cor = []
for _, sample in samples.iterrows():
scor = []
for i in range(2):
for j in range(i + 1, 3):
scor.append(
sample['sigma.{}.{}'.format(i + 1, j + 1)]/\
(sample['scale.{}'.format(i + 1)]*sample['scale.{}'.format(j + 1)]))
cor.append(scor)
cols = ('corArho', 'corAtheta', 'corrhotheta')
cor = pd.DataFrame(cor, columns=cols)
for col in cols:
print(
col, f2s(
cor[col].mean(), cor[col].quantile(0.025),
cor[col].quantile(0.975), hpd(cor[col])))
def pretraining_wd(self, indm, ind_doc, indr_doc):
'''
SGD update (pre-training on wd graph).
'''
w = W[indm, :]
d = D[indc, :]
wR = W[indr, :]
cost = - 1 * np.log(expit(np.dot(w, d)))
for k in range(wR.shape[0]):
cost -= np.log(expit(- np.dot(w, wR[k, :])))
return self.fun_wd(self, indm, ind_doc, indr_doc)
def compute_lambda(u, size):
fwd = 0
bwd = 0
bwd_count = 0
for v in xrange(size):
if labels[u] > labels[v]:
fwd -= expit(scores[v] - scores[u])
elif labels[u] < labels[v]:
bwd_count += 1
bwd += expit(scores[v] - scores[u])
return fwd - bwd_count + bwd
def f_gradient(w, Xb, yb):
denom = scspec.expit(-yb * Xb.dot(w))
return -Xb.T.dot(yb * denom) / Xb.shape[0]
def f_IL_6_AMP(self, NF_xB, IL6_STAT3, N):
y = self.betas['IL6_AMP_NF_xB_w'] * NF_xB + self.betas['IL6_AMP_IL6_STAT3_w'] * IL6_STAT3 + \
+ self.betas['IL6_AMP_b']
IL6_AMP = self.max_abundance['IL6_AMP'] * expit(y) + N
return IL6_AMP
# Plots
fig = plt.figure(figsize=(19, 10), facecolor='white')
ax = fig.add_subplot(1, 6, 1, xticks=[], yticks=[])
z = bst1.predict(dtest)
z = np.array([row for row in z])
ax.set_title("XGBoost with 50 trees")
plotsurf(ax, z)
ax = fig.add_subplot(1, 6, 2, xticks=[], yticks=[])
z = bst2.predict(dtest)
z = np.array([row for row in z])
ax.set_title("XGBoost with 1 tree")
plotsurf(ax, z)
ax = fig.add_subplot(1, 6, 3, xticks=[], yticks=[])
z = linbst.predict(test_X)
ax.set_title("LinXGBoost with 1 tree")
plotsurf(ax, expit(z))
ax = fig.add_subplot(1, 6, 4, xticks=[], yticks=[])
z = cart.predict_proba(test_X)[:, 1]
ax.set_title("CART")
plotsurf(ax, z)
ax = fig.add_subplot(1, 6, 5, xticks=[], yticks=[])
z = rf.predict_proba(test_X)[:, 1]
ax.set_title("RandomForest")
plotsurf(ax, z)
ax = fig.add_subplot(1, 6, 6, xticks=[], yticks=[])
z = rrPLF.predict(test_X)
ax.set_title("random rotation PL-Tree Forest")
plotsurf(ax, expit(z))
plt.show()
##---experiment1---#
def sigmoid(x):
#return 1 / (1 + np.expit(-x))
return expit(x) # expit (x) = 1 / (1 + exp (-x))
def sigmoidMotor(val):
return (expit(val) - 0.5) / 50
plt.scatter(x_new, y_new)
# %%
np.corrcoef(x_new, y_new)
# %%
num_data = 200
x_1 = np.random.randint(15, 76, num_data)
x_2 = np.random.randint(0, 2, num_data)
# %%
from scipy.special import expit # 1.
e_x = randn(num_data)
z_base = x_1 + (1 - x_2)*10 - 40 + 5*e_z
z_prob = expit(0.1 * z_base)
# %%
Z = np.array([])
for i in range(num_data):
Z_i = np.random.choice(2, size=1, p=[1-z_prob[i], z_prob[i]])[0]
Z = np.append(Z, Z_i)
# %%
e_y = randn(num_data)
Y = -x_1 + 30*x_2 + 10*Z + 80 + 10*e_y
# %%
import pandas as pd
df = pd.DataFrame({'age':x_1,
'Sex':x_2,
'WatchedCM':Z,
'PurchaseAmount':Y,
def train(f_dataset_train, h_prev, m_prev, t_index):
inp_unit = f_dataset_train[:, 0].reshape(
input_size, 1) # input vector xt taken from image data
inp_whole = numpy.vstack(
(inp_unit, h_prev)) # input vector It combined from xt and ht-1
hidden_whole = numpy.dot(curr_model.get_wm_hidden(),
inp_whole) # raw hidden value for all gates zt
a_gate = numpy.tanh(hidden_whole[:curr_model.get_hidden_unit_size(), :]
) # split, squashed value from zt
i_gate = expit(hidden_whole[curr_model.get_hidden_unit_size():2 *
curr_model.get_hidden_unit_size(), :])
f_gate = expit(hidden_whole[2 * curr_model.get_hidden_unit_size():3 *
curr_model.get_hidden_unit_size(), :] + f_bias)
o_gate = expit(hidden_whole[3 * curr_model.get_hidden_unit_size():, :])
m_curr = i_gate * a_gate + f_gate * m_prev # memory cell value ct calculated from at, it, ft, and ct-1
h_curr = o_gate * numpy.tanh(
m_curr) # current timestep output value calculated from ot and ct
if f_dataset_train[:,
1:].size == 0: # on the last step, calculate score, errhc
score = numpy.dot(curr_model.get_wm_score(),
h_curr) # output layer producing 24 length vector s
score_prob = numpy.exp(score) / numpy.sum(
numpy.exp(score)) # softmax layer producing probability vector P
loss = -numpy.log(
score_prob[t_index]) # negative log loss error value E
jac = numpy.diagflat(score_prob) - numpy.dot(
score_prob, score_prob.T) # matrix J for deriving softmax layer
err_score = numpy.dot(
jac, score_prob -
target_array[t_index]) # error vector ds from J and audio target
err_wm_score = numpy.dot(
err_score, h_curr.T) # matrix error dws calculated from ds
curr_model.set_wm_score(
curr_model.get_wm_score() -
curr_model.get_learning_rate() * err_wm_score) # ws update
err_h_curr = numpy.dot(
curr_model.get_wm_score().T,
err_score) # error vector dht for deriving previous steps
err_m_curr = 0 # last step has no next memory cell, dct = 0
err_wm_hidden = 0 # last step has 0 accumulated weight error, dw = 0
else: # if not on last step, send ht, ct for forward prop, wait for dht, dct, dw for backward prop
err_h_curr, err_m_curr, err_wm_hidden, loss = train(
f_dataset_train[:, 1:], h_curr, m_curr, t_index)
err_m_curr = err_m_curr + (err_h_curr * o_gate *
(1 - numpy.power(numpy.tanh(m_curr), 2))
) # accumulative dct
err_a_gate = err_m_curr * i_gate # error vector dat calculated from dct
err_i_gate = err_m_curr * a_gate # error vector dat calculated from dct
err_f_gate = err_m_curr * m_prev # error vector dat calculated from dct
err_o_gate = err_h_curr * numpy.tanh(
m_curr) # error vector dot calculated from dht
err_m_prev = err_m_curr * f_gate # error vector dct-1 calculated from dct
# calculate error vector for raw, unsplit, unsquashed hidden values for all at, it, ft, and ot
err_a_inp = err_a_gate * (1 - numpy.power(
numpy.tanh(hidden_whole[:curr_model.get_hidden_unit_size(), :]), 2))
err_i_inp = err_i_gate * i_gate * (1 - i_gate)
err_f_inp = err_f_gate * f_gate * (1 - f_gate)
err_o_inp = err_o_gate * o_gate * (1 - o_gate)
err_inp_whole = numpy.vstack(
(err_a_inp, err_i_inp, err_f_inp,
err_o_inp)) # dzt concated from hidden value errors
err_wm_hidden = err_wm_hidden + numpy.dot(
err_inp_whole, inp_whole.T) # accumulative dw for weight update
err_inp_unit = numpy.dot(curr_model.get_wm_hidden().T,
err_inp_whole) # whole input error vector dIt
err_h_prev = err_inp_unit[input_size:, 0].reshape(
(curr_model.get_hidden_unit_size(), 1)) # dht-1 split from dIt
return err_h_prev, err_m_prev, err_wm_hidden, loss # returns dht-1, dct-1, dw for backprop
def pmf(self, phi):
return expit(np.sum(self.weights[phi]))
ax.set_xlabel('LFMC(%)')
ax.set_yticks([0, 1])
ax.set_yticklabels(['No fire', 'Fire'])
#%% logit
X = np.append(no_fire, yes_fire)[np.newaxis].T
y = np.append(np.repeat(0, len(no_fire)), np.repeat(1, len(yes_fire)))
clf = LogisticRegression(random_state=0).fit(X, y)
clf.predict(X[:2, :])
clf.predict_proba(X[:2, :])
clf.score(X, y)
X_test = np.linspace(0, 250, 300)
loss = expit(X_test * clf.coef_ + clf.intercept_).ravel()
fig, ax = plt.subplots(figsize=(4, 4))
ax.scatter(x=no_fire,
y=np.repeat(0, len(no_fire)),
marker='o',
color='grey',
alpha=0.01)
ax.scatter(x=yes_fire,
y=np.repeat(1, len(yes_fire)),
marker='o',
color='crimson',
alpha=0.01)
ax.plot(X_test, loss, color='orange', linewidth=3, label='Prediction')
def grad(self, x):
m = self.b.shape[0]
return (-1 / m) * self.matvec_ATx(
self.b * expit(-self.b * self.matvec_Ax(x))) + self.regcoef * x
def f_TNF(self, ADAM17, N):
y = self.betas['TNF_ADAM17_w'] * ADAM17 + self.betas['TNF_b']
TNF = self.max_abundance['TNF'] * expit(y) + N
return TNF
def predict_ci(fitted, df, alpha=0.05):
"""
Compute predicted probabilities with confidence intervals based on a
logistic regression model
Parameters
----------
fitted : Logistic regression model fitted using the statsmodels formula interface
df : Pandas dataframe with input data for prediction
alpha : float
Significance level (0-1). Default is 0.05
Returns
-------
Pandas dataframe with probability predictions and lower and upper confidence bounds
Example
-------
import numpy as np
import statsmodels.formula.api as smf
import pandas as pd
# simulate data
np.random.seed(1)
x1 = np.arange(100)
x2 = pd.Series(["a", "b", "c", "a"], dtype="category").sample(100, replace=True)
y = (x1 * 0.5 + np.random.normal(size=100, scale=10) > 30).astype(int)
df = pd.DataFrame({"y": y, "x1": x1, "x2": x2})
# estimate the model
model = smf.logit(formula="y ~ x1 + x2", data=df).fit()
model.summary()
pred = predict_ci(model, df)
plt.clf()
plt.plot(x1, pred["prediction"])
plt.plot(x1, pred["2.5%"], color='black', linestyle="--", linewidth=0.5)
plt.plot(x1, pred["97.5%"], color='black', linestyle="--", linewidth=0.5)
plt.show()
"""
if alpha < 0 or alpha > 1:
raise ValueError("alpha must be a numeric value between 0 and 1")
# generate predictions
prediction = fitted.predict(df)
# set up the data in df in the same was as the exog data
# that is part of fitted.model.exog
# use a fake response variable
df = df.assign(__rvar__=1).copy()
form = "__rvar__ ~ " + fitted.model.formula.split("~", 1)[1]
exog = smf.logit(formula=form, data=df).exog
low, high = [alpha / 2, 1 - (alpha / 2)]
Xb = np.dot(exog, fitted.params)
se = np.sqrt((exog.dot(fitted.cov_params()) * exog).sum(-1))
me = stats.norm.ppf(high) * se
return pd.DataFrame({
"prediction": prediction,
f"{low*100}%": expit(Xb - me),
f"{high*100}%": expit(Xb + me),
})
def hess(self, x):
m = self.b.shape[0]
Ax = self.matvec_Ax(x)
return (1 / m) * self.matmat_ATsA(self.b * self.b * expit(
self.b * Ax) * expit(-self.b * Ax)) + self.regcoef * np.eye(len(x))
def sigmoid(x, out):
# out[:] = 1 / (1 + np.exp(-x))
# np.reciprocal(np.add(np.exp(np.negative(x, out), out), 1, out), out)
expit(x, out)
def predict(self,
question: str,
documents: List[Document],
top_k: Optional[int] = None):
"""
Use loaded QA model to find answers for a question in the supplied list of Document.
Returns dictionaries containing answers sorted by (desc.) probability
Example:
{'question': 'Who is the father of Arya Stark?',
'answers': [
{'answer': 'Eddard,',
'context': " She travels with her father, Eddard, to King's Landing when he is ",
'offset_answer_start': 147,
'offset_answer_end': 154,
'probability': 0.9787139466668613,
'score': None,
'document_id': '1337'
},
...
]
}
:param question: question string
:param documents: list of Document in which to search for the answer
:param top_k: the maximum number of answers to return
:return: dict containing question and answers
"""
# convert input to FARM format
inputs = []
for doc in documents:
cur = QAInput(doc_text=doc.text,
questions=Question(text=question, uid=doc.id))
inputs.append(cur)
# get answers from QA model
predictions = self.inferencer.inference_from_objects(
objects=inputs, return_json=False, multiprocessing_chunksize=1)
# assemble answers from all the different documents & format them.
# For the "no answer" option, we collect all no_ans_gaps and decide how likely
# a no answer is based on all no_ans_gaps values across all documents
answers = []
no_ans_gaps = []
best_score_answer = 0
for pred in predictions:
answers_per_document = []
no_ans_gaps.append(pred.no_answer_gap)
for ans in pred.prediction:
# skip "no answers" here
if self._check_no_answer(ans):
pass
else:
cur = {
"answer": ans.answer,
"score": ans.score,
# just a pseudo prob for now
"probability": float(expit(
np.asarray([ans.score]) / 8)), # type: ignore
"context": ans.context_window,
"offset_start": ans.offset_answer_start -
ans.offset_context_window_start,
"offset_end": ans.offset_answer_end -
ans.offset_context_window_start,
"offset_start_in_doc": ans.offset_answer_start,
"offset_end_in_doc": ans.offset_answer_end,
"document_id": pred.id
}
answers_per_document.append(cur)
if ans.score > best_score_answer:
best_score_answer = ans.score
# only take n best candidates. Answers coming back from FARM are sorted with decreasing relevance.
answers += answers_per_document[:self.top_k_per_candidate]
# Calculate the score for predicting "no answer", relative to our best positive answer score
no_ans_prediction, max_no_ans_gap = self._calc_no_answer(
no_ans_gaps, best_score_answer)
if self.return_no_answers:
answers.append(no_ans_prediction)
# sort answers by their `probability` and select top-k
answers = sorted(answers, key=lambda k: k["probability"], reverse=True)
answers = answers[:top_k]
result = {
"question": question,
"no_ans_gap": max_no_ans_gap,
"answers": answers
}
return result
def GxSample(x, nCores, pop, sampleKernel, popFlag, nBin, bw, nEcc, Tobs,
output):
def untransform(dat, datSet):
datMin = min(datSet)
datMax = max(datSet)
datUntransformed = dat * (datMax - datMin)
datUnzeroed = datUntransformed + datMin
return datUnzeroed
# CONSTANTS
##############################################################################
G = 6.67384 * math.pow(10, -11.0)
c = 2.99792458 * math.pow(10, 8.0)
parsec = 3.08567758 * math.pow(10, 16)
Rsun = 6.955 * math.pow(10, 8)
Msun = 1.9891 * math.pow(10, 30)
day = 86400.0
rsun_in_au = 215.0954
day_in_year = 365.242
sec_in_day = 86400.0
sec_in_hour = 3600.0
hrs_in_day = 24.0
sec_in_year = 3.15569 * 10**7.0
Tobs = 3.15569 * 10**7.0
geo_mass = G / c**2
m_in_AU = 1.496 * 10**11.0
##### UNITS EXPECTED FOR POPULATION ###################
# mass: Msun, orbital period: days, Tobs: seconds #
#######################################################
# seed the random generator
########################################
np.random.seed()
# solar coordinates in the galaxy: in parsecs from
# (Chaper 8 of Galactic Structure and stellar Pops book) Yoshii (2013)
############################################################################
x_sun = 8000.0
y_sun = 0.0
z_sun = 25
# sample the number of binaries given by nBin
#################################################
power = []
freq = []
n = 0
nWrite = 0
eccBin = 0
# open the file to save the gx data
#################################################
gxFile = 'gxRealization_' + str(x) + '_' + str(popFlag) + '.dat'
binDat = []
nSample = int(nBin / float(nCores))
dataSample = sampleKernel.resample(nSample)
# check to see if the population has BHs or is ecc
###########################################################
m1T = ss.expit(dataSample[0, :])
m2T = ss.expit(dataSample[1, :])
porbT = ss.expit(dataSample[2, :])
eccT = ss.expit(dataSample[3, :])
rad1T = ss.expit(dataSample[4, :])
rad2T = ss.expit(dataSample[5, :])
Lum1T = ss.expit(dataSample[6, :])
Lum2T = ss.expit(dataSample[7, :])
m1 = untransform(m1T, pop['m1'])
m2 = untransform(m2T, pop['m2'])
porb = untransform(porbT, np.log10(pop['porb']))
ecc = eccT
ii = 0
for e in ecc:
if e < 0:
ecc[ii] = abs(e)
elif e > 1:
ecc[ii] = 1 - (ecc - 1)
ii += 1
rad1 = 10**(untransform(rad1T, pop['rad1']))
rad2 = 10**(untransform(rad2T, pop['rad2']))
Lum1 = 10**(untransform(Lum1T, pop['Lum1']))
Lum2 = 10**(untransform(Lum2T, pop['Lum2']))
print('you should see me!')
# COMPUTE THE POSITION AND ORIENTATION OF EACH BINARY
##############################################################################
# First assign the position relative to the galactic center
# Assign the radial position of the binary
norm_r = 1 / 2.5
a_0_r = np.random.uniform(0, 1.0, len(m1))
r = -2.5 * np.log(1 - a_0_r)
# Assign the azimuthal position of the star
phi = np.random.uniform(0, 2 * np.pi, len(m1))
# Assign the z position of the star with r as a parameter
norm_zr = 0.71023
a_0_zr = np.random.uniform(0, 1.0, len(m1))
z = 1 / 1.42 * np.arctanh(a_0_zr / (0.704) - 1)
# convert to cartesian and parsecs
xGX = r * np.cos(phi) * 1000.0
yGX = r * np.sin(phi) * 1000.0
zGX = z * 1000.0
# compute the distance to Earth/LISA/us in kiloparsecs
dist = ((xGX - x_sun)**2 + (yGX - y_sun)**2 + (zGX - z_sun)**2)**(1 / 2.0)
dist_kpc = dist / 1000.0
inc = np.random.uniform(0, math.pi / 2, len(m1))
OMEGA = np.random.uniform(0, 2 * math.pi, len(m1))
omega = np.random.uniform(0, 2 * math.pi, len(m1))
binDat = np.vstack((m1, m2, porb, ecc, rad1, rad2, Lum1, Lum2, xGX, yGX,
zGX, dist_kpc, inc, OMEGA, omega)).T
radTotAU = (rad1 + rad2) / rsun_in_au
radAng = radTotAU / dist
binEclipseIndex, = np.where(radAng > inc * 4.8e-6)
print(len(binEclipseIndex), len(binDat))
np.savetxt(gxFile, binDat, delimiter=',')
#gxFile.close()
output.put(np.shape(binDat))
def activation(self, batch, weights, offset):
return expit(numpy.dot(batch, weights) + offset)
k = 10
n_dim = 4
n_usr = 5000
n_itm = 5000
n_rat = int(1e5)
# Generate fake data
np.random.seed(42)
b_usr = np.random.randn(n_usr)
b_itm = np.random.randn(n_itm)
v_usr = np.random.randn(n_usr, n_dim)
v_itm = np.random.randn(n_itm, n_dim)
i_usr = np.random.randint(0, n_usr, size=n_rat)
i_itm = np.random.randint(0, n_itm, size=n_rat)
y = expit(b_usr[i_usr] + b_itm[i_itm] +
np.sum(v_usr[i_usr, :] * v_itm[i_itm, :], axis=1))
y = np.rint(y)
print(b_usr)
print(v_itm)
print(y)
model = CFM(n_usr + n_itm, k, threshold=1e5, do_svd=True)
# optim = Adam(model.parameters(), lr=1e-4)
optim = SGD(model.parameters(), lr=1e-2)
callbacks = {'auc': auc_callback}
trainer = Trainer(model,
optim,
batchsize=512,
window=32,
callbacks=callbacks,
def find_best_split_parallel(batch, X, y, s, z, idx, theta,
learning_rate):
y_auc = roc_auc_score(y, z)
if len(s.shape) > 1:
ovr_s_auc = []
for j in range(s.shape[1]):
s_auc = roc_auc_score(s[:, j], z)
s_auc = max(1 - s_auc, s_auc)
ovr_s_auc.append(s_auc)
s_auc = max(ovr_s_auc)
else:
s_auc = roc_auc_score(s, z)
s_auc = max(1 - s_auc, s_auc)
base_score = (1 - theta) * y_auc - theta * s_auc
best_score = copy(base_score)
for split in batch:
variable, value = split
left_idx = idx[X[idx, variable] < value]
right_idx = idx[X[idx, variable] >= value]
left_n, right_n = len(left_idx), len(right_idx)
if (left_n > 0) and (right_n > 0):
left_y, right_y = y[left_idx], y[right_idx]
left_s, right_s = s[left_idx], s[right_idx]
left_z, right_z = z[left_idx], z[right_idx]
left_p, right_p = expit(left_z), expit(right_z)
left_z_increase = np.mean(left_y - left_p) * learning_rate
right_z_increase = np.mean(right_y -
right_p) * learning_rate
left_new_z = left_z + left_z_increase
right_new_z = right_z + right_z_increase
y_auc = roc_auc_score(
left_y.tolist() + right_y.tolist(),
left_new_z.tolist() + right_new_z.tolist())
if len(s.shape) > 1:
ovr_s_auc = []
for j in range(s.shape[1]):
s_auc = roc_auc_score(
s[left_idx, j].tolist() +
s[right_idx, j].tolist(),
left_new_z.tolist() + right_new_z.tolist())
s_auc = max(1 - s_auc, s_auc)
ovr_s_auc.append(s_auc)
s_auc = max(ovr_s_auc)
else:
s_auc = roc_auc_score(
s[left_idx].tolist() + s[right_idx].tolist(),
left_new_z.tolist() + right_new_z.tolist())
s_auc = max(1 - s_auc, s_auc)
score = (1 - theta) * y_auc - theta * s_auc
if score > best_score:
best_split = split
best_score = score
best_left_n = left_n
best_right_n = right_n
best_left_idx = left_idx
best_right_idx = right_idx
best_left_z_increase = left_z_increase
best_right_z_increase = right_z_increase
if best_score == base_score:
best_split = np.nan
best_score = -np.inf
best_left_idx = np.nan
best_right_idx = np.nan
best_left_z_increase = np.nan
best_right_z_increase = np.nan
return best_left_z_increase, best_left_idx, best_right_z_increase, best_right_idx, best_split, best_score
def naive_log_logistic(x):
return np.log(expit(x))
# x_select_col_idxs = [26, 40, 31, 24, 3, 41, 54, 55, 45, 52, 28, 22, 53, 34] x_test_data_rows = x_test_data_rows[:, x_select_col_idxs] # read model model = np.load(MODEL_FILE_PATH) w = model['w'] x_train_means = model['x_train_means'] x_train_stds = model['x_train_stds'] # In[ ]: # normalize x_test_data_rows = (x_test_data_rows - x_train_means) / x_train_stds # add 1 x_test_data_rows = np.c_[x_test_data_rows, np.ones(x_test_data_rows.shape[0])] # In[ ]: # test y_test_rows = to_bool(special.expit(np.dot(x_test_data_rows, w))) y_test_series = pd.Series(y_test_rows.flatten()) # concat id and y output = pd.concat([_id, y_test_series], axis=1) output.columns = ["id", "label"] # write file output.to_csv(OUTPUT_FILE_PATH, index=False)
for x in version_string.split('.'):
try:
version.append(int(x))
except ValueError:
# x may be of the form dev-1ea1592
version.append(x)
return tuple(version)
np_version = _parse_version(np.__version__)
sp_version = _parse_version(scipy.__version__)
try:
from scipy.special import expit # SciPy >= 0.10
with np.errstate(invalid='ignore', over='ignore'):
if np.isnan(expit(1000)): # SciPy < 0.14
raise ImportError("no stable expit in scipy.special")
except ImportError:
def expit(x, out=None):
"""Logistic sigmoid function, ``1 / (1 + exp(-x))``.
See sklearn.utils.extmath.log_logistic for the log of this function.
"""
if out is None:
out = np.empty(np.atleast_1d(x).shape, dtype=np.float64)
out[:] = x
# 1 / (1 + exp(-x)) = (1 + tanh(x / 2)) / 2
# This way of computing the logistic is both fast and stable.
out *= .5
def sett(self, M, b):
""" compute answer labels based on model M,b """
self.t = s.expit(z(self.q, M, self.a,
b)[0]) # answer labels as estimated by the model
def result():
if request.method == 'POST' :
start_time = time.time()
arctic_circle_lat = 64.0 #it's actually 66.3 degrees, but give a bit of wiggle room
route_dl = 100 #km, the spatial resolution of the great circle geometry
airport_code = request.form['airport'].upper()
slider_value = request.form['value_slider']
print(slider_value)
#if airport_code in ['LAX', 'ORD', 'JFK']: #speed up code for busy airports
# route_dl = 200
current_date = time.strftime('%m/%d/%Y') #make date format a constant
sp_api = nf.SkyPickerApi()
today_date = datetime.today()
dec1 = today_date.replace(year=2018,month=12,day=1).strftime('%m/%d/%Y')
dec31 = today_date.replace(year=2018,month=12,day=31).strftime('%m/%d/%Y')
sp_results = sp_api.search_flights_from_airport(airport_code, datetime.strptime(dec1, '%m/%d/%Y'),datetime.strptime(dec31, '%m/%d/%Y'))
dest_iata_codes = []
for i in range(len(sp_results)):
dest_iata_code = sp_results[i]['legs'][0]['to'][-4:-1]
dest_iata_codes.append(dest_iata_code)
unique_dest_iata_codes = set(dest_iata_codes)
sp_loc_api = nf.SkyPickerApi()
unique_dest_lons = []
unique_dest_lats = []
elapsed_time = time.time() - start_time
print('time to search flights from airport: '+str(elapsed_time))
start_time = time.time()
lat_dict = nf.load_airport_lat_dictionary()
lon_dict = nf.load_airport_lon_dictionary()
for iata_code in unique_dest_iata_codes:
unique_dest_lats.append(lat_dict[iata_code])
unique_dest_lons.append(lon_dict[iata_code])
try:
origin_lat = lat_dict[airport_code]
origin_lon = lon_dict[airport_code]
except:
abort(400,'Unknown airport code. Please enter a valid three-letter IATA airport code.')
if origin_lat < 10:
abort(400,'This airport is too far South. Please choose an airport at higher latitude.')
elapsed_time = time.time() - start_time
print('time to get airport coordinates: '+str(elapsed_time))
start_time = time.time()
dest_airports = pd.DataFrame(data=np.transpose([list(unique_dest_iata_codes),unique_dest_lons,unique_dest_lats]),columns=['code','lon','lat'])
if origin_lat > arctic_circle_lat:
does_this_route_pass_through_arctic = np.ones(len(dest_airports))
else:
does_this_route_pass_through_arctic = np.zeros(len(dest_airports))
for i in range(len(dest_airports)):
folderpath = 'gis_temp/'
layername = airport_code+'-'+dest_airports.loc[i].code
gtg = gc.GeodesicLine2Gisfile()
lons_lats = (origin_lon, origin_lat, np.float(dest_airports.loc[i].lon), np.float(dest_airports.loc[i].lat))
cd = gtg.gdlComp(lons_lats, km_pts=route_dl)
gtg.gdlToGisFile(cd, folderpath, layername, fmt="GeoJSON") # geojson output
gc_lat = []; gc_lon = []
with open(folderpath+'/'+layername+'.geojson') as json_file:
data = json.load(json_file)
if len(data['features'][0]['geometry']['coordinates'][0]) == 2: #if everything is working as expected
for j in range(len(data['features'][0]['geometry']['coordinates'][:])):
lon,lat = data['features'][0]['geometry']['coordinates'][j]
gc_lat.append(lat)
gc_lon.append(lon)
else: #anti-meridian crossing, they split the results into two groups so the len will be a lot longer
for j in range(len(data['features'][0]['geometry']['coordinates'][0][:])):
lon,lat = data['features'][0]['geometry']['coordinates'][0][j]
gc_lat.append(lat)
gc_lon.append(lon)
for j in range(len(data['features'][0]['geometry']['coordinates'][1][:])):
lon,lat = data['features'][0]['geometry']['coordinates'][1][j]
gc_lat.append(lat)
gc_lon.append(lon)
if np.max(gc_lat) > arctic_circle_lat:
does_this_route_pass_through_arctic[i] = 1
elapsed_time = time.time() - start_time
if does_this_route_pass_through_arctic.sum() == 0:
abort(400, 'No direct flights that have a chance to see the Northern Lights are found from this airport. Please try another airport. \n If this airport should have direct flights over the Arctic, please check https://status.kiwi.com to verify the status of the API, as an API outage can also cause this error. ')
print('time to get flight routes: '+str(elapsed_time))
start_time = time.time()
figdata_path = nf.make_plot(arctic_circle_lat,origin_lon,origin_lat,does_this_route_pass_through_arctic,dest_airports,airport_code)
elapsed_time = time.time() - start_time
print('time to make the globe plot: '+str(elapsed_time))
start_time = time.time()
final_airport_codes = []
final_airport_lats = []
final_airport_lons = []
for i in range(len(does_this_route_pass_through_arctic)):
if does_this_route_pass_through_arctic[i]:
final_airport_codes.append(airport_code+'-'+dest_airports.loc[i].code)
final_airport_lats.append(dest_airports.loc[i].lat)
final_airport_lons.append(dest_airports.loc[i].lon)
### make a giant dataframe that contains all the routes which pass through the arctic
### and it also contains the flight path, sampled every 10 km
### in both (lat,lon) and in magnetic (lat,lon)
N_routes = 0 #Number of routes, this will get filled in by the code
routes = pd.DataFrame(data=None)
for i in range(len(does_this_route_pass_through_arctic)):
df = nf.get_flight_route(i,does_this_route_pass_through_arctic,airport_code,dest_airports)
if df is not None:
routes = routes.append(df)
N_routes += 1
elapsed_time = time.time() - start_time
print('time to get the final flight routes prepped: '+str(elapsed_time))
start_time = time.time()
### load aurora model fit to NOAA data, defined in make_aurora_model.ipynb
### strictly valid until Jan 4, 2019 (end of current Solar cycle)
### and extrapolated out to Jan 4, 2021 assuming the next Solar cycle begins like the current one
x_model,bf_model = np.loadtxt('./aurora_model.dat',skiprows=1,usecols=[0,1],unpack=True)
### aurora prob lookup table (from https://www.ngdc.noaa.gov/stp/geomag/kp_ap.html)
### to convert ap values into kp values
kp_lookup = np.array([0,0.33,0.67,1.0,1.33,1.67,2.0,2.33,2.67,3.0,3.33,3.67,4.0,4.33,4.67,5.0,5.33,5.67,6.0,6.33,6.67])
ap_lookup = np.array([0,2,3,4,5,6,7,9,12,15,18,22,27,32,39,48,56,67,80,94,111])
kp_model = np.interp(bf_model,ap_lookup,kp_lookup)
elapsed_time = time.time() - start_time
print('time to load precomputed aurora data '+str(elapsed_time))
start_time = time.time()
N_days = 26
aurora_p = np.zeros([N_days,N_routes])
for i in range(N_routes):
p,dts = nf.get_aurora_prob_over_time(routes.iloc[4*i][1:].values,routes.iloc[4*i+1][1:].values,routes.iloc[4*i+2][1:].values,routes.iloc[4*i+3][1:].values,x_model,kp_model,route_dl,N_days=N_days,day_max=500)
aurora_p[:,i] = p
elapsed_time = time.time() - start_time
print('time to estimate aurora probabilities: '+str(elapsed_time))
start_time = time.time()
aurora_plot_path = nf.make_aurora_plot(N_routes,dts,aurora_p,final_airport_codes)
elapsed_time = time.time() - start_time
print('time to make aurora plot: '+str(elapsed_time))
start_time = time.time()
rf_model,rf_df = rf.load_random_forest_data('k_interp_direct_rf')
prices = np.zeros([N_days,N_routes])
for i in range(N_routes):
pr,dts = rf.predict_airfares(origin_lat,origin_lon,final_airport_lats[i],final_airport_lons[i],rf_model,N_days=N_days,day_max=500)
prices[:,i] = pr
#print(prices)
elapsed_time = time.time() - start_time
print('time to predict flight prices with neural network: '+str(elapsed_time))
start_time = time.time()
price_plot_path = nf.make_prices_plot(N_routes,dts,prices,final_airport_codes,rf_df)
elapsed_time = time.time() - start_time
print('time to make flight prices plot: '+str(elapsed_time))
start_time = time.time()
best_aurora = np.where(aurora_p == aurora_p.max())
best_aurora_price = prices[best_aurora][0]
best_aurora_month = (datetime.today() + timedelta(days=dts[best_aurora[0]][0])).month
best_aurora_year = (datetime.today() + timedelta(days=dts[best_aurora[0]][0])).year
best_aurora_text = final_airport_codes[best_aurora[1][0]]
best_aurora_text +=' in '+str(nf.NumtoMonth(np.int(best_aurora_month)))+' '+str(best_aurora_year)
best_aurora_text +='. Estimated price ~ $%.f' % (best_aurora_price)
best_aurora_text +=', and estimated aurora probability = %0.2f' % (aurora_p[best_aurora][0])
best_price = np.where(prices == prices.min())
best_price_aurora = aurora_p[best_price][0]
best_price_month = (datetime.today() + timedelta(days=dts[best_price[0]][0])).month
best_price_year = (datetime.today() + timedelta(days=dts[best_price[0]][0])).year
best_price_text = final_airport_codes[best_price[1][0]]
best_price_text +=' in '+str(nf.NumtoMonth(np.int(best_price_month)))+' '+str(best_price_year)
best_price_text +='. Estimated price ~ $%.f' % (prices[best_price][0])
best_price_text +=', and estimated aurora probability = %0.2f' % (best_price_aurora)
### use sigmoid function to translate slider value to weights
aurora_weight = expit(-1.0*np.float(slider_value))
price_weight = expit(np.float(slider_value))
figure_of_merit = (aurora_p * aurora_weight) + (prices.min() / prices)*price_weight * aurora_p.max()
best_overall = np.where(figure_of_merit == figure_of_merit.max())
best_aurora_overall = aurora_p[best_overall][0]
best_price_overall = prices[best_overall][0]
best_overall_month = (datetime.today() + timedelta(days=dts[best_overall[0]][0])).month
best_overall_year = (datetime.today() + timedelta(days=dts[best_overall[0]][0])).year
best_overall_text = final_airport_codes[best_overall[1][0]]
best_overall_text +=' in '+str(nf.NumtoMonth(np.int(best_overall_month)))+' '+str(best_overall_year)
best_overall_text +='. Estimated price ~ $%.f' % (best_price_overall)
best_overall_text +=', and estimated aurora probability = %0.2f' % (best_aurora_overall)
elapsed_time = time.time() - start_time
print('time to compute best values: '+str(elapsed_time))
start_time = time.time()
kiwi_url = nf.make_kiwi_url(airport_code, final_airport_codes[best_overall[1][0]][4:7], best_overall_month, best_overall_year)
print(kiwi_url)
### housecleaning - clear out temp json files in gis_temp
#for filename in os.listdir(folderpath):
# if filename.endswith('.geojson'):
# try:
# os.unlink(folderpath+filename)
# #print(folderpath+filename)
# except:
# pass
print('time to finish code: '+str(elapsed_time))
return render_template('result.html',plot_code=figdata_path,aurora_plot_code=aurora_plot_path,price_plot_code=price_plot_path,best_aurora_text=best_aurora_text,
best_price_text=best_price_text,best_overall_text=best_overall_text,kiwi_url=kiwi_url)
def load_img(img_path):
img = PIL.Image.open(img_path)
img = np.array(img, dtype=np.uint8)
return img
def transform_img(img):
mean_bgr = np.array([104.00699, 116.66877, 122.67892])
img_copy = img.copy()
img_copy = img_copy[:, :, ::-1] # RGB -> BGR
img_copy = img_copy.astype(np.float32)
img_copy -= mean_bgr
img_copy = img_copy.transpose(2, 0, 1) # C x H x W
img_copy = torch.from_numpy(img_copy).float()
return img_copy
img_path = os.path.join(img_dir, img_name)
img = load_img(img_path)
img_transformed = transform_img(img)[np.newaxis]
model = models.FCN32s() if fcn_type == 'fcn32' else models.FCN16s()
model_weight = torch.load(pretrained_model)
model.load_state_dict(model_weight)
with torch.no_grad():
img_transformed = Variable(img_transformed)
score = model(img_transformed)
lbl_pred = (expit(score.data.cpu().numpy()) * 255).astype(np.uint8)[0][0]
save_path = os.path.join(save_dir, 'test_' + img_name)
utils.overlay_imp_on_img(img, lbl_pred, save_path, colormap='jet')
def f_NF_xB(self, PRR, EGFR, TNF, N):
y = self.betas['NF_xB_PRR_w'] * PRR + self.betas['NF_xB_TNF_w'] * TNF + \
self.betas['NF_xB_EGFR_w'] * EGFR + self.betas['NF_xB_b']
NF_xB = self.max_abundance['NF_xB'] * expit(y) + N
return NF_xB
def f_ADAM17(self, AGTR1, N):
y = self.betas['ADAM17_AGTR1_w'] * AGTR1 + self.betas['ADAM17_b']
ADAM17 = self.max_abundance['ADAM17'] * expit(y) + N
return ADAM17
def f_EGF(self, ADAM17, N):
y = self.betas['EGF_ADAM17_w'] * ADAM17 + self.betas['EGF_b']
EGF = self.max_abundance['EGF'] * expit(y) + N
return EGF
def f_AGTR1(self, AngII, N):
y = self.betas['AGTR1_AngII_w'] * AngII + self.betas['AGTR1_b']
AGTR1 = self.max_abundance['AGTR1'] * expit(y) + N
return AGTR1
def evaluate(weight_file_path,
data_dir,
output_dir,
prob_thresh=0.5,
nms_thresh=0.1,
lw=3,
display=False):
# placeholder of input images. Currently batch size of one is supported.
x = tf.placeholder(tf.float32, [1, None, None, 3]) # n, h, w, c
# Create the tiny face model which weights are loaded from a pretrained model.
model = tiny_face_model.Model(weight_file_path)
score_final = model.tiny_face(x)
# Find image files in data_dir.
filenames = []
for ext in ('*.png', '*.gif', '*.jpg', '*.jpeg'):
filenames.extend(glob.glob(os.path.join(data_dir, ext)))
# Load an average image and clusters(reference boxes of templates).
with open(weight_file_path, "rb") as f:
_, mat_params_dict = pickle.load(f)
average_image = model.get_data_by_key("average_image")
clusters = model.get_data_by_key("clusters")
clusters_h = clusters[:, 3] - clusters[:, 1] + 1
clusters_w = clusters[:, 2] - clusters[:, 0] + 1
normal_idx = np.where(clusters[:, 4] == 1)
# main
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
_results = []
for filename in filenames:
fname = filename.split(os.sep)[-1]
raw_img = cv2.imread(filename)
raw_img = cv2.cvtColor(raw_img, cv2.COLOR_BGR2RGB)
raw_img_f = raw_img.astype(np.float32)
def _calc_scales():
raw_h, raw_w = raw_img.shape[0], raw_img.shape[1]
min_scale = min(
np.floor(np.log2(np.max(clusters_w[normal_idx] / raw_w))),
np.floor(np.log2(np.max(clusters_h[normal_idx] / raw_h))))
max_scale = min(1.0,
-np.log2(max(raw_h, raw_w) / MAX_INPUT_DIM))
scales_down = pl.frange(min_scale, 0, 1.)
scales_up = pl.frange(0.5, max_scale, 0.5)
scales_pow = np.hstack((scales_down, scales_up))
scales = np.power(2.0, scales_pow)
return scales
scales = _calc_scales()
start = time.time()
# initialize output
bboxes = np.empty(shape=(0, 5))
# process input at different scales
for s in scales:
print("Processing {} at scale {:.4f}".format(fname, s))
img = cv2.resize(raw_img_f, (0, 0),
fx=s,
fy=s,
interpolation=cv2.INTER_LINEAR)
img = img - average_image
img = img[np.newaxis, :]
# we don't run every template on every scale ids of templates to ignore
tids = list(range(
4, 12)) + ([] if s <= 1.0 else list(range(18, 25)))
ignoredTids = list(
set(range(0, clusters.shape[0])) - set(tids))
# run through the net
score_final_tf = sess.run(score_final, feed_dict={x: img})
# collect scores
score_cls_tf, score_reg_tf = score_final_tf[:, :, :, :
25], score_final_tf[:, :, :,
25:
125]
prob_cls_tf = expit(score_cls_tf)
prob_cls_tf[0, :, :, ignoredTids] = 0.0
def _calc_bounding_boxes():
# threshold for detection
_, fy, fx, fc = np.where(prob_cls_tf > prob_thresh)
# interpret heatmap into bounding boxes
cy = fy * 8 - 1
cx = fx * 8 - 1
ch = clusters[fc, 3] - clusters[fc, 1] + 1
cw = clusters[fc, 2] - clusters[fc, 0] + 1
# extract bounding box refinement
Nt = clusters.shape[0]
tx = score_reg_tf[0, :, :, 0:Nt]
ty = score_reg_tf[0, :, :, Nt:2 * Nt]
tw = score_reg_tf[0, :, :, 2 * Nt:3 * Nt]
th = score_reg_tf[0, :, :, 3 * Nt:4 * Nt]
# refine bounding boxes
dcx = cw * tx[fy, fx, fc]
dcy = ch * ty[fy, fx, fc]
rcx = cx + dcx
rcy = cy + dcy
rcw = cw * np.exp(tw[fy, fx, fc])
rch = ch * np.exp(th[fy, fx, fc])
scores = score_cls_tf[0, fy, fx, fc]
tmp_bboxes = np.vstack((rcx - rcw / 2, rcy - rch / 2,
rcx + rcw / 2, rcy + rch / 2))
tmp_bboxes = np.vstack((tmp_bboxes / s, scores))
tmp_bboxes = tmp_bboxes.transpose()
return tmp_bboxes
tmp_bboxes = _calc_bounding_boxes()
bboxes = np.vstack(
(bboxes, tmp_bboxes)) # <class 'tuple'>: (5265, 5)
print("time {:.2f} secs for {}".format(time.time() - start, fname))
# non maximum suppression
# refind_idx = util.nms(bboxes, nms_thresh)
refind_idx = tf.image.non_max_suppression(
tf.convert_to_tensor(bboxes[:, :4], dtype=tf.float32),
tf.convert_to_tensor(bboxes[:, 4], dtype=tf.float32),
max_output_size=bboxes.shape[0],
iou_threshold=nms_thresh)
refind_idx = sess.run(refind_idx)
refined_bboxes = bboxes[refind_idx]
_result = []
for r in refined_bboxes:
_score = expit(r[4])
_r = [int(x) for x in r[:4]]
print("{} {} {} {} {}".format(_score, _r[0], _r[1], _r[2],
_r[3]))
_result.append([_r[0], _r[1], _r[2], _r[3], _score])
pass
overlay_bounding_boxes(raw_img, refined_bboxes, lw)
if display:
# plt.axis('off')
plt.imshow(raw_img)
plt.show()
# save image with bounding boxes
raw_img = cv2.cvtColor(raw_img, cv2.COLOR_RGB2BGR)
cv2.imwrite(os.path.join(output_dir, fname), raw_img)
_results.append(_result)
pass
return _results
pass
