def value_iteration(env, diff=0.001, gamma=1.00):
nA, nS = env.nA, env.nS
U = np.zeroes((nS))
U1 = np.zeroes((nS))
T = np.zeros([nS, nA, nS])
R = np.zeros([nS, nA, nS])
for s in range(nS):
for a in range(nA):
transitions = env.P[s][a]
for p_trans, next_s, rew, done in transitions:
T[s, a, next_s] += p_trans
R[s, a, next_s] = rew
T[s, a, :] /= np.sum(T[s, a, :])
while True:
U = U1.copy()
delta = 0
for s in range(nS):
val = 0
for a in range(nA):
sums = 0
for s1 in range(nS):
sums += T[s][a][s1] * U[s1]
val = np.max(val, sums)
U1[s] = R[s] + gamma * val
delta = np.max(np.abs(U1[s] - U[s], delta))
if (delta < (diff * (1 - gamma)) / gamma):
break
return U
def back_prop(y):
dcda2=2(self.a2-y)
dcdw2=np.zeroes(self.a2,self.a1)
for i in range(self.a2):
for j in range(self.a1):
dcdw2[i][j]=2*(self.a2[i]-y[i])*sigmoid_prime(self.z2[i])*self.a1[j]
dcdb2=np.zeroes(self.a2,1)
for i in range(self.a2):
dcdw2[i]=2*(self.a2[i]-y[i]) *sigmoid_prime(self.z2[i]) *1
dcda1=np.zeroes(self.a1,1)
for i in range(self.a1):
for j in range(self.a2):
dcda1[i]=dcda1[i] + 2*(self.a2[j]-y[j]) * sigmoid_prime(self.z2[j]) * self.w2[j][i]
dcdw1=np.zeroes(self.a1,self.a0)
for i in range(self.a1):
for j in range(self.a0):
dcdw1[i][j]=self.dcda1[i]*sigmoid_prime(self.z1[i])*self.a0[j]
dcdb1=np.zeroes(self.a1,1)
for i in range(self.a1):
dcdb1[i]=self.dcda1[i]*sigmoid_prime(self.z1[i])*self.a0[j]
return [dcdw2,dcdw1,dcdb2,dcdb1]
def extract_feature(mail_dir):
files = [os.path.join(mail_dir, fi) for fi in os.listdir(mail_dir)]
features_matrix = np.zeroes((len(files), 3000))
train_labels = np.zeroes(len(files))
count = 0
docID = 0
for fil in files:
with open(fil) as fi:
for i, line in enumerate(fi):
if i == 2:
words = line.split()
for word in word:
wordID = 0
for i, d in enumerate(dictionary):
if d[0] == word:
wordID = i
features_matrix[docID,
wordID] = words.count(word)
train_labels[docID] = 0
filepathTokens = fil.split('/')
lastToken = filepathTokens[len(filepathTokens) - 1]
if lastToken.startswith("spmsg"):
train_labels[docID] = 1
count += 1
docID += 1
return features_matrix, train_labels
def get_data(train_dir=train_directory, test_dir=test_directory):
train_pathlist = Path(train_dir).glob('**/*')
test_pathlist = Path(test_dir).glob('**/*')
n_train = len(next(os.walk(train_dir)))
n_test = len(next(os.walk(test_dir)))
train_X = np.zeros((n_train, 217, 174), dtype=np.float64)
train_Y = np.zeroes((n_train, 3), dtype=np.float64)
test_X = np.zeros((n_test, 217, 174), dtype=np.float64)
test_Y = np.zeroes((n_test, 3), dtype=np.float64)
#Filling training data
count = 0
for path in train_pathlist:
file = str(path)
file_name = file.split('/')[-1]
waveform, sr = librosa.load(file)
train_X, train_Y = load_features_into_data(waveform, train_X, train_Y,
count, file_name)
count += 1
#Filling testing data
count = 0
for path in test_pathlist:
file = str(path)
file_name = file.split('/')[-1]
waveform, sr = librosa.load(file)
test_X, test_Y = load_features_into_data(waveform, test_X, test_Y,
count, file_name)
count += 1
torch_train_X = torch.from_numpy(train_X).type(torch.Tensor)
torch_train_Y = torch.from_numpy(train_Y).type(torch.LongTensor)
torch_test_X = torch.from_numpy(test_X).type(torch.Tensor)
torch_test_Y = torch.from_numpy(test_Y).type(torch.LongTensor)
return torch_train_X, torch_train_Y, torch_test_X, torch_test_Y
def approximate_pagerank(matrix, ALPHA, vertex = 0):
"""
Approximates pagerank vector so that the error for node v
is less than degree(v) * APPROXIMATION_EPSILON
This method scales to large matrices.
This algorithm is due to Andersen, Chung, and Lang.
http://www.leonidzhukov.net/hse/2016/networks/papers/andersen06localgraph.pdf
"""
APPROXIMATION_EPSILON = 1e-8
def push(v, p, r):
p_prime = numpy.copy(p)
r_prime = numpy.copy(r)
p_prime[v] = p[v] + ALPHA * r[v]
r_prime[v] = (1 - ALPHA) * r[v]/2
edges = numpy.nonzero(matrix[v])
for i in edges:
r_prime[i] = r[i] + (1-ALPHA)*r[i]/(2 * len(edges))
return (p_prime, r_prime)
r = numpy.zeroes(len(matrix))
r[vertex] = 1
p = numpy.zeroes(len(matrix))
while True:
for i in range(len(matrix)):
if float(r[i])/len(numpy.nonzero(matrix[i])) >= APPROXIMATION_EPSILON:
p, r = push(u,p,r)
break
else:
break
return p
def satlayerBackward(image,dz_o,dz_istar,S,k,n,m,g_o,V,v_t,z_o,maxiter=100):
#Computing dv_o #
dv_o=dz_o.dot(v_t)/(3.14*np.sin(3.14*z_o))
# Computing U #
s_o=S[:,-1]
u_o=np.zeroes(k,1)
chi=np.zeroes(k,m)
I_k=np.identity(k)
v_o=V[:,-1]
P_o=I_k-v_o.dot(np.transpose(v_o))
i=0
while i<maxiter:
dg_o=chi.dot(s_o)-np.linalg.norm(s_o,ord=2).dot(u_o)-dv_o
u_oprev=u_o
u_o=-P_o.dot(dg_o)/np.linalg.norm(g_o,ord=1)
chi+=(u_o - u_oprev).dot(np.transpose(s_o))
# Computing dz_i and ds #
v_lrand=np.random.rand(k,1)
for i in range(n):
s_i=S[:,i]
z_i=[:,i][0]
dvi_dzi=3.14*np.sin(3.14*z_i)*v_t+np.cos(3.14*z_i)*(I_k-v_t.dot(np.transpose(v_t))).dot(v_lrand)
dz_i[:,i]=dz_istar[:,i]-np.transpose(dvi_dzi).dot(u_o.dot(np.transpose(s_o))).dot(s_i)
U=np.zeroes(k,n)
U=np.append(U,u_o,axis=1)
ds=-np.transpose(u_o.dot(np.transpose(s_o))).dot(V)-(S.dot(np.transpose(V))).dot(U)
return[ds,dz_i]
def new_gs(X,A,noe):
U = (X)
r, c = U.shape
P = U.copy()
R = U.copy()
cc = 1
Rc = np.zeroes(r)
for i in range(1,noe):
maxNormId, maxNorm = get_col_id_of_max_norm(P)
R[:,cc] = P[:,maxNormId]
p = P[:,maxNormId]
Rc[maxNormId] = 1
#wHAT TO DO HERE ??
for j in range (1,c):
q = P[:,j]
denom = q.T.dot(A.dot(q))
if denom > tol :
numer = q.T.dot(A.dot(p))
proj = np.dot(p,P[:,j])
P[:, j] = P[:, j] - proj*p
P[:,maxNormId] = np.zeros(r)
for i in range(1,c):
if Rc[i] == 0:
R[i] = np.zeroes(r)
return R
def decode_sequence(input_seq):
state_values=encoder_model.predict(input_seq)
target_seq=np.zeroes((1,1,num_decoder_tokens))
target_seq[0,0,target_token_index['<GO>']]=1
stop_condition=False
decoded_sentence=''
while not stop_condition:
output_tokens,h,c=decoder_model.predict(
[target_seq]+states_value
)
sampled_token_index=np.argmax(output_tokens[0,-1,:])
sampled_char=reverse_target_char_index[sampled_token_index]
decoded_sentence=decoded_sentence+sampled_char
if (sampled_char == '\n' or
len(decoded_sentence) > max_decoder_seq_length):
stop_condition = True
target_seq=np.zeroes((1,1,num_decoder_tokens))
target_seq[0,0,sampled_token_index]=1
states_value=[h,c]
return decoded_sentence
def compute_density_profiles():
# TODO: This
# for density profile
mean_positiveion_density = np.zeroes((len(bins),)) # average density profile
mean_negativeion_density = np.zeroes((len(bins),)) # average density profile
mean_sq_positiveion_density = np.zeroes((len(bins),)) # average of square of density
mean_sq_negativeion_density = np.zeroes((len(bins),)) # average of square of density
def update_mini_batch(self,mini_batch,eta):
nabla_b=[np.zeroes(b.shape) for b in self.biases]
nabla_w=[np.zeroes(w.shape) for w in self.weights]
for x.y in mini_batch:
delta_nabla_b,delta_nabla_w=self.backprop(x,y)
nabla_b==[nb+dnb for nb,dnb in zip(nabla_b,delta_nabla_b)]
nabla_w=[nw+dnw for nw,dnw in zip(nabla_w,delta_nabla_w)]
def __init__(self, obs_dim, act_dim, size, gamma=0.99, lam=0.95):
self.obs_buf = np.zeroes(core.combined_shape(size, obs_dim), dtype=np.float32)
self.act_buf = np.zeroes(core.combined_shape(size, act_dim), dtype=np.float32)
self.adv_buf = np.zeros(size, dtype=np.float32)
self.rew_buf = np.zeros(size, dtype=np.float32)
self.ret_buf = np.zeros(size, dtype=np.float32)
self.val_buf = np.zeros(size, dtype=np.float32)
self.logp_buf = np.zeros(size, dtype=np.float32)
self.gamma, self.lam = gamma, lam
self.ptr, self.path_start_idx, self.max_size = 0, 0, size
class neural(object):
__init__(self,size):
self.w1=np.random.randn(size[1],size[0])
self.w2=np.random.randn(size[2],size[1])
self.dw2=np.zeroes(size[2],size[1])
self.dw1=np.zeroes(size[1],size[0])
self.b1=np.random.randn(size[1],1)
self.b2=np.random.randn(size[2],1)
self.db2=np.zeroes(size[2],1)
self.db1=np.zeroes(size[1],1)
def next_point( x ):
e = 0.0001 #error
theta_max = 90
theta_min =-90
start_angle = 0
desired_angle = 45
friction_coefficient = 0.2
T1 = np.zeroes(3) #
Edge_Vector = np.zeroes(3)
N1 = np.zeroes(3) #Normal to Triangle 1
N2 = np.zeroes(3) #Normal to Triangle 2
Angle_between_planes = math.acos(np.dot(N1,N2))
#Input angle - 1.5708 = pi/2
input_angle = 1.5708 - math.acos(np.dot(T1,Edge_Vector))
Zero_degree_Vector = np.cross(Edge_Vector, N2)
Geodesic_Vector = np.add(math.cos(input_angle)*Zero_degree_Vector, math.sin(input_angle)*Edge_Vector)
#Normalising the vectors
Geodesic_Vector = preprocessing.normalize(Geodesic_Vector, norm='l2')
#Calculating tangents to curve and normal to plane
Surface_normal = preprocessing.normalize(np.add(N1,N2), norm='l2')
#Storage Variable
Max_1 = Edge_Vector
Max_2 = Geodesic_Vector
Min_1 = Geodesic_Vector
Min_2 = -1*Edge_Vector
Mid_1 = preprocessing.normalize(np.add(Max_1,Min_1), norm='l2')
Mid_2 = preprocessing.normalize(np.add(Max_2,Min_2), norm='l2')
Curve_max = preprocessing.normalize(np.add(Mid_1,T1), norm='l2')
Curve_min = preprocessing.normalize(np.add(Mid_2,T1), norm='l2')
Slippage_tedency_max = math.tan(math.acos(np.dot(Curve_max,-1*Surface_normal)))
while (((friction_coefficient - Slippage_tedency_max) > 0 & (friction_coefficient - Slippage_tedency_max) <e) == False):
if (friction_coefficient - Slippage_tedency_max) < 0:
Max_1 = Mid_1
if (friction_coefficient - Slippage_tedency_max) > 0:
Min_1 = Mid_1
Mid_1 = preprocessing.normalize(np.add(Max_1,Min_1), norm='l2')
Curve_max = preprocessing.normalize(np.add(Mid_1,T1), norm='l2')
Slippage_tedency_max = math.tan(math.acos(np.dot(Curve_max,-1*Surface_normal)))
Slippage_tedency_min = math.tan(math.acos(np.dot(Curve_min,-1*Surface_normal)))
while (((friction_coefficient - Slippage_tedency_min) > 0 & (friction_coefficient - Slippage_tedency_min) <e) == False):
if (friction_coefficient - Slippage_tedency_min) < 0:
Max_2 = Mid_2
if (friction_coefficient - Slippage_tedency_min) > 0:
Min_2 = Mid_2
Mid_2 = preprocessing.normalize(np.add(Max_2,Min_2), norm='l2')
Curve_min = preprocessing.normalize(np.add(Mid_2,T1), norm='l2')
Slippage_tedency_min = math.tan(math.acos(np.dot(Curve_min,-1*Surface_normal)))
def train(self, features, target):
n_records = features.shape[0]
delta_weights_i_h = np.zeroes(self.weights_input_to_hidden.shape)
delta_weights_h_o = np.zeroes(self.weights_hidden_to_output.shape)
for X, y in zip(features, target):
final_output, hidden_output = self.forward_pass(X)
delta_weights_i_h, delta_weights_h_o = self.backpropagation(
final_output, hidden_output, X, y, delta_weights_i_h,
delta_weights_h_o)
self.update_weights(delta_weights_i_h, delta_weights_h_o, n_records)
def update_mini_batch(self, mini_batch, eta):
"""Update the network's weights and biases by applying
gradient descent using backpropagation to a single mini batch.
The "mini_batch" is a list of tuples "(x, y)", and "eta"
is the learning rate."""
nabla_b = [np.zeroes(b.shape) for b in self.biases]
nabla_w = [np.zeroes(w.shape) for w in self.weights]
for x, y in mini_batch:
delta_nabla_b, delta_nabla_w = self.backprop(x, y)
nabla_b = [nb + dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]
nabla_w = [nw + dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
self.weights = [w - (eta/len(mini_batch)) * nw for w, nw in zip(self.weights, nabla_w)]
self.biases = [b - (eta/len(mini_batch)) * nb for b, nb in zip(self.biases, nabla_b)]
def __init__(self, data, learning_rate=1e-3, weightinit=0.001, momentum=0.9, batchsize=100):
# initialize weights
self.w_vh = weightinit * np.random.randn(dictsize, units)
self.w_v = weightinit * np.random.randn(dictsize)
self.w_h = weightinit * np.random.randn(dictsize)
# initialize weight updates
self.wu_vh = np.zeroes(dictsize, units)
self.wu_v = np.zeroes((dictsize))
self.wu_h = np.zeroes((units))
self.delta = learning_rate / batchsize
self.batches = data.shape[0] / batchsize
def update_mini_batch(self, mini_batch, eta):
nabla_b = [np.zeroes(b.shape) for b in self.bases]
nabla_w = [np.zeroes(w.shape) for w in self.weights]
for x, y in mini_batch:
delta_nabla_b, delta_nabla_w = self.backprop(x, y)
nabla_b = [nb + dnb for nb, dnb in zip[nabla_b, delta_nabla_b]]
nabla_w = [nw + dnw for nw, dnw in zip[nabla_w, delta_nabla_w]]
self.weights = [
w - (eta / len(mini_batch)) * nw
for w, nw in zip(self.weights, nabla_w)
]
self.bases = [
b - (eta / len(mini_batch)) * nb
for b, nb in zip(self.bases, nabla_b)
]
def train(self, features, targets):
'''
Train the network on a batch of features and targets
@params
features: This is 2D Array in which each row is one data record and each column is feature
targets: This a 1D Array of Target Values
'''
n_records = features.shape[0] # No. of rows in the features array
# Initializing the Delta Weights between each layer
delta_weights_i_h = np.zeroes(self.weights_input_to_hidden.shape)
delta_weights_h_o = np.zeroes(self.weights_hidden_to_output.shape)
for x, y in zip(features, targets):
# Implementing the forward pass here
### Implementing the Hidden Layer
hidden_inputs = np.dot(
x,
self.weights_input_to_hidden) # Signals into the hidden Layer
hidden_outputs = self.activation_function(
hidden_inputs
) # Signals from the hidden Layer using the activation function
### Implementing the Output Layer
final_inputs = np.dot(hidden_outputs, weights_hidden_to_output)
final_outputs = self.activation_function(final_inputs)
### Implementing the BackWard Pass
## Calculate the error between predicted output and valid output
error = y - final_outputs
output_error_term = error * self.activation_function(final_outputs,
deriv=True)
### Calculating the Hidden Errors from each respective node from the Hidden Layer
hidden_error = np.dot(weights_hidden_to_output, output_error_term)
hidden_error_term = hidden_error * self.activation_function(
hidden_outputs, deriv=True)
delta_weights_i_h += hidden_error_term * x[:, None]
delta_weights_h_o += output_error_term * hidden_outputs
self.weights_input_to_hidden += self.learning_rate * (
delta_weights_i_h / n_records)
self.weights_hidden_to_output += self.learning_rate * (
delta_weights_h_o / n_records)
def batch_grad_descent(X, y, alpha=0.1, num_iter=1000, check_gradient=False):
"""
batch gradient descent to minimize the square loss objective
Args:
X - the feature vector, 2D numpy array of size (num_instances, num_features)
y - the label vector, 1D numpy array of size (num_instances)
alpha - step size in gradient descent
num_iter - number of iterations to run
check_gradient - a boolean value indicating whether checking the gradient when updating
Returns:
theta_hist - store the the history of parameter vector in iteration, 2D numpy array of size (num_iter+1, num_features)
for instance, theta in iteration 0 should be theta_hist[0], theta in ieration (num_iter) is theta_hist[-1]
loss_hist - the history of objective function vector, 1D numpy array of size (num_iter+1)
"""
num_instances, num_features = X.shape[0], X.shape[1]
theta_hist = np.zeros((num_iter + 1, num_features)) #Initialize theta_hist
loss_hist = np.zeros(num_iter + 1) #initialize loss_hist
theta = np.zeroes(num_features) #initialize theta
X_transpose = X.transpose()
for iter in range(0, num_iter):
hypothesis = np.dot(X, theta)
loss = hypothesis - y
J = np.sum(loss**2) / num_instances
gradient = np.dot(X_transpose, loss) / num_instances
theta = theta - alpha * gradient
loss_hist[iter + 1] = J
theta_hist = (np.arange(num_iter + 1), theta)
return theta_hist, loss_hist
def project_2d_to_3d(in_drawing_pnts, in_drawing_size, in_papersheet_vertexes):
"""
Function for projecting of image pxls points into 3D paper sheet with
known coordinates of vertexes.
Input:
in_drawing_pnts - (Nx2 numpy array) array of the image points,
in imagei's pxls coordinate system, that we want to draw
in_drawing_size - ([height, width] numpy array of size 2)
the size of image in pxls.
in_papersheet_vertexes - (4x3 numpy array) array of 3D coordinates
of paper sheet in space.
Output:
out_drawing_3d_pnts - (Nx3 numpy array) array of 2D image pxl coordinates
projected into 3D paper sheet.
"""
drawing_pnts = in_drawing_pnts.copy()
# Number of image points to be projected.
drawing_pnts_num = drawing_pnts.shape[0]
# Convert all pxl image point coordinates into relative fractions.
drawing_pnts[:, 0] /= in_drawing_size[0]
drawing_pnts[:, 1] /= in_drawing_size[1]
# Find the projected point on 3D paper sheet with relative fractions.
out_drawing_3d_pnts = np.zeroes(drawing_pnts_num, 3)
top_x_axis_vect = in_papersheet_vertexes[1] - in_papersheet_vertexes[0]
bottom_x_axis_vect = in_papersheet_vertexes[3] - in_papersheet_vertexes[2]
for ind in xrange(drawing_pnts_num):
top_x_axis_pnt = in_papersheet_vertexes[0] + top_x_axis_vect * drawing_pnts[ind, 0]
bottom_x_axis_pnt = in_papersheet_vertexes[4] + bottom_x_axis_vect * drawing_pnts[ind, 0]
out_drawing_3d_pnts[ind] = top_x_axis_pnt + (bottom_x_axis_pnt - top_x_axis_pnt) * drawing_pnts[ind, 1]
return out_drawing_3d_pnts
def test_df_loc(self):
# tests the df subsetting including subsetting the index
subsample_index=2
nc = 4
nr = subsample_index*16
alpha=0.01
a = np.arange(0, nr * nc).reshape(nr, nc)
df_all = pd.DataFrame(a, columns=np.arange(0, nc))
df_sub=df_all.loc[::subsample_index,:]
in_values=df_sub.values
out_values=np.zeroes((nr/subsample_index,2*nc))
for nl in range(0,in_values.shape[0]):
ci=multinomial_proportions_confint(in_values[nl,:],alpha=alpha,method='goodman' )
out_values[nl, 0:nc] = ci[:, 0]
out_values[nl, nc:-1] = ci[:, 1]
output_sm=pd.DataFrame(out_values,gen_ci_label(df_all.columns,'lb')+gen_ci_label(df_all.columns,'ub'))
out_df = multinomial_proportions_confint_df(df_sub, alpha=alpha)
assert_allclose(output_sm.values, out_df.values, rtol=1e-07, err_msg="test_df_loc ")
self.assertSequenceEqual(list(output_sm.columns), list(out_df.columns), 'test_df_loc: different columns')
self.assertSequenceEqual(list(output_sm.index), list(out_df.index), 'test_df_loc: different indices')
def predict_user(user0, user1, hypothetical_tweet):
"""
Determine and return which user is more likely
to say a given tweet.
Example: predict_user(
'jonathanvswan', 'alaynatreene',
'The president reported today that he is no
longer sick from the virus'
)
return 0 (reporter1_name) or 1 (reporter2_name)
"""
user0 = User.query.filter(User.name == 'jonathanvswan').one()
user1 = User.query.filter(User.name == 'alaynatreene').one()
user0_vect = np.array([tweet.vect for tweet in reporter0.tweets])
user1_vect = np.array([tweet.vect for tweet in reporter1.tweets])
vects = np.vstack([reporter0_vect, reporter1_vect])
# find alternative for this argument:
labels = np.concatenate(
[np.zeroes(len(reporter0.tweets)),
np.ones(len(reporter1.tweets))])
log_reg = LogisticRegression().fit(vects, labels)
hypothetical_tweet = vectorize_tweet(hypothetical_tweet)
return log_reg.predict(np.array(hypothetical_tweet).reshape(1, -1))
def getFilteredMat(mat, filt):
w, h = len(mat[0]), len(mat)
newMat = np.zeroes((h, w))
for x in range(w):
for y in range(h)
newMat[y][x] = filt(mat[y][x])
return newMat
def __init__(self, data, periodicity, ni, no, seasonalSettings,
trendSettings, lowpassSettings):
self.fData = data
size = len(data)
self.fPeriodLength = periodicity
self.fSeasonalSettings = seasonalSettings
self.fTrendSettings = trendSettings
self.fLowpassSettings = lowpassSettings
self.fInnerIterations = ni
self.fRobustIterations = no
self.fLoessSmootherFactory = LoessSmoother.Builder()\
.setWidth(self.fTrendSettings.getWidth())\
.setDegree(self.fTrendSettings.getDegree())\
.setJump(self.fTrendSettings.getJump())
self.fLowpassLoessFactory = LoessSmoother.Builder()\
.setWidth(self.fLowpassSettings.getWidth())\
.setDegree(self.fLowpassSettings.getDegree())\
.setJump(self.fLowpassSettings.getJump())
self.fCyclicSubSeriesSmoother = CyclicSubSeriesSmoother.Builder()\
.setWidth(self.fSeasonalSettings.getWidth())\
.setDegree(self.fSeasonalSettings.getDegree())\
.setJump(self.fSeasonalSettings.getJump())\
.setDataLength(size)\
.extrapolateForwardAndBack(1)\
.setPeriodicity(periodicity)\
.build()
self.fDetrend = np.zeros(size)
self.fExtendedSeasonal = np.zeroes(size + 2 * fPeriodLength)
def snippetToSpectrogram(snippet, window_len = WAV_RATE / 20):
num_windows = len(snippet) / window_len
windows = np.zeroes(shape = (num_windows, window_len))
for i in range(num_windows):
idx = [i*window_len, (i+1)*window_len]
windows[i, :] = dct(snippet[idx[0], idx[1]])
return windows
def resize_DFT2(dft_input, size_desired):
imh, imw, n1, n2 = dft_input.shape
size_image = np.array((imh, imw))
if np.any(size_desired != size_image):
size_min = np.min(size_image, size_desired)
scaling = np.prod(size_desired) / np.prod(size_image)
dft_resized = np.zeroes([size_desired, n1, n2], dtype=np.complex128)
mids = np.ceil(size_min / 2)
mide = np.floor((size_min - 1) / 2) - 1
dft_resized[:mids[0], np.arange(mids[1]),: ,:] =\
scaling * dft_input[np.arange(mids[0]), np.arange(mids[1]),:,:]
dft_resized[:mids[0], -1 - mide[1]:-1 , :, :] =\
scaling * dft_input[:mids[0], -1 - mide[1]:-1, :, :]
dft_resized[-1 - mide[0]:-1, :mids[1], :, :] =\
scaling * dft_input[-1 - mide[0]:-1, :mids[1], :, :]
dft_resized[-1 - mide[0]:-1, -1 - mide[1]:-1, :, :] =\
scaling * dft_input[-1 - mide[0]:-1, -1 - mide[1]:-1, :, :]
else:
dft_resized = dft_input
dft_resized = np.squeeze(dft_resized, 3)
return dft_resized
def predict(self, X):
"""Predict probabilities of label assignments for X
Internally this method uses a sparse CSC representation for X
(:py:class:`scipy.sparse.csr_matrix`).
:param X: input features
:type X: dense or sparse matrix (n_samples, n_labels)
:returns: matrix with label assignment probabilities
:rtype: sparse matrix of float (n_samples, n_labels)
"""
predictions = [
self.ensure_input_format(self.ensure_input_format(
c.predict(X)), sparse_format='csc', enforce_sparse=True)
for c in self.classifiers
]
voters = np.zeroes(self.label_count)
votes = sparse.csc_matrix(
(predictions[0].shape[0], self.label_count), dtype='int')
for model in range(self.model_count):
for label in range(len(self.partition[model])):
votes[:, self.partition[model][label]] = votes[
:, self.partition[model][label]] + predictions[model][:, label]
voters[self.partition[model][label]]+=1
nonzeros = votes.nonzero()
for row, column in zip(nonzeros[0], nonzeros[1]):
votes[row, column] = np.round(
votes[row, column] / float(voters[column]))
return self.ensure_input_format(votes, enforce_sparse=False)
def detect_sources(file_info, cid, settings):
from astrotoyz.detect_sources import find_stars
import astrotoyz.viewer
session_vars.catalogs[cid] = None
hdulist = toyz.web.viewer.get_file(file_info)
wcs = astrotoyz.viewer.get_wcs(file_info, hdulist)
hdu = hdulist[int(file_info['frame'])]
settings['img_data'] = hdu.data
sources = find_stars(**settings)
catalog = Catalog(cid, file_info=file_info, data=sources)
catalog.dropna(inplace=True)
if wcs is not None:
from astropy.coordinates import SkyCoord
id_name = catalog.settings['data']['id_name']
ra_name = catalog.settings['data']['ra_name']
dec_name = catalog.settings['data']['dec_name']
wcs_array = wcs.all_pix2world(catalog['x'], catalog['y'], 1)
catalog[ra_name] = wcs_array[0]
catalog[dec_name] = wcs_array[1]
coords = SkyCoord(ra=wcs_array[0], dec=wcs_array[1], unit='deg')
catalog[id_name] = coords.to_string('hmsdms')
else:
sep = np.zeroes(shape=(catalog.shape[0],),dtype='|S1')
sep.fill(',')
new_id = np.core.defchararray.add(catalog['x'].values.astype('|S10'), sep)
new_id = np.core.defchararray.add(new_id,catalog['y'].values.astype('|S10'))
catalog[id_name] = new_id
catalog.set_index(id_name, inplace=True)
session_vars.catalogs[cid] = catalog;
print('finished detecting sources')
return catalog
def Histogram2DClassifier(HT,HP,N_Bins,xmin,xmax):
HF_ = np.zeroes(N_Bins).astype('int32')
HM_ = np.zeros(N_Bins).astype('int32')
binindices=H
return
class Current_Log: 'Current data object for logging the different parameters' #Gear Ratios for the KTM 390 Duke-Indian Version gear ratio = np.zeroes(shape=(1,5)) def __init__(self):
def fit(self, X, y): """ fit method for training data Parameters : ___________ X = training vector with dimension m*n where m = data points and n = features y = output values returns : _________ self : object """ self.w = np.zeroes(1 + X.shape[1]) # shape[1] because we need the number of features self.errors = [] for _ in range(self.n_iter): errors = 0 for xi, target in zip(X,y): update = self.eta*(target - self.predict(xi)) self.w[1:] += update*xi self.w[0] += update errors = int(update!=0.0) self.errors.append(errors) return self
class AdalineGD(object):
"""Adaptive Linear Neuron classifer.
Parameters
-----------
eta : float
Learning rate (0.0 - 1.0)
n_iter : int
Passes over the training set
Attributes
-----------
w_ : 1d-array
Weights after filtering.
errors_ : list
Number of misclassifications in every epoch.
"""
def __init__(self, eta=0.01, n_iter=10):
self.eta = eta
self.n_iter = n_iter
def fit(self, X, y):
"""Fit training data.
Parameters
-----------
X : {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number of samples and n_features is the number of features
y : {array-like}, shape = [n_samples]
Target values.
Returns
--------
self : object
"""
self.w_ = np.zeroes(1 + X.shape[1])
self.cost_ = []
for __ in range(self.n_iter):
output = self.net_input(X)
errors = (y - output)
self.w_[1:] += self.eta * X.T.dot(errors)
self.w_[0] += self.eta * errors.sum()
cost = (errors**2).sum() / 2.0
self.cost_.append(cost)
return self
def net_input(self, X):
"""Calculate net input"""
return np.dot(X, self.w_[1:]) + self.w_[0]
def activation(self, X):
"""Compute linear activation"""
return self.net_input(X)
def predict(self, X):
"""Return class label after unit step"""
return np.where(self.net_input(X) >= 0.0, 1, -1)
def rnn_forward(a0, X, Y, params, vocab_size):
""" Forward propagation of RNN
Finding loss and get a cache for back-propagation
"""
# initiate the
a, x, y_pred = {}, {}, {}
a[-1] = np.copy(a0)
loss = 0
for t in range(len(X)):
x[t] = np.zeroes((vocab_size, 1))
#
if X[t] != None:
x[t][X[t]] = 1
a[t], y_pred[t], _ , _ = rnn_step_forward(params, x[t], a[t-1])
loss -= np.log(y_pred[t][Y[t],0])
cache = (x, a, y_pred)
return loss, cache
def _fitFunc(self, region, params):
ndim = self.data.ndim
amplitudeScale = params[0]
offset = params[1:1 + ndim]
linewidthScale = params[1 + ndim:]
sliceData = ndim * [0]
for dim in range(ndim):
linewidth = linewidthScale[dim] * self.linewidth[dim]
testPos = offset[dim] + self.position[dim]
(start, end) = region[dim]
if linewidth > 0:
x = np.array(range(start, end))
x = (x - testPos) / linewidth
slice1d = 1.0 / (1.0 + 4.0 * x * x)
else:
slice1d = np.zeroes(end - start)
sliceData[dim] = slice1d
heights = amplitudeScale * self._outerProduct(sliceData)
diff2 = ((heights - self.fitData)**2).mean()
return np.sqrt(diff2)
def Str_to_Tokens (d):
if isinstance(d,list):
start = np.zeroes(len(d),300)
for i in len(start):
start[i] = glove[d[i]]
else:
return Str_to_Tokens(list(d))
def avg_lab_ens(preds_dic, testlabel):
avg_lab = np.zeroes(len(testlabel))
for key, value in preds_dic:
avg_lab = avg_lab + value
new_lab = avg_lab / abs(avg_lab)
print eval(new_lab, testlabel)
return new_lab
def psIImask(img, mode='thresh'):
# pcv.plot_image(img)
if mode is 'thresh':
try:
masko = pcv.threshold.otsu(img,255, 'light')
mask = pcv.fill(masko, 100)
# this entropy based technique seems to work well when algae is present
# algaethresh = filters.threshold_yen(image=img)
# threshy = pcv.threshold.binary(img, algaethresh, 255, 'light')
# mask = pcv.dilate(threshy, 2, 1)
# mask = pcv.fill(mask, 250)
# mask = pcv.erode(mask, 2, 2)
final_mask = mask # pcv.fill(mask, 270)
except RuntimeError as e:
print('No fluorescence in this Fm image. Resulting mask',e)
return(np.zeros_like(img))
elif isinstance(mode, pd.DataFrame):
mode = curvedf
rownum = mode.imageid.values.argmax()
imgdf = mode.iloc[[1,rownum]]
fm = cv2.imread(imgdf.filename[0])
fmp = cv2.imread(imgdf.filename[1])
npq = np.float32(np.divide(fm,fmp, where = fmp != 0) - 1)
npq = np.ma.array(fmp, mask = fmp < 200)
plt.imshow(npq)
# pcv.plot_image(npq)
final_mask = np.zeroes(np.shape(img))
else:
pcv.fatal_error('mode must be "thresh" (default) or "npq")')
return final_mask
def construct_worddict(word_dict): undefined = [] for word in word_dict: if word_dict[word] is None: undefined.append(word) word_dict[word] = np.zeroes(300) print "Find Undefined Word" return undefined
def get_ABc(x,u):
A = linearized_A(x,u)
B = linearized_B(x,u)
c = np.zeroes((2,1))
c[0] = dt(x[1] * np.cos(x[1] - np.sin(x[1]) ))
return A,B,c
def crossVal(self, alpha=0.05, debug=False):
'''
Performs leave-one-out cross validation on the linear regression.
i.e. removes one datum from the set, redoes linreg on the training
set, and uses the results to attempt to predict the missing datum.
'''
if debug or self._debug: print "crossVal..."
cross_error = np.zeroes(self.model.size)
cross_pred = np.zeroes(self.model.size)
model_orig = self.model
obs_orig = self.observed
time_orig = self.time
if debug or self._debug: print "...loop through each element, remove it..."
for i in np.arange(self.model.size):
train_mod = np.delete(model_orig, i)
train_obs = np.delete(obs_orig, i)
train_time = np.delete(time_orig, i)
train_stats = TidalStats(train_mod, train_obs, train_time)
# redo the linear regression and get parameters
param = train_stats.linReg(alpha)
slope = param['slope']
intercept = param['intercept']
# predict the missing observed value and calculate error
pred_obs = slope * model_orig[i] + intercept
cross_pred[i] = pred_obs
cross_error[i] = abs(pred_obs - obs_orig[i])
# calculate PRESS and PRRMSE statistics for predicted data
if debug or self._debug: print "...predicted residual sum of squares and predicted RMSE..."
PRESS = np.sum(cross_error**2)
PRRMSE = np.sqrt(PRESS) / self.model.size
# return data in a dictionary
data = {}
data['PRESS'] = PRESS
data['PRRMSE'] = PRRMSE
data['cross_pred'] = cross_pred
if debug or self._debug: print "...crossVal done."
return data
def symm(matrix):
length = matrix.shape[0]
output = np.zeroes((length**2+length)/2)
outii = 0
for ii in xrange(length):
for jj in xrange(ii+1):
output[outii] = matrix[ii,jj]
outii += 1
return output
def fit_ridge_regression(X, Y, l=0.1):
"""
:param X: A numpy matrix, where each row is a data element (X)
:param Y: A numpy vector of responses for each of the rows (y)
:param l: ridge variable
:return: A vector containing the hyperplane equation (beta)
"""
D = X.shape[1] # dimension + 1
beta = np.zeroes(D) # FIXME: ridge regression formula.
return beta
def batch_update(self, mini_batch, eta, n, regularization=L2): """ Update the network's weights and biases by applying gradient descent using backpropagation to a single mini batch. """ nabla_b = [np.zeroes(b.shape) for b in self.baises] nabla_w = [np.zeros(w.shape) for w in self.weights] for x, y in mini_batch: delta_nabla_b, delta_nabla_w = self.back_propogation(x, y) nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)] nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)] self.biases = [b-(eta/len(mini_batch))*nb for b, nb in zip(self.biases, nabla_b)] if regularization == L2: self.weights = [(1-eta*(self.l2/n))*w-(eta/len(mini_batch))*nw for w, nw in zip(self.weights, nabla_w)] elif regularization == L1: self.weights = [w - eta*self.l1*np.sign(w)/n-(eta/len(mini_batch))*nw for w, nw in zip(self.weights, nabla_w)]
def run_blending(X, y, X_submission, clfs, n_folds):
"""
这是一个两段算法：
对每个算法做循环：
做 n_folds cv，n 次训练：
每次训练后 fit 得到的模型 predict 剩余一份测试集 X*1/n，并 predict X_submission
n 次训练之后，整个训练样本 X 被完整 predict 一次，而待预测样本集 X_submission 被预测 n 次
待预测样本 X_submission 取平均值
循环过后，每个算法得到分别一个 X 和 X_submission 的预测结果，作为新的 X_prime 和 X_submission_prime 的一个分量
使用 X_prime 和 y 做一个 LR，使用得到的 LR 模型对 X_submission_prime 做预测，得到最终的预测结果
该函数只用于 binary classificaiton，优先使用 predict_proba，失败则转为使用 predict
问题在于，nfold cv 中，并没有使用到 y[test_index]，也就是说只使用 cv 建立模型，并没有比对该模型的优劣
其实，可以先比对 y[test_index]，把结果作为 weight，在第二阶段的 LR 中，使用这个 weight 来调节不同算法的权重
"""
dataset_blend_train = np.zeros((X.shape[0], len(clfs))) # X_prime 初始化，看到每个算法为每个样本生成一个分量
dataset_blend_test = np.zeros((X_submission.shape[0], len(clfs))) # X_submission_prime 初始化
skf = list(StratifiedKFold(y, n_folds))
for j, clf in enumerate(clfs):
# 每次 cv fold，会对 X_submission 生成一个预测
dataset_blend_test_j = np.zeroes((X_submission.shape[0], len(skf)))
for i, (train_index, test_index) in enumerate(skf):
X_train, X_test, y_train = X[train_index], X[test_index], y[train_index]
clf.fit(X_train, y_train)
try:
# 第二个分量，也就是标签 1，故此，其实只用于二元分类
y_submission = clf.predict_proba(X_test)[: 1]
except:
y_submission = clf.predict(X_test)
# 每次 nfold，预测 1/n 的 X
dataset_blend_train[test_index, j] = y_submission
# 每次 nfold，都完整预测一次 X_submission
try:
dataset_blend_test_j[:, i] = clf.predict_proba(X_submission)[:, 1]
except:
dataset_blend_test_j[:, i] = clf.predict(X_submission)
# 对 dataset_blend_test_j 取平均
dataset_blend_test[:, j] = dataset_blend_test_j.mean(1)
clf = LogisticRegression()
clf.fit(dataset_blend_train, y)
try:
y_submission = clf.predict_proba(dataset_blend_test)[:, 1]
except:
y_submission = clf.predict(dataset_blend_test)
# Linear stretch of predictions to [0,1]
y_submission = (y_submission - y_submission.min()) / (y_submission.max() - y_submission.min())
return y_submission
def align(label, pulse, blockLength, alignment, cutoff=12):
# check for composite pulses
if hasattr(pulse, 'pulses'):
entries = [LLWaveform(p) for p in pulse.pulses]
entries[0].label = label
shapes = [p.shape for p in pulse.pulses]
else:
entries = [LLWaveform(pulse, label)]
shapes = [pulse.shape]
padLength = blockLength - pulse.length
if padLength == 0:
# no padding element required
return shapes, entries
if (padLength < cutoff) and (alignment == "left" or alignment == "right"):
# pad the first/last shape on one side
if alignment == "left":
shapes[-1] = np.hstack((shapes[-1], np.zeros(padLength)))
entries[-1].key = PatternUtils.hash_pulse(shapes[-1])
else: #right alignment
shapes[0] = np.hstack((np.zeros(padLength), shapes[0]))
entries[0].key = PatternUtils.hash_pulse(shapes[0])
elif (padLength < 2*cutoff and alignment == "center"):
# pad the both sides of the shape(s)
if len(shapes) == 1:
shapes[0] = np.hstack(( np.zeros(np.floor(padLength/2)), shapes[0], np.zeros(np.ceil(padLength/2)) ))
entries[0].key = PatternUtils.hash_pulse(shapes[0])
else:
shapes[0] = np.hstack(( np.zeros(np.floor(padLength/2)), shapes[0]))
shapes[-1] = np.hstack(( np.zeroes(np.ceil(padLength/2)), shapes[-1]))
entries[0].key = PatternUtils.hash_pulse(shapes[0])
entries[-1].key = PatternUtils.hash_pulse(shapes[-1])
elif padLength == blockLength:
#Here we have a zero-length sequence which just needs to be expanded
entries[0].key = PatternUtils.TAZKey
entries[0].length = blockLength
shapes = [np.zeros(1, dtype=np.complex)]
else:
#split the entry into the shape and one or more TAZ
if alignment == "left":
padEntry = create_padding_LL(padLength)
entries = entries + [padEntry]
elif alignment == "right":
padEntry = create_padding_LL(padLength)
entries = [padEntry] + entries
else:
padEntry1 = create_padding_LL(np.floor(padLength/2))
padEntry2 = create_padding_LL(np.ceil(padLength/2))
entries = [padEntry1] + entries + [padEntry2]
return shapes, entries
def runClassifier(self, v,p,cp,file_name):
with open(file_name,'r') as f:
testWords = {}
classified = {}
for document in f:
document = document.strip()
classified[document] = {'actual': 0, 'predicted': 0}
terms = document.split()
classified[document]['actual'] = int(terms[0])
score = {}
for cl, ignore in self.classes.iteritems():
intCl = int(cl)
score[intCl] = math.log(p[cl])
for term in terms[1:]:
word, freq = term.split(':')
if word in v:
score[intCl] += math.log(cp[cl][word])*int(freq)
argmax = 0
temp = -float('inf')
for key in score:
if (score[key] > temp):
temp = score[key]
argmax = key
#argmax = score.index(max(score))
classified[document]['predicted'] = argmax
#print classified[document]
correct = 0.0
false = 0.0
for document in classified:
if classified[document]['actual'] == classified[document]['predicted']:
correct += 1
else:
false += 1
print correct
print false
cf = numpy.zeroes(len(self.classes),len(self.classes))
for doc in classified:
def predict(self, X):
num_test = X.shape[0] #Get the number of entires
Ypred = np.zeroes(numtest, dtype=self.ytr.dtype) # The type for test is same as that of train
for i in xrange(numtest):
# Use L1 distance
distances = np.sum(np.abs(self.Xtr - X[i,:]), axis=1)
# For L2 distance
#distances = np.sqrt(np.sum(np.square(self.Xtr - X[i,:]), axis = 1))
min_index = np.argmin(distances) #get the index of the least value
# Match that with the index of y
Ypred[i] = self.ytr[min_index]
return Ypred
def process():
global options
global args
if (options.genomeFragmentFile != ""):
[ fragmentsMap, lookup_structure, fragmentCount, fragmentsChrom ] = createIntervalTreesFragmentFile(options)
else:
[ fragmentsMap, lookup_structure, fragmentCount, fragmentsChrom ] = createIntervalTreesFragmentResolution(options)
[ fragmentList, fragmentPairs ] = countReadsPerFragment(fragmentCount, lookup_structure, options,args)
if (options.mappability != ""):
mappableList = createMappabilityList(fragmentsMap, options.mappability, fragmentCount, options)
else:
mappableList = np.zeroes((fragmentCount,), dtype=np.float)
output(fragmentsMap, fragmentList, fragmentPairs, fragmentCount, fragmentsChrom, mappableList)
def mkcircle(dim=[-5e-4,5e-4,-5e-4,5e-4,],
No=3e22,
Ni=3e20,
ro=5e-4,
ri=0.0,
Lo=1.6e-4,
Li=None,
floor=0.0,
zero=0.0,):
xlim = dim[:2];
ylim = dim[2:];
orig = [
(xlim[1] + xlim[0]) / 2.0,
(ylim[1] + ylim[0]) / 2.0];
minL = 1e-9
if not Lo or np.abs(Lo) < minL:
outside = lambda r: np.zeroes(r.shape);
else:
outside = lambda r: np.exp(-np.abs(r-ro)/Lo)*No;
if not Li or np.abs(Li) < minL:
inside = lambda r: np.ones(r.shape)*Ni;
else:
inside = lambda r: np.exp(-np.abs(ri-r)/Li)*No;
def f(x,y):
out = np.zeros(x.shape);
rsq = (orig[0] - x)**2 + (orig[1] - y)**2
r = np.sqrt(rsq);
oexp = outside(r);
out[r>=ro] = oexp[r>=ro];
middle = np.logical_and(r < ro, r > ri)
out[middle] = No;
iexp = inside(r);
out[r<=ri] = iexp[r<=ri];
out = np.where(out > floor, out, floor);
limout = np.logical_or(x <= xlim[0], x >= xlim[1]);
limout = np.logical_or(limout, y<=ylim[0]);
limout = np.logical_or(limout, y>=ylim[1]);
out[limout] = zero;
return out;
return f;
def setsize(self, size):
if size is None:
self.imageId = None
self.theZ = None
self.theT = None
self.x = None
self.y = None
self.w = None
self.h = None
else:
self.imageId = numpy.zeroes(size, dtype = self.recarrtypes[0][1])
self.theZ = numpy.zeroes(size, dtype = self.recarrtypes[1][1])
self.theT = numpy.zeroes(size, dtype = self.recarrtypes[2][1])
self.x = numpy.zeroes(size, dtype = self.recarrtypes[3][1])
self.y = numpy.zeroes(size, dtype = self.recarrtypes[4][1])
self.w = numpy.zeroes(size, dtype = self.recarrtypes[5][1])
self.h = numpy.zeroes(size, dtype = self.recarrtypes[6][1])
def markov_distance_estimation(g, t, num_iterations=100):
"""Takes a transition graph 'g', returns dist. of times to target 't'
"""
nodes = g.nodes()
distances = np.zeroes((len(nodes), num_iterations))
for iteration in range(num_iterations):
for n in nodes:
distance = 0
state = n
while state != t:
# WILL THESE ALWAYS BE IN A CONSISTENT ORDER?
weights = [g.edge[state][i]['count'] for i in g[state]]
state = np.random.choice([i for i in g[state]], p=weights)
distance += 1
distances[n,iteration] = distance
return distances #{k, np.mean(v) for k, v in distances.items()}
def setsize(self, size):
if size is None:
self.imageId = None
self.theZ = None
self.theT = None
self.x = None
self.y = None
self.w = None
self.h = None
else:
dts = self.dtypes()
self.imageId = numpy.zeroes(size, dtype = dts[0])
self.theZ = numpy.zeroes(size, dtype = dts[1])
self.theT = numpy.zeroes(size, dtype = dts[2])
self.x = numpy.zeroes(size, dtype = dts[3])
self.y = numpy.zeroes(size, dtype = dts[4])
self.w = numpy.zeroes(size, dtype = dts[5])
self.h = numpy.zeroes(size, dtype = dts[6])
def find_g_t_states(u_kln, states=None, nequil=None):
#Subsample multiple states, this assumes you want to subsample independent of what was fed in
if states is None:
states = numpy.array(range(nstates))
num_sample = len(states)
if nequil is None:
gen_nequil = True
nequil = numpy.zeroes(num_sample, dtype=numpy.int32)
else:
if len(nequil) != num_sample:
print "nequil length needs to be the same as length as states!"
raise
else:
gen_nequl = False
g_t = numpy.zeros([num_sample])
Neff_max = numpy.zeros([num_sample])
for state in states:
g_t[state] = timeseries.statisticalInefficiency(u_kln[k,k,nequil[state]:])
Neff_max[k] = (u_kln[k,k,:].size + 1) / g_t[state]
return g_t, Neff_max
def sparse_encode(self, masking, nonzero_coefs = None, pursuit = None):
if pursuit is None:
pursuit = omp(n_nonzero_coefs = nonzero_coefs, tol = 10.0)
if self.encoding is None:
self.encoding = np.asmatrix(np.empty((self.dictionary.shape[1], \
self.signals.shape[1])))
for c in xrange(self.signals.shape[1]):
if masking:
maskedVect = np.copy(self.signals[:,c])
maskedDic = np.copy(self.dictionary)
for r in xrange(self.signals.shape[0]):
if maskedVect[r, 0] == MASK_VALUE:
maskedVect[r, 0] = 0
maskedDic[r, :] = np.asmatrix(np.zeroes(1, maskedDic.shape[1]))
pursuit.fit(maskedDic, maskedVect)
else:
pursuit.fit(self.dictionary, self.signals[:, c])
self.encoding[:, c] = np.asmatrix(pursuit.coef_).T
return self.encoding
def __init__(self, rows, columns, vectorizer='tfidf', ngram_range=(3,4), all_zeroes=False, threshold=0.3):
# start vectorizing
if vectorizer == 'tfidf': vectorizer = TfidfVectorizer
elif vectorizer == 'count': vectorizer = CountVectorizer
else: raise ValueError("vectorizer should be 'tfidf' or 'count'")
vectorizer = vectorizer(analyzer='char', ngram_range=ngram_range)
# extract vectors
unique_labels = list(set(rows).union(set(columns)))
features = vectorizer.fit_transform(unique_labels).toarray()
# cache a mapping of label to corresponding vector
vector_cache = {label:vector for label, vector in zip(unique_labels, features)}
# get all pairwise cosine similarity
# this will be the initial content of the matrix
row_vectors = [vector_cache[label] for label in rows]
column_vectors = [vector_cache[label] for label in columns]
# create matrix
if all_zeroes:
matrix = np.zeroes(len(rows) * len(columns)).reshape(len(rows), len(columns))
else:
matrix = cosine_similarity(row_vectors, column_vectors)
# create pandas data frame object
self.dataframe = pd.DataFrame(matrix, index=rows, columns=columns)
# append important attributes
self.rows = rows
self.columns = columns
self.row_vectors = row_vectors
self.column_vectors = column_vectors
self.vectorizer = vectorizer
self.vector_cache = vector_cache
self.threshold = threshold
self.matrix = self.dataframe.values
def main():
max_balls=20
results = np.zeroes(n,n)
for i,j in itertools.product(*[range(max_balls)]):
states=[(i,j,1)]
saturated =1
while staturated:
balls = states[0][0] + states[0][1] -1
new_states=[(k,balls-k, 0 ) for k in range(balls)]]
for state in states:
a,b,c=state
if a,b >1:
new_states[a-2] =(a-2,b+1,new_states[a-2][2]+c*(a*(a-1)/((a+b)*((a-1)+b))))
new_states[a] =(a,b-1,new_states[a][2]+c*(b*(b-1)/((a+b)*((b-1)+a))+(b*a/((a+b)*((b-1)+a)))+(b*a/((a+b)*((a-1)+b)))))
elif a >1:
new_states[a-2] =(a-2,b+1,new_states[a-2][2]+c*(a*(a-1)/((a+b)*((a-1)+b))))
new_states[a] =(a,b-1,new_states[a][2]+c*((b*a/((a+b)*((b-1)+a)))+(b*a/((a+b)*((a-1)+b)))))
elif b>1:
new_states[a] =(a,b-1,new_states[a][2]+c*((b*(b-1)/((a+b)*((b-1)+a)))+((b*a/((a+b)*((b-1)+a)))+(b*a/((a+b)*((a-1)+b)))))
elif a==1 and b==1:
new_states[a] =(1,0,new_states[a][2]+c)
def initialize_trajectories(self,signalInit='gw',trajectories=[None]):
'''Initialize the initial generation of trajectories. The purpose of this
metod ist that it can be expanded to implement other kinds of initial signals.
Input:
* signalInit : value corresponds to different types of signals
- 'gw' : gaussian white noise
* trajectories : input trajectories if provided
Output:
* self.trajectories : initial trajectories are saved to internal variable
'''
k = self.params.k
N = self.params.NReal
if np.any(trajectories):
if np.shape(trajectories) == (N,k):
self.trajectories = trajectories
else:
raise InitializationError("Shape of input trajectories incompatible with parameters. The shape is %s and should be %s" %(str(np.shape(trajectories)),str((N,k))))
elif signalInit == 'gw':
self.trajectories = self.get_signal_gaussian_white()
else:
self.trajectories = np.zeroes((N,k)) # initialize as zeros, since it needs to be there
#create a mouse callback function which is executed when a mouse event takes place. it gives us (x,y) for every mouse event.
import cv2
import numpy as np
#mouse callback function
def draw_circle(event,x,y,flags,param):
if event==cv2.EVENT_LBUTTONDBLCLK:
cv2.circle(img, (x,y), 100, (255,0,0), -1)
#create a black image, a window and bind the function to window
img=np.zeroes((512,512,3), np.uint8)
cv2.namedWindow('image')
cv2.setMouseCallback('image', draw_circle)
while(1):
cv2.imshow('image',img)
if cv2.waitKey(20) & 0xFF == 27:
break
cv2.destroyAllWindows()
import numpy as np
import matplotlib.pyplot as plt
f = open("blah", "r")
rows = len(f.readLines())
f.seek(0)
columns = len(f.readLines().split(","))
f.seek(0)
data = np.zeroes((columns, rows))
for rowdex in range(0, rows):
s = f.readLines().split(",")
for coldex in range(0, columns):
data[coldex][rowdex] = s[coldex]
print data
def overlayPoesBnd(basemap_obj, ax, start_time, coords='geo', hemi=1,
equBnd=True, polBnd=False ) :
"""Overlay POES data on a map
Parameters
----------
basemap_obj : (class 'mpl_toolkits.basemap.Basemap')
the map object you want data to be overlayed on.
ax : (class 'matplotlib.axes._subplots.AxesSubplot')
the Axis Handle used.
start_time : (datetime)
the starttime you want data for. If endTime is not given overlays data
from satellites with in +/- 45 min of the start_time
coords : (list or None)
Coordinates of the map object on which you want data to be overlayed on.
(default='geo')
hemi : (list or None)
Hemisphere of the map object on which you want data to be overlayed on.
Value is 1 for northern hemisphere and -1 for the southern hemisphere.
(default=1)
equBnd : (list or None]
If this is True the equatorward auroral oval boundary fit from the TED
data is overlayed on the map object. (default=True)
polBnd : (list or None]
If this is True the poleward auroral oval boundary fit from the TED
data is overlayed on the map object. (default=False)
Returns
-------
Nothing
Notes
---------
This function reads POES TED data with in +/- 45 min of the given time, fits
the auroral oval boundaries and overlays them on a map object. The poleward
boundary is not accurate all the times due to lesser number of satellite
passes identifying it.
Example
-------
import datetime as dt
poesList = gme.sat.overlayPoesTed(MapObj, sTime=dt.datetime(2011,3,4,4))
written by Bharat Kunduri, 20130216
"""
import datetime as dt
import numpy as np
import math
import matplotlib.cm as cm
from scipy import optimize
import models
from davitpy import rcParams
igrf_file = rcParams['IGRF_DAVITPY_COEFF_FILE']
# check all the inputs for validity
assert isinstance(start_time, dt.datetime), \
logging.error('stime must be a datetime object')
# Check the hemisphere and get the appropriate folat
folat = [45.0 * hemi, 90.0 * hemi]
# Get the time range we choose +/- 45 minutes....
plt_time_int = np.array(dt.timedelta(minutes=45))
time_range = np.array([start_time-plt_time_int, start_time+plt_time_int])
# We set the TED cut-off value to -0.75,
# From observed cases this appeared to do well...
# though fails sometimes especially during geomagnetically quiet times...
# However this is version 1.0 and there always is room for improvement
equBndCutoffVal = -0.75
# If any particular satellite number is not chosen by user loop through all
# the available ones. Writer preferred numpy arrays over lists
sat_num = np.array([15, 16, 17, 18, 19])
snum_size = sat_num.shape[0]
lat_poes_all = [[] for j in range(snum_size)]
lon_poes_all = [[] for j in range(snum_size)]
ted_poes_all = [[] for j in range(snum_size)]
time_poes_all = [[] for j in range(snum_size)]
len_data_all = [[] for j in range(snum_size)]
for sn in range(snum_size):
curr_poes = readPoes(time_range[0], eTime=time_range[1],
satnum=int(sat_num[sn]), folat=folat)
# Check if the data is loaded...
if curr_poes == None:
logging.warning('No data found')
continue
# Loop through the list and store the data into arrays
len_data_all.append(len(curr_poes))
for l in range(len_data_all[-1]):
# Store our data in arrays if the TED data value is > than the
# cutoff value
try:
x = math.log10(curr_poes[l].ted)
except:
continue
if x > equBndCutoffVal:
if coords == 'mag' or coords == 'mlt':
lat, lon, _ = models.aacgm.convert_latlon_arr( \
curr_poes[l].folat, curr_poes[l].folon, \
np.zeroes(shape=curr_poes[l].folat.shape), \
curr_poes[l].time.year, 'G2A')
lat_poes_all[sn].append(lat)
if coords == 'mag':
lon_poes_all[sn].append(lon)
else:
tt = curr_poes[l].time
mlt = models.aacgm.mlt_convert(tt.year, tt.month,
tt.day, tt.hour,
tt.minute, tt.second,
lon, igrf_file)
lon_poes_all[sn].append(mlt * 360.0 / 24.0)
else:
lat_poes_all[sn].append(curr_poes[l].folat)
lon_poes_all[sn].append(curr_poes[l].folon)
ted_poes_all[sn].append(math.log10(curr_poes[l].ted))
time_poes_all[sn].append(curr_poes[l].time)
lat_poes_all = np.array(lat_poes_all)
lon_poes_all = np.array(lon_poes_all)
ted_poes_all = np.array(ted_poes_all)
time_poes_all = np.array(time_poes_all)
len_data_all = np.array(len_data_all)
# Now to identify the boundaries...
# Also need to check if the boundary is equatorward or poleward..
# When satellite is moving from high-lat to low-lat decrease in flux would
# mean equatorward boundary. When satellite is moving from low-lat to
# high-lat increase in flux would mean equatorward boundary that is what we
# are trying to check here
eq_bnd_lats = []
eq_bnd_lons = []
po_bnd_lats = []
po_bnd_lons = []
for n1 in range(snum_size):
curr_sat_lats = lat_poes_all[n1]
curr_sat_lons = lon_poes_all[n1]
curr_sat_teds = ted_poes_all[n1]
test_lat_ltoh = []
test_lon_ltoh = []
test_lat_htol = []
test_lon_htol = []
test_lat_ltohp = []
test_lon_ltohp = []
test_lat_htolp = []
test_lon_htolp = []
for n2 in range(len(curr_sat_lats)-1):
# Check if the satellite is moving from low-lat to high-lat or not
if math.fabs(curr_sat_lats[n2]) < math.fabs(curr_sat_lats[n2+1]):
if curr_sat_teds[n2] < curr_sat_teds[n2+1]:
test_lat_ltoh.append(curr_sat_lats[n2])
test_lon_ltoh.append(curr_sat_lons[n2])
if curr_sat_teds[n2] > curr_sat_teds[n2+1]:
test_lat_ltohp.append(curr_sat_lats[n2])
test_lon_ltohp.append(curr_sat_lons[n2])
if math.fabs(curr_sat_lats[n2]) > math.fabs(curr_sat_lats[n2+1]):
if curr_sat_teds[n2] > curr_sat_teds[n2+1]:
test_lat_htol.append(curr_sat_lats[n2])
test_lon_htol.append(curr_sat_lons[n2])
if curr_sat_teds[n2] < curr_sat_teds[n2+1]:
test_lat_htolp.append(curr_sat_lats[n2])
test_lon_htolp.append(curr_sat_lons[n2])
# I do this to find the index of the min lat...
if test_lat_ltoh != []:
test_lat_ltoh = np.array(test_lat_ltoh)
test_lon_ltoh = np.array(test_lon_ltoh)
var_eq_lat1 = test_lat_ltoh[np.where(test_lat_ltoh ==
min(test_lat_ltoh))]
var_eq_lon1 = test_lon_ltoh[np.where(test_lat_ltoh ==
min(test_lat_ltoh))]
eq_bnd_lats.append(var_eq_lat1[0])
eq_bnd_lons.append(var_eq_lon1[0])
if test_lat_htol != []:
test_lat_htol = np.array(test_lat_htol)
test_lon_htol = np.array(test_lon_htol)
var_eq_lat2 = test_lat_htol[np.where(test_lat_htol ==
min(test_lat_htol))]
var_eq_lon2 = test_lon_htol[np.where(test_lat_htol ==
min(test_lat_htol))]
eq_bnd_lats.append(var_eq_lat2[0])
eq_bnd_lons.append(var_eq_lon2[0])
if test_lat_ltohp != []:
test_lat_ltohp = np.array(test_lat_ltohp)
test_lon_ltohp = np.array(test_lon_ltohp)
var_eq_latP1 = test_lat_ltohp[np.where(test_lat_ltohp ==
min(test_lat_ltohp))]
var_eq_lonP1 = test_lon_ltohp[np.where(test_lat_ltohp ==
min(test_lat_ltohp))]
if var_eq_latP1[0] > 64.0:
po_bnd_lats.append(var_eq_latP1[0])
po_bnd_lons.append(var_eq_lonP1[0])
if test_lat_htolp != []:
test_lat_htolp = np.array(test_lat_htolp)
test_lon_htolp = np.array(test_lon_htolp)
var_eq_latP2 = test_lat_htolp[np.where(test_lat_htolp ==
min(test_lat_htolp))]
var_eq_lonP2 = test_lon_htolp[np.where(test_lat_htolp ==
min(test_lat_htolp))]
if var_eq_latP2[0] > 64:
po_bnd_lats.append(var_eq_latP2[0])
po_bnd_lons.append(var_eq_lonP2[0])
eq_bnd_lats = np.array(eq_bnd_lats)
eq_bnd_lons = np.array(eq_bnd_lons)
po_bnd_lats = np.array(po_bnd_lats)
po_bnd_lons = np.array(po_bnd_lons)
# Now we do the fitting part. We start by establishing a target function.
# The error is the distance to the target function.
fitfunc = lambda p, x: p[0] + p[1] * np.cos(np.radians(x) + p[2])
errfunc = lambda p, x, y: fitfunc(p, x) - y
# Initial guess for the parameters
# Equatorward boundary
p0_eq = [1.0, 1.0, 1.0]
p1_eq, success_eq = optimize.leastsq(errfunc, p0_eq[:], args=(eq_bnd_lons,
eq_bnd_lats))
if pol_bnd == True :
p0_pol = [ 1., 1., 1.]
p1_pol, success_pol = optimize.leastsq(errfunc, p0_pol[:],
args=(po_bnd_lons, po_bnd_lats))
all_plot_lons = np.linspace(0.0, 360.0, 25.0)
all_plot_lons[-1] = 0.0
eq_plot_lats = []
if pol_bnd == True :
po_plot_lats = []
for xx in all_plot_lons :
if equ_bnd == True :
eq_plot_lats.append(p1_eq[0] + p1_eq[1] * np.cos(np.radians(xx)
+ p1_eq[2]))
if pol_bnd == True :
po_plot_lats.append(p1_pol[0] + p1_pol[1]*np.cos(np.radians(xx)
+ p1_pol[2]))
x_eq, y_eq = basemap_obj(all_plot_lons, eq_plot_lats)
bpltpoes = basemap_obj.plot(x_eq, y_eq, zorder=7.0, color='b')
if pol_bnd == True :
x_pol, y_pol = basemap_obj(all_plot_lons, po_plot_lats)
bpltpoes = basemap_obj.plot(x_pol, y_pol, zorder=7.0, color='r')
return
