def group_ref_color_atom_overlaps(results):
"""
Create a 3D masked array containing all overlap scores.
Parameters
----------
scores : array_like
2D array containing reference molecule color atom overlap results.
"""
# get maximum number of ref color atoms
# don't use `for result in it` because that gives an array of size 1
max_size = 0
it = np.nditer(results, flags=['multi_index', 'refs_ok'])
for _ in it:
max_size = max(max_size, len(results[it.multi_index]))
# build a masked array containing results
# don't use data[it.multi_index][:result.size] because that assigns
# to a view and not to data
data = np.ma.masked_all((results.shape[:2] + (max_size,)), dtype=float)
it = np.nditer(results, flags=['multi_index', 'refs_ok'])
for _ in it:
i, j = it.multi_index
result = results[i, j]
data[i, j, :result.size] = result
return data
def search(self, fn, top_n=10, sim_thresh=None):
"""
retrieval face from database,
return top_n similar faces' imgIDs, return None if failed
"""
if top_n > len(self.data):
top_n = len(self.data)
aligned_fn = send2align(fn)
aligned_arr = path2arr(aligned_fn)
if aligned_arr is None:
print "align none."
return None
deepIDfea = self.model.getID([aligned_arr])[0]
sims = [cosine_similarity(deepIDfea, item[1])[0][0] for item in self.data]
# print len(self.data), len(sims)
for i in range(len(sims)):
print sims[i], self.data[i][0]
sort_index = np.argsort(-np.array(sims))
result = []
if sim_thresh is None:
for index in np.nditer(sort_index):
cur_id = self.data[index][0].split("-")[0]
if cur_id not in result and len(result) < top_n:
result.append(cur_id)
return result
else:
for index in np.nditer(sort_index):
if sims[index] < sim_thresh:
break
cur_id = self.data[index][0].split("-")[0]
if cur_id not in result:
result.append(cur_id)
return result
def nabla(self,cost,n,c=1,q=0.001,a=0.001,A=100,alpha=0.602,gamma=0.101):
cn=(c+0.0)/(n+A)**gamma
an=a/(n+1+A)**alpha
qk=math.sqrt(q/(n+A)*math.log(math.log(n+A)))
wk=normal()
dv=[]
sess=self.sess
g=[]
orig=self.var
for m in self.var:
shape=m.shape
nm=np.ones(shape=shape)
for x in np.nditer(nm, op_flags=['readwrite']):
x[...]=dist.bernoulli() * 2 * cn
dv.append(nm)
del l=[:]
for m,d,t in zip(self.var,dv,self.var_t):
l.append(t.assign(m+d))
sess.run(tf.group(*l))
f1=sess.run(cost,self.feed)
del l=[:]
for m,d,t in zip(self.var,dv,self.var_t):
l.append(t.assign(m-d))
sess.run(tf.group(*l))
f0=sess.run(cost,self.feed)
df=f1-f0
for m in dv:
for x in np.nditer(m, op_flags=['readwrite']):
x[...]=-(df+0.0)/x/2
return dv
def test_external_loop(self):
from numpy import arange, nditer, array
a = arange(24).reshape(2, 3, 4)
import sys
if '__pypy__' in sys.builtin_module_names:
raises(NotImplementedError, nditer, a, flags=['external_loop'])
skip('nditer external_loop not implmented')
r = []
n = 0
for x in nditer(a, flags=['external_loop']):
r.append(x)
n += 1
assert n == 1
assert (array(r) == range(24)).all()
r = []
n = 0
for x in nditer(a, flags=['external_loop'], order='F'):
r.append(x)
n += 1
assert n == 12
assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21], [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23]]).all()
e = raises(ValueError, 'r[0][0] = 0')
assert str(e.value) == 'assignment destination is read-only'
r = []
for x in nditer(a.T, flags=['external_loop'], order='F'):
r.append(x)
array_r = array(r)
assert len(array_r.shape) == 2
assert array_r.shape == (1,24)
assert (array(r) == arange(24)).all()
def process(self):
# counts
self.count += 1
# calculate eta
eta_sum = np.float64(0)
dt_ = np.zeros((2, 2, 2, 2, 2, 2), np.float64)
it = np.nditer(dt_, flags=['multi_index'])
while not it.finished:
i, j, k, l, m, n = it.multi_index
dt_[i, j, k, l, m, n] = __eta_dt(self, i, j, k, l, m, n)
eta_sum += dt_[i, j, k, l, m, n]
it.iternext()
############################
# normalization
############################
self.checker = np.float64(0)
# 4-body
self.m41_ = np.zeros((2, 2, 2, 2), np.float64)
# 3-body
self.m31_ = np.zeros((2, 2, 2), np.float64)
# pair
self.m21_ = np.zeros((2, 2), np.float64)
self.m22_ = np.zeros((2, 2), np.float64)
m22_ = np.zeros((2, 2), np.float64)
# point
self.x_ = np.zeros((2), np.float64)
it = np.nditer(dt_, flags=['multi_index'])
while not it.finished:
i, j, k, l, m, n = it.multi_index
# print('self.zt_{} is: {}'.format(it.multi_index, self.zt_[i, j, k]))
dt_[i, j, k, l, m, n] /= eta_sum
self.checker += np.absolute(dt_[i, j, k, l, m, n] -
self.dt_[i, j, k, l, m, n])
# dt_
self.dt_[i, j, k, l, m, n] = dt_[i, j, k, l, m, n]
# m41_
self.m41_[i, j, k, l] += self.dt_[i, j, k, l, m, n]
# m31_
self.m31_[i, m, k] += self.dt_[i, j, k, l, m, n]
# m21_
self.m21_[i, j] += self.dt_[i, j, k, l, m, n]
# m22_
self.m22_[j, n] += self.dt_[i, j, k, l, m, n]
m22_[i, m] += self.dt_[i, j, k, l, m, n]
# x_
self.x_[i] += self.dt_[i, j, k, l, m, n]
it.iternext()
def buildDistanceMatrix(self):
for head, ngrams in self.head_clusters.iteritems():
word_indices = []
stmt_indices = []
priority_indices = []
feature_words = []
sections = []
dm_w_rows = []
dm_s_rows = []
dm_p_rows = []
for ngram in ngrams:
word_indices.append(ngram[3][1])
stmt_indices.append(ngram[3][0])
priority_indices.append(ngram[1])
feature_words.append(ngram[0])
sections.append(ngram[-1])
word_indices_clone = word_indices
stmt_indices_clone = stmt_indices
priority_indices_clone = priority_indices
for word_index, stmt_index, priority_index in zip(word_indices, stmt_indices, priority_indices):
dm_w_row = []
dm_s_row = []
dm_p_row = []
for word_index_clone, stmt_index_clone, priority_index_clone in zip(word_indices_clone, stmt_indices_clone, priority_indices_clone):
dm_w_row.append(fabs(((1 + word_index) * (1 + stmt_index)) - ((1 + word_index_clone) * (1 + stmt_index_clone))))
dm_s_row.append(fabs((1 + stmt_index) - (1 + stmt_index_clone)))
dm_p_row.append(fabs(float(priority_index) - float(priority_index_clone)))
dm_w_rows.append(dm_w_row)
dm_s_rows.append(dm_s_row)
dm_p_rows.append(dm_p_row)
dm_w = np.array(dm_w_rows)
dm_s = np.array(dm_s_rows)
dm_p = np.array(dm_p_rows)
#print dm_w
#print dm_s
#print dm_p
prox_mat = []
for w_dist, s_dist, PI in zip(np.nditer(dm_w), np.nditer(dm_s), np.nditer(dm_p)):
if PI == 0.0:
proximity_score = ((w_dist + len(np.unique(dm_s) * s_dist)) / (dm_w.shape[0] * len(np.unique(dm_s))))
prox_mat.append(proximity_score)
else:
proximity_score = ((w_dist + len(np.unique(dm_s) * s_dist)) / (dm_w.shape[0] * len(np.unique(dm_s)))) * log10(10 * PI)
prox_mat.append(proximity_score)
ps = np.array(prox_mat)
ps = np.reshape(ps, dm_w.shape)
#print ps
for r, row in enumerate(ps):
for i, ele in enumerate(row):
if ele == min(row):
self.f2.writerow([feature_words[r], priority_indices[r], 1 - np.min(row), feature_words[i], sections[r]])
def test_external_loop(self):
from numpy import arange, nditer, array
a = arange(24).reshape(2, 3, 4)
import sys
r = []
for x in nditer(a, flags=['external_loop']):
r.append(x)
assert len(r) == 1
assert r[0].shape == (24,)
assert (array(r) == range(24)).all()
r = []
for x in nditer(a, flags=['external_loop'], order='F'):
r.append(x)
assert len(r) == 12
assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21],
[ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23],
]).all()
e = raises(ValueError, 'r[0][0] = 0')
assert str(e.value) == 'assignment destination is read-only'
r = []
for x in nditer(a.T, flags=['external_loop'], order='F'):
r.append(x)
array_r = array(r)
assert len(array_r.shape) == 2
assert array_r.shape == (1,24)
assert (array(r) == arange(24)).all()
def rvs(self, loc=0, scale=1, size=1):
"""Random variates.
Parameters
----------
loc : float or np.ndarray
0-D or 1-D tensor.
scale : float or np.ndarray
0-D or 1-D tensor, with all elements constrained to
:math:`scale > 0`.
size : int
Number of random variable samples to return.
Returns
-------
np.ndarray
A np.ndarray of dimensions size x shape.
"""
if not isinstance(loc, np.ndarray):
loc = np.asarray(loc)
if not isinstance(scale, np.ndarray):
scale = np.asarray(scale)
if len(loc.shape) == 0:
return stats.norm.rvs(loc, scale, size=size)
x = []
for locidx, scaleidx in zip(np.nditer(loc), np.nditer(scale)):
x += [stats.norm.rvs(locidx, scaleidx, size=size)]
# Note this doesn't work for multi-dimensional sizes.
x = np.asarray(x).transpose()
return x
def rvs(self, n, p, size=1):
"""Random variates.
Parameters
----------
n : int or np.ndarray
0-D or 1-D tensor, with all elements constrained to
:math:`n > 0`.
p : float or np.ndarray
0-D or 1-D tensor, with all elements constrained to
:math:`p\in(0,1)`.
size : int
Number of random variable samples to return.
Returns
-------
np.ndarray
A np.ndarray of dimensions size x shape.
"""
if not isinstance(n, np.ndarray):
n = np.asarray(n)
if not isinstance(p, np.ndarray):
p = np.asarray(p)
if len(n.shape) == 0:
return stats.nbinom.rvs(n, p, size=size)
x = []
for nidx, pidx in zip(np.nditer(n), np.nditer(p)):
x += [stats.nbinom.rvs(nidx, pidx, size=size)]
# Note this doesn't work for multi-dimensional sizes.
x = np.asarray(x).transpose()
return x
def rvs(self, a, scale=1, size=1):
"""Random variates.
Parameters
----------
a : float or np.ndarray
**Shape** parameter. 0-D or 1-D tensor, with all elements
constrained to :math:`a > 0`.
scale : float or np.ndarray
**Scale** parameter. 0-D or 1-D tensor, with all elements
constrained to :math:`scale > 0`.
size : int
Number of random variable samples to return.
Returns
-------
np.ndarray
A np.ndarray of dimensions size x shape.
"""
if not isinstance(a, np.ndarray):
a = np.asarray(a)
if not isinstance(scale, np.ndarray):
scale = np.asarray(scale)
if len(a.shape) == 0:
return stats.gamma.rvs(a, scale=scale, size=size)
x = []
for aidx, scaleidx in zip(np.nditer(a), np.nditer(scale)):
x += [stats.gamma.rvs(aidx, scale=scaleidx, size=size)]
# Note this doesn't work for multi-dimensional sizes.
x = np.asarray(x).transpose()
return x
def clustering(self):
"""
Class the GSOM world into the right number of clusters, update cluster association dictionary as well as the
representative of each cluster.
"""
updated_representative = {}
for cell in np.nditer(self._world):
distance = {}
for key, value in self._representative.iteritems():
distance[key] = np.linalg.norm(cell - value)
belong_group = min(distance, key=distance.get)
self._association[belong_group].append(cell)
for key in self._representative.keys():
updated_representative[key] = np.mean(self._association[key], axis=0)
while self._representative != updated_representative:
self._representative = updated_representative
for cell in np.nditer(self._world):
distance = {}
for key, value in self._representative.iteritems():
distance[key] = np.linalg.norm(cell - value)
belong_group = min(distance, key=distance.get)
self._association[belong_group].append(cell)
for key in self._representative.keys():
updated_representative[key] = np.mean(self._association[key], axis=0)
def test_itershape(self):
# Check that allocated outputs work with a specified shape
from numpy import nditer, arange
import sys
if '__pypy__' in sys.builtin_module_names:
skip("op_axes not totally supported yet")
a = arange(6, dtype='i2').reshape(2,3)
i = nditer([a, None], [], [['readonly'], ['writeonly','allocate']],
op_axes=[[0,1,None], None],
itershape=(-1,-1,4))
assert i.operands[1].shape == (2,3,4)
assert i.operands[1].strides, (24,8,2)
i = nditer([a.T, None], [], [['readonly'], ['writeonly','allocate']],
op_axes=[[0,1,None], None],
itershape=(-1,-1,4))
assert i.operands[1].shape, (3,2,4)
assert i.operands[1].strides, (8,24,2)
i = nditer([a.T, None], [], [['readonly'], ['writeonly','allocate']],
order='F',
op_axes=[[0,1,None], None],
itershape=(-1,-1,4))
assert i.operands[1].shape, (3,2,4)
assert i.operands[1].strides, (2,6,12)
# If we specify 1 in the itershape, it shouldn't allow broadcasting
# of that dimension to a bigger value
raises(ValueError, nditer, [a, None], [],
[['readonly'], ['writeonly','allocate']],
op_axes=[[0,1,None], None],
itershape=(-1,1,4))
def launch_boolean_query(self, query, num_results):
doc_relevance_vector = np.zeros(len(self.doc_index.index))
query_feature_vector = \
helpers.create_doc_index(self.dictionary, helpers.docs2bows([query], self.dictionary)).index[0]
iter_count = 0
for doc_feature_vector in self.doc_index.index:
if np.sum(query_feature_vector) > 0 and np.array_equal(
np.where((query_feature_vector > 0) & (doc_feature_vector > 0)),
np.where(query_feature_vector > 0)):
doc_relevance_vector[iter_count] = 1
iter_count += 1
relevant_docs = np.where(doc_relevance_vector == 1)[0]
if relevant_docs.size == 0:
return []
else:
results_shown = 0
for doc in np.nditer(relevant_docs):
if results_shown < num_results:
print('[ID: ' + str(doc + 1) + '] ' + self.corpus[doc])
results_shown += 1
ranking = []
for doc in np.nditer(relevant_docs):
ranking.append((doc, 1))
return ranking
def _form_slip_xyz_file_string(self):
_txt = ''
for lon, lat, s in zip(np.nditer(self.lons),
np.nditer(self.lats),
np.nditer(self.slip)):
_txt +='%f %f %f\n'%(lon, lat, s)
return _txt
def write_array(fp, array, version=None):
"""
Write an array to an NPY file, including a header.
If the array is neither C-contiguous nor Fortran-contiguous AND the
file_like object is not a real file object, this function will have to
copy data in memory.
Parameters
----------
fp : file_like object
An open, writable file object, or similar object with a ``.write()``
method.
array : ndarray
The array to write to disk.
version : (int, int) or None, optional
The version number of the format. None means use the oldest supported
version that is able to store the data. Default: None
Raises
------
ValueError
If the array cannot be persisted.
Various other errors
If the array contains Python objects as part of its dtype, the
process of pickling them may raise various errors if the objects
are not picklable.
"""
_check_version(version)
used_ver = _write_array_header(fp, header_data_from_array_1_0(array),
version)
# this warning can be removed when 1.9 has aged enough
if version != (2, 0) and used_ver == (2, 0):
warnings.warn("Stored array in format 2.0. It can only be"
"read by NumPy >= 1.9", UserWarning)
# Set buffer size to 16 MiB to hide the Python loop overhead.
buffersize = max(16 * 1024 ** 2 // array.itemsize, 1)
if array.dtype.hasobject:
# We contain Python objects so we cannot write out the data directly.
# Instead, we will pickle it out with version 2 of the pickle protocol.
pickle.dump(array, fp, protocol=2)
elif array.flags.f_contiguous and not array.flags.c_contiguous:
if isfileobj(fp):
array.T.tofile(fp)
else:
for chunk in numpy.nditer(
array, flags=['external_loop', 'buffered', 'zerosize_ok'],
buffersize=buffersize, order='F'):
fp.write(chunk.tobytes('C'))
else:
if isfileobj(fp):
array.tofile(fp)
else:
for chunk in numpy.nditer(
array, flags=['external_loop', 'buffered', 'zerosize_ok'],
buffersize=buffersize, order='C'):
fp.write(chunk.tobytes('C'))
def var_inner(self,var_v1,var_v2):
v1=[]
v2=[]
for m1,m2 in zip(var_v1,var_v2):
v1=v1+[x for x in np.nditer(m1, op_flags=['readwrite'])]
v2=v2+[x for x in np.nditer(m2, op_flags=['readwrite'])]
return np.inner(v1,v2)
def run(self):
# temperature iteration
for dmu in np.nditer(self.delta_mu):
data = []
self.mu[0] += dmu
self.mu[1] = -self.mu[0]
self.x_[1] = self.x_1
self.x_[0] = 1 - self.x_1
print(' mu = {:06.4f}:'.format(self.mu[0].item(0)))
# delta mu iteration
for temp in np.nditer(self.temp):
self.beta = np.float64(pow(self.bzc * temp, -1))
# calculate
self.__run()
# push result into data
data.append({'temp': temp.item(0), 'c': self.x_[1].item(0)})
print(' T = {:06.3f}K, c = {:06.6f}, count = {}'.
format(temp.item(0), self.x_[1].item(0), self.count))
print('\n')
# save result to output
self.output['Results'].append(
{'mu': self.mu[0].item(0), 'data': data})
self.mu[0] -= dmu
def numerical_gradients(self):
"""Compute numerical gradients of f with respect to self.Wh, self.bh, self.Ws, and self.bs
Returns approximation for df/dWh, df/dbh, df/dWs, df/dbs
"""
dWh, dbh, dWs, dbs = np.zeros_like(self.Wh), np.zeros_like(self.bh), np.zeros_like(self.Ws), np.zeros_like(self.bs)
Wh, bh, Ws, bs = self.Wh, self.bh, self.Ws, self.bs
step = 1e-5
# df/dWh
h = np.zeros_like(self.Wh)
it = np.nditer(Wh, flags=['multi_index'])
while not it.finished:
ix = it.multi_index
h[ix] = step
dWh[ix] = (self.forward_backward_prop(Wh+h, bh, Ws, bs).loss - self.forward_backward_prop(Wh-h, bh, Ws, bs).loss) / (2*step)
h[ix] = 0
it.iternext()
# df/dbh
h = np.zeros_like(self.bh)
it = np.nditer(bh, flags=['multi_index'])
while not it.finished:
ix = it.multi_index
h[ix] = step
dbh[ix] = (self.forward_backward_prop(Wh, bh+h, Ws, bs).loss - self.forward_backward_prop(Wh, bh-h, Ws, bs).loss) / (2*step)
h[ix] = 0
it.iternext()
# df/dWh
h = np.zeros_like(self.Ws)
it = np.nditer(Ws, flags=['multi_index'])
while not it.finished:
ix = it.multi_index
h[ix] = step
dWs[ix] = (self.forward_backward_prop(Wh, bh, Ws+h, bs).loss - self.forward_backward_prop(Wh, bh, Ws-h, bs).loss) / (2*step)
h[ix] = 0
it.iternext()
# df/dbs
h = np.zeros_like(self.bs)
it = np.nditer(bs, flags=['multi_index'])
while not it.finished:
ix = it.multi_index
h[ix] = step
dbs[ix] = (self.forward_backward_prop(Wh, bh, Ws, bs+h).loss - self.forward_backward_prop(Wh, bh, Ws, bs-h).loss) / (2*step)
h[ix] = 0
it.iternext()
return dWh, dbh, dWs, dbs
def calc(self, input):
"""
Calculates the network output for the given input
@param input A array of inputs [in1, in2,..]
@return lastNetResult
"""
lastNetResult = np.array(input)
# save each layer in/output for training
self.inputs = []
self.outputs = []
for i in range(len(self.layout) - 1):
# append bias
# self.outputFun(lastNetResult)
lastNetResult = np.hstack((lastNetResult, [1]))
self.inputs.append(lastNetResult)
# calc result
lastNetResult = np.dot(self.weights[i], lastNetResult)
if i == len(self.layout) - 2:
# different activation function for last layer
lastNetResult = np.array(list(map(
self.last_layer_transfer, np.nditer(lastNetResult))))
else:
# lastNetResult = self.layer_transfer(lastNetResult)
lastNetResult = np.array(list(map(
self.layer_transfer, np.nditer(lastNetResult))))
self.outputs.append(lastNetResult)
return lastNetResult
def get_rmss(fc):
import numpy as np
from python_gen import get_fn_sd
if hasattr(fc[(0,)*fc.ndim], '__call__'): #for forecasts that are functions
rmss=np.zeros(fc.shape)
for i in range(fc.shape[1]): #loop over forecast lead times
fc_temp=fc[:,i,...]
fc_temp_iter=np.nditer(fc_temp, flags=['multi_index','refs_ok'])
while not fc_temp_iter.finished: #loop over other indices
ind=fc_temp_iter.multi_index
rmss[ind]=get_fn_sd(fc_temp[ind])
fc_temp_iter.iternext()
else:
fc_anom=np.zeros(fc.shape)
for i in range(fc.shape[0]):
fc_anom[i,...]=fc[i,...]-np.mean(fc[i,...]) #anomaly from the mean forecast over all forecast start times and ensemble members for each lead time.
ens_mean=np.mean(fc_anom,axis=-1)
rmss=np.zeros(fc.shape[:-1])
rmss_iter=np.nditer(rmss, flags=['multi_index'])
while not rmss_iter.finished:
ind=rmss_iter.multi_index
rmss[ind]=np.mean((fc_anom[ind]-ens_mean[ind])**2)
rmss_iter.iternext()
rmss=np.mean(rmss,axis=1)
rmss=np.sqrt(rmss)
return rmss
def test_iter_allocate_output_subtype():
# Make sure that the subtype with priority wins
# 2018-04-29: moved here from core.tests.test_nditer, given the
# matrix specific shape test.
# matrix vs ndarray
a = np.matrix([[1, 2], [3, 4]])
b = np.arange(4).reshape(2, 2).T
i = np.nditer([a, b, None], [],
[['readonly'], ['readonly'], ['writeonly', 'allocate']])
assert_(type(i.operands[2]) is np.matrix)
assert_(type(i.operands[2]) is not np.ndarray)
assert_equal(i.operands[2].shape, (2, 2))
# matrix always wants things to be 2D
b = np.arange(4).reshape(1, 2, 2)
assert_raises(RuntimeError, np.nditer, [a, b, None], [],
[['readonly'], ['readonly'], ['writeonly', 'allocate']])
# but if subtypes are disabled, the result can still work
i = np.nditer([a, b, None], [],
[['readonly'], ['readonly'],
['writeonly', 'allocate', 'no_subtype']])
assert_(type(i.operands[2]) is np.ndarray)
assert_(type(i.operands[2]) is not np.matrix)
assert_equal(i.operands[2].shape, (1, 2, 2))
def __init__(self, maxResult=10, gridSpec=None, verbose=True):
self.gridSpec = gridSpec
self.maxResult = maxResult
self.enableGrid = False
self.verbose = verbose
# Calculate exact grid
self.grid = []
gsTau = self.gridSpec[0]
gsS = self.gridSpec[1]
if len(gsTau) > 1 and len(gsS) > 1:
self.enableGrid = True
countTau = 5
countS = 5
if len(gsTau) > 2:
countTau = int(gsTau[2])
if len(gsS) > 2:
countS = int(gsS[2])
minTau = gsTau[0] - gsTau[1]
maxTau = (gsTau[0] + gsTau[1]) * (1+ (1/ (2*countTau)))
minS = gsS[0] - gsS[1]
maxS = (gsS[0] + gsS[1]) * (1+ (1/ (2*countS)))
tau = np.arange(minTau, maxTau, (gsTau[1] * 2.0) / countTau)
S = np.arange(minS, maxS, (gsS[1] * 2.0) / countS)
for t in np.nditer(tau):
for s in np.nditer(S):
self.grid.append( np.array([t, s]) )
self.dTau = tau[1] - tau[0]
self.dS = S[1] - S[0]
self.bounds = [ [minTau, maxTau], [minS, maxS] ]
def descend_weights_numeric(cost, weights, reg, learn, step):
"""
Gradient descent, for weights
cost - objective function, not requiring parameters, without regularisation
weights - their derivative will be approximated
reg - regularisation factor
learn - (negative) learning rate
step - step size for derivative
"""
derivative = []
for arr in weights:
der = zeros(arr.shape)
it = nditer(arr, flags=['multi_index'], op_flags=['readwrite'])
for value in it:
old_val = value.copy()
old_obj = cost()
value[...] += step
new_obj = cost()
value[...] = old_val
grad = (new_obj - old_obj)/step
grad = add_reg(old_val, grad, reg)
der[it.multi_index] = grad
derivative.append(der)
for n, arr in enumerate(weights):
der = derivative[n]
it = nditer(arr, flags=['multi_index'], op_flags=['readwrite'])
for value in it:
value[...] = descend(value[...], der[it.multi_index]*learn)
def z_normalization(time_series):
time_series = np.asarray(time_series, dtype=np.float64)
mean = np.mean(time_series)
stdev = np.std(time_series)
if stdev > 0:
return [(val - mean) / stdev for val in np.nditer(time_series)]
else:
return [0.0 for _ in np.nditer(time_series)]
def _new_lines_string(kml, lons, lats, name=None):
ls = kml.newlinestring(name=name)
coords = []
for lati, loni in zip(np.nditer(lats),
np.nditer(lons)):
coords.append([loni, lati])
ls.coords = coords
ls.extrude = 1
ls.altitudemode = sk.AltitudeMode.relativetoground
def normalize(im_data):
normalized = np.zeros(im_data.shape)
dmax = np.amax(im_data)
dmin = np.amin(im_data)
for i,j in zip(np.nditer(im_data), np.nditer(normalized, op_flags = ['readwrite'])):
j[...] = (i - dmin) / (dmax - dmin)
return (normalized, dmax, dmin)
def shuffle_labels(label_map):
"Randomize labels in a label map."
# TODO write this with iterators
t = time.time()
labels = np.unique(label_map)
labels = sorted(labels)
old_labels = copy.deepcopy(labels)
old_labels_dict = dict((label, index) for index, label in enumerate(old_labels))
np.random.seed(12)
np.random.shuffle(labels)
for i in xrange(len(labels)):
if labels[i] == 0:
temp = labels[i]
labels[i] = labels[0]
labels[0] = temp
if labels[i] == 1:
temp = labels[i]
labels[i] = labels[1]
labels[1] = temp
if labels[0] == 0 and labels[1] == 1:
break
# reverse_index = {label: [] for label in set_of_labels}
# it = np.nditer(label_map, flags=['multi_index'])
# while not it.finished:
# label = int(it[0])
# if label in set_of_labels:
# reverse_index[label].append(it.multi_index)
# it.iternext()
it = np.nditer(label_map, op_flags=['readwrite'])
for x in np.nditer(label_map, op_flags =["readwrite"]):
label = int(x)
x[...] = labels[old_labels_dict[label]]
# for i in xrange(label_map.shape[0]):
# for j in xrange(label_map.shape[1]):
# for k in xrange(label_map.shape[2]):
# label_map[i,j,k] = labels[old_labels_dict[label_map[i,j,k]]]
print "label scramble:" , time.time() - t
return label_map
def write_array(fp, array, version=(1, 0)):
"""
Write an array to an NPY file, including a header.
If the array is neither C-contiguous nor Fortran-contiguous AND the
file_like object is not a real file object, this function will have to
copy data in memory.
Parameters
----------
fp : file_like object
An open, writable file object, or similar object with a ``.write()``
method.
array : ndarray
The array to write to disk.
version : (int, int), optional
The version number of the format. Default: (1, 0)
Raises
------
ValueError
If the array cannot be persisted.
Various other errors
If the array contains Python objects as part of its dtype, the
process of pickling them may raise various errors if the objects
are not picklable.
"""
if version != (1, 0):
msg = "we only support format version (1,0), not %s"
raise ValueError(msg % (version,))
fp.write(magic(*version))
write_array_header_1_0(fp, header_data_from_array_1_0(array))
# Set buffer size to 16 MiB to hide the Python loop overhead.
buffersize = max(16 * 1024 ** 2 // array.itemsize, 1)
if array.dtype.hasobject:
# We contain Python objects so we cannot write out the data directly.
# Instead, we will pickle it out with version 2 of the pickle protocol.
pickle.dump(array, fp, protocol=2)
elif array.flags.f_contiguous and not array.flags.c_contiguous:
if isfileobj(fp):
array.T.tofile(fp)
else:
for chunk in numpy.nditer(
array, flags=['external_loop', 'buffered', 'zerosize_ok'],
buffersize=buffersize, order='F'):
fp.write(chunk.tostring('C'))
else:
if isfileobj(fp):
array.tofile(fp)
else:
for chunk in numpy.nditer(
array, flags=['external_loop', 'buffered', 'zerosize_ok'],
buffersize=buffersize, order='C'):
fp.write(chunk.tostring('C'))
def ComputeClusterPosition(self,clusterArray):
""" this will compute the weighted average of the cluster position
'clusterArray' should be normalized
note that the means can be computed by just multiplying the clusterArray with row or column vectors like [0,1,2....] and then taking the sum
"""
# setup variables first
mean=[0.,0.]
covariance=numpy.zeros((2,2))
countsSquared=0.0
# Compute the mean. Do this quickly by using projections -------------------------------------
# project in two dimensions
projectX=numpy.sum(clusterArray,axis=1) # note the ordering of the axis here!
projectY=numpy.sum(clusterArray,axis=0)
# compute the means from the projections
# get mean x
arrayIterator=numpy.nditer(projectX,flags=['multi_index'])
while not arrayIterator.finished:
position=arrayIterator.multi_index # this will return the indices of the SLICED array!
counts=arrayIterator[0]
if counts:
mean[0]+=float(position[0])*counts
arrayIterator.iternext()
# get mean y
arrayIterator=numpy.nditer(projectY,flags=['multi_index'])
while not arrayIterator.finished:
position=arrayIterator.multi_index # this will return the indices of the SLICED array!
counts=arrayIterator[0]
if counts:
mean[1]+=float(position[0])*counts # note the indexing of the position array: this is correct
arrayIterator.iternext()
# mean has been computed
# now compute the covariance matrix-------------------------------------------------------------
arrayIterator=numpy.nditer(clusterArray,flags=['multi_index'])
while not arrayIterator.finished:
position=arrayIterator.multi_index # this will return the indices of the SLICED array!
counts=arrayIterator[0]
if counts:
countsSquared+=counts*counts
covariance[0,0]+=counts* (float(position[0])-mean[0])*(float(position[0]-mean[0]))
covariance[0,1]+=counts* (float(position[0])-mean[0])*(float(position[1]-mean[1]))
covariance[1,1]+=counts* (float(position[1])-mean[1])*(float(position[1]-mean[1]))
arrayIterator.iternext()
covariance[1,0] = covariance[0,1]
covarianceFactor=1./(1.-countsSquared) # because we've already normalized
covariance=covariance*covarianceFactor
variance=[covariance[0,0],covariance[1,1]]
return (mean,covariance)
def jsonify(a, b):
assert a.shape[1] == b.shape[0]
it = np.nditer(a, flags=['multi_index'])
while not it.finished:
yield json.dumps((0,)+it.multi_index)+'\t'+json.dumps(int(it[0]))
it.iternext()
it = np.nditer(b, flags=['multi_index'])
while not it.finished:
yield json.dumps((1,)+it.multi_index)+'\t'+json.dumps(int(it[0]))
it.iternext()
import numpy as np
from sklearn import datasets # sklearn模块封装了常用的递归、降维、分类、聚类等方法,并提供一些标准数据
import pymysql
# numpy的测试
arr = np.arange(6).reshape(2, 3) # arange()创建一个2*3数组
print('原数组：')
print(arr) # [[0 1 2][3 4 5]]
print('\n')
print('数组的基本式遍历：')
for i in np.nditer(arr):
print(i, end=', ')
print('\n')
# 控制遍历顺序
# Fortran order,即数组列序优先
print('改为列序优先的数组：')
for s in np.nditer(arr, order = 'F'):
print(s, end=', ') # 0, 3, 1, 4, 2, 5,
print('\n')
# C Order,即数组行序优先
print('改为行序优先的数组：')
for l in np.nditer(arr, order = 'C'):
print(l, end=', ') # 0, 1, 2, 3, 4, 5,
print('\n')
# nditer对象有另一个可选参数 op_flags。
# 默认 nditer 将视待迭代遍历的数组为只读对象（read-only）
Y = pcl[:, 1]
Z = pcl[:, 2]
distance = pcl[:, 3]
# make A matrix (x y z)
size = len(X)
X1 = np.matrix.transpose(X)
Y1 = np.matrix.transpose(Y)
Z1 = np.matrix.transpose(Z)
A = [X1, Y1, Z1]
A = np.matrix([X1, Y1, Z1])
T1 = np.matrix.transpose(T)
T2 = np.repeat(T1, size, axis=0)
T2 = np.matrix.transpose(T2)
c2 = np.matmul((F), (R))
c2 = .25 * np.matmul((c2), (A + T2))
# Plot points on frame
for x in np.nditer(c2, flags=['external_loop'], order='F'):
cv2.circle(frame, (int(x[0]), int(x[1])), 1, (255, 0, 0), thickness=-1)
#print(x)
cv2.imshow('frame', frame)
c = cv2.waitKey(1)
if c == 27:
break
cap.release()
cap.destroyAllWindows()
#Iterates through directory only considering 'png' and 'jpeg' files.
for file in os.listdir(source_directory_path):
filename = os.fsdecode(file)
if filename.endswith(".png") or filename.endswith(".jpeg"):
filepath = ''.join([source_directory, "\\", filename])
#Open reworked and supersampled images
img = Image.open(filepath)
#Transform image to array and cast it to float type
arr = np.array(img)
arr = arr.astype(np.float32, copy=False)
#Change array values from 0 - 255 to 0 - 1 (0 == black, 1 == white)
for x in np.nditer(arr, op_flags=['readwrite']):
x[...] = x / 255
arr = arr.ravel()
combinedArray.append(arr)
#Test prints to see if everything is working correctly
#print("Shape: ",arr.shape)
#print(arr)
#print(img.format, img.size, img.mode)
#print(filepath)
continue
else:
continue
def multiannotator(args, writer=None):
#load annotator confidences for each language from json file
lang_lang_scores = json.load(open(config.langtaglang_file, 'r'))
model_created = False
nermodel = None
lang_scores = {}
unsup_lang_scores = {}
lr_unsup = config.unsuplr
lr_sup = config.unsupsuplr
nepochs_backup = config.nepochs
test_batch_size_backup = config.test_batch_size
scores = {}
with tf.Graph().as_default() as graph:
with tf.Session() as sess:
for lang in config.lowres_langs:
config.test_batch_size = test_batch_size_backup
set_seed(args.seed)
config.load(lang)
#very long sentences OOM error
if lang in ['hu', 'en', 'th', 'pl', 'sl']:
config.test_batch_size = 20
if not model_created:
nermodel = NERModelMultiAnnotator(config)
merged, dev_acc_ph, dev_f1_ph, test_acc_ph, test_f1_ph = merge_summaries(lang)
model_created = True
else:
nermodel.config = config
#normalise annotator expertise
annotators = lang_lang_scores[lang]
#decide which partition of the multiannotated datasets to train on train versus test
partition = 'test' if args.testannotations else 'train'
annotator_expertise = {f'{lang}_{k}_{partition}': v['train']['f1'] for k, v in lang_lang_scores[lang].items() if k not in ['ja', 'ko', 'zh', 'th']}
annotator_sorted = sorted(annotator_expertise)
expertise = [annotator_expertise[ann] for ann in annotator_sorted] if not args.uniformexpertise else [1.0] * len(annotator_sorted)
expertise = np.array(expertise)
#only use topk annotators
topk = config.unsuptopk
#assert topk <= len(annotator_sorted)
sorted_indices = np.argsort(expertise)
zero_out_num = len(annotator_sorted) - topk
if zero_out_num > 0:
zero_out_indices = sorted_indices[0:zero_out_num] if not args.uniformexpertise else np.random.choice(sorted_indices, size=zero_out_num, replace=False)
expertise[zero_out_indices] = 0
expertise = expertise / np.sum(expertise)
#load annotations (unlabelled data labelled by highresource models from diff. languages
lang_datasets = read_dataset_panx_multiannotated(config, lang, max_iter=hr_samples)
#load lr dataset
lrdataset = read_dataset_panx_individual(config, lang, max_iter=lr_samples)
nermodel.set_annotation_info(annotator_sorted, expertise, lang_datasets)
sess.run(tf.global_variables_initializer())
model_state = VariableState(sess, tf.trainable_variables())
model_wembed_state = VariableState(sess, [v for v in tf.global_variables() if
'words/_word_embeddings:0' == v.name])
model_wembed_state.import_variables([config.embeddings[lang]])
config.lr = lr_unsup
if args.lowtrainasdev:
#note we don't pass development set to unsup training, train set plays the role of dev set here because
#of scarsity of annotated data.
best_score, best_params = nermodel.train_unsup(lrdataset['train'], lrdataset['train'], sess, model_state, verbose=args.verbose)
else:
dev_annotator_sorted = [ann.replace(partition, 'dev') for ann in annotator_sorted]
total_dev_samples = 1000
num_dev_samples = np.random.multinomial(total_dev_samples, expertise)
#now sample from annotations based on num_samples
dev_samples = []
for i, n in enumerate(np.nditer(num_dev_samples)):
n = int(n)
if n == 0:
continue
dev_annotator = dev_annotator_sorted[i]
samples = lang_datasets[dev_annotator].sample(n)
dev_samples.extend(samples)
best_score, best_params = nermodel.train_unsup(lrdataset['train'], dev_samples, sess, model_state, verbose=args.verbose)
model_state.import_variables(best_params)
msgtest = nermodel.evaluate(lrdataset['test'], sess)
test_acc = msgtest['acc']
test_f1 = msgtest['f1']
logging.info('test lang:{} acc:{} f1:{}'.format(lang, test_acc, test_f1))
msgdev = nermodel.evaluate(lrdataset['dev'], sess, isDev=False)
acc = msgdev['acc']
f1 = msgdev['f1']
score = {'dev': msgdev, 'test': msgtest}
logging.info('dev lang:{} acc:{} f1:{}'.format(lang, acc, f1))
unsup_lang_scores[lang] = {'acc': test_acc, 'f1': test_f1}
#after training on data labelled by other languages, train on little supervision from target language
train_on_gold = True
if train_on_gold:
#now train on gold train after training on bad annotators
config.lr = lr_sup
#could we not use dev data here for early stopping?
#e.g. only run for 5 epochs and set dev to None instead of lrdataset['dev']
if args.unsupsupnepochs:
logging.info(f'no dev for early stopping, stop after {args.unsupsupnepochs} iters.')
nermodel.config.nepochs = args.unsupsupnepochs
best_score, best_params = nermodel.train(train=lrdataset['train'], dev=None, session=sess,
model_state=model_state, verbose=args.verbose)
nermodel.config.nepochs = nepochs_backup
else:
#use devset and early stopping
best_score, best_params = nermodel.train(train=lrdataset['train'], dev=lrdataset['dev'], session=sess,
model_state=model_state, verbose=args.verbose)
model_state.import_variables(best_params)
msgtest = nermodel.evaluate(lrdataset['test'], sess)
test_acc = msgtest['acc']
test_f1 = msgtest['f1']
logging.info('test lang:{} acc:{} f1:{}'.format(lang, test_acc, test_f1))
msgdev = nermodel.evaluate(lrdataset['dev'], sess, isDev=False)
acc = msgdev['acc']
f1 = msgdev['f1']
score = {'dev': msgdev, 'test': msgtest}
logging.info('dev lang:{} acc:{} f1:{}'.format(lang, acc, f1))
lang_scores[lang] = {'acc': test_acc, 'f1': test_f1}
logging.info('results after unsup pretrain')
logging.info(f'unsup{args.seed}={unsup_lang_scores}')
for lang, score in unsup_lang_scores.items():
logging.info('{}\t{}\t{}'.format(lang, int(score['acc']), int(score['f1'])))
logging.info('fine-tune results after unsup pretrain')
logging.info(lang_scores)
for lang, score in lang_scores.items():
logging.info('{}\t{}\t{}'.format(lang, int(score['acc']), int(score['f1'])))
scores[lang] = {'dev': msgdev, 'test': msgtest}
if writer:
summary = sess.run(merged,
feed_dict={dev_f1_ph: f1, dev_acc_ph: acc, test_acc_ph: test_acc,
test_f1_ph: test_f1})
writer.add_summary(summary)
logging.info(f'sup{args.seed}={lang_scores}')
return scores
def visualize(exp_dir, step, model, iterator_by_seqs, seqs, args):
"""
visualize reconstruction, factorization, sequence-translation
"""
logging.info("Starting vizualization...")
if len(seqs) > 10:
logging.info(">25 seqs. randomly select 25 seqs for visualization")
seqs = sorted(list(np.random.choice(seqs, 10, replace=False)))
if not os.path.exists("%s/img" % exp_dir):
os.makedirs("%s/img" % exp_dir)
if not os.path.exists("%s/spec" % exp_dir):
os.makedirs("%s/spec" % exp_dir)
if not os.path.exists("%s/wav" % exp_dir):
os.makedirs("%s/wav" % exp_dir)
if not os.path.exists("%s/txt" % exp_dir):
os.makedirs("%s/txt" % exp_dir)
saver = tf.train.Saver()
with tf.Session(config=SESS_CONF) as sess:
stime = time.time()
restore_model(sess, saver, "%s/models" % exp_dir, step)
logging.info("restore model takes %.2f s" % (time.time() - stime))
# infer z1, z2, mu2
z1_by_seq = defaultdict(list)
z2_by_seq = defaultdict(list)
mu2_by_seq = dict()
regpost_by_seq = dict()
xin_by_seq = defaultdict(list)
xout_by_seq = defaultdict(list)
xoutv_by_seq = defaultdict(list)
z1reg_by_seq = defaultdict(list)
for i, seq in enumerate(seqs):
if i % 10 == 0:
logging.info("encoding sequence %d/%d" % (i, len(seqs)))
for x, _, _, _, _ in iterator_by_seqs([seq]):
z2 = z2_mu_fn(sess, model, x)
z1 = z1_mu_fn(sess, model, x, z2)
xout = x_mu_fn(sess, model, z1, z2)
xoutv = x_logvar_fn(sess, model, z1, z2)
z1reg = reg_posteriors_z1(sess, model, z1)
z1_by_seq[seq].append(z1)
z2_by_seq[seq].append(z2)
xin_by_seq[seq].append(x)
xout_by_seq[seq].append(xout)
xoutv_by_seq[seq].append(xoutv)
z1reg_by_seq[seq] = z1reg
z1_by_seq[seq] = np.concatenate(z1_by_seq[seq], axis=0)
z2_by_seq[seq] = np.concatenate(z2_by_seq[seq], axis=0)
xin_by_seq[seq] = np.concatenate(xin_by_seq[seq], axis=0)
xout_by_seq[seq] = np.concatenate(xout_by_seq[seq], axis=0)
xoutv_by_seq[seq] = np.concatenate(xoutv_by_seq[seq], axis=0)
# z1reg_by_seq[seq] = np.concatenate(z1reg_by_seq[seq], axis=0)
mu2_by_seq[seq] = map_mu2_z2(model, z2_by_seq[seq])
d2 = mu2_by_seq[seq].shape[0]
z2 = np.asarray(mu2_by_seq[seq]).reshape([1, d2])
regpost_by_seq[seq] = reg_posteriors_z2(sess, model, z2)
# # save the mu2
# f = open("mu2_by_seq.txt","w")
# for seq in seqs:
# f.write( ' '.join (map(str,mu2_by_seq[seq])) )
# f.write('\n')
# f.close()
#
# save the mean mu2
if not os.path.exists("%s/mu2_by_seq.npy" % exp_dir):
mumu = np.zeros([mu2_by_seq[seqs[1]].size])
for seq in seqs:
mumu += mu2_by_seq[seq]
mumu /= len(seqs)
with open("%s/mu2_by_seq.npy" % exp_dir, "wb") as fnp:
np.save(fnp, mumu)
if args.write_gender:
names = ["reg1", "gender"]
for i, name in enumerate(names):
with open("%s/txt/%s.scp" % (exp_dir, name), "wb") as f:
for seq in seqs:
f.write(seq + " [ ")
for e in np.nditer(regpost_by_seq[seq][i]):
f.write("%10.3f " % e)
f.write("]\n")
if args.write_phon:
names = ["pho"]
for i, name in enumerate(names):
try:
os.makedirs("%s/txt/%s" % (exp_dir, name))
except OSError:
pass
for seq in seqs:
np.save("%s/txt/%s/%s" % (exp_dir, name, seq),
z1reg_by_seq[seq][i])
seq_names = ["%02d_%s" % (i, seq) for i, seq in enumerate(seqs)]
if args.viz_reconstruction:
# visualize reconstruction
logging.info("visualizing reconstruction")
plot_x([xin_by_seq[seq] for seq in seqs], seq_names,
"%s/img/xin.png" % exp_dir)
plot_x([xout_by_seq[seq] for seq in seqs], seq_names,
"%s/img/xout.png" % exp_dir)
plot_x([xoutv_by_seq[seq] for seq in seqs],
seq_names,
"%s/img/xout_logvar.png" % exp_dir,
clim=(None, None))
if args.viz_factorization:
# factorization: use the centered segment from each sequence
logging.info("visualizing factorization")
cen_z1 = np.array(
[z1_by_seq[seq][len(z1_by_seq[seq]) / 2] for seq in seqs])
cen_z2 = np.array(
[z2_by_seq[seq][len(z2_by_seq[seq]) / 2] for seq in seqs])
xfac = []
for z1 in cen_z1:
z1 = np.tile(z1, (len(cen_z2), 1))
xfac.append(x_mu_fn(sess, model, z1, cen_z2))
plot_x(xfac, seq_names, "%s/img/xfac.png" % exp_dir, sep=True)
if args.write_spec or args.translate:
logging.info("saving mel spectrograms to \"%s/spec/\"" % exp_dir)
# with open( "./datasets/cgn_np_fbank/train/mvn.pkl" ) as f:
# with open( "./datasets/timit_np_fbank/train/mvn.pkl" ) as f:
# with open( "./datasets/cgn_per_speaker/train/mvn.pkl" ) as f:
# with open( "./datasets/grabo_np_fbank/train/mvn.pkl" ) as f:
with open("./datasets/cgn_per_speaker_afgklno/train/mvn.pkl") as f:
mvn_params = cPickle.load(f)
nb_mel = mvn_params["mean"].size
if args.write_spec:
for src_seq, src_seq_name in zip(seqs, seq_names):
with open("%s/spec/xin_%s.npy" % (exp_dir, src_seq),
"wb") as fnp:
np.save(
fnp,
np.reshape(xin_by_seq[src_seq],
(-1, nb_mel)) * mvn_params["std"] +
mvn_params["mean"])
with open("%s/spec/xout_%s.npy" % (exp_dir, src_seq),
"wb") as fnp:
np.save(
fnp,
np.reshape(xout_by_seq[src_seq],
(-1, nb_mel)) * mvn_params["std"] +
mvn_params["mean"])
if args.write_neutral:
# sequence neutralisation
logging.info("visualizing neutral sequences...")
neu_by_seq = dict()
with open("%s/mu2_by_seq.npy" % exp_dir, "rb") as fnp:
mumu = np.load(fnp)
for src_seq, src_seq_name in zip(seqs, seq_names):
del_mu2 = mumu - mu2_by_seq[src_seq]
src_z1, src_z2 = z1_by_seq[src_seq], z2_by_seq[src_seq]
neu_by_seq[src_seq] = _seq_translate(sess, model, src_z1,
src_z2, del_mu2)
with open("%s/spec/neu_%s.npy" % (exp_dir, src_seq),
"wb") as fnp:
np.save(
fnp,
np.reshape(neu_by_seq[src_seq],
(-1, nb_mel)) * mvn_params["std"] +
mvn_params["mean"])
plot_x([neu_by_seq[seq] for seq in seqs], seq_names,
"%s/img/neutral.png" % exp_dir, False)
if args.translate:
# sequence translation
logging.info("visualizing sequence translation...")
xtra_by_seq = dict()
for src_seq, src_seq_name in zip(seqs, seq_names):
xtra_by_seq[src_seq] = dict()
src_z1, src_z2 = z1_by_seq[src_seq], z2_by_seq[src_seq]
for tar_seq in seqs:
del_mu2 = mu2_by_seq[tar_seq] - mu2_by_seq[src_seq]
xtra_by_seq[src_seq][tar_seq] = _seq_translate(
sess, model, src_z1, src_z2, del_mu2)
with open(
"%s/spec/src_%s_tar_%s.npy" %
(exp_dir, src_seq, tar_seq), "wb") as fnp:
np.save(
fnp,
np.reshape(xtra_by_seq[src_seq][tar_seq],
(-1, nb_mel)) * mvn_params["std"] +
mvn_params["mean"])
# plot_x([xtra_by_seq[src_seq][seq] for seq in seqs], seq_names,
# "%s/img/%s_tra.png" % (exp_dir, src_seq_name), True)
if False:
# random z1 z2
logging.info("visualizing sequence translation...")
xtra_by_seq = dict()
for src_seq, src_seq_name in zip(seqs, seq_names):
xtra_by_seq[src_seq] = dict()
src_z1, src_z2 = z1_by_seq[src_seq], z2_by_seq[src_seq]
np.random.rand(src_z1.shape[0], src_z1.shape[1]) * 2 - 1
np.random.rand(src_z2.shape[0], src_z2.shape[1]) * 2 - 1
del_mu2 = mu2_by_seq[src_seq] - mu2_by_seq[src_seq]
xtra_by_seq[src_seq][src_seq] = _seq_translate(
sess, model, src_z1, src_z2, del_mu2)
with open("./spec/src_%s.npy" % (src_seq), "wb") as fnp:
np.save(
fnp,
np.reshape(xtra_by_seq[src_seq][src_seq],
(-1, nb_mel)) * mvn_params["std"] +
mvn_params["mean"])
plot_x([xtra_by_seq[src_seq][seq] for seq in seqs], seq_names,
"./spec/%s_tra.png" % (src_seq_name), True)
# for tar_seq in seqs: # TODO: don't loop over all n^2 combinations
# del_mu2 = mu2_by_seq[tar_seq] - mu2_by_seq[src_seq]
# xtra_by_seq[src_seq][tar_seq] = _seq_translate(
# sess, model, src_z1, src_z2, del_mu2)
# with open("%s/spec/src_%s_tar_%s.npy" % (exp_dir, src_seq, tar_seq), "wb") as fnp:
# np.save(fnp, np.reshape(
# xtra_by_seq[src_seq][tar_seq], (-1, nb_mel)) * mvn_params["std"] + mvn_params["mean"])
# plot_x([xtra_by_seq[src_seq][seq] for seq in seqs], seq_names,
# "%s/img/%s_tra.png" % (exp_dir, src_seq_name), True)
if args.tsne:
# tsne z1 and z2
logging.info("t-SNE analysis on latent variables")
n = [len(z1_by_seq[seq]) for seq in seqs]
z1 = np.concatenate([z1_by_seq[seq] for seq in seqs], axis=0)
z2 = np.concatenate([z2_by_seq[seq] for seq in seqs], axis=0)
p = 30
logging.info("perplexity = %s" % p)
tsne = TSNE(n_components=2, verbose=0, perplexity=p, n_iter=1000)
z1_tsne = _unflatten(tsne.fit_transform(z1), n)
scatter_plot(z1_tsne, seq_names, "z1_tsne_%03d" % p,
"%s/img/z1_tsne_%03d.png" % (exp_dir, p))
z2_tsne = _unflatten(tsne.fit_transform(z2), n)
scatter_plot(z2_tsne, seq_names, "z2_tsne_%03d" % p,
"%s/img/z2_tsne_%03d.png" % (exp_dir, p))
logging.info("Finished.")
def generate_force_profile(self,
R,
V,
name=None,
progress_bar=False,
**kwargs):
"""
Map out the equilibrium force vs. position and velocity
Parameters
----------
R : array_like, shape(3, ...)
Position vector. First dimension of the array must be length 3, and
corresponds to :math:`x`, :math:`y`, and :math`z` components,
repsectively.
V : array_like, shape(3, ...)
Velocity vector. First dimension of the array must be length 3, and
corresponds to :math:`v_x`, :math:`v_y`, and :math`v_z` components,
repsectively.
name : str, optional
Name for the profile. Stored in profile dictionary in this object.
If None, uses the next integer, cast as a string, (i.e., '0') as
the name.
progress_bar : boolean, optional
Displays a progress bar as the proceeds. Default: False
kwargs :
Any additional keyword arguments to be passed to
rateeq.find_equilibrium_force()
Returns
-------
profile : pylcp.rateeq.force_profile
Resulting force profile.
"""
if not name:
name = '{0:d}'.format(len(self.profile))
self.profile[name] = force_profile(R, V, self.laserBeams,
self.hamiltonian)
it = np.nditer([R[0], R[1], R[2], V[0], V[1], V[2]],
flags=['refs_ok', 'multi_index'],
op_flags=[['readonly'], ['readonly'], ['readonly'],
['readonly'], ['readonly'], ['readonly']])
if progress_bar:
progress = progressBar()
for (x, y, z, vx, vy, vz) in it:
# Construct the rate equations:
r = np.array([x, y, z])
v = np.array([vx, vy, vz])
# Update position (use multi_index), populations, and forces.
self.set_initial_position_and_velocity(r, v)
try:
F, f, Neq, Rijl, f_mag = self.find_equilibrium_force(
return_details=True, **kwargs)
except:
raise ValueError(
"Unable to find solution at " +\
"r=({0:0.2f},{1:0.2f},{2:0.2f})".format(x, y, z) + " and " +
"v=({0:0.2f},{1:0.2f},{2:0.2f})".format(vx, vy, vz)
)
if progress_bar:
progress.update((it.iterindex + 1) / it.itersize)
self.profile[name].store_data(it.multi_index, Neq, F, f, f_mag,
Rijl)
def test_dense_to_sparse_indicies(shape, sparse, mask, axes, tiling=True):
for order in ['C', 'F']:
# create matrix
arr = np.arange(1, np.prod(shape) + 1).reshape(shape, order=order)
def __slicer(x, y):
slicer = [slice(None)] * arr.ndim
slicer[1:] = x, y
return tuple(slicer)
def apply_sparse(x, y):
arr[__slicer(*np.where(~sparse(x, y)))] = 0
# sparsify
np.fromfunction(apply_sparse, arr.shape[1:], dtype=kint_type)
matrix = csr_matrix if order == 'C' else csc_matrix
matrix = matrix(arr[0])
# next, create a sparse copy of the matrix
sparse_arr = np.zeros((arr.shape[0], matrix.nnz), dtype=arr.dtype)
it = np.nditer(np.empty(shape[1:]), flags=['multi_index'], order=order)
i = 0
while not it.finished:
if not sparse(*it.multi_index):
it.iternext()
continue
sparse_arr[:, i] = arr[__slicer(*it.multi_index)]
it.iternext()
i += 1
# get the sparse indicies
row, col = (matrix.indptr, matrix.indices) if order == 'C' \
else (matrix.indices, matrix.indptr)
sparse_axes, sparse_inds = dense_to_sparse_indicies(mask,
axes,
col,
row,
order,
tiling=tiling)
sparse_inds = sparse_inds[-1]
# and check
it = np.nditer(np.empty(shape[1:]), flags=['multi_index'], order=order)
i = 0
while not it.finished:
if not sparse(*it.multi_index):
it.iternext()
continue
if not tiling:
if not (it.multi_index[0] in mask[-2] and it.multi_index[1]
== mask[-1][np.where(it.multi_index[0] == mask[-2])]):
it.iternext()
continue
if not (it.multi_index[0] in mask[-2]
and it.multi_index[1] in mask[-1]):
it.iternext()
continue
# check that the sparse indicies match what we expect
assert np.all(sparse_arr[:, sparse_inds[i]] == arr[__slicer(
*it.multi_index)])
it.iternext()
i += 1
data = pd.read_csv(directory + '/' + filename)
vars = ['cpuutil']
anomaly = ['label']
data_vars = np.array(data[vars])
data_anomaly = np.array(data[anomaly])
data_vars = np.transpose(data_vars)
anomaly_data = anomaly_data + np.transpose(data_anomaly)
anomaly_data_each[i] = np.transpose(data_anomaly)
all_data[i] = data_vars
with open("nodename.pkl","wb") as f:
pickle.dump(file, f)
anomaly_data = np.array([1 if j == 2 else j for j in np.nditer(anomaly_data)])
all_data[-1] = anomaly_data
file.extend(['label_all'])
df = pd.DataFrame(all_data.transpose(), columns = file)
df.to_csv('science_data_all.csv')
all_data = np.delete(all_data,-1,0)
np.save('science_data', all_data)
np.save('science_anomaly_all',anomaly_data)
np.save('science_anomally_each',anomaly_data_each)
print(all_data.shape)
print(anomaly_data.shape)
print(anomaly_data_each.shape)
return int(m)
# 0 < tolerance <= 0.3
# transition_width: 1.0 --> fs/2
def param(tolerance, transition_width):
tol_db = -20.0 * np.log10(tolerance)
window_size = size(tol_db, transition_width)
window_beta = beta(tol_db)
return window_size, window_beta
tol_list = np.arange(1e-6, 0.2, 1e-4)
tol_db_list = -20.0 * np.log10(tol_list)
beta_list = np.empty_like(tol_db_list)
for i, tol in enumerate(np.nditer(tol_db_list)):
beta_list[i] = beta(tol)
beta_list_ref = np.empty_like(tol_db_list)
for i, tol in enumerate(np.nditer(tol_db_list)):
beta_list_ref[i] = kaiser_beta(tol)
print('max abs beta error:', np.abs(beta_list - beta_list_ref).max())
hdf5util.write_ndarray(data=tol_db_list, file_path='kaiser_tol_db.h5', dataset_name='v')
hdf5util.write_ndarray(data=beta_list_ref, file_path='kaiser_beta.h5', dataset_name='v')
plt.figure()
plt.plot(tol_db_list, beta_list, label='local')
plt.plot(tol_db_list, beta_list_ref, label='scipy')
plt.title('Beta list')
plt.legend(loc='upper right', labelspacing=0.2)
def Array(self, f, dim, **args):
x = np.ndarray(dim, dtype=object)
for i in np.nditer(x, flags=["refs_ok"], op_flags=['readwrite']):
i[...] = f(**args)
return x
]
trxy = np.tile(trx, (y_plates, 1))
print "treatment matrix:\n", trxy
# new matrix for cell types
#trx=conditions*2 #not this
ct = cell_types
print ct
cty = np.tile(ct, (x_plates, 1))
ctxy = np.transpose(cty)
print "cell type matrix:\n", ctxy
#os.remove("image.svg")
y = 0
for n in np.nditer(ctxy, order='C'):
y = (y + 1)
print(n)
print(y)
for tr, ct in itertools.izip(trxy, ctxy):
print(tr, ct)
fi = open("output.svg", "w")
fi.write("<svg>\n")
for n in xx:
row, col = np.where(xy == n)
name = xy[row, col]
name = str(name)[1:-1]
print name
file_list = []
r = img_shape[0]
c = img_shape[1]
for i in range(len(class_name)):
file_list.append(os.listdir(path + "\\" + class_name[i]))
file_pointer = 0
while file_pointer != file_no:
finnal_mask = np.zeros(img_shape, dtype=np.uint8)
calc_mask = np.zeros((r, c, len(class_name)), dtype=np.uint8)
for i in range(len(class_name)):
img_tmp = cv_imread(path + "\\" + class_name[i] + "\\" +
file_list[i][file_pointer])
if len(img_tmp.shape) >= 3: img_tmp = img_tmp[:, :, 0]
calc_mask[:, :, i] = img_tmp.copy()
count = 0
for x in np.nditer(calc_mask[:, :, 0]):
chanel_val = np.sum(calc_mask[count // c][count % c])
final_pixel_val = -1
if chanel_val >= 2:
suc = 0
else:
suc = 1
if chanel_val == 0:
final_pixel_val = 0
else:
final_pixel_val = np.where(
calc_mask[count // c][count % c])[0][0] + 1
if not suc:
ro = count // c
co = count % c
inner_count = 0
T_wall = td['T_wall'] # [K] Wall temperature
p_inlet = td['p_inlet'] # [Pa] Inlet pressure
m_dot = td['m_dot'] # [kg/s] Mass flow (through all channels if multiple)
channel_amount = td['channel_amount'] # [-] Amount of channels
h_channel = td['h_channel'] # [m] Channel height/depth
fp = FluidProperties(
td['propellant']) # Object from which fluid properties can be accessed
# Calculate mass flow for one single channel
m_dot_channel = m_dot / channel_amount # [kg/s] Mass flow through one single channel
## Chamber temperature is unknown, so instead a range is chosen from T_sat + 1 to T_wall-1
T_sat = fp.get_saturation_temperature(p=p_inlet) # [K]
T_chamber = np.linspace(start=T_sat + 1, stop=T_wall - 1, num=250) # [K]
it = np.nditer(T_chamber, flags=['c_index'])
L_channel_1 = np.zeros_like(T_chamber) # [m]
L_channel_2 = np.zeros_like(T_chamber) # [m]
L_channel_3 = np.zeros_like(T_chamber) # [m]
for T in it:
## First set of Nusselt relations
res_1 = zD.two_phase_single_channel( T_wall=T_wall, w_channel=w_channel, Nu_func_gas=Nu_func_gas_1, Nu_func_liquid=Nu_func_liquid_1,\
T_inlet=T_inlet, T_chamber=float(T), p_ref=p_inlet, m_dot=m_dot_channel,\
h_channel=h_channel, fp=fp,print_info=False)
# Store results
L_channel_1[it.index] = res_1['L_channel']
## Second set of Nusselt relations
res_2 = zD.two_phase_single_channel( T_wall=T_wall, w_channel=w_channel, Nu_func_gas=Nu_func_gas_2, Nu_func_liquid=Nu_func_liquid_2,\
def cond_mutual_information(x, y, z, algorithm='EQQ', bins=8, log2=True):
"""
Computes conditional mutual information between two time series x and y
conditioned on a third z (which can be multi-dimensional) as
I(x; y | z) = sum( p(x,y,z) * log( p(z)*p(x,y,z) / p(x,z)*p(y,z) ),
where p(z), p(x,z), p(y,z) and p(x,y,z) are probability distributions.
The probability distributions could be estimated using these algorithms:
equiquantal binning - algorithm keyword 'EQQ' or 'EQQ2'
EQQ - equiquantality is forced (even if many samples have the same value
at and near the bin edge), can happen that samples with same value fall
into different bin
EQQ2 - if more than one sample has the same value at the bin edge, the edge is shifted,
so that all samples with the same value fall into the same bin, can happen that bins
do not necessarily contain the same amount of samples
equidistant binning - algorithm keyword 'EQD'
Gaussian correlation matrix - algorithm keyword 'GCM' (for phase - amplitude dependence)
(preparing more...)
If log2 is True (default), the units of cond. mutual information are bits, if False
the mutual information is estimated using natural logarithm and therefore
units are nats.
"""
log_f = np.log2 if log2 else np.log
# binning algorithms
if 'EQ' in algorithm:
# for multi-dimensional condition -- create array from list as (dim x length of ts)
x = np.atleast_2d(x)
y = np.atleast_2d(y)
z = np.atleast_2d(z)
if algorithm == 'EQQ':
# create EQQ bins -- condition [possibly multidimensional]
z_bins = [] # arrays of bins for all conditions
for cond in range(z.shape[0]):
z_sorted = np.sort(z[cond, :])
z_bin = [z_sorted.min()]
[
z_bin.append(z_sorted[i * z_sorted.shape[0] / bins])
for i in range(1, bins)
]
z_bin.append(z_sorted.max())
z_bins.append(np.array(z_bin))
# create EQQ bins -- variables
x_bins = []
for cond in range(x.shape[0]):
x_sorted = np.sort(x[cond, :])
x_bin = [x_sorted.min()]
[
x_bin.append(x_sorted[i * x_sorted.shape[0] / bins])
for i in range(1, bins)
]
x_bin.append(x_sorted.max())
x_bins.append(np.array(x_bin))
y_bins = []
for cond in range(y.shape[0]):
y_sorted = np.sort(y[cond, :])
y_bin = [y_sorted.min()]
[
y_bin.append(y_sorted[i * y_sorted.shape[0] / bins])
for i in range(1, bins)
]
y_bin.append(y_sorted.max())
y_bins.append(np.array(y_bin))
# create multidim bins for histogram
xyz_bins = x_bins + y_bins + z_bins
xz_bins = x_bins + z_bins
yz_bins = y_bins + z_bins
elif algorithm == 'EQQ2':
# create EQQ bins -- condition [possibly multidimensional]
z_bins = [] # arrays of bins for all conditions
for cond in range(z.shape[0]):
z_sorted = np.sort(z[cond, :])
z_bin = [z_sorted.min()]
one_bin_count = z_sorted.shape[0] / bins
for i in range(1, bins):
idx = i * one_bin_count
if np.all(np.diff(z_sorted[idx - 1:idx + 2]) != 0):
z_bin.append(z_sorted[idx])
elif np.any(np.diff(z_sorted[idx - 1:idx + 2]) == 0):
where = np.where(
np.diff(z_sorted[idx - 1:idx + 2]) != 0)[0]
expand_idx = 1
while where.size == 0:
where = np.where(
np.diff(z_sorted[idx - expand_idx:idx + 1 +
expand_idx]) != 0)[0]
expand_idx += 1
if where[0] == 0:
z_bin.append(z_sorted[idx - expand_idx])
else:
z_bin.append(z_sorted[idx + expand_idx])
z_bin.append(z_sorted.max())
z_bin = np.array(z_bin)
z_bins.append(np.array(z_bin))
# create EQQ bins -- variables
x_bins = [] # arrays of bins for all conditions
for cond in range(x.shape[0]):
x_sorted = np.sort(x[cond, :])
x_bin = [x_sorted.min()]
one_bin_count = x_sorted.shape[0] / bins
for i in range(1, bins):
idx = i * one_bin_count
if np.all(np.diff(x_sorted[idx - 1:idx + 2]) != 0):
x_bin.append(x_sorted[idx])
elif np.any(np.diff(x_sorted[idx - 1:idx + 2]) == 0):
where = np.where(
np.diff(x_sorted[idx - 1:idx + 2]) != 0)[0]
expand_idx = 1
while where.size == 0:
where = np.where(
np.diff(x_sorted[idx - expand_idx:idx + 1 +
expand_idx]) != 0)[0]
expand_idx += 1
if where[0] == 0:
x_bin.append(x_sorted[idx - expand_idx])
else:
x_bin.append(x_sorted[idx + expand_idx])
x_bin.append(x_sorted.max())
x_bin = np.array(x_bin)
x_bins.append(np.array(x_bin))
y_bins = [] # arrays of bins for all conditions
for cond in range(y.shape[0]):
y_sorted = np.sort(y[cond, :])
y_bin = [y_sorted.min()]
one_bin_count = y_sorted.shape[0] / bins
for i in range(1, bins):
idx = i * one_bin_count
if np.all(np.diff(y_sorted[idx - 1:idx + 2]) != 0):
y_bin.append(y_sorted[idx])
elif np.any(np.diff(y_sorted[idx - 1:idx + 2]) == 0):
where = np.where(
np.diff(y_sorted[idx - 1:idx + 2]) != 0)[0]
expand_idx = 1
while where.size == 0:
where = np.where(
np.diff(y_sorted[idx - expand_idx:idx + 1 +
expand_idx]) != 0)[0]
expand_idx += 1
if where[0] == 0:
y_bin.append(y_sorted[idx - expand_idx])
else:
y_bin.append(y_sorted[idx + expand_idx])
y_bin.append(y_sorted.max())
y_bin = np.array(y_bin)
y_bins.append(np.array(y_bin))
# create multidim bins for histogram
xyz_bins = x_bins + y_bins + z_bins
xz_bins = x_bins + z_bins
yz_bins = y_bins + z_bins
elif algorithm == 'EQD':
# bins are just integer
xyz_bins = bins
xz_bins = bins
yz_bins = bins
z_bins = bins
# histo
count_z = np.histogramdd(z.T, bins=z_bins)[0]
xyz = np.vstack((x, y, z))
count_xyz = np.histogramdd(xyz.T, bins=xyz_bins)[0]
xz = np.vstack((x, z))
count_xz = np.histogramdd(xz.T, bins=xz_bins)[0]
yz = np.vstack((y, z))
count_yz = np.histogramdd(yz.T, bins=yz_bins)[0]
# normalise
count_z /= np.float(np.sum(count_z))
count_xyz /= np.float(np.sum(count_xyz))
count_xz /= np.float(np.sum(count_xz))
count_yz /= np.float(np.sum(count_yz))
# sum
cmi = 0
iterator = np.nditer(count_xyz, flags=['multi_index'])
while not iterator.finished:
idx = iterator.multi_index
xz_idx = tuple([
item for sublist in [idx[:len(x)], idx[-len(z):]]
for item in sublist
]) # creates index for xz histo
yz_idx = idx[-len(z) - len(y):]
z_idx = idx[-len(z):]
if count_xyz[idx] == 0 or count_z[z_idx] == 0 or count_xz[
xz_idx] == 0 or count_yz[yz_idx] == 0:
iterator.iternext()
continue
else:
cmi += count_xyz[idx] * log_f(
count_z[z_idx] * count_xyz[idx] /
(count_xz[xz_idx] * count_yz[yz_idx]))
iterator.iternext()
elif algorithm == 'GCM':
if len(z) <= 1:
raise Exception(
"Gaussian correlation matrix method should be used with multidimensional condition."
)
# center time series - zero mean, unit variance
x = _center_ts(x)
y = _center_ts(y)
for cond_ts in z:
cond_ts = _center_ts(cond_ts)
# get CMI
Hall = _get_corr_entropy([x, y] + list(z), log2=log2)
Hxz = _get_corr_entropy([x] + list(z), log2=log2)
Hyz = _get_corr_entropy([y] + list(z), log2=log2)
Hz = _get_corr_entropy(z, log2=log2)
cmi = Hall - Hxz - Hyz + Hz
return cmi
def write_array(fp,
array,
version=None,
allow_pickle=True,
pickle_kwargs=None):
"""
Write an array to an NPY file, including a header.
If the array is neither C-contiguous nor Fortran-contiguous AND the
file_like object is not a real file object, this function will have to
copy data in memory.
Parameters
----------
fp : file_like object
An open, writable file object, or similar object with a
``.write()`` method.
array : ndarray
The array to write to disk.
version : (int, int) or None, optional
The version number of the format. None means use the oldest
supported version that is able to store the data. Default: None
allow_pickle : bool, optional
Whether to allow writing pickled data. Default: True
pickle_kwargs : dict, optional
Additional keyword arguments to pass to pickle.dump, excluding
'protocol'. These are only useful when pickling objects in object
arrays on Python 3 to Python 2 compatible format.
Raises
------
ValueError
If the array cannot be persisted. This includes the case of
allow_pickle=False and array being an object array.
Various other errors
If the array contains Python objects as part of its dtype, the
process of pickling them may raise various errors if the objects
are not picklable.
"""
_check_version(version)
used_ver = _write_array_header(fp, header_data_from_array_1_0(array),
version)
# this warning can be removed when 1.9 has aged enough
if version != (2, 0) and used_ver == (2, 0):
warnings.warn(
"Stored array in format 2.0. It can only be"
"read by NumPy >= 1.9",
UserWarning,
stacklevel=2)
if array.itemsize == 0:
buffersize = 0
else:
# Set buffer size to 16 MiB to hide the Python loop overhead.
buffersize = max(16 * 1024**2 // array.itemsize, 1)
if array.dtype.hasobject:
# We contain Python objects so we cannot write out the data
# directly. Instead, we will pickle it out with version 2 of the
# pickle protocol.
if not allow_pickle:
raise ValueError("Object arrays cannot be saved when "
"allow_pickle=False")
if pickle_kwargs is None:
pickle_kwargs = {}
pickle.dump(array, fp, protocol=2, **pickle_kwargs)
elif array.flags.f_contiguous and not array.flags.c_contiguous:
if isfileobj(fp):
array.T.tofile(fp)
else:
for chunk in numpy.nditer(
array,
flags=['external_loop', 'buffered', 'zerosize_ok'],
buffersize=buffersize,
order='F'):
fp.write(chunk.tobytes('C'))
else:
if isfileobj(fp):
array.tofile(fp)
else:
for chunk in numpy.nditer(
array,
flags=['external_loop', 'buffered', 'zerosize_ok'],
buffersize=buffersize,
order='C'):
fp.write(chunk.tobytes('C'))
# Add the following two NumPy arrays and Modify a result array by calculating the square root of each element
import numpy as np
arrOne = np.array([[5, 6, 9], [21, 18, 27]])
arrTwo = np.array([[15, 33, 24], [4, 7, 1]])
result = arrOne + arrTwo
print(result)
for numbers in np.nditer(result, op_flags=['readwrite']):
numbers[...] = numbers*numbers
print("result array after calculating the sqrt of all elements")
# [[ 400 1521 1089]
# [ 625 625 784]]
print(result)
def propagation(file_conll, sigma, deep_or_surf):
surf, deep = conll_parser(file_conll)
vec_dico = dico("vecs100-linear-frwiki")
pos_dico = pos_list(file_conll)
random = np.random.rand(1, 100)
random = random[0]
if deep_or_surf == "deep":
surf_copy = surf
surf = deep
surf_exemples = []
class_list = []
for every_v in surf:
class_list.append(every_v[0][1])
surf_exemples.append(single_vec(every_v, pos_dico, vec_dico, random))
uniq = list(set(class_list))
train_array = np.linspace(0,
len(surf_exemples),
num=30,
dtype=int,
endpoint=False)
y_matrix = np.zeros((len(surf_exemples), len(uniq))) # array of indexes
for id in np.nditer(train_array):
class_id = surf[id][0][1]
indice = uniq.index(class_id)
y_matrix[id] = np.zeros(len(uniq))
y_matrix[id, indice] = 1
t_matrix = np.zeros((len(surf_exemples), len(surf_exemples)))
surf_key = np.asarray(surf_exemples, dtype=np.float32)
j_sim = []
x = float(sigma)
#essentiel step to construct the T matrix
for j in range(len(surf_exemples)):
kj_sim = 0
for k in range(len(surf_exemples)):
kj_sim += np.exp(-np.linalg.norm(surf_key[k] - surf_key[j])**2 /
x**2)
j_sim.append(kj_sim)
j_sim_array = np.asarray(j_sim)
for i in range(len(surf_exemples)):
summ = 0
for j in range(len(surf_exemples)):
ij_sim = np.exp(-np.linalg.norm(surf_key[i] - surf_key[j])**2 /
x**2)
if i != j:
summ += ij_sim
t_matrix[i, j] = ij_sim / j_sim_array[j]
#print(summ) # can be used to check whether the sum of no reflexive similarities is equal to zero when sigma is too small
new_y_matrix = np.copy(y_matrix)
last_matrix = np.copy(y_matrix)
while True:
diff_abs = 0
#propagation
for index in range(len(last_matrix)):
if index not in train_array:
for c in range(len(uniq)):
rate = 0
for j in range(len(t_matrix)):
rate += t_matrix[index, j] * last_matrix[j, c]
new_y_matrix[index, c] = rate
#normalise Y
for index in range(len(new_y_matrix)):
sum_row = sum(new_y_matrix[index])
for c in range(len(uniq)):
new_y_matrix[index,
c] = round(new_y_matrix[index, c] / sum_row, 4)
diff_abs += abs(new_y_matrix[index, c] - last_matrix[index, c])
#condition d'arrêt
if diff_abs < 0.1:
break
last_matrix = np.copy(new_y_matrix)
l = 0
sentence = 0
base = 0
# calculate precision
for i in range(len(new_y_matrix)):
sentence += 1
if max(new_y_matrix[i]) != 0 and i not in train_array:
ind = new_y_matrix[i].tolist().index(max(new_y_matrix[i]))
c = uniq[ind]
else:
c = "-1"
if c == class_list[i]:
l += 1
if i not in train_array and class_list[
i] == "2": # you can use this to calculate MFS
base += 1
accuracy = "{:.2%}".format(l / (len(new_y_matrix) - 30))
base_line = "{:.2%}".format(base /
(len(new_y_matrix) - 30)) # calculate MFS
print(accuracy)
def object_einsum(string, *arrays):
"""Simplified object einsum, not as much error checking
does not support "..." or list input and will see "...", etc. as three times
an axes identifier, tries normal einsum first!
NOTE: This is untested, and not fast, but object type is
never really fast anyway...
"""
try:
return np.einsum(string, *arrays)
except TypeError:
pass
s = string.split('->')
in_op = s[0].split(',')
out_op = None if len(s) == 1 else s[1].replace(' ', '')
in_op = [axes.replace(' ', '') for axes in in_op]
all_axes = set()
for axes in in_op:
all_axes.update(axes)
if out_op is None:
out_op = sorted(all_axes)
else:
all_axes.update(out_op)
perm_dict = {_[1]: _[0] for _ in enumerate(all_axes)}
dims = len(perm_dict)
op_axes = []
for axes in (in_op + list((out_op, ))):
op = [-1] * dims
for i, ax in enumerate(axes):
op[perm_dict[ax]] = i
op_axes.append(op)
op_flags = [('readonly', )] * len(in_op) + [('readwrite', 'allocate')]
dtypes = [np.object_] * (len(in_op) + 1) # cast all to object
nditer = np.nditer(arrays + (None, ),
op_axes=op_axes,
flags=[
'buffered', 'delay_bufalloc', 'reduce_ok',
'grow_inner', 'refs_ok'
],
op_dtypes=dtypes,
op_flags=op_flags)
nditer.operands[-1][...] = 0
nditer.reset()
for vals in nditer:
out = vals[-1]
prod = copy.deepcopy(vals[0])
for value in vals[1:-1]:
prod *= value
out += prod
return nditer.operands[-1]
def _interp_rbf(self):
"""Interpolate using radial basis function"""
# re-initialise vols
self._init_vols()
self._define_index_xy()
if self._n_threads is 0:
cores = max(1, mp.cpu_count() - 1)
else:
cores = self._n_threads
print('Starting pool [%d processes]' % cores)
pool = mp.Pool(cores)
t1 = time.time()
for ww in range(1, self._nstates + 1):
print('Interpolating resp window: %d' % ww)
tt = time.time()
slice_idx = np.where(self._states == ww)
self._zs = (self._slice_locations[slice_idx, ]
).flatten() # z-sample points
sub_arrays = pool.starmap_async(
self._rbf_interp_line,
[(self._get_training_y(
xx, yy, slice_idx, kernel_dim=self._kernel_dims),
self._get_training_x(
xx, yy, slice_idx, kernel_dim=self._kernel_dims),
self._get_zq(xx, yy, kernel_dim=self._kernel_dims))
for xx in np.nditer(self._xi)
for yy in np.nditer(self._yi)]).get()
print('%s duration: %.1fs' % ('_interp_rbf', (time.time() - tt)))
# insert interpolated data into pre-allocated volume
print('\ncollecting data')
index = 0
for xx in np.nditer(self._xi):
for yy in np.nditer(self._yi):
self._img_4d[xx, yy, :,
ww - 1] = sub_arrays[index].flatten()
index = index + 1
# save single resp state volumes
self._image_resp_3d.set_data(self._img_4d[:, :, :, ww - 1])
self._write_resampled_data(
self._image_resp_3d,
self._write_paths.path_interpolated_3d(ww))
print('---')
pool.close()
# reset python environ to normalise threading
os.environ.clear()
# write full 4D interp volume
self._image_4d.set_data(self._img_4d)
self._write_resampled_data(self._image_4d,
self._write_paths.path_interpolated_4d())
# print function duration info
print('%s duration: %.1fs' % ('_interp_rbf', (time.time() - t1)))
FREQ = 800000
DMA = 5
INVERT = False # Invert required when using inverting buffer
BRIGHTNESS = 255
strip = Adafruit_NeoPixel(LEDCOUNT, GPIOPIN, FREQ, DMA, INVERT, BRIGHTNESS)
# Intialize the library (must be called once before other functions).
strip.begin()
# get the last values to set the neopixels
print("flashing LED" + str(args.i))
for _ in range(LEDCOUNT):
for x in np.nditer(LED):
x = x + 0 #! for some reason i have to add zero to this to get it to work
strip.setPixelColor(x, Color(0, 0, 255))
strip.show()
time.sleep(.8)
for x in np.nditer(LED):
x = x + 0
strip.setPixelColor(x, Color(0, 0, 0))
strip.show()
time.sleep(.05)
#. leave the lights on
for x in np.nditer(LED):
x = x + 0 #! for some reason i have to add zero to this to get it to work
strip.setPixelColor(x, Color(0, 0, 255))
strip.show()
nb = load_interpolation_data('nb')
q = np.random.rand(10000, 3) * 10 - 5
senb = BrEu(nb, nest=True, parallel=True, max_volume=0.0001, max_branchings=5)
tictoc.tic()
while not (tictoc.elapsed() > 60 or tictoc.relative_uncertainty() < 0.05):
senb.w_q(q, interpolate=False)
tictoc.toc()
eu_time_point = tictoc.average() / q.shape[0] * 10**6
eu_delt_point = tictoc.uncertainty() / q.shape[0] * 10**6
nthreads = np.arange(1, os.cpu_count() // 2 + 1, dtype='int')
it = np.nditer([nthreads, None, None], op_dtypes=('int', 'double', 'double'))
for n, t, u in it:
tictoc.tic()
while not (tictoc.elapsed() > 20 or tictoc.relative_uncertainty() < 0.05):
senb.w_q(q, threads=n)
tictoc.toc()
print('{} threads: {:6.3f}/{} -> {:5.3f} +/- {:5.3f}'.format(
n, tictoc.elapsed(), tictoc.times_called, tictoc.average(),
tictoc.uncertainty()))
t[...] = tictoc.average()
u[...] = tictoc.uncertainty()
time_point = it.operands[1] / q.shape[0] * 10**6
uncertainty_point = it.operands[2] / q.shape[0] * 10**6
pp.figure()
pp.errorbar([1],
def matrix_from_ndarray(m, a):
m.values.extend(map(float, np.nditer(a, order='C')))
m.shape.extend(a.shape)
return m
def main(radar, snd_input, hca_hail_idx, dzdr, ref_name, zdr_name, rhv_name,
phi_name, snr_name, cbb_name, hca_name):
"""
Wrapper function for HSDA processing
Parameters:
===========
radar: struct
Py-ART radar object.
snd_input: string
sounding full filename (inc path)
hca_hail_idx: list
index of hail related fields in classification to apply HSDA
dzdr:
offset for differential reflectivity
####_name: string
field name from radar object
Returns:
========
hsda: ndarray
hsda classe array (1 = small < 25, 2 = large 25-50, 3 = giant > 50
"""
#build sounding data
snd_data = netCDF4.Dataset(snd_input)
snd_temp = snd_data.variables["temp"][:]
snd_geop = snd_data.variables["height"][:]
snd_rh = snd_data.variables["rh"][:]
snd_data.close()
#calc wbt
snd_wbt = common.wbt(snd_temp, snd_rh)
#run interpolation
wbt_minus25C = common.sounding_interp(snd_wbt, snd_geop, -25) / 1000
wbt_0C = common.sounding_interp(snd_wbt, snd_geop, 0) / 1000
#building consts
const = {
'wbt_minus25C': wbt_minus25C,
'wbt_0C': wbt_0C,
'dzdr': dzdr,
'hca_hail_idx': hca_hail_idx
}
#load data
zh_cf = radar.fields[ref_name]['data']
zdr_cf = radar.fields[zdr_name]['data']
rhv_cf = radar.fields[rhv_name]['data']
phi_cf = radar.fields[phi_name]['data']
snr_cf = radar.fields[snr_name]['data']
cbb_cf = radar.fields[cbb_name]['data']
hca = radar.fields[hca_name]['data']
#smooth radar data
zh_cf_smooth = common.smooth_ppi_rays(zh_cf, 5)
zdr_cf_smooth = common.smooth_ppi_rays(zdr_cf, 5)
rhv_cf_smooth = common.smooth_ppi_rays(rhv_cf, 5)
#build membership functions
w, mf = hsda_mf.build_mf()
#generate quality vector
q = hsda_q(zh_cf_smooth,
phi_cf,
rhv_cf_smooth,
phi_cf,
cbb_cf,
cbb_threshold=0.5)
#calc pixel alt
rg, azg = np.meshgrid(radar.range['data'], radar.azimuth['data'])
rg, eleg = np.meshgrid(radar.range['data'], radar.elevation['data'])
_, _, alt = pyart.core.antenna_to_cartesian(rg / 1000.0, azg, eleg)
#find all pixels in hca which match the hail classes
#for each pixel, apply transform
hail_mask = np.isin(hca, const['hca_hail_idx'])
hail_idx = np.where(hail_mask)
#loop through every pixel
hsda = np.zeros(hca.shape)
try:
for i in np.nditer(hail_idx):
tmp_alt = alt[i]
tmp_zh = zh_cf_smooth[i]
tmp_zdr = zdr_cf_smooth[i]
tmp_rhv = rhv_cf_smooth[i]
tmp_q_zh = q['zh'][i]
tmp_q_zdr = q['zdr'][i]
tmp_q_rhv = q['rhv'][i]
tmp_q = {'zh': tmp_q_zh, 'zdr': tmp_q_zdr, 'rhv': tmp_q_rhv}
if np.ma.is_masked(tmp_zh) or np.ma.is_masked(
tmp_zdr) or np.ma.is_masked(tmp_rhv):
continue
pixel_hsda = h_sz(tmp_alt, tmp_zh, tmp_zdr, tmp_rhv, mf, tmp_q, w,
const)
hsda[i] = pixel_hsda
except:
print('Error processing HSDA')
pass
#generate meta
the_comments = "1: Small Hail (< 25 mm); 2: Large Hail (25 - 50 mm); 3: Giant Hail (> 50 mm)"
hsda_meta = {
'data': hsda,
'units': ' ',
'long_name': 'Hail Size Discrimination Algorithm',
'standard_name': 'HSDA',
'comments': the_comments
}
#return radar object
return hsda_meta
def vector_from_ndarray(m, a):
m.values.extend(map(float, np.nditer(a, order='C')))
return m
print(idx)
print(np.max(ind),np.min(ind))
data = np.genfromtxt('./data/data.csv',delimiter=',',dtype='float32',usecols=[i*5+1 for i in range(483)])
act = data.transpose()
act = act[:,-future:]
buy_threshold = np.arange(0,0.056,0.00001)
sell_threshold = np.arange(0,-0.056,-0.00001)
in_hand_stocks = {}
rate = np.empty(shape=(5600,5600))
for X,x in enumerate(np.nditer(buy_threshold)):
for Y,y in enumerate(np.nditer(sell_threshold)):
pay = 0
back = 0
for i in range(future):
sell_stocks = []
for k in in_hand_stocks:
if ind[k,i] < y or i == future - 1:
# print('Sell stock %d in day %d at %f with buy price %f, profit_rate=%f' % (k, i, act[k,i], in_hand_stocks[k], (act[k,i] - in_hand_stocks[k])/in_hand_stocks[k]))
back += act[k,i]
sell_stocks.append(k)
for j in sell_stocks:
in_hand_stocks.pop(j)
for j in range(n):
if ind[j,i] > x and j not in in_hand_stocks and (i != future - 1):
# print('Buy stock %d in day %d at %f' % (j, i, act[j, i],))
def create_bins_array(image):
bins_array = [0] * 32
for x in np.nditer(image):
bins_array[int(math.floor(x / 8))] += 1
return bins_array
def half_life(self):
self.new_x = np.array([0])
self.hl_eval = np.array([0])
self.hl_array = np.array([0])
self.hl_coord = np.array([0])
self.bestfit = self.exp_out.best_fit
self.idx = np.argmin(np.abs(self.bestfit - 0.5))
if self.idx == 5 and self.bestfit[self.idx - 1] < 0.5:
self.bestfit = self.exp_out.best_fit[:-1]
self.idx = np.argmin(np.abs(self.bestfit - 0.5))
if self.bestfit[self.idx] == 0.5:
self.half_life_y = self.bestfit[self.idx]
self.half_life_x = self.idx
elif 0.5 > self.bestfit[self.idx] and self.bestfit[self.idx - 1] > 0.5:
self.max = self.x[self.idx]
self.min = self.x[self.idx - 1]
elif 0.5 < self.bestfit[self.idx] and self.bestfit[
self.idx] == self.bestfit[5]:
self.min = 0
self.max = 0
elif 0.5 < self.bestfit[self.idx] and self.bestfit[self.idx + 1] < 0.5:
self.min = self.x[self.idx]
self.max = self.x[self.idx + 1]
elif 0.5 < self.bestfit[self.idx] and self.bestfit[
self.idx + 1] > 0.5 and self.bestfit[self.idx + 2] < 0.5:
self.min = self.x[self.idx + 1]
self.max = self.x[self.idx + 2]
elif 0.5 > self.bestfit[self.idx] and self.bestfit[
self.idx + 1] < 0.5 and self.bestfit[self.idx - 2] > 0.5:
self.min = self.x[self.idx - 2]
self.max = self.x[self.idx]
self.ranging = np.arange(self.min, self.max, 0.001)
if self.max > 0:
# if self.min > 0 and self.max > 0:
for j in np.nditer(self.ranging):
self.new_x = np.array([j])
self.hl_eval = self.exp_out.eval(self.exp_out.params,
x=self.new_x)
if self.hl_eval >= 0.50 and self.hl_eval <= 0.51:
self.hl_array = np.append(self.hl_array, self.hl_eval)
self.hl_coord = np.append(self.hl_coord, self.new_x)
self.half_life_id = np.argmin(np.abs(self.hl_array - 0.5))
self.half_life_y = self.hl_array[self.half_life_id]
self.half_life_x = self.hl_coord[self.half_life_id]
self.bestfit = self.exp_out.best_fit
else:
self.half_life_y = 0
self.half_life_x = 0
self.bestfit = self.exp_out.best_fit
return self.half_life_y, self.half_life_x
def process_data(data_dir, data_identifier, winSize=30):
RUL_01 = np.loadtxt(data_dir + 'RUL_' + data_identifier + '.txt')
# read training data - It is the aircraft engine run-to-failure data.
train_01_raw = pd.read_csv(data_dir + '/train_' + data_identifier + '.txt',
sep=" ",
header=None)
train_01_raw.drop(train_01_raw.columns[[26, 27]], axis=1, inplace=True)
train_01_raw.columns = [
'id', 'cycle', 'setting1', 'setting2', 'setting3', 's1', 's2', 's3',
's4', 's5', 's6', 's7', 's8', 's9', 's10', 's11', 's12', 's13', 's14',
's15', 's16', 's17', 's18', 's19', 's20', 's21'
]
train_01_raw = train_01_raw.sort_values(['id', 'cycle'])
# read test data - It is the aircraft engine operating data without failure events recorded.
test_01_raw = pd.read_csv(data_dir + '/test_' + data_identifier + '.txt',
sep=" ",
header=None)
test_01_raw.drop(test_01_raw.columns[[26, 27]], axis=1, inplace=True)
test_01_raw.columns = [
'id', 'cycle', 'setting1', 'setting2', 'setting3', 's1', 's2', 's3',
's4', 's5', 's6', 's7', 's8', 's9', 's10', 's11', 's12', 's13', 's14',
's15', 's16', 's17', 's18', 's19', 's20', 's21'
]
test_01_raw = test_01_raw.sort_values(['id', 'cycle'])
if data_identifier == 'FD002' or data_identifier == 'FD004':
max_RUL = 130
print("--multi operating conditions--")
#standard the setting1 values
train_01_raw.loc[
train_01_raw['setting1'].between(0.00000e+00, 3.00000e-03),
'setting1'] = 0.0
train_01_raw.loc[
train_01_raw['setting1'].between(9.99800e+00, 1.00080e+01),
'setting1'] = 10.0
train_01_raw.loc[
train_01_raw['setting1'].between(1.99980e+01, 2.00080e+01),
'setting1'] = 20.0
train_01_raw.loc[
train_01_raw['setting1'].between(2.49980e+01, 2.50080e+01),
'setting1'] = 25.0
train_01_raw.loc[
train_01_raw['setting1'].between(3.49980e+01, 3.50080e+01),
'setting1'] = 35.0
train_01_raw.loc[
train_01_raw['setting1'].between(4.19980e+01, 4.20080e+01),
'setting1'] = 42.0
test_01_raw.loc[
test_01_raw['setting1'].between(0.00000e+00, 3.00000e-03),
'setting1'] = 0.0
test_01_raw.loc[
test_01_raw['setting1'].between(9.99800e+00, 1.00080e+01),
'setting1'] = 10.0
test_01_raw.loc[
test_01_raw['setting1'].between(1.99980e+01, 2.00080e+01),
'setting1'] = 20.0
test_01_raw.loc[
test_01_raw['setting1'].between(2.49980e+01, 2.50080e+01),
'setting1'] = 25.0
test_01_raw.loc[
test_01_raw['setting1'].between(3.49980e+01, 3.50080e+01),
'setting1'] = 35.0
test_01_raw.loc[
test_01_raw['setting1'].between(4.19980e+01, 4.20080e+01),
'setting1'] = 42.0
######
# Normalising sensor and settings data
######
# skip the first 2 columns, id and cycle
train_sensor = train_01_raw.iloc[:, 2:]
test_sensor = test_01_raw.iloc[:, 2:]
# Obtain the 21 column names from 's1' to 's21'
Train_Norm = pd.DataFrame(columns=train_sensor.columns[3:])
Test_Norm = pd.DataFrame(columns=test_sensor.columns[3:])
grouped_train = train_sensor.groupby('setting1')
grouped_test = test_sensor.groupby('setting1')
for train_idx, train in grouped_train:
scaled_train = scaler.fit_transform(train.iloc[:, 3:])
scaled_train_combine = pd.DataFrame(
data=scaled_train,
index=train.index,
columns=train_sensor.columns[3:])
Train_Norm = pd.concat([Train_Norm, scaled_train_combine])
for test_idx, test in grouped_test:
if train_idx == test_idx:
scaled_test = scaler.transform(test.iloc[:, 3:])
scaled_test_combine = pd.DataFrame(
data=scaled_test,
index=test.index,
columns=test_sensor.columns[3:])
Test_Norm = pd.concat([Test_Norm, scaled_test_combine])
Train_Norm = Train_Norm.sort_index()
Test_Norm = Test_Norm.sort_index()
train_01_raw.iloc[:,
2:5] = scaler.fit_transform(train_01_raw.iloc[:,
2:5])
test_01_raw.iloc[:, 2:5] = scaler.transform(test_01_raw.iloc[:, 2:5])
Train_Settings = pd.DataFrame(data=train_01_raw.iloc[:, :5],
index=train_01_raw.index,
columns=train_01_raw.columns[:5])
Test_Settings = pd.DataFrame(data=test_01_raw.iloc[:, :5],
index=test_01_raw.index,
columns=test_01_raw.columns[:5])
#adding the column of 'time'
train_01_nor = pd.concat([Train_Settings, Train_Norm], axis=1)
test_01_nor = pd.concat([Test_Settings, Test_Norm], axis=1)
train_01_nor = train_01_nor.values
test_01_nor = test_01_nor.values
train_01_nor = np.delete(train_01_nor, [5, 9, 10, 14, 20, 22, 23],
axis=1) # sensor 1 for index 5 2,3,4
test_01_nor = np.delete(test_01_nor, [5, 9, 10, 14, 20, 22, 23],
axis=1)
else:
print("--single operating conditions--")
max_RUL = 130.0
with np.nditer(train_01_raw['setting1'], op_flags=['readwrite']) as it:
for x in it:
x[...] = 0.0
#skip the first 2 columns, id and cycle
train_01_raw.iloc[:, 2:] = scaler.fit_transform(train_01_raw.iloc[:,
2:])
test_01_raw.iloc[:, 2:] = scaler.transform(test_01_raw.iloc[:, 2:])
train_01_nor = train_01_raw
test_01_nor = test_01_raw
train_01_nor = train_01_nor.values
test_01_nor = test_01_nor.values
train_01_nor = np.delete(train_01_nor, [5, 9, 10, 14, 20, 22, 23],
axis=1) # sensor 1 for index 5 2,3,4
test_01_nor = np.delete(test_01_nor, [5, 9, 10, 14, 20, 22, 23],
axis=1)
train_data, train_labels, valid_data, valid_labels = get_train_valid(
train_01_nor, winSize, max_RUL)
testX = []
testY = []
testLen = []
for i in range(1, int(np.max(test_01_nor[:, 0])) + 1):
ind = np.where(test_01_nor[:, 0] == i)
ind = ind[0]
testLen.append(len(ind))
data_temp = test_01_nor[ind, :]
if len(data_temp) < winSize:
data_temp_a = []
for myi in range(data_temp.shape[1]):
x1 = np.linspace(0, winSize - 1, len(data_temp))
x_new = np.linspace(0, winSize - 1, winSize)
tck = interpolate.splrep(x1, data_temp[:, myi])
a = interpolate.splev(x_new, tck)
data_temp_a.append(a.tolist())
data_temp_a = np.array(data_temp_a)
data_temp = data_temp_a.T
data_temp = data_temp[:, 1:]
else:
data_temp = data_temp[-winSize:, 1:]
# print('np.shape(data_temp)', np.shape(data_temp))
# print('data_temp[:,1]', data_temp[:,0])
# break
data_temp = np.reshape(data_temp,
(1, data_temp.shape[0], data_temp.shape[1]))
if i == 1:
testX = data_temp
else:
testX = np.concatenate((testX, data_temp), axis=0)
if RUL_01[i - 1] > max_RUL:
testY.append(max_RUL)
else:
testY.append(RUL_01[i - 1])
testX = np.array(testX)
train_data[:, :, 0] = scaler.fit_transform(train_data[:, :, 0]) #normalize
valid_data[:, :, 0] = scaler.fit_transform(valid_data[:, :, 0]) #normalize
# train_labels = np.array(train_labels)/max_RUL # normalize to 0-1
train_data = train_data[:, :, 4:] # remove wokring setting parameters.
valid_data = valid_data[:, :, 4:] # remove working setting parameters.
# train_labels = np.array(train_labels)/max_RUL # normalize to 0-1
testX[:, :, 0] = scaler.transform(testX[:, :, 0])
testX = testX[:, :, 4:]
# return trainX, testX, trainY, testY#, trainAux, testAux
return train_data, valid_data, testX, train_labels, valid_labels, testY
temp_list = [2]
for temp in range(0, num_arrays):
temp_list.append(2)
return np.zeros((temp_list))
if __name__ == "__main__":
NUM_VAR = int(input("How many inputs do you have: "))
k_map_array = create_2x2_arrays(NUM_VAR-1)
user_input = input(
"Enter 0 to fill a truth table or enter your circuit in the form x+y*z")
if user_input == "0":
it = np.nditer(k_map_array, op_flags=[
'readwrite'], flags=['multi_index'])
print("(", end='')
for x in range(0, NUM_VAR):
print(chr(65+x), end='')
if x >= NUM_VAR-1:
print(") | ")
else:
print(", ", end='')
while not it.finished:
print(str(it.multi_index) + " | ", end='')
it[0] = input()
it.iternext()
print(k_map_array)
else:
def calculate_precipitable_water(ds,
temp_name='tdry',
rh_name='rh',
pres_name='pres'):
"""
Function to calculate precipitable water vapor from ARM sondewnpn b1 data.
Will first calculate saturation vapor pressure of all data using Arden-Buck
equations, then calculate specific humidity and integrate over all pressure
levels to give us a precipitable water value in centimeters.
ds : ACT object
Object as read in by the ACT netCDF reader.
temp_name : str
Name of temperature field to use. Defaults to 'tdry' for sondewnpn b1
level data.
rh_name : str
Name of relative humidity field to use. Defaults to 'rh' for sondewnpn
b1 level data.
pres_name : str
Name of atmospheric pressure field to use. Defaults to 'pres' for
sondewnpn b1 level data.
"""
temp = ds[temp_name].values
rh = ds[rh_name].values
pres = ds[pres_name].values
# Get list of temperature values for saturation vapor pressure calc
temperature = []
for t in np.nditer(temp):
temperature.append(t)
# Apply Arden-Buck equation to get saturation vapor pressure
sat_vap_pres = []
for t in temperature:
# Over liquid water, above freezing
if t >= 0:
sat_vap_pres.append(0.61121 * np.exp(
(18.678 - (t / 234.5)) * (t / (257.14 + t))))
# Over ice, below freezing
else:
sat_vap_pres.append(0.61115 * np.exp(
(23.036 - (t / 333.7)) * (t / (279.82 + t))))
# convert rh from % to decimal
rel_hum = []
for r in np.nditer(rh):
rel_hum.append(r / 100.)
# get vapor pressure from rh and saturation vapor pressure
vap_pres = []
for i in range(0, len(sat_vap_pres)):
es = rel_hum[i] * sat_vap_pres[i]
vap_pres.append(es)
# Get list of pressure values for mixing ratio calc
pressure = []
for p in np.nditer(pres):
pressure.append(p)
# Mixing ratio calc
mix_rat = []
for i in range(0, len(vap_pres)):
mix_rat.append(0.622 * vap_pres[i] / (pressure[i] - vap_pres[i]))
# Specific humidity
spec_hum = []
for rat in mix_rat:
spec_hum.append(rat / (1 + rat))
# Integrate specific humidity
pwv = 0.0
for i in range(1, len(pressure) - 1):
pwv = pwv + 0.5 * (spec_hum[i] + spec_hum[i - 1]) * (pressure[i - 1] -
pressure[i])
pwv = pwv / 0.098
return pwv
