def fade( from_color, to_color, steps ):
from_color = narray(from_color)
to_color = narray(to_color)
diff = (to_color-from_color)/float(steps)
for n in range(steps):
color = from_color + n*diff
yield color.round().astype(int).tolist()
def read(self,file):
iclose=False
try:
# If file is not open, open it as a fileobject
fo=open(file,'rb')
iclose=True
except:
# If already a fileobject, we will append to it
fo=file
ti=array('i')
ti.read(fo,2)
self.nboot=ti[0]
self.ndim=ti[1]
self.values.resize(self.nboot,self.ndim)
tf=array('d')
tf.read(fo,self.ndim)
self.Avg=narray(tf)
tf=array('d')
tf.read(fo,self.ndim)
self.Std=narray(tf)
for iboot in xrange(self.nboot):
tf=array('d')
tf.read(fo,self.ndim)
self.values[iboot]=narray(tf)
# If file was a new file, close it
if iclose:
fo.close()
def _blob(x,y,area,colour): """ Draws a square-shaped blob with the given area (< 1) at the given coordinates. """ hs = sqrt(area) / 2 xcorners = narray([x - hs, x + hs, x + hs, x - hs]) ycorners = narray([y - hs, y - hs, y + hs, y + hs]) P.fill(xcorners, ycorners, colour, edgecolor=colour)
def load_test_data(self, test_data):
# Validate dataset dimensions
if(len(test_data[0]) == self.data_point_size):
for data_point in test_data:
assert(len(data_point) == self.data_point_size)
self.test_data = narray(test_data)
else:
# Extra step for converting binary data into one-hot encoding for tf
for data_point in test_data:
assert(len(data_point) == (self.data_point_size-1))
self.test_data = narray(self.binary_2_one_hot(test_data))
def process_chunk(self, chunk):
chunk = chunk.replace({"subcate": self.l2table})
text = chunk["desc"]
label = chunk["subcate"]
np.random.seed(17)
indices = np.random.permutation(np.arange(len(text)))
text = text[indices].tolist()
label = label[indices].tolist()
text = np.narray(text)
label = np.narray(label)
return text, label
def tolist(self, fill_value=None):
"""
Return the data portion of the array as a list.
Data items are converted to the nearest compatible Python type.
Masked values are converted to fill_value. If fill_value is None,
the corresponding entries in the output list will be ``None``.
"""
if fill_value is not None:
return self.filled(fill_value).tolist()
result = narray(self.filled().tolist(), dtype=object)
mask = narray(self._mask.tolist())
result[mask] = None
return result.tolist()
def tolist(self, fill_value=None):
"""
Return the data portion of the array as a list.
Data items are converted to the nearest compatible Python type.
Masked values are converted to fill_value. If fill_value is None,
the corresponding entries in the output list will be ``None``.
"""
if fill_value is not None:
return self.filled(fill_value).tolist()
result = narray(self.filled().tolist(), dtype=object)
mask = narray(self._mask.tolist())
result[mask] = None
return result.tolist()
def __array_finalize__(self, obj):
# Make sure we have a _fieldmask by default ..
_fieldmask = getattr(obj, '_fieldmask', None)
if _fieldmask is None:
mdescr = [(n, '|b1') for (n, _) in self.dtype.descr]
_mask = getattr(obj, '_mask', nomask)
if _mask is nomask:
_fieldmask = np.empty(self.shape, dtype=mdescr).view(recarray)
_fieldmask.flat = tuple([False] * len(mdescr))
else:
_fieldmask = narray([tuple([m] * len(mdescr)) for m in _mask],
dtype=mdescr).view(recarray)
# Update some of the attributes
if obj is not None:
_baseclass = getattr(obj, '_baseclass', type(obj))
else:
_baseclass = recarray
attrdict = dict(_fieldmask=_fieldmask,
_hardmask=getattr(obj, '_hardmask', False),
_fill_value=getattr(obj, '_fill_value', None),
_sharedmask=getattr(obj, '_sharedmask', False),
_baseclass=_baseclass)
self.__dict__.update(attrdict)
# Finalize as a regular maskedarray .....
# Update special attributes ...
self._basedict = getattr(obj, '_basedict',
getattr(obj, '__dict__', {}))
self.__dict__.update(self._basedict)
return
def __array_finalize__(self,obj):
# Make sure we have a _fieldmask by default ..
_fieldmask = getattr(obj, '_fieldmask', None)
if _fieldmask is None:
mdescr = _make_mask_dtype(ndarray.__getattribute__(self, 'dtype'))
_mask = getattr(obj, '_mask', nomask)
if _mask is nomask:
_fieldmask = np.empty(self.shape, dtype=mdescr).view(recarray)
_fieldmask.flat = tuple([False]*len(mdescr))
else:
_fieldmask = narray([tuple([m]*len(mdescr)) for m in _mask],
dtype=mdescr).view(recarray)
# Update some of the attributes
if obj is not None:
_baseclass = getattr(obj,'_baseclass',type(obj))
else:
_baseclass = recarray
attrdict = dict(_fieldmask=_fieldmask,
_hardmask=getattr(obj,'_hardmask',False),
_fill_value=getattr(obj,'_fill_value',None),
_sharedmask=getattr(obj,'_sharedmask',False),
_baseclass=_baseclass)
self.__dict__.update(attrdict)
# Finalize as a regular maskedarray .....
# Update special attributes ...
self._basedict = getattr(obj, '_basedict', getattr(obj,'__dict__',{}))
self.__dict__.update(self._basedict)
return
def mquantiles(data, prob=list([.25,.5,.75]), alphap=.4, betap=.4, axis=None):
"""Computes empirical quantiles for a *1xN* data array.
Samples quantile are defined by:
*Q(p) = (1-g).x[i] +g.x[i+1]*
where *x[j]* is the jth order statistic,
with *i = (floor(n*p+m))*, *m=alpha+p*(1-alpha-beta)* and *g = n*p + m - i)*.
Typical values of (alpha,beta) are:
- (0,1) : *p(k) = k/n* : linear interpolation of cdf (R, type 4)
- (.5,.5) : *p(k) = (k+1/2.)/n* : piecewise linear function (R, type 5)
- (0,0) : *p(k) = k/(n+1)* : (R type 6)
- (1,1) : *p(k) = (k-1)/(n-1)*. In this case, p(k) = mode[F(x[k])].
That's R default (R type 7)
- (1/3,1/3): *p(k) = (k-1/3)/(n+1/3)*. Then p(k) ~ median[F(x[k])].
The resulting quantile estimates are approximately median-unbiased
regardless of the distribution of x. (R type 8)
- (3/8,3/8): *p(k) = (k-3/8)/(n+1/4)*. Blom.
The resulting quantile estimates are approximately unbiased
if x is normally distributed (R type 9)
- (.4,.4) : approximately quantile unbiased (Cunnane)
- (.35,.35): APL, used with PWM
:Parameters:
x : Sequence
Input data, as a sequence or array of dimension at most 2.
prob : Sequence *[(0.25, 0.5, 0.75)]*
List of quantiles to compute.
alpha : Float (*[0.4]*)
Plotting positions parameter.
beta : Float (*[0.4]*)
Plotting positions parameter.
axis : Integer *[None]*
Axis along which to compute quantiles. If *None*, uses the whole
(flattened/compressed) dataset.
"""
def _quantiles1D(data,m,p):
x = numpy.sort(data.compressed())
n = len(x)
if n == 0:
return masked_array(numpy.empty(len(p), dtype=float_), mask=True)
elif n == 1:
return masked_array(numpy.resize(x, p.shape), mask=nomask)
aleph = (n*p + m)
k = numpy.floor(aleph.clip(1, n-1)).astype(int_)
gamma = (aleph-k).clip(0,1)
return (1.-gamma)*x[(k-1).tolist()] + gamma*x[k.tolist()]
# Initialization & checks ---------
data = masked_array(data, copy=False)
p = narray(prob, copy=False, ndmin=1)
m = alphap + p*(1.-alphap-betap)
# Computes quantiles along axis (or globally)
if (axis is None):
return _quantiles1D(data, m, p)
else:
assert data.ndim <= 2, "Array should be 2D at most !"
return apply_along_axis(_quantiles1D, axis, data, m, p)
def cov(x, y=None, rowvar=True, bias=False, strict=False):
"""
Estimate the covariance matrix.
If x is a vector, return the variance. For matrices, returns the covariance
matrix.
If y is given, it is treated as an additional (set of) variable(s).
Normalization is by (N-1) where N is the number of observations (unbiased
estimate). If bias is True then normalization is by N.
If rowvar is non-zero (default), then each row is a variable with observations
in the columns, otherwise each column is a variable and the observations are
in the rows.
If strict is True, masked values are propagated: if a masked value appears in
a row or column, the whole row or column is considered masked.
"""
X = narray(x, ndmin=2, subok=True, dtype=float)
if X.shape[0] == 1:
rowvar = True
if rowvar:
axis = 0
tup = (slice(None),None)
else:
axis = 1
tup = (None, slice(None))
#
if y is not None:
y = narray(y, copy=False, ndmin=2, subok=True, dtype=float)
X = concatenate((X,y),axis)
#
X -= X.mean(axis=1-axis)[tup]
n = X.count(1-axis)
#
if bias:
fact = n*1.0
else:
fact = n-1.0
#
if not rowvar:
return (dot(X.T, X.conj(), strict=False) / fact).squeeze()
else:
return (dot(X, X.T.conj(), strict=False) / fact).squeeze()
def load_data(path):
data = []
seq_data = []
target = []
with open(path) as f:
for line in f.readlines():
lines = line.strip().split()
data_tmp = [lines[0], lines[1], lines[2]]
label_tmp = lines[4]
[seq_tmp.append(1)
for _ in range(len(data_tmp))] # 表示真实的数据存在，防止rnn 发生梯度消失
data.append(data_tmp)
target.append(label_tmp)
data = np.narray(data)
target = np.narray(target)
seq_data = np.narray(seq_data)
return data, target, seq_data
def AddValue(self, value):
"""supply one more numerical value"""
if (self.previous_index == self.size - 1 and not (self.buffer_full)):
self.buffer_full = True
self.previous_index = (self.previous_index + 1) % self.size
self.buf[self.previous_index * 2 + 1] = value
self.vbo[self.previous_index * 2 + 1:self.previous_index * 2 +
2] = narray([value], 'f')
self.maxValue = max(self.maxValue, value)
def initLayers(self, layers):
self.layers = []
for i in range(len(layers) - 1):
layer = np.narray(layers[i], layers[i + 1])
#TODO randomly initialize the layer
for j in range(layers[j]):
for k in range(layers[j + 1]):
layer[j][k] = self.randInit(0.12)
self.layers.append(layer)
def write(self,file):
iclose=False
try:
# If file is not open, open it as a fileobject
fo=open(file,'wb')
iclose=True
except:
# If already a fileobject, we will append to it
fo=file
narray(self.nboot).tofile(fo)
narray(self.ndim).tofile(fo)
self.Avg.tofile(fo)
self.Std.tofile(fo)
self.values.tofile(fo)
# If file was a new file, close it
if iclose:
fo.close()
def train(self,train_data,train_data_index,valid_data_index,valid_data,train_label,valid_label,early_stopping=True):
''' train '''
shuffle_indices = np.random.permutation(np.arange(len(train_data)))
for epoch in self.epochs:
train_prob_list = []
total_batch = len(train_label)/self.batch_size
for ind in range(total_batch):
start = ind * self.batch_size
end = (ind+1)* self.batch_size
end = end if end < len(train_label) else len(train_label)
train_batch_index = train_data_index[shuffle_indices[start:end],:]
train_batch_data = train_data[shuffle_indices[start:end],]
train_batch_label = train_label[shuffle_indices[start:end],:]
feed_dict = {self.feat_index: train_batch_index, self.feat_value : train_batch_data , self.label : train_batch_label }
tf.sess.run(optimizer,feed_dict = feed_dict) # 希望从网络中得到的数据，填写在 run（）中 ,训练 optimizer
# train_loss, train_ks etc.
block = 10000
for j in range(len(train_label)/block):
start = j*block
end = (j+1)*block if (j+1)*block < len(train_label) else len(train_label) # run(loss,prob)
train_loss, prob = self.sess.run([self.loss,self.prob ], \
feed_dict = {self.feat_index: train_data_index[start:end],\
self.feat_value : train_data[start:end] , \
self.label : train_label[start:end] })
for i in range(end-start):
train_prob_list.append(prob[i,0])
train_ks = cal_ks(train_label,np.narray(train_prob_list))
wf.write(" block: %d , train dataset : loss=%f , ks=%f " % (j, train_loss, train_ks))
eval_prob_list = []
total_batch = len(valid_label)/ block
for j in range(total_batch):
start = ind * block
end = (ind+1)* block
end = end if end < len(valid_label) else len(valid_label) # run(loss,prob)
valid_loss, prob = self.sess.run([self.loss,self.prob ], \
feed_dict = {self.feat_index: valid_data_index[start:end],\
self.feat_value : valid_data[start:end] , \
self.label : valid_label[start:end] })
for i in range(end-start):
eval_prob_list.append(prob[i,0])
valid_ks = cal_ks(valid_ks,np.narray(eval_prob_list))
wf.write(" block: %d , train dataset : loss=%f , ks=%f " % (j, train_loss, train_ks))
def AddValue(self, value):
"""supply one more numerical value"""
if(self.previous_index == self.size - 1
and not(self.buffer_full)):
self.buffer_full = True
self.previous_index = (self.previous_index + 1) % self.size
self.buf[self.previous_index * 2 + 1] = value
self.vbo[self.previous_index * 2 + 1:
self.previous_index * 2 + 2] = narray([value], 'f')
self.maxValue = max(self.maxValue, value)
def _UpdateObject(self, _id, obj):
"""Updates an object in the VBO"""
if self._vbo is None:
return
index = self._indices.get(_id, -1)
# have to check since object could have been deleted since marked as
# dirty
if index < 0:
return
num_values = type(self).__num_values
self._vbo[index * num_values : (index + 1) * num_values] = narray(self.__descToArray(obj), "f")
def load_dataset(self, dataset_obj):
# Load training and testing data from dataset object
if(len(dataset_obj.train_data[0]) == self.data_point_size):
# Validate dataset dimensions
for data_point in dataset_obj.train_data:
assert(len(data_point) == self.data_point_size)
for data_point in dataset_obj.test_data:
assert(len(data_point) == self.data_point_size)
# Load dataset
self.train_data = narray(dataset_obj.train_data)
self.test_data = narray(dataset_obj.test_data)
else:
# Extra step for converting binary data into one-hot encoding for tf
for data_point in dataset_obj.train_data:
assert(len(data_point) == (self.data_point_size-1))
for data_point in dataset_obj.test_data:
assert(len(data_point) == (self.data_point_size-1))
# Load dataset
self.train_data = narray(self.binary_2_one_hot(dataset_obj.train_data))
self.test_data = narray(self.binary_2_one_hot(dataset_obj.test_data))
def plot_values(teamsdict, teamslist, vals):
vals = narray(vals).T
v = []
for team in teamslist:
if team in teamsdict:
v.append(vals[teamsdict[team]])
v = zip(*v)
plt(v[0], v[1], 'rx')
show()
def _UpdateObject(self, _id, obj):
"""Updates an object in the VBO"""
if self._vbo is None:
return
index = self._indices.get(_id, -1)
# have to check since object could have been deleted since marked as
# dirty
if index < 0:
return
num_values = type(self).__num_values
self._vbo[index * num_values:(index + 1) * num_values] = narray(
self.__descToArray(obj), "f")
def hinton(W, length, name, maxWeight=None):
"""
Draws a Hinton diagram for visualizing a weight matrix.
Temporarily disables matplotlib interactive mode if it is on,
otherwise this takes forever.
"""
reenable = False
if P.isinteractive():
P.ioff()
P.clf()
height, width = W.shape
if not maxWeight:
#maxWeight = 2**N.ceil(N.log(N.max(N.abs(W)))/N.log(2))
maxWeight = max(abs(W))*1.2
P.fill(narray([0,width,width,0]),narray([0,0,height,height]),'gray')
P.axis('off')
P.axis('equal')
correlationcounter = 0
for x in xrange(width):
for y in xrange(height):
_x = x+1
_y = y+1
w = W[y,x]
if w > 0:
_blob(_x - 0.5, height - _y + 0.5, min(1,w/maxWeight*w/maxWeight),'white')
correlationcounter += w - 0.5
elif w < 0:
_blob(_x - 0.5, height - _y + 0.5, min(1,w/maxWeight*w/maxWeight),'black')
correlationcounter += w + 0.5
correlationindex = abs(correlationcounter/length*100 + 50)
titlestring = 'My votes versus ' + name + ': ' + str(correlationindex) + '% agreement'
P.title(titlestring)
if reenable:
P.ion()
P.show()
def __getitem__(self, indx):
"""Returns all the fields sharing the same fieldname base.
The fieldname base is either `_data` or `_mask`."""
_localdict = self.__dict__
_fieldmask = _localdict['_fieldmask']
_data = self._data
# We want a field ........
if isinstance(indx, basestring):
obj = _data[indx].view(MaskedArray)
obj._set_mask(_fieldmask[indx])
# Force to nomask if the mask is empty
if not obj._mask.any():
obj._mask = nomask
# Force to masked if the mask is True
if not obj.ndim and obj._mask:
return masked
return obj
# We want some elements ..
# First, the data ........
obj = narray(_data[indx], copy=False).view(mrecarray)
obj._fieldmask = narray(_fieldmask[indx], copy=False).view(recarray)
return obj
def correlate(self):
""" inst.correlate() -> None
This must be overloaded to populate the '_correlations'
member dictionary with valid correlation functions for
every included (i.e. tracked) baseline. This should be a
numpy array.
"""
with RLock():
itime = self.server._integration_time
self._stopevent.wait(itime)
self._last_correlation = time()
for baseline in self._include_baselines:
self._correlations[baseline] = narray([0]*self._lags*2)
def __getitem__(self, indx):
"""Returns all the fields sharing the same fieldname base.
The fieldname base is either `_data` or `_mask`."""
_localdict = self.__dict__
_fieldmask = _localdict['_fieldmask']
_data = self._data
# We want a field ........
if isinstance(indx, basestring):
obj = _data[indx].view(MaskedArray)
obj._set_mask(_fieldmask[indx])
# Force to nomask if the mask is empty
if not obj._mask.any():
obj._mask = nomask
# Force to masked if the mask is True
if not obj.ndim and obj._mask:
return masked
return obj
# We want some elements ..
# First, the data ........
obj = narray(_data[indx], copy=False).view(mrecarray)
obj._fieldmask = narray(_fieldmask[indx], copy=False).view(recarray)
return obj
def Import(self,data,bin=1,weights=[],randlist = []):
if len(weights)>0 and bin!=1:
print "Weights only currently supported for binsize=1"
assert 1==0
self.nconf=len(data)/bin
if bin>1:
tmpdata=narray([average(ien) for ien in split(narray(data[0:bin*(len(data)/bin)]),self.nconf)])
else:
tmpdata=narray(data)
# self.Raw=append(self.Raw,tmpdata)
# self.nconf=len(self.Raw)
# rint=zeros( (self.nboot,self.nconf))
# if randlist == []:
myseed=startseed*self.nconf/self.nboot
random.seed(myseed)
# locranint=random.randint
locranint=random.random_integers
# else:
# locranint = randlist
if len(weights)>0:
tmpweight=narray(weights)
self.Avg=average(multiply(tmpdata,tmpweight))/sum(tmpweight)
else:
self.Avg=average(tmpdata)
for iboot in range(self.nboot):
rint=locranint(0,self.nconf-1,self.nconf)
# tmp=0.0
# for iconf in range(self.nconf):
# rint=locranint(0,self.nconf-1)
# tmp+=tmpdata[rint]/self.nconf
# self.values[iboot]=tmp
if len(weights)>0:
tw2=tmpweight[rint]
td2=multiply(tmpdata[rint],tw2)
self.values[iboot]=average(td2)/sum(tw2)
#average(multiply(tmpdata[rint],tmpweight[rint]))/sum(tmpweight[rint])
else:
self.values[iboot]=average(tmpdata[rint])
return locranint
def dict_to_numpy(csv_data, columns_to_exclude = []):
target = []
data = []
attributes = None
for row in csv_data:
target.append(row.pop('quality'))
[row.pop(col) for col in columns_to_exclude]
if attributes is None:
attributes = row.keys()
data.append(row.values())
target = narray(target)
data = narray(data)
return {
'target': target,
'data': data,
'attributes': attributes
}
def MakePalette():
"Generate a palette suitable for PIL.Image.ImageData.putpalette"
return flatten((i,i,i) for i in xrange(256))
# The below code mimicks the source from the original
# If you want to use this, you need numpy. "easy_install numpy" or whatever
# for your distribution
from numpy import array as narray
# Some possible destination colours (sepia, or blueish, etc)
#dst = narray([18,74,155])
#dst = narray([155,74,18])
#dst = narray([17,51,96])
dst = narray([96,51,17])
high = narray([255,255,255])
firsthalf = [high + (dst-high)/255*i for i in xrange(0,256,2)]
secondhalf = [dst + (-dst)/255*i for i in xrange(0,256,2)]
result = flatten(firsthalf+secondhalf)
result = map(int, result)
return result
def Plot4D( x, y, z, t, markerstyle = 20, markersize = 1 ):
data = narray( [0.] * 4 )
tree = TTree('DummyTree','DummyTree')
tree.Branch('xyzt', data, 'x/D:y:z:t')
for datai in zip(x,y,z,t):
data[0], data[1], data[2], data[3] = datai
#print data
tree.Fill()
tree.SetMarkerStyle( markerstyle )
tree.SetMarkerSize( markersize )
c = TCanvas()
tree.Draw('x:y:z:t','','zcol')
return c, tree
def __init__(self, size):
self.size = size
initialBuffer = []
for x, y in enumerate(range(size)):
initialBuffer.append(x)
initialBuffer.append(y)
self.buf = narray(initialBuffer, 'f')
self.vbo = vbo.VBO(self.buf,
usage=GL_STREAM_DRAW,
target=GL_ARRAY_BUFFER,
size=4 * 2 * size)
self.previous_index = -1
self.buffer_full = False
self.maxValue = 0
def Plot4D(x, y, z, t, markerstyle=20, markersize=1):
data = narray([0.] * 4)
tree = TTree('DummyTree', 'DummyTree')
tree.Branch('xyzt', data, 'x/D:y:z:t')
for datai in zip(x, y, z, t):
data[0], data[1], data[2], data[3] = datai
#print data
tree.Fill()
tree.SetMarkerStyle(markerstyle)
tree.SetMarkerSize(markersize)
c = TCanvas()
tree.Draw('x:y:z:t', '', 'zcol')
return c, tree
def __init__(self, size):
self.size = size
initialBuffer = []
for x, y in enumerate(range(size)):
initialBuffer.append(x)
initialBuffer.append(y)
self.buf = narray(initialBuffer, 'f')
self.vbo = vbo.VBO(self.buf,
usage=GL_STREAM_DRAW,
target=GL_ARRAY_BUFFER,
size=4 * 2 * size)
self.previous_index = -1
self.buffer_full = False
self.maxValue = 0
def _AddObject(self, _id, obj):
"""Adds an object to the VBO"""
if self._vbo is None:
return
index = -1
if len(self._empty_indices) > 0:
index = self._empty_indices.pop()
elif self._max_index < self._data_size - 1:
self._max_index += 1
index = self._max_index
if index > 0:
values = self.__descToArray(obj)
num_values = len(values)
self._vbo[index * num_values : (index + 1) * num_values] = narray(values, "f")
self._indices[_id] = index
else:
self._vbo = None
def _AddObject(self, _id, obj):
"""Adds an object to the VBO"""
if self._vbo is None:
return
index = -1
if len(self._empty_indices) > 0:
index = self._empty_indices.pop()
elif self._max_index < self._data_size - 1:
self._max_index += 1
index = self._max_index
if index > 0:
values = self.__descToArray(obj)
num_values = len(values)
self._vbo[index * num_values:(index + 1) * num_values] = narray(
values, "f")
self._indices[_id] = index
else:
self._vbo = None
def __array_finalize__(self, obj):
# Make sure we have a _fieldmask by default ..
_mask = getattr(obj, "_mask", None)
if _mask is None:
objmask = getattr(obj, "_mask", nomask)
_dtype = ndarray.__getattribute__(self, "dtype")
if objmask is nomask:
_mask = ma.make_mask_none(self.shape, dtype=_dtype)
else:
mdescr = ma.make_mask_descr(_dtype)
_mask = narray([tuple([m] * len(mdescr)) for m in objmask], dtype=mdescr).view(recarray)
# Update some of the attributes
_dict = self.__dict__
_dict.update(_mask=_mask, _fieldmask=_mask)
self._update_from(obj)
if _dict["_baseclass"] == ndarray:
_dict["_baseclass"] = recarray
return
def __array_finalize__(self, obj):
# Make sure we have a _fieldmask by default
_mask = getattr(obj, '_mask', None)
if _mask is None:
objmask = getattr(obj, '_mask', nomask)
_dtype = ndarray.__getattribute__(self, 'dtype')
if objmask is nomask:
_mask = ma.make_mask_none(self.shape, dtype=_dtype)
else:
mdescr = ma.make_mask_descr(_dtype)
_mask = narray([tuple([m] * len(mdescr)) for m in objmask],
dtype=mdescr).view(recarray)
# Update some of the attributes
_dict = self.__dict__
_dict.update(_mask=_mask)
self._update_from(obj)
if _dict['_baseclass'] == ndarray:
_dict['_baseclass'] = recarray
return
def __getattribute__(self, attr):
"Returns the given attribute."
try:
# Returns a generic attribute
return object.__getattribute__(self,attr)
except AttributeError:
# OK, so attr must be a field name
pass
# Get the list of fields ......
_names = self.dtype.names
if attr in _names:
_data = self._data
_mask = self._fieldmask
# obj = masked_array(_data.__getattribute__(attr), copy=False,
# mask=_mask.__getattribute__(attr))
# Use a view in order to avoid the copy of the mask in MaskedArray.__new__
obj = narray(_data.__getattribute__(attr), copy=False).view(MaskedArray)
obj._mask = _mask.__getattribute__(attr)
if not obj.ndim and obj._mask:
return masked
return obj
raise AttributeError,"No attribute '%s' !" % attr
def read(self,file):
iclose=False
try:
# If file is not open, open it as a fileobject
fo=open(file,'rb')
iclose=True
except:
# If already a fileobject, we will append to it
fo=file
ti=array('i')
ti.read(fo,1)
self.nboot=ti[0]
tf=array('d')
tf.read(fo,2)
self.Avg=tf[0]
self.Std=tf[1]
tf=array('d')
tf.read(fo,self.nboot)
self.values=narray(tf)
# If file was a new file, close it
if iclose:
fo.close()
def __getattribute__(self, attr):
"Returns the given attribute."
try:
# Returns a generic attribute
return object.__getattribute__(self, attr)
except AttributeError:
# OK, so attr must be a field name
pass
# Get the list of fields ......
_names = self.dtype.names
if attr in _names:
_data = self._data
_mask = self._fieldmask
# obj = masked_array(_data.__getattribute__(attr), copy=False,
# mask=_mask.__getattribute__(attr))
# Use a view in order to avoid the copy of the mask in MaskedArray.__new__
obj = narray(_data.__getattribute__(attr),
copy=False).view(MaskedArray)
obj._mask = _mask.__getattribute__(attr)
if not obj.ndim and obj._mask:
return masked
return obj
raise AttributeError, "No attribute '%s' !" % attr
# for now, define your own puzzle:
'''
grid = narray.[[0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0]]
'''
size = 9 # 9 by 9 game by default but changeable
game = narray([[0, 0, 0, 0, 0, 0, 0, 3, 1], [7, 0, 0, 9, 0, 0, 0, 4, 0],
[0, 0, 0, 0, 0, 0, 5, 0, 0], [0, 0, 0, 0, 3, 1, 0, 0, 0],
[9, 0, 0, 0, 0, 0, 0, 0, 8], [0, 0, 0, 4, 0, 5, 0, 0, 0],
[0, 0, 0, 6, 0, 0, 2, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 6],
[0, 5, 3, 0, 0, 0, 0, 0, 0]])
# (y, x) = (r, c)
# matrix index c = coordinate x = column
# matrix index r = coordinate y = row
def possible(r, c, num):
# given a number num, check if it's possible to put it in (r, c)
global game
global size
zone = int(sqrt(size))
for i in range(0, size): # check within row
if game[r][i] == num:
return False
def addRegisteredBlocks(blocks):
to = time()
result = sum([narray(block)**2 for block in blocks])
print "Time needed for adding blocks:", time() - to
return result
def _get_bytes_size(name, batch):
_validate_batch(name, batch)
if not isinstance(batch, np.ndarray):
batch = np.narray(batch)
return batch.nbytes
def do3(app_nums: Iterable=(5,), house_n: int=1,
no_save: bool=False, no_load: bool=True,
):
'''
'''
global D1
if not 'D1' in dir():
# if `D1` has not defined
D1 = DataSet(r'D:\dataset\REFIT\refit.h5')
p1 = PKMap(D1.buildings[house_n], no_count=True, no_load=no_load)
# no_old = OrderedDict.fromkeys(app_nums) # not get loaded data
is_old = {k:False for k in app_nums} # not get loaded data
bm0 = {k:None for k in app_nums}
sbm0 = {k:None for k in app_nums}
BMs = {k:[] for k in app_nums}
D0s = {k:[] for k in app_nums}
D1s = {k:[] for k in app_nums}
dates = []
# re-use data by caching
cache_dir = os.path.join(os.getcwd(), 'BMcache')
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
if not no_save:
cache_name0 = '_'.join(['BMcache', p1.dataset, p1.house_name, ''])
for an in app_nums:
cache_name = ''.join([cache_name0, p1.ins_name['active'][an-1], '.csv'])
for dirp, dirn, files in os.walk(cache_dir):
# print('dirpath is {}'.format(dirp))
# print('dirname is {}'.format(dirn))
# print('have {} files'.format(len(files)))
files_l = narray([x.lower() for x in files])
cache_l = cache_name.lower()
if cache_l in files_l:
cache_name2 = narray(files)[files_l == cache_l][0]
with open(os.path.join(cache_dir, cache_name2), 'r') as f:
data_old = read_csv(f, index_col=0, header=0,
names=('D0', 'D1', 'bm'),
parse_dates=True,
)
D0s[an] = data_old.D0
D1s[an] = data_old.D1
BMs[an] = data_old.bm
is_old[an] = True
break
else:
is_old[an] = False
print('cache_name is {}'.format(cache_name), ' '*16)
else:
cache_name0 = ""
# init
print('initializing ...', ' '*64, end='\r')
if not isinstance(app_nums, Iterable):
app_nums = (app_nums, )
with NoPrints():
for an in app_nums:
# an = int(an)
bm0[an], sbm0[an] = p1.BM(obj=an, sel_ac='active', no_plot=True)
p1.appQ['dd'] = 9
p1.app_name['dd'] = p1.app_name['active']
p1.ins_name['dd'] = p1.ins_name['active']
datas = []
d0 = p1.data0['active']
# if not any([is_old[an] for an in app_nums]):
if all([is_old[an] for an in app_nums]):
print('{0} all `{1}` App exist! {0}'.format('='*6, len(app_nums)))
years = ()
else:
print('app {} not exist'.format(', '.join([str(an) for an in app_nums if not is_old[an]])))
years = set(d0.index.year)
sleep(1)
t0 = time()
app_rems = [an for an in app_nums if not is_old[an]]
for _year in years:
# for _year in (2014, ):
print('\r\t\tcutting ... ', ' '*8, end='\r')
# the hash of int is neurally in order
d0_y = d0.loc[d0.index.year==_year]
months = set(d0_y.index.month)
for _month in months:
# for _month in (9, ):
d0_m = d0_y.loc[d0_y.index.month==_month]
days = set(d0_m.index.day)
days = (1, )
for _day in days:
# for _day in range(5, 11):
# d0_x = d0_m.loc[d0_m.index.day==_day]
d0_x = d0_m.copy()
# dates.append('-'.join(narray([_year, _month, _day], dtype=str)))
dates.append('-'.join(narray([_year, _month], dtype=str)))
# set `end` to steady the cursor
print('\r\t\treading {}'.format(dates[-1]), end='')
# if d0_x.index[0].hour < 1 and d0_x.index[-1].hour > 22:
if True:
datas.append(d0_x)
with NoPrints():
p1.data0['dd'] = d0_x.copy()
p1.data2['dd'] = gen_PKMap(p1, data0=d0_x, key='dd')
for an in app_rems:
bm, sbm = p1.BM(obj=an, sel_ac='dd', no_plot=False,
no_show=True, fig_types=('ty.jpg',))
BMs[an].append(bm)
D1s[an].append(Hellinger(sbm, sbm0[an]))
D0s[an].append(Hellinger(sbm))
else:
# imcomplete day, discard
d0id0, d0id1 = d0_x.index[0], d0_x.index[-1]
print('\tfind invalid time: {}, {:0>2}:{:0>2} to {:0>2}:{:0>2}'.format(
d0id0.date(), d0id0.hour, d0id0.minute, d0id1.hour, d0id1.minute
), end='')
del d0_y, d0_m, d0_x
# ring to alart the finishing
if len(app_rems) < 2:
print('\r\a `{}` app in {} of {} days cost {} '.format(
len(app_rems), len(datas),
len(set(d0.index.date)), beauty_time(time()-t0)+' '*128))
else:
print('\r\a `{}` apps in {} of {} days cost {} '.format(
len(app_rems), len(datas),
len(set(d0.index.date)), beauty_time(time()-t0)+' '*128))
if datas:
d1 = pd.concat(datas, )
# plt.plot(d1.index, range(len(d1.index)))
# plt.show()
# sleep(1e-15)
for an in app_nums:
fig = plt.figure(figsize=(6, 5))
ax = fig.add_subplot(121)
vio1 = ax.violinplot([D0s[an], D1s[an]], )
h0 = Hellinger(sbm0[an])
xlim = ax.get_xlim()
print(xlim)
ax.plot(xlim, [h0, h0], color='cornflowerblue')
ax.plot(xlim, [0, 0], 'k')
ax.set_xlim(xlim)
ax.set_xticks([1,2])
ax.set_xticklabels(['$D_0$', '$D_1$'])
ylim = ax.get_ylim()
tick = [k for k in ax.get_yticks() if k>ylim[0] and k<ylim[1]]
ax.set_yticks(nappend(tick, h0))
ntick = ['{:.1f}'.format(k) for k in tick]
ax.set_yticklabels(nappend(ntick, r'$D_{pH}$'+' is{0}\n{1:.2f}{0}'.format(' '*9, h0)))
plt.grid()
ax2 = fig.add_subplot(122)
vio2 = ax2.violinplot(BMs[an], )
# vio2['bodies'][0].set_facecolor('r')
# vio2['bodies'][0].set_edgecolor('violet')
xlim2 = ax2.get_xlim()
print(xlim2)
ax2.plot(xlim2, [bm0[an], bm0[an]], 'b')
ax2.set_xlim(xlim2)
ylim2 = ax2.get_ylim()
ax2.set_ylim(top=1.06, bottom=0)
ax2.yaxis.tick_right()
ax2.set_xticks([1])
ax2.set_xticklabels(['BM'])
tick = [k for k in ax2.get_yticks() if k < 1.1]
ax2.set_yticks(nappend(tick, bm0[an]))
print(ax2.get_yticks())
ntick = ['{:.1f}'.format(k) for k in tick]
ax2.set_yticklabels(nappend(ntick, '{0}BM is\n{0}{1:.3f}'.format(' '*7, bm0[an])))
plt.grid()
plt.savefig('./figs/D(bm)_{}_{}_A{}_{}.png'.format(
p1.dataset, p1.house_name, an, p1.ins_name['active'][an-1]
), bbox_inches='tight')
# plt.show()
# save data
t0 = time()
if not no_save and not is_old[an]:
cache_name = ''.join([cache_name0, p1.ins_name['active'][an-1], '.csv'])
print('='*6, 'saving into `{}`'.format(cache_name), '='*6, end='\r')
with open(os.path.join(cache_dir, cache_name), 'w') as f:
f.write('date, D0, D1, BM')
for _date, _d0, _d1, _bm in zip(dates, D0s[an], D1s[an], BMs[an]):
f.write('\n')
f.write(','.join(narray([_date, _d0, _d1, _bm], dtype=str)))
print('='*6, 'done saving'.format(beauty_time(time()-t0)))
pass
fitsobj = maputils.FITSimage("m101.fits")
fitsobj.set_limits((200,400), (200,400))
annim = fitsobj.Annotatedimage()
pos = "300 300"
print("Is " + pos + " inside?", annim.inside(pos=pos))
# Mode has no effect on parameter pos
print("Is " + pos + " inside?", annim.inside(pos=pos, mode='p'))
print("Is " + pos + " inside?", annim.inside(pos=pos, mode='w'))
print("Is 300, 300 inside?", annim.inside(x=300, y=300, mode='p'))
print("Is 300, 300 inside?", annim.inside(x=300, y=300, mode='w'))
print("Is 300, 300, 20,200 inside?", annim.inside(x=[300,20], y=[300,200],
mode='p'))
x = narray([100,200,300,20])
y = narray([100,200,270,0])
print("Two numpy arrays inside?", annim.inside(x=x, y=y, mode='p'))
crval1 = annim.projection.crval[0]
crval2 = annim.projection.crval[1]
print("Is %f %f" % (crval1, crval2) + " inside?", annim.inside(x=crval1,
y=crval2, mode='w'))
pos = '{} ' + str(crval1) + ' {} ' + str(crval2)
print("Is " + pos + " inside?", annim.inside(pos=pos))
# Raise an exception
try:
fitsobj = maputils.FITSimage("m101.fits")
fitsobj.set_limits((200, 400), (200, 400))
annim = fitsobj.Annotatedimage()
pos = "300 300"
print("Is " + pos + " inside?", annim.inside(pos=pos))
# Mode has no effect on parameter pos
print("Is " + pos + " inside?", annim.inside(pos=pos, mode='p'))
print("Is " + pos + " inside?", annim.inside(pos=pos, mode='w'))
print("Is 300, 300 inside?", annim.inside(x=300, y=300, mode='p'))
print("Is 300, 300 inside?", annim.inside(x=300, y=300, mode='w'))
print("Is 300, 300, 20,200 inside?",
annim.inside(x=[300, 20], y=[300, 200], mode='p'))
x = narray([100, 200, 300, 20])
y = narray([100, 200, 270, 0])
print("Two numpy arrays inside?", annim.inside(x=x, y=y, mode='p'))
crval1 = annim.projection.crval[0]
crval2 = annim.projection.crval[1]
print("Is %f %f" % (crval1, crval2) + " inside?",
annim.inside(x=crval1, y=crval2, mode='w'))
pos = '{} ' + str(crval1) + ' {} ' + str(crval2)
print("Is " + pos + " inside?", annim.inside(pos=pos))
# Raise an exception
try:
def score_english(s):
s = s.lower() # compare everything case insensitive
test_values = narray(relative_frequency(s))
score = norm(LETTER_FREQUENCIES - test_values)
num_non_english = [x not in string.printable for x in s].count(True)
return score * exp(num_non_english / float(len(s)))
import string
import urllib2
from collections import namedtuple
from numpy import array as narray, exp
from numpy.linalg import norm
Result = namedtuple('Result', 'message score')
SINGLE_BITS = map(chr, range(256))
# from https://en.wikipedia.org/wiki/Letter_frequency
LETTER_FREQUENCIES = narray([.08167, .01492, .02782, .04253, .02702, .02228, .02015, .06094, .06966, .00153, .00772, .04025, .02406, .06749, .07507, .01929, .00095, .05987, .06327, .09056, .02758, .00978, .02360, .00150, .01974, .0007])
# challenge 1.1
def hex_to_b64(s):
return s.decode("hex").encode("base64").strip() # has a trailing newline
def xor(a, b):
return chr(ord(a) ^ ord(b))
# challenge 1.2
def fixed_xor(a, b):
return ''.join([xor(i, j) for i, j in zip(a, b)])
def score_english(s):
s = s.lower() # compare everything case insensitive
test_values = narray(relative_frequency(s))
score = norm(LETTER_FREQUENCIES - test_values)
num_non_english = [x not in string.printable for x in s].count(True)
return score * exp(num_non_english / float(len(s)))
def _itemsize(self):
return narray([],dtype=self.dtype).itemsize
def __len__(self):
return int(path.getsize(self.name)/narray([],self.dtype).itemsize)
def __init__(self, size):
self.item = narray(-1, dtype=object)
for i in range(size - 1):
self.item = np.insert(self.item, 1, -1)
def _read_lag(self, other, sideband):
bram_real = self.bram_format.format(other=other, sideband=sideband, type='real')
bram_imag = self.bram_format.format(other=other, sideband=sideband, type='imag')
real = self.bee2.bramread(bram_real, self._lags)
imag = self.bee2.bramread(bram_imag, self._lags)
return narray([real[i]+imag[i]*1j for i in range(self._lags)])
def squareandhinton(correlationlist,name): lengthint = ceil(sqrt(len(correlationlist))) array = narray(splitarr(correlationlist,lengthint)) hinton(array,len(correlationlist),name)
