def fade( from_color, to_color, steps ): from_color = narray(from_color) to_color = narray(to_color) diff = (to_color-from_color)/float(steps) for n in range(steps): color = from_color + n*diff yield color.round().astype(int).tolist()
def read(self,file): iclose=False try: # If file is not open, open it as a fileobject fo=open(file,'rb') iclose=True except: # If already a fileobject, we will append to it fo=file ti=array('i') ti.read(fo,2) self.nboot=ti[0] self.ndim=ti[1] self.values.resize(self.nboot,self.ndim) tf=array('d') tf.read(fo,self.ndim) self.Avg=narray(tf) tf=array('d') tf.read(fo,self.ndim) self.Std=narray(tf) for iboot in xrange(self.nboot): tf=array('d') tf.read(fo,self.ndim) self.values[iboot]=narray(tf) # If file was a new file, close it if iclose: fo.close()
def _blob(x,y,area,colour): """ Draws a square-shaped blob with the given area (< 1) at the given coordinates. """ hs = sqrt(area) / 2 xcorners = narray([x - hs, x + hs, x + hs, x - hs]) ycorners = narray([y - hs, y - hs, y + hs, y + hs]) P.fill(xcorners, ycorners, colour, edgecolor=colour)
def load_test_data(self, test_data): # Validate dataset dimensions if(len(test_data[0]) == self.data_point_size): for data_point in test_data: assert(len(data_point) == self.data_point_size) self.test_data = narray(test_data) else: # Extra step for converting binary data into one-hot encoding for tf for data_point in test_data: assert(len(data_point) == (self.data_point_size-1)) self.test_data = narray(self.binary_2_one_hot(test_data))
def process_chunk(self, chunk): chunk = chunk.replace({"subcate": self.l2table}) text = chunk["desc"] label = chunk["subcate"] np.random.seed(17) indices = np.random.permutation(np.arange(len(text))) text = text[indices].tolist() label = label[indices].tolist() text = np.narray(text) label = np.narray(label) return text, label
def tolist(self, fill_value=None): """ Return the data portion of the array as a list. Data items are converted to the nearest compatible Python type. Masked values are converted to fill_value. If fill_value is None, the corresponding entries in the output list will be ``None``. """ if fill_value is not None: return self.filled(fill_value).tolist() result = narray(self.filled().tolist(), dtype=object) mask = narray(self._mask.tolist()) result[mask] = None return result.tolist()
def __array_finalize__(self, obj): # Make sure we have a _fieldmask by default .. _fieldmask = getattr(obj, '_fieldmask', None) if _fieldmask is None: mdescr = [(n, '|b1') for (n, _) in self.dtype.descr] _mask = getattr(obj, '_mask', nomask) if _mask is nomask: _fieldmask = np.empty(self.shape, dtype=mdescr).view(recarray) _fieldmask.flat = tuple([False] * len(mdescr)) else: _fieldmask = narray([tuple([m] * len(mdescr)) for m in _mask], dtype=mdescr).view(recarray) # Update some of the attributes if obj is not None: _baseclass = getattr(obj, '_baseclass', type(obj)) else: _baseclass = recarray attrdict = dict(_fieldmask=_fieldmask, _hardmask=getattr(obj, '_hardmask', False), _fill_value=getattr(obj, '_fill_value', None), _sharedmask=getattr(obj, '_sharedmask', False), _baseclass=_baseclass) self.__dict__.update(attrdict) # Finalize as a regular maskedarray ..... # Update special attributes ... self._basedict = getattr(obj, '_basedict', getattr(obj, '__dict__', {})) self.__dict__.update(self._basedict) return
def __array_finalize__(self,obj): # Make sure we have a _fieldmask by default .. _fieldmask = getattr(obj, '_fieldmask', None) if _fieldmask is None: mdescr = _make_mask_dtype(ndarray.__getattribute__(self, 'dtype')) _mask = getattr(obj, '_mask', nomask) if _mask is nomask: _fieldmask = np.empty(self.shape, dtype=mdescr).view(recarray) _fieldmask.flat = tuple([False]*len(mdescr)) else: _fieldmask = narray([tuple([m]*len(mdescr)) for m in _mask], dtype=mdescr).view(recarray) # Update some of the attributes if obj is not None: _baseclass = getattr(obj,'_baseclass',type(obj)) else: _baseclass = recarray attrdict = dict(_fieldmask=_fieldmask, _hardmask=getattr(obj,'_hardmask',False), _fill_value=getattr(obj,'_fill_value',None), _sharedmask=getattr(obj,'_sharedmask',False), _baseclass=_baseclass) self.__dict__.update(attrdict) # Finalize as a regular maskedarray ..... # Update special attributes ... self._basedict = getattr(obj, '_basedict', getattr(obj,'__dict__',{})) self.__dict__.update(self._basedict) return
def mquantiles(data, prob=list([.25,.5,.75]), alphap=.4, betap=.4, axis=None): """Computes empirical quantiles for a *1xN* data array. Samples quantile are defined by: *Q(p) = (1-g).x[i] +g.x[i+1]* where *x[j]* is the jth order statistic, with *i = (floor(n*p+m))*, *m=alpha+p*(1-alpha-beta)* and *g = n*p + m - i)*. Typical values of (alpha,beta) are: - (0,1) : *p(k) = k/n* : linear interpolation of cdf (R, type 4) - (.5,.5) : *p(k) = (k+1/2.)/n* : piecewise linear function (R, type 5) - (0,0) : *p(k) = k/(n+1)* : (R type 6) - (1,1) : *p(k) = (k-1)/(n-1)*. In this case, p(k) = mode[F(x[k])]. That's R default (R type 7) - (1/3,1/3): *p(k) = (k-1/3)/(n+1/3)*. Then p(k) ~ median[F(x[k])]. The resulting quantile estimates are approximately median-unbiased regardless of the distribution of x. (R type 8) - (3/8,3/8): *p(k) = (k-3/8)/(n+1/4)*. Blom. The resulting quantile estimates are approximately unbiased if x is normally distributed (R type 9) - (.4,.4) : approximately quantile unbiased (Cunnane) - (.35,.35): APL, used with PWM :Parameters: x : Sequence Input data, as a sequence or array of dimension at most 2. prob : Sequence *[(0.25, 0.5, 0.75)]* List of quantiles to compute. alpha : Float (*[0.4]*) Plotting positions parameter. beta : Float (*[0.4]*) Plotting positions parameter. axis : Integer *[None]* Axis along which to compute quantiles. If *None*, uses the whole (flattened/compressed) dataset. """ def _quantiles1D(data,m,p): x = numpy.sort(data.compressed()) n = len(x) if n == 0: return masked_array(numpy.empty(len(p), dtype=float_), mask=True) elif n == 1: return masked_array(numpy.resize(x, p.shape), mask=nomask) aleph = (n*p + m) k = numpy.floor(aleph.clip(1, n-1)).astype(int_) gamma = (aleph-k).clip(0,1) return (1.-gamma)*x[(k-1).tolist()] + gamma*x[k.tolist()] # Initialization & checks --------- data = masked_array(data, copy=False) p = narray(prob, copy=False, ndmin=1) m = alphap + p*(1.-alphap-betap) # Computes quantiles along axis (or globally) if (axis is None): return _quantiles1D(data, m, p) else: assert data.ndim <= 2, "Array should be 2D at most !" return apply_along_axis(_quantiles1D, axis, data, m, p)
def cov(x, y=None, rowvar=True, bias=False, strict=False): """ Estimate the covariance matrix. If x is a vector, return the variance. For matrices, returns the covariance matrix. If y is given, it is treated as an additional (set of) variable(s). Normalization is by (N-1) where N is the number of observations (unbiased estimate). If bias is True then normalization is by N. If rowvar is non-zero (default), then each row is a variable with observations in the columns, otherwise each column is a variable and the observations are in the rows. If strict is True, masked values are propagated: if a masked value appears in a row or column, the whole row or column is considered masked. """ X = narray(x, ndmin=2, subok=True, dtype=float) if X.shape[0] == 1: rowvar = True if rowvar: axis = 0 tup = (slice(None),None) else: axis = 1 tup = (None, slice(None)) # if y is not None: y = narray(y, copy=False, ndmin=2, subok=True, dtype=float) X = concatenate((X,y),axis) # X -= X.mean(axis=1-axis)[tup] n = X.count(1-axis) # if bias: fact = n*1.0 else: fact = n-1.0 # if not rowvar: return (dot(X.T, X.conj(), strict=False) / fact).squeeze() else: return (dot(X, X.T.conj(), strict=False) / fact).squeeze()
def load_data(path): data = [] seq_data = [] target = [] with open(path) as f: for line in f.readlines(): lines = line.strip().split() data_tmp = [lines[0], lines[1], lines[2]] label_tmp = lines[4] [seq_tmp.append(1) for _ in range(len(data_tmp))] # 表示真实的数据存在,防止rnn 发生梯度消失 data.append(data_tmp) target.append(label_tmp) data = np.narray(data) target = np.narray(target) seq_data = np.narray(seq_data) return data, target, seq_data
def AddValue(self, value): """supply one more numerical value""" if (self.previous_index == self.size - 1 and not (self.buffer_full)): self.buffer_full = True self.previous_index = (self.previous_index + 1) % self.size self.buf[self.previous_index * 2 + 1] = value self.vbo[self.previous_index * 2 + 1:self.previous_index * 2 + 2] = narray([value], 'f') self.maxValue = max(self.maxValue, value)
def initLayers(self, layers): self.layers = [] for i in range(len(layers) - 1): layer = np.narray(layers[i], layers[i + 1]) #TODO randomly initialize the layer for j in range(layers[j]): for k in range(layers[j + 1]): layer[j][k] = self.randInit(0.12) self.layers.append(layer)
def write(self,file): iclose=False try: # If file is not open, open it as a fileobject fo=open(file,'wb') iclose=True except: # If already a fileobject, we will append to it fo=file narray(self.nboot).tofile(fo) narray(self.ndim).tofile(fo) self.Avg.tofile(fo) self.Std.tofile(fo) self.values.tofile(fo) # If file was a new file, close it if iclose: fo.close()
def train(self,train_data,train_data_index,valid_data_index,valid_data,train_label,valid_label,early_stopping=True): ''' train ''' shuffle_indices = np.random.permutation(np.arange(len(train_data))) for epoch in self.epochs: train_prob_list = [] total_batch = len(train_label)/self.batch_size for ind in range(total_batch): start = ind * self.batch_size end = (ind+1)* self.batch_size end = end if end < len(train_label) else len(train_label) train_batch_index = train_data_index[shuffle_indices[start:end],:] train_batch_data = train_data[shuffle_indices[start:end],] train_batch_label = train_label[shuffle_indices[start:end],:] feed_dict = {self.feat_index: train_batch_index, self.feat_value : train_batch_data , self.label : train_batch_label } tf.sess.run(optimizer,feed_dict = feed_dict) # 希望从网络中得到的数据,填写在 run()中 ,训练 optimizer # train_loss, train_ks etc. block = 10000 for j in range(len(train_label)/block): start = j*block end = (j+1)*block if (j+1)*block < len(train_label) else len(train_label) # run(loss,prob) train_loss, prob = self.sess.run([self.loss,self.prob ], \ feed_dict = {self.feat_index: train_data_index[start:end],\ self.feat_value : train_data[start:end] , \ self.label : train_label[start:end] }) for i in range(end-start): train_prob_list.append(prob[i,0]) train_ks = cal_ks(train_label,np.narray(train_prob_list)) wf.write(" block: %d , train dataset : loss=%f , ks=%f " % (j, train_loss, train_ks)) eval_prob_list = [] total_batch = len(valid_label)/ block for j in range(total_batch): start = ind * block end = (ind+1)* block end = end if end < len(valid_label) else len(valid_label) # run(loss,prob) valid_loss, prob = self.sess.run([self.loss,self.prob ], \ feed_dict = {self.feat_index: valid_data_index[start:end],\ self.feat_value : valid_data[start:end] , \ self.label : valid_label[start:end] }) for i in range(end-start): eval_prob_list.append(prob[i,0]) valid_ks = cal_ks(valid_ks,np.narray(eval_prob_list)) wf.write(" block: %d , train dataset : loss=%f , ks=%f " % (j, train_loss, train_ks))
def AddValue(self, value): """supply one more numerical value""" if(self.previous_index == self.size - 1 and not(self.buffer_full)): self.buffer_full = True self.previous_index = (self.previous_index + 1) % self.size self.buf[self.previous_index * 2 + 1] = value self.vbo[self.previous_index * 2 + 1: self.previous_index * 2 + 2] = narray([value], 'f') self.maxValue = max(self.maxValue, value)
def _UpdateObject(self, _id, obj): """Updates an object in the VBO""" if self._vbo is None: return index = self._indices.get(_id, -1) # have to check since object could have been deleted since marked as # dirty if index < 0: return num_values = type(self).__num_values self._vbo[index * num_values : (index + 1) * num_values] = narray(self.__descToArray(obj), "f")
def load_dataset(self, dataset_obj): # Load training and testing data from dataset object if(len(dataset_obj.train_data[0]) == self.data_point_size): # Validate dataset dimensions for data_point in dataset_obj.train_data: assert(len(data_point) == self.data_point_size) for data_point in dataset_obj.test_data: assert(len(data_point) == self.data_point_size) # Load dataset self.train_data = narray(dataset_obj.train_data) self.test_data = narray(dataset_obj.test_data) else: # Extra step for converting binary data into one-hot encoding for tf for data_point in dataset_obj.train_data: assert(len(data_point) == (self.data_point_size-1)) for data_point in dataset_obj.test_data: assert(len(data_point) == (self.data_point_size-1)) # Load dataset self.train_data = narray(self.binary_2_one_hot(dataset_obj.train_data)) self.test_data = narray(self.binary_2_one_hot(dataset_obj.test_data))
def plot_values(teamsdict, teamslist, vals): vals = narray(vals).T v = [] for team in teamslist: if team in teamsdict: v.append(vals[teamsdict[team]]) v = zip(*v) plt(v[0], v[1], 'rx') show()
def _UpdateObject(self, _id, obj): """Updates an object in the VBO""" if self._vbo is None: return index = self._indices.get(_id, -1) # have to check since object could have been deleted since marked as # dirty if index < 0: return num_values = type(self).__num_values self._vbo[index * num_values:(index + 1) * num_values] = narray( self.__descToArray(obj), "f")
def hinton(W, length, name, maxWeight=None): """ Draws a Hinton diagram for visualizing a weight matrix. Temporarily disables matplotlib interactive mode if it is on, otherwise this takes forever. """ reenable = False if P.isinteractive(): P.ioff() P.clf() height, width = W.shape if not maxWeight: #maxWeight = 2**N.ceil(N.log(N.max(N.abs(W)))/N.log(2)) maxWeight = max(abs(W))*1.2 P.fill(narray([0,width,width,0]),narray([0,0,height,height]),'gray') P.axis('off') P.axis('equal') correlationcounter = 0 for x in xrange(width): for y in xrange(height): _x = x+1 _y = y+1 w = W[y,x] if w > 0: _blob(_x - 0.5, height - _y + 0.5, min(1,w/maxWeight*w/maxWeight),'white') correlationcounter += w - 0.5 elif w < 0: _blob(_x - 0.5, height - _y + 0.5, min(1,w/maxWeight*w/maxWeight),'black') correlationcounter += w + 0.5 correlationindex = abs(correlationcounter/length*100 + 50) titlestring = 'My votes versus ' + name + ': ' + str(correlationindex) + '% agreement' P.title(titlestring) if reenable: P.ion() P.show()
def __getitem__(self, indx): """Returns all the fields sharing the same fieldname base. The fieldname base is either `_data` or `_mask`.""" _localdict = self.__dict__ _fieldmask = _localdict['_fieldmask'] _data = self._data # We want a field ........ if isinstance(indx, basestring): obj = _data[indx].view(MaskedArray) obj._set_mask(_fieldmask[indx]) # Force to nomask if the mask is empty if not obj._mask.any(): obj._mask = nomask # Force to masked if the mask is True if not obj.ndim and obj._mask: return masked return obj # We want some elements .. # First, the data ........ obj = narray(_data[indx], copy=False).view(mrecarray) obj._fieldmask = narray(_fieldmask[indx], copy=False).view(recarray) return obj
def correlate(self): """ inst.correlate() -> None This must be overloaded to populate the '_correlations' member dictionary with valid correlation functions for every included (i.e. tracked) baseline. This should be a numpy array. """ with RLock(): itime = self.server._integration_time self._stopevent.wait(itime) self._last_correlation = time() for baseline in self._include_baselines: self._correlations[baseline] = narray([0]*self._lags*2)
def Import(self,data,bin=1,weights=[],randlist = []): if len(weights)>0 and bin!=1: print "Weights only currently supported for binsize=1" assert 1==0 self.nconf=len(data)/bin if bin>1: tmpdata=narray([average(ien) for ien in split(narray(data[0:bin*(len(data)/bin)]),self.nconf)]) else: tmpdata=narray(data) # self.Raw=append(self.Raw,tmpdata) # self.nconf=len(self.Raw) # rint=zeros( (self.nboot,self.nconf)) # if randlist == []: myseed=startseed*self.nconf/self.nboot random.seed(myseed) # locranint=random.randint locranint=random.random_integers # else: # locranint = randlist if len(weights)>0: tmpweight=narray(weights) self.Avg=average(multiply(tmpdata,tmpweight))/sum(tmpweight) else: self.Avg=average(tmpdata) for iboot in range(self.nboot): rint=locranint(0,self.nconf-1,self.nconf) # tmp=0.0 # for iconf in range(self.nconf): # rint=locranint(0,self.nconf-1) # tmp+=tmpdata[rint]/self.nconf # self.values[iboot]=tmp if len(weights)>0: tw2=tmpweight[rint] td2=multiply(tmpdata[rint],tw2) self.values[iboot]=average(td2)/sum(tw2) #average(multiply(tmpdata[rint],tmpweight[rint]))/sum(tmpweight[rint]) else: self.values[iboot]=average(tmpdata[rint]) return locranint
def dict_to_numpy(csv_data, columns_to_exclude = []): target = [] data = [] attributes = None for row in csv_data: target.append(row.pop('quality')) [row.pop(col) for col in columns_to_exclude] if attributes is None: attributes = row.keys() data.append(row.values()) target = narray(target) data = narray(data) return { 'target': target, 'data': data, 'attributes': attributes }
def MakePalette(): "Generate a palette suitable for PIL.Image.ImageData.putpalette" return flatten((i,i,i) for i in xrange(256)) # The below code mimicks the source from the original # If you want to use this, you need numpy. "easy_install numpy" or whatever # for your distribution from numpy import array as narray # Some possible destination colours (sepia, or blueish, etc) #dst = narray([18,74,155]) #dst = narray([155,74,18]) #dst = narray([17,51,96]) dst = narray([96,51,17]) high = narray([255,255,255]) firsthalf = [high + (dst-high)/255*i for i in xrange(0,256,2)] secondhalf = [dst + (-dst)/255*i for i in xrange(0,256,2)] result = flatten(firsthalf+secondhalf) result = map(int, result) return result
def Plot4D( x, y, z, t, markerstyle = 20, markersize = 1 ): data = narray( [0.] * 4 ) tree = TTree('DummyTree','DummyTree') tree.Branch('xyzt', data, 'x/D:y:z:t') for datai in zip(x,y,z,t): data[0], data[1], data[2], data[3] = datai #print data tree.Fill() tree.SetMarkerStyle( markerstyle ) tree.SetMarkerSize( markersize ) c = TCanvas() tree.Draw('x:y:z:t','','zcol') return c, tree
def __init__(self, size): self.size = size initialBuffer = [] for x, y in enumerate(range(size)): initialBuffer.append(x) initialBuffer.append(y) self.buf = narray(initialBuffer, 'f') self.vbo = vbo.VBO(self.buf, usage=GL_STREAM_DRAW, target=GL_ARRAY_BUFFER, size=4 * 2 * size) self.previous_index = -1 self.buffer_full = False self.maxValue = 0
def Plot4D(x, y, z, t, markerstyle=20, markersize=1): data = narray([0.] * 4) tree = TTree('DummyTree', 'DummyTree') tree.Branch('xyzt', data, 'x/D:y:z:t') for datai in zip(x, y, z, t): data[0], data[1], data[2], data[3] = datai #print data tree.Fill() tree.SetMarkerStyle(markerstyle) tree.SetMarkerSize(markersize) c = TCanvas() tree.Draw('x:y:z:t', '', 'zcol') return c, tree
def _AddObject(self, _id, obj): """Adds an object to the VBO""" if self._vbo is None: return index = -1 if len(self._empty_indices) > 0: index = self._empty_indices.pop() elif self._max_index < self._data_size - 1: self._max_index += 1 index = self._max_index if index > 0: values = self.__descToArray(obj) num_values = len(values) self._vbo[index * num_values : (index + 1) * num_values] = narray(values, "f") self._indices[_id] = index else: self._vbo = None
def _AddObject(self, _id, obj): """Adds an object to the VBO""" if self._vbo is None: return index = -1 if len(self._empty_indices) > 0: index = self._empty_indices.pop() elif self._max_index < self._data_size - 1: self._max_index += 1 index = self._max_index if index > 0: values = self.__descToArray(obj) num_values = len(values) self._vbo[index * num_values:(index + 1) * num_values] = narray( values, "f") self._indices[_id] = index else: self._vbo = None
def __array_finalize__(self, obj): # Make sure we have a _fieldmask by default .. _mask = getattr(obj, "_mask", None) if _mask is None: objmask = getattr(obj, "_mask", nomask) _dtype = ndarray.__getattribute__(self, "dtype") if objmask is nomask: _mask = ma.make_mask_none(self.shape, dtype=_dtype) else: mdescr = ma.make_mask_descr(_dtype) _mask = narray([tuple([m] * len(mdescr)) for m in objmask], dtype=mdescr).view(recarray) # Update some of the attributes _dict = self.__dict__ _dict.update(_mask=_mask, _fieldmask=_mask) self._update_from(obj) if _dict["_baseclass"] == ndarray: _dict["_baseclass"] = recarray return
def __array_finalize__(self, obj): # Make sure we have a _fieldmask by default _mask = getattr(obj, '_mask', None) if _mask is None: objmask = getattr(obj, '_mask', nomask) _dtype = ndarray.__getattribute__(self, 'dtype') if objmask is nomask: _mask = ma.make_mask_none(self.shape, dtype=_dtype) else: mdescr = ma.make_mask_descr(_dtype) _mask = narray([tuple([m] * len(mdescr)) for m in objmask], dtype=mdescr).view(recarray) # Update some of the attributes _dict = self.__dict__ _dict.update(_mask=_mask) self._update_from(obj) if _dict['_baseclass'] == ndarray: _dict['_baseclass'] = recarray return
def __getattribute__(self, attr): "Returns the given attribute." try: # Returns a generic attribute return object.__getattribute__(self,attr) except AttributeError: # OK, so attr must be a field name pass # Get the list of fields ...... _names = self.dtype.names if attr in _names: _data = self._data _mask = self._fieldmask # obj = masked_array(_data.__getattribute__(attr), copy=False, # mask=_mask.__getattribute__(attr)) # Use a view in order to avoid the copy of the mask in MaskedArray.__new__ obj = narray(_data.__getattribute__(attr), copy=False).view(MaskedArray) obj._mask = _mask.__getattribute__(attr) if not obj.ndim and obj._mask: return masked return obj raise AttributeError,"No attribute '%s' !" % attr
def read(self,file): iclose=False try: # If file is not open, open it as a fileobject fo=open(file,'rb') iclose=True except: # If already a fileobject, we will append to it fo=file ti=array('i') ti.read(fo,1) self.nboot=ti[0] tf=array('d') tf.read(fo,2) self.Avg=tf[0] self.Std=tf[1] tf=array('d') tf.read(fo,self.nboot) self.values=narray(tf) # If file was a new file, close it if iclose: fo.close()
def __getattribute__(self, attr): "Returns the given attribute." try: # Returns a generic attribute return object.__getattribute__(self, attr) except AttributeError: # OK, so attr must be a field name pass # Get the list of fields ...... _names = self.dtype.names if attr in _names: _data = self._data _mask = self._fieldmask # obj = masked_array(_data.__getattribute__(attr), copy=False, # mask=_mask.__getattribute__(attr)) # Use a view in order to avoid the copy of the mask in MaskedArray.__new__ obj = narray(_data.__getattribute__(attr), copy=False).view(MaskedArray) obj._mask = _mask.__getattribute__(attr) if not obj.ndim and obj._mask: return masked return obj raise AttributeError, "No attribute '%s' !" % attr
# for now, define your own puzzle: ''' grid = narray.[[0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0]] ''' size = 9 # 9 by 9 game by default but changeable game = narray([[0, 0, 0, 0, 0, 0, 0, 3, 1], [7, 0, 0, 9, 0, 0, 0, 4, 0], [0, 0, 0, 0, 0, 0, 5, 0, 0], [0, 0, 0, 0, 3, 1, 0, 0, 0], [9, 0, 0, 0, 0, 0, 0, 0, 8], [0, 0, 0, 4, 0, 5, 0, 0, 0], [0, 0, 0, 6, 0, 0, 2, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 6], [0, 5, 3, 0, 0, 0, 0, 0, 0]]) # (y, x) = (r, c) # matrix index c = coordinate x = column # matrix index r = coordinate y = row def possible(r, c, num): # given a number num, check if it's possible to put it in (r, c) global game global size zone = int(sqrt(size)) for i in range(0, size): # check within row if game[r][i] == num: return False
def addRegisteredBlocks(blocks): to = time() result = sum([narray(block)**2 for block in blocks]) print "Time needed for adding blocks:", time() - to return result
def _get_bytes_size(name, batch): _validate_batch(name, batch) if not isinstance(batch, np.ndarray): batch = np.narray(batch) return batch.nbytes
def do3(app_nums: Iterable=(5,), house_n: int=1, no_save: bool=False, no_load: bool=True, ): ''' ''' global D1 if not 'D1' in dir(): # if `D1` has not defined D1 = DataSet(r'D:\dataset\REFIT\refit.h5') p1 = PKMap(D1.buildings[house_n], no_count=True, no_load=no_load) # no_old = OrderedDict.fromkeys(app_nums) # not get loaded data is_old = {k:False for k in app_nums} # not get loaded data bm0 = {k:None for k in app_nums} sbm0 = {k:None for k in app_nums} BMs = {k:[] for k in app_nums} D0s = {k:[] for k in app_nums} D1s = {k:[] for k in app_nums} dates = [] # re-use data by caching cache_dir = os.path.join(os.getcwd(), 'BMcache') if not os.path.exists(cache_dir): os.mkdir(cache_dir) if not no_save: cache_name0 = '_'.join(['BMcache', p1.dataset, p1.house_name, '']) for an in app_nums: cache_name = ''.join([cache_name0, p1.ins_name['active'][an-1], '.csv']) for dirp, dirn, files in os.walk(cache_dir): # print('dirpath is {}'.format(dirp)) # print('dirname is {}'.format(dirn)) # print('have {} files'.format(len(files))) files_l = narray([x.lower() for x in files]) cache_l = cache_name.lower() if cache_l in files_l: cache_name2 = narray(files)[files_l == cache_l][0] with open(os.path.join(cache_dir, cache_name2), 'r') as f: data_old = read_csv(f, index_col=0, header=0, names=('D0', 'D1', 'bm'), parse_dates=True, ) D0s[an] = data_old.D0 D1s[an] = data_old.D1 BMs[an] = data_old.bm is_old[an] = True break else: is_old[an] = False print('cache_name is {}'.format(cache_name), ' '*16) else: cache_name0 = "" # init print('initializing ...', ' '*64, end='\r') if not isinstance(app_nums, Iterable): app_nums = (app_nums, ) with NoPrints(): for an in app_nums: # an = int(an) bm0[an], sbm0[an] = p1.BM(obj=an, sel_ac='active', no_plot=True) p1.appQ['dd'] = 9 p1.app_name['dd'] = p1.app_name['active'] p1.ins_name['dd'] = p1.ins_name['active'] datas = [] d0 = p1.data0['active'] # if not any([is_old[an] for an in app_nums]): if all([is_old[an] for an in app_nums]): print('{0} all `{1}` App exist! {0}'.format('='*6, len(app_nums))) years = () else: print('app {} not exist'.format(', '.join([str(an) for an in app_nums if not is_old[an]]))) years = set(d0.index.year) sleep(1) t0 = time() app_rems = [an for an in app_nums if not is_old[an]] for _year in years: # for _year in (2014, ): print('\r\t\tcutting ... ', ' '*8, end='\r') # the hash of int is neurally in order d0_y = d0.loc[d0.index.year==_year] months = set(d0_y.index.month) for _month in months: # for _month in (9, ): d0_m = d0_y.loc[d0_y.index.month==_month] days = set(d0_m.index.day) days = (1, ) for _day in days: # for _day in range(5, 11): # d0_x = d0_m.loc[d0_m.index.day==_day] d0_x = d0_m.copy() # dates.append('-'.join(narray([_year, _month, _day], dtype=str))) dates.append('-'.join(narray([_year, _month], dtype=str))) # set `end` to steady the cursor print('\r\t\treading {}'.format(dates[-1]), end='') # if d0_x.index[0].hour < 1 and d0_x.index[-1].hour > 22: if True: datas.append(d0_x) with NoPrints(): p1.data0['dd'] = d0_x.copy() p1.data2['dd'] = gen_PKMap(p1, data0=d0_x, key='dd') for an in app_rems: bm, sbm = p1.BM(obj=an, sel_ac='dd', no_plot=False, no_show=True, fig_types=('ty.jpg',)) BMs[an].append(bm) D1s[an].append(Hellinger(sbm, sbm0[an])) D0s[an].append(Hellinger(sbm)) else: # imcomplete day, discard d0id0, d0id1 = d0_x.index[0], d0_x.index[-1] print('\tfind invalid time: {}, {:0>2}:{:0>2} to {:0>2}:{:0>2}'.format( d0id0.date(), d0id0.hour, d0id0.minute, d0id1.hour, d0id1.minute ), end='') del d0_y, d0_m, d0_x # ring to alart the finishing if len(app_rems) < 2: print('\r\a `{}` app in {} of {} days cost {} '.format( len(app_rems), len(datas), len(set(d0.index.date)), beauty_time(time()-t0)+' '*128)) else: print('\r\a `{}` apps in {} of {} days cost {} '.format( len(app_rems), len(datas), len(set(d0.index.date)), beauty_time(time()-t0)+' '*128)) if datas: d1 = pd.concat(datas, ) # plt.plot(d1.index, range(len(d1.index))) # plt.show() # sleep(1e-15) for an in app_nums: fig = plt.figure(figsize=(6, 5)) ax = fig.add_subplot(121) vio1 = ax.violinplot([D0s[an], D1s[an]], ) h0 = Hellinger(sbm0[an]) xlim = ax.get_xlim() print(xlim) ax.plot(xlim, [h0, h0], color='cornflowerblue') ax.plot(xlim, [0, 0], 'k') ax.set_xlim(xlim) ax.set_xticks([1,2]) ax.set_xticklabels(['$D_0$', '$D_1$']) ylim = ax.get_ylim() tick = [k for k in ax.get_yticks() if k>ylim[0] and k<ylim[1]] ax.set_yticks(nappend(tick, h0)) ntick = ['{:.1f}'.format(k) for k in tick] ax.set_yticklabels(nappend(ntick, r'$D_{pH}$'+' is{0}\n{1:.2f}{0}'.format(' '*9, h0))) plt.grid() ax2 = fig.add_subplot(122) vio2 = ax2.violinplot(BMs[an], ) # vio2['bodies'][0].set_facecolor('r') # vio2['bodies'][0].set_edgecolor('violet') xlim2 = ax2.get_xlim() print(xlim2) ax2.plot(xlim2, [bm0[an], bm0[an]], 'b') ax2.set_xlim(xlim2) ylim2 = ax2.get_ylim() ax2.set_ylim(top=1.06, bottom=0) ax2.yaxis.tick_right() ax2.set_xticks([1]) ax2.set_xticklabels(['BM']) tick = [k for k in ax2.get_yticks() if k < 1.1] ax2.set_yticks(nappend(tick, bm0[an])) print(ax2.get_yticks()) ntick = ['{:.1f}'.format(k) for k in tick] ax2.set_yticklabels(nappend(ntick, '{0}BM is\n{0}{1:.3f}'.format(' '*7, bm0[an]))) plt.grid() plt.savefig('./figs/D(bm)_{}_{}_A{}_{}.png'.format( p1.dataset, p1.house_name, an, p1.ins_name['active'][an-1] ), bbox_inches='tight') # plt.show() # save data t0 = time() if not no_save and not is_old[an]: cache_name = ''.join([cache_name0, p1.ins_name['active'][an-1], '.csv']) print('='*6, 'saving into `{}`'.format(cache_name), '='*6, end='\r') with open(os.path.join(cache_dir, cache_name), 'w') as f: f.write('date, D0, D1, BM') for _date, _d0, _d1, _bm in zip(dates, D0s[an], D1s[an], BMs[an]): f.write('\n') f.write(','.join(narray([_date, _d0, _d1, _bm], dtype=str))) print('='*6, 'done saving'.format(beauty_time(time()-t0))) pass
fitsobj = maputils.FITSimage("m101.fits") fitsobj.set_limits((200,400), (200,400)) annim = fitsobj.Annotatedimage() pos = "300 300" print("Is " + pos + " inside?", annim.inside(pos=pos)) # Mode has no effect on parameter pos print("Is " + pos + " inside?", annim.inside(pos=pos, mode='p')) print("Is " + pos + " inside?", annim.inside(pos=pos, mode='w')) print("Is 300, 300 inside?", annim.inside(x=300, y=300, mode='p')) print("Is 300, 300 inside?", annim.inside(x=300, y=300, mode='w')) print("Is 300, 300, 20,200 inside?", annim.inside(x=[300,20], y=[300,200], mode='p')) x = narray([100,200,300,20]) y = narray([100,200,270,0]) print("Two numpy arrays inside?", annim.inside(x=x, y=y, mode='p')) crval1 = annim.projection.crval[0] crval2 = annim.projection.crval[1] print("Is %f %f" % (crval1, crval2) + " inside?", annim.inside(x=crval1, y=crval2, mode='w')) pos = '{} ' + str(crval1) + ' {} ' + str(crval2) print("Is " + pos + " inside?", annim.inside(pos=pos)) # Raise an exception try:
fitsobj = maputils.FITSimage("m101.fits") fitsobj.set_limits((200, 400), (200, 400)) annim = fitsobj.Annotatedimage() pos = "300 300" print("Is " + pos + " inside?", annim.inside(pos=pos)) # Mode has no effect on parameter pos print("Is " + pos + " inside?", annim.inside(pos=pos, mode='p')) print("Is " + pos + " inside?", annim.inside(pos=pos, mode='w')) print("Is 300, 300 inside?", annim.inside(x=300, y=300, mode='p')) print("Is 300, 300 inside?", annim.inside(x=300, y=300, mode='w')) print("Is 300, 300, 20,200 inside?", annim.inside(x=[300, 20], y=[300, 200], mode='p')) x = narray([100, 200, 300, 20]) y = narray([100, 200, 270, 0]) print("Two numpy arrays inside?", annim.inside(x=x, y=y, mode='p')) crval1 = annim.projection.crval[0] crval2 = annim.projection.crval[1] print("Is %f %f" % (crval1, crval2) + " inside?", annim.inside(x=crval1, y=crval2, mode='w')) pos = '{} ' + str(crval1) + ' {} ' + str(crval2) print("Is " + pos + " inside?", annim.inside(pos=pos)) # Raise an exception try:
def score_english(s): s = s.lower() # compare everything case insensitive test_values = narray(relative_frequency(s)) score = norm(LETTER_FREQUENCIES - test_values) num_non_english = [x not in string.printable for x in s].count(True) return score * exp(num_non_english / float(len(s)))
import string import urllib2 from collections import namedtuple from numpy import array as narray, exp from numpy.linalg import norm Result = namedtuple('Result', 'message score') SINGLE_BITS = map(chr, range(256)) # from https://en.wikipedia.org/wiki/Letter_frequency LETTER_FREQUENCIES = narray([.08167, .01492, .02782, .04253, .02702, .02228, .02015, .06094, .06966, .00153, .00772, .04025, .02406, .06749, .07507, .01929, .00095, .05987, .06327, .09056, .02758, .00978, .02360, .00150, .01974, .0007]) # challenge 1.1 def hex_to_b64(s): return s.decode("hex").encode("base64").strip() # has a trailing newline def xor(a, b): return chr(ord(a) ^ ord(b)) # challenge 1.2 def fixed_xor(a, b): return ''.join([xor(i, j) for i, j in zip(a, b)]) def score_english(s): s = s.lower() # compare everything case insensitive test_values = narray(relative_frequency(s)) score = norm(LETTER_FREQUENCIES - test_values) num_non_english = [x not in string.printable for x in s].count(True) return score * exp(num_non_english / float(len(s)))
def _itemsize(self): return narray([],dtype=self.dtype).itemsize
def __len__(self): return int(path.getsize(self.name)/narray([],self.dtype).itemsize)
def __init__(self, size): self.item = narray(-1, dtype=object) for i in range(size - 1): self.item = np.insert(self.item, 1, -1)
def _read_lag(self, other, sideband): bram_real = self.bram_format.format(other=other, sideband=sideband, type='real') bram_imag = self.bram_format.format(other=other, sideband=sideband, type='imag') real = self.bee2.bramread(bram_real, self._lags) imag = self.bee2.bramread(bram_imag, self._lags) return narray([real[i]+imag[i]*1j for i in range(self._lags)])
def squareandhinton(correlationlist,name): lengthint = ceil(sqrt(len(correlationlist))) array = narray(splitarr(correlationlist,lengthint)) hinton(array,len(correlationlist),name)