def pcprint(self): if self.doBoth==1: subplot(1,2,2) fs=12 else: fs=15 self.c1=self.data1.copy() #for copying histogram value data2=np.isfinite(self.data1) data3=self.data1[data2] nn=str(self.lineEdit_4.text()) b=nn.split(',') if len(b)==1: m=int(b[0]) n=int(b[0]) else: m=int(b[0]) n=int(b[1]) dmax=nanmax(data3) dmin=nanmin(data3) total=dmax-dmin low=dmin+(total*m)/100 high=dmax-(total*n)/100 datal = np.where(data3 < low ,low,data3) datah = np.where(datal > high ,high,datal) if self.checkBox.isChecked(): # for fix Legend self.c1[0][0]=self.maxdata self.c1[0][1]=self.mindata np.place(self.c1,data2,datah) plt.imshow(self.c1,cmap=get_cmap(self.colormap),extent=[self.originX,self.lastX,self.lastY,self.originY]) plt.text(79.2,31,self.image,fontsize=fs) plt.title('Percentile Clipped') plt.colorbar()
def remove_wrongly_sized_connected_components(a, min_size, max_size=None, in_place=False, bin_out=False): """ Copied from lazyflow.operators.opFilterLabels.py Originally adapted from http://github.com/jni/ray/blob/develop/ray/morpho.py (MIT License) """ original_dtype = a.dtype if not in_place: a = a.copy() if min_size == 0 and (max_size is None or max_size > np.prod(a.shape)): # shortcut for efficiency if (bin_out): np.place(a,a,1) return a try: component_sizes = np.bincount( a.ravel() ) except TypeError: # On 32-bit systems, must explicitly convert from uint32 to int # (This fix is just for VM testing.) component_sizes = np.bincount( np.asarray(a.ravel(), dtype=int) ) bad_sizes = component_sizes < min_size if max_size is not None: np.logical_or( bad_sizes, component_sizes > max_size, out=bad_sizes ) bad_locations = bad_sizes[a] a[bad_locations] = 0 if (bin_out): # Replace non-zero values with 1 np.place(a,a,1) return np.array(a, dtype=original_dtype)
def matrixDiscreteMaker(t): ''' this function converts the frequencies matrix t into an integer valued-matrix rules used: less than 1% or nan --> nan between 1% and 5% --> 1 between 5% and 10% --> 2 between 10% and 20% --> 3 more than 20% --> 4 ''' # nan, convert to 0 to avoid numerical warnings numpy.place(t,numpy.isnan(t),0) # between 1% and 5% --> 1 t[numpy.where(numpy.logical_and(t>=0.01, t<0.05))]=1 # between 5% and 10% --> 2 t[numpy.where(numpy.logical_and(t>=0.05, t<0.1))]=2 # between 10% and 20% --> 3 t[numpy.where(numpy.logical_and(t>=0.1, t<0.2))]=3 # more than 20% --> 4 t[numpy.where(numpy.logical_and(t>=0.2, t<1-1e-10))]=4 # less than 1% or nan --> nan, which maps as white. this line should be last in order to avoid numerical warnings numpy.place(t,t<0.01,float('nan')) return t
def sdprint(self): if self.doBoth==1: subplot(1,2,2) fs=12 else: fs=15 self.c1=self.data1.copy() #for copying histogram value data2=np.isfinite(self.data1) data3=self.data1[data2] mean=np.mean(data3) sd=np.std(data3) nn=str(self.lineEdit_4.text()) b=nn.split(',') if len(b)==1: m=int(b[0]) n=int(b[0]) else: m=int(b[0]) n=int(b[1]) low=mean-m*sd high=mean+n*sd ## print sd,mean,low,high datal = np.where(data3 < low ,low,data3) datah = np.where(datal > high ,high,datal) np.place(self.c1,data2,datah) if self.checkBox.isChecked(): # for Fix-Legend self.c1[0][0]=self.maxdata self.c1[0][1]=self.mindata plt.imshow(self.c1,cmap=get_cmap(self.colormap),extent=[self.originX,self.lastX,self.lastY,self.originY]) plt.text(79.2,31,self.image,fontsize=fs) plt.title('Standard Deviation') plt.colorbar()
def IDCTnDequantize(prop): #To dequantize img3 = prop.image iHeight, iWidth = img3.shape[:2] img2 = np.zeros((iHeight,iWidth,3), np.uint8) #print img2.dtype for startY in range(0, iHeight, 8): for startX in range(0, iWidth, 8): for c in range(0, 3): block = img3[startY:startY+8, startX:startX+8, c:c+1].reshape(8,8) blockf = np.float32(block) # float conversion dst = cv2.idct(blockf) # inverse dct np.place(dst, dst>255.0, 255.0) # saturation np.place(dst, dst<0.0, 0.0) # grounding block = np.uint8(np.around(dst)) # store the results for y in range(8): for x in range(8): img2[startY+y, startX+x, c] = block[y, x] # convert to BGR img2 = cv2.cvtColor(img2, cv2.COLOR_YCR_CB2BGR) prop.image = img2
def micro_step(self): """ Defining microphysical step """ print "qc min, max przed mikro", self.state["rc"].min(), self.state["rc"].max() print "nc min, max przed mikro", self.state["nc"].min(), self.state["nc"].max() # dot_ variables have to be zero before rhs_cellwise for k in ("dot_th_d", "dot_rv", "dot_rc", "dot_nc", "dot_rr", "dot_nr"): self.state_dot[k] *= 0. #pdb.set_trace() libcl.blk_2m.rhs_cellwise(self.opts, self.state_dot["dot_th_d"], self.state_dot["dot_rv"], self.state_dot["dot_rc"], self.state_dot["dot_nc"], self.state_dot["dot_rr"], self.state_dot["dot_nr"], self.state["rho_d"], self.state["th_d"], self.state["rv"], self.state["rc"], self.state["nc"], self.state["rr"], self.state["nr"], self.dt) print "qc min, max po mikro", self.state["rc"].min(), self.state["rc"].max() print "nc min, max po mikro", self.state["nc"].min(), self.state["nc"].max() for k in ("th_d", "rv", "rc", "nc", "rr", "nr"): self.state[k] += self.state_dot["dot_"+k] * self.dt # rc, nc can be sometimes smaller than zero -- TODO!! np.place(self.state["rc"], self.state["rc"]<0, 0) np.place(self.state["nc"], self.state["nc"]<0, 0) print "qc min, max po place", self.state["rc"].min(), self.state["rc"].max() print "nc min, max po place", self.state["nc"].min(), self.state["nc"].max()
def generate(input): """ Testing function, all parameters are in a single tuple """ seed,gen,p0,p1,n,method,MCsize= input npran.seed(seed) if gen.lower() == "ising": X = ising_X(p1+p0,n) ynor = norm_y(X,p1) elif gen.lower() == "genetic": genes = np.genfromtxt('data/SNPdata.txt', delimiter=',') np.place(genes,genes!=0,1) X = given_X(p1+p0,n,genes) ybin = bern_y(X,p1) ynor = norm_y(X,p1) # Logit bin_logit = ko.knockoff_logit(ybin,X,.2, knockoff='binary', method=method, MCsize=MCsize, intercept=True ) bin_logit.fit() ori_logit = ko.knockoff_logit(ybin,X,.2, knockoff='original', intercept=False ) ori_logit.fit() trueS = (np.arange(p0+p1)<p1).astype(int) bin_FDR = np.dot(bin_logit.S,1-trueS)/max(np.sum(bin_logit.S),1) bin_power = np.dot(bin_logit.S,trueS) /max(p1,1) ori_FDR = np.dot(ori_logit.S,1-trueS)/max(np.sum(ori_logit.S),1) ori_power = np.dot(ori_logit.S,trueS) /max(p1,1) corr = np.corrcoef(ori_logit.S,bin_logit.S)[0,1] ko_corr = [cor for cor in bin_logit.emp_ko_corr if not np.isnan(cor)] with open('data/logit_test_'+str(p0+p1)+'_w_n.txt','a') as f: f.write("%d\t%s\t%d\t%d\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\n" % (seed, gen, p1, n, bin_logit.M_distortion, np.mean(ko_corr), bin_FDR, bin_power, np.mean(ori_logit.emp_ko_corr), ori_FDR, ori_power, corr)) # LASSO bin_lasso = ko.knockoff_lasso(ynor,X,.2, knockoff='binary', method=method, MCsize=MCsize, intercept=True ) bin_lasso.fit(bin_logit.X_lrg) ori_lasso = ko.knockoff_lasso(ynor,X,.2, knockoff='original', intercept=False ) ori_lasso.fit() trueS = (np.arange(p0+p1)<p1).astype(int) bin_FDR = np.dot(bin_lasso.S,1-trueS)/max(np.sum(bin_lasso.S),1) bin_power = np.dot(bin_lasso.S,trueS) /max(p1,1) ori_FDR = np.dot(ori_lasso.S,1-trueS)/max(np.sum(ori_lasso.S),1) ori_power = np.dot(ori_lasso.S,trueS) /max(p1,1) corr = np.corrcoef(ori_lasso.S,bin_lasso.S)[0,1] with open('data/lasso_test_'+str(p0+p1)+'_w_n.txt','a') as f: f.write("%d\t%s\t%d\t%d\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\t%.5f\n" % (seed, gen, p1, n, bin_logit.M_distortion, np.mean(ko_corr), bin_FDR, bin_power, np.mean(ori_lasso.emp_ko_corr), ori_FDR, ori_power, corr))
def logsumexp_array(x, axis=None): """ Compute log(sum(exp(x))) along a particular axis for Numpy arrays. """ # This implementation hasn't been tested on arrays with dimension > 2! if axis is None: max_ent = x.max() bias = max_ent else: max_ent = x.max(axis) bias = max_ent if axis == 0 else max_ent[:, numpy.newaxis] # In the no-axis case, if -Inf is the max value, it means *all* the entries # were -Inf, so we already know what's going to happen and we can skip doing # real work. if axis is None and max_ent == LOG_ZERO: return LOG_ZERO # Otherwise, there's a bit of trickiness here - subtracting -Inf results in # Nan, so we never want to use that as a bias. mask = (max_ent == LOG_ZERO) if mask.any(): # If some rows or columns have only -Inf values, use a bias of 0 in just # those rows or cols. numpy.place(max_ent, mask, 0.0) numpy.place(bias, bias == LOG_ZERO, 0.0) return max_ent + quiet_log(numpy.sum(numpy.exp(x - bias), axis=axis))
def _lazywhere(cond, arrays, f, fillvalue=None, f2=None): """ np.where(cond, x, fillvalue) always evaluates x even where cond is False. This one only evaluates f(arr1[cond], arr2[cond], ...). For example, >>> a, b = np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]) >>> def f(a, b): return a*b >>> _lazywhere(a > 2, (a, b), f, np.nan) array([ nan, nan, 21., 32.]) Notice it assumes that all `arrays` are of the same shape, or can be broadcasted together. """ if fillvalue is None: if f2 is None: raise ValueError("One of (fillvalue, f2) must be given.") else: fillvalue = np.nan else: if f2 is not None: raise ValueError("Only one of (fillvalue, f2) can be given.") arrays = np.broadcast_arrays(*arrays) temp = tuple(np.extract(cond, arr) for arr in arrays) tcode = np.mintypecode([a.dtype.char for a in arrays]) out = _valarray(np.shape(arrays[0]), value=fillvalue, typecode=tcode) np.place(out, cond, f(*temp)) if f2 is not None: temp = tuple(np.extract(~cond, arr) for arr in arrays) np.place(out, ~cond, f2(*temp)) return out
def __init__ (self, raw, lambdas_in): self.lambdas = lambdas_in self.previous_prediction = None self.num_obs_fall = 0 self.num_obs_rise = 0 self.num_predictions = 0 self.num_correct_short_predictions = 0 self.num_correct_long_predictions = 0 self.num_uncertain_predictions = 0 self.alpha_hat = {} lambdas = self.lambdas for key in lambdas: split = lambdas[key]['split'] A = np.array (lambdas[key]['A']) B = np.array (lambdas[key]['B']) Pi = np.array (lambdas[key]['Pi']) N = Pi.shape[0] M = B.shape[1] T = len (raw) O = np.zeros (T, dtype=np.int8) np.place (O, raw >= split, 1) alpha_bar = np.multiply (Pi, B[:, O[0]]) alpha_hat = alpha_bar / alpha_bar.sum () for t in range (0, T - 1): # multiply is elementwise, dot is matrix multiplication alpha_bar = np.multiply (np.dot (alpha_hat, A), B[:, O[t+1]]) alpha_hat = alpha_bar / alpha_bar.sum () self.alpha_hat[key] = alpha_hat
def __init__(self, V_nM, S_MM, No, pwfmask, lcaoindices=None): Nw = V_nM.shape[1] self.V_oM = V_nM[:No] dtype = V_nM.dtype # Do PWF optimization for pwfbasis submatrix only! Npwf = len(pwfmask.nonzero()[0]) pwfmask2 = np.outer(pwfmask, pwfmask) s_MM = S_MM[pwfmask2].reshape(Npwf, Npwf) v_oM = self.V_oM[:, pwfmask] f_MM = s_MM - np.dot(v_oM.T.conj(), v_oM) nw = len(s_MM) assert No <= nw u_ow, u_lw, u_Ml = get_rot(f_MM, v_oM, nw - No) u_Mw = np.dot(u_Ml, u_lw) u_ow = u_ow - np.dot(v_oM, u_Mw) # Determine U for full lcao basis self.U_ow = np.zeros((No, Nw), dtype) for U_w, u_w in zip(self.U_ow, u_ow): np.place(U_w, pwfmask, u_w) self.U_Mw = np.identity(Nw, dtype) np.place(self.U_Mw, pwfmask2, u_Mw.flat) if lcaoindices is not None: for i in lcaoindices: self.U_ow[:, i] = 0.0 self.U_Mw[:, i] = 0.0 self.U_Mw[i, i] = 1.0 self.S_ww = self.rotate_matrix(np.ones(1), S_MM) self.norms_n = None
def make_mask_map(data, infile, outfile, index=None): from neurosynth.base.mask import Masker from neurosynth.base import imageutils # Load image with masker masker = Masker(infile) img = imageutils.load_imgs(infile, masker) data = list(data) if index is None: index = np.arange(0, len(data)) rev_index = None else: all_reg = np.arange(0, img.max()) rev_index = all_reg[np.invert(np.in1d(all_reg, index))] min_val = img.min() for num, value in enumerate(data): n = index[num] np.place(img, img == n + min_val, [value]) if rev_index is not None: for value in rev_index: np.place(img, img == value + min_val, 0) img = img.astype('float32') imageutils.save_img(img, outfile, masker)
def conditional_logsumexp(where, axis): masked = -np.ones(a.shape) * np.inf np.copyto(masked, a, where = where) masked_sum = logsumexp(masked, axis = axis) #np.copyto(masked_sum, -np.ones(masked_sum.shape) * np.inf, where = np.isnan(masked_sum)) np.place(masked_sum, np.isnan(masked_sum), -np.inf) return masked_sum
def factorial2(n,exact=0): """n!! = special.gamma(n/2+1)*2**((m+1)/2)/sqrt(pi) n odd = 2**(n) * n! n even If exact==0, then floating point precision is used, otherwise exact long integer is computed. Notes: - Array argument accepted only for exact=0 case. - If n<0, the return value is 0. """ if exact: if n < -1: return 0L if n <= 0: return 1L val = 1L for k in xrange(n,0,-2): val *= k return val else: from scipy import special n = asarray(n) vals = zeros(n.shape,'d') cond1 = (n % 2) & (n >= -1) cond2 = (1-(n % 2)) & (n >= -1) oddn = extract(cond1,n) evenn = extract(cond2,n) nd2o = oddn / 2.0 nd2e = evenn / 2.0 place(vals,cond1,special.gamma(nd2o+1)/sqrt(pi)*pow(2.0,nd2o+0.5)) place(vals,cond2,special.gamma(nd2e+1) * pow(2.0,nd2e)) return vals
def remove_wrongly_sized_connected_components(self, a, min_size, max_size, in_place): """ Adapted from http://github.com/jni/ray/blob/develop/ray/morpho.py (MIT License) """ bin_out = self.BinaryOut.value original_dtype = a.dtype if not in_place: a = a.copy() if min_size == 0 and (max_size is None or max_size > numpy.prod(a.shape)): # shortcut for efficiency return a try: component_sizes = numpy.bincount( a.ravel() ) except TypeError: # On 32-bit systems, must explicitly convert from uint32 to int # (This fix is just for VM testing.) component_sizes = numpy.bincount( numpy.asarray(a.ravel(), dtype=int) ) bad_sizes = component_sizes < min_size if max_size is not None: numpy.logical_or( bad_sizes, component_sizes > max_size, out=bad_sizes ) bad_locations = bad_sizes[a] a[bad_locations] = 0 if (bin_out): # Replace non-zero values with 1 numpy.place(a,a,1) return numpy.array(a, dtype=original_dtype)
def investment(positions, num_trials): '''this program accepts two inputs: positions and num_trials: positions: list of number of shares to buy in parallel num_trials: how many times to randomly repeat the test the program returns the outcomes of num_trials times''' final_ret = np.zeros((num_trials,1)) '''for every position, using the position value: 1. calculate cumulative return that day with 51% chance of doubling and 49% of losing everythin 2. calculate daily return based on cumulative return if there are 10 positions, each would be a random independent trial and the cumulative return is the sum of the outcome of the 10 positions''' for position in positions: position_value = 1000 / position win = position_value * 2 odds = np.random.rand(num_trials,position) np.place(odds, odds > 0.51, 0) np.place(odds, odds != 0, win) cumu_ret = np.sum(odds, axis = 1) daily_ret = cumu_ret / 1000 -1 daily_ret = np.expand_dims(daily_ret, axis =1) final_ret = np.concatenate((final_ret, daily_ret),axis = 1) final_ret = np.delete(final_ret,0,1) return final_ret
def __call__(self, x): x_ = np.asarray(x) if x_.size == 1: x_.shape = (1,) np.place(x_, x_ > 0, [1]) np.place(x_, x_ < 0, [-1]) return x_
def solve_tangency(R, C, rf, fit_func=lambda mean, var, rf: mean-rf/(var**.5)): """Calculates the tangency portfolio given a set of expected returns (R), covariances (C), risk-free rate (rf), and a fitness function (fit_func) which defaults to the Sharpe ratio. Returns the weights of the tangency portfolio. """ n = len(R) #Begin with equal weights W = np.ones([n])/n # Replace expected returns that are less than rf with rf + a super small value # since if it's less than rf, the sharpe ratio is negative, which ruins minimization np.place(R, R<rf, rf+0.00001) # Set boundaries on weights - no shorting or leverage allowed. Can probably incorporate # this functionality easily, though. bounds = [(0., 1.) for i in xrange(n)] # Set constraints as defined in SciPy's documentation for minimize. 'fun' forces the weights to sum to 1 constraints = ({'type':'eq', 'fun': lambda W: sum(W)-1.}) # Minimize fitness by changing W with (R, C, rf, fit_func) as given using the SLQSP method # based on the above defined constraints and bounds tangency = spo.minimize(fun=fitness, x0=W, args=(R, C, rf, fit_func), method='SLSQP', constraints=constraints, bounds=bounds) if not tangency.success: raise BaseException(tangency.message) return tangency.x
def get_normalized_data(): print("Reading in and transforming data...") if not os.path.exists('../large_files/train.csv'): print('Looking for ../large_files/train.csv') print('You have not downloaded the data and/or not placed the files in the correct location.') print('Please get the data from: https://www.kaggle.com/c/digit-recognizer') print('Place train.csv in the folder large_files adjacent to the class folder') exit() df = pd.read_csv('../large_files/train.csv') data = df.values.astype(np.float32) np.random.shuffle(data) X = data[:, 1:] Y = data[:, 0] Xtrain = X[:-1000] Ytrain = Y[:-1000] Xtest = X[-1000:] Ytest = Y[-1000:] # normalize the data mu = Xtrain.mean(axis=0) std = Xtrain.std(axis=0) np.place(std, std == 0, 1) Xtrain = (Xtrain - mu) / std Xtest = (Xtest - mu) / std return Xtrain, Xtest, Ytrain, Ytest
def _getdatafromsql(connection, tmp_table, query): """ Private function creating a ndarray from the current table. Parameters ---------- connection: sqlite3.Connection Current SQL connection. tmp_table: string Name of the temporary table created for the purpose of keeping ids when WHERE is used query: string SQL query. """ # Transforms the typestr into dtypes # Define and execute the query connection.execute("CREATE TEMPORARY TABLE %s AS %s"%(tmp_table, query)) # Get the list of names and types from the pragma pragmastr = "PRAGMA TABLE_INFO(%s)"%tmp_table (names, typestr) = zip(*(_[1:3] for _ in connection.execute(pragmastr).fetchall())) ndtype = [] for (i, (n, t)) in enumerate(zip(names, typestr)): # Transform the name into a regular string (not unicode) n = str(n) if t =='INTEGER': ndtype.append((n, int)) elif t =='TEXT': ndtype.append((n, '|S30')) elif t == 'BLOB': ndtype.append((n, object)) else: ndtype.append((n, float)) # Construct the ndarray connection.row_factory = sqlite3.Row data = connection.execute("SELECT * FROM %s"%tmp_table).fetchall() try: return np.array(data, dtype=ndtype) except TypeError: output = ma.empty(len(data), dtype=ndtype) # Find the index of the first row (0 or 1)? rowidref = connection.execute("SELECT rowid FROM %s LIMIT 1"%tmp_table).fetchone()[0] # Loop through the different fields identifying the null fields to mask maskstr_template = "SELECT rowid FROM %s WHERE %%s IS NULL"%tmp_table datastr_template = "SELECT %%s FROM %s WHERE %%s IS NOT NULL"%tmp_table for (i, field) in enumerate(names): current_output = output[field] current_mask = current_output._mask maskstr = maskstr_template % field maskidx = [_[0] - rowidref for _ in connection.execute(maskstr).fetchall()] current_mask[maskidx] = True datastr = datastr_template % (field, field) np.place(current_output._data, ~current_mask, [_[0] for _ in connection.execute(datastr).fetchall()]) connection.execute("DROP TABLE %s"%tmp_table) return output
def testDropNegatives(self): # Note: the test is done by replacing segment_ids with 8 to -1 # for index and replace values generated by numpy with 0. dtypes = [ dtypes_lib.float32, dtypes_lib.float64, dtypes_lib.int64, dtypes_lib.int32, dtypes_lib.complex64, dtypes_lib.complex128 ] indices_flat = np.array([0, 4, 0, 8, 3, 8, 4, 7, 7, 3]) num_segments = 12 for indices in indices_flat, indices_flat.reshape(5, 2): shape = indices.shape + (2,) for dtype in dtypes: with self.test_session(use_gpu=True): tf_x, np_x = self._input(shape, dtype=dtype) np_ans = self._segmentReduce( indices, np_x, np.add, op2=None, num_segments=num_segments) # Replace np_ans[8] with 0 for the value np_ans[8:] = 0 # Replace 8 with -1 in indices np.place(indices, indices == 8, [-1]) s = math_ops.unsorted_segment_sum( data=tf_x, segment_ids=indices, num_segments=num_segments) tf_ans = s.eval() self.assertAllClose(np_ans, tf_ans) self.assertShapeEqual(np_ans, s)
def substitute_values(self, vect): """ Internal method to substitute integers into the vector, and construct metadata to convert back to the original vector. np.nan is always given -1, all other objects are given integers in order of apperence. Parameters ---------- vect : np.array the vector in which to substitute values in """ try: unique = np.unique(vect) except: unique = set(vect) unique = [ x for x in unique if not isinstance(x, float) or not isnan(x) ] arr = np.copy(vect) for new_id, value in enumerate(unique): np.place(arr, arr==value, new_id) self.metadata[new_id] = value arr = arr.astype(np.float) np.place(arr, np.isnan(arr), -1) self.arr = arr if -1 in arr: self.metadata[-1] = self._missing_id
def add_noise_batch(self, shingle_imgs, shingle_dim, rng): # assumes RGBA stain > shingle in both dim # assumes shingle is 'L' assert shingle_imgs is not None assert shingle_dim is not None assert rng is not None randseq = rng.randint(len(self.images), size=len(shingle_imgs)) return_images = np.zeros(shingle_imgs.shape) # Choose a random image and a random section inside of the image for i, randnoise in enumerate(randseq): stain_img = self.images[randnoise] stain_w, stain_h = stain_img.size max_rand_x = stain_w - shingle_dim[1] max_rand_y = stain_h - shingle_dim[0] startx = rng.randint(max_rand_x) starty = rng.randint(max_rand_y) # Crop the image rand_cropped_stain = stain_img.crop((startx, starty, startx + shingle_dim[1], starty + shingle_dim[0])) # Invert both images and add them together shingle_inv = (255 - shingle_imgs[i]) # .astype(np.uint16) stain_inv = (255 - np.asarray(rand_cropped_stain)) # .astype(np.uint16) final_img = shingle_inv + stain_inv np.place(final_img, final_img > 255, 255) assert final_img is not None # Invert image back to 255 return_images[i] = 255 - final_img return return_images
def equalise_histogram(image): histo = generate_histogram(image) new_image = np.zeros(image.shape) height, width = image.shape total_pixels = height * width freq_sum = np.sum(histo) for i, freq in enumerate(reversed(histo)): intensity = 255 - i new_intensity = round(255 * freq_sum / total_pixels) temp = image.copy() np.place(temp, temp != intensity, 0) np.place(temp, temp == intensity, new_intensity) new_image += temp freq_sum -= freq new_image = np.array(new_image, dtype=np.uint8) new_histo = generate_histogram(new_image) plot_histogram(new_histo) return new_image
def __call__(self, wave): """ Return dimensionless throughput of bandpass at given wavelength in nanometers. Note that outside of the wavelength range defined by the `blue_limit` and `red_limit` attributes, the throughput is assumed to be zero. @param wave Wavelength in nanometers. @returns the dimensionless throughput. """ # figure out what we received, and return the same thing # option 1: a NumPy array if isinstance(wave, np.ndarray): wgood = (wave >= self.blue_limit) & (wave <= self.red_limit) ret = np.zeros(wave.shape, dtype=np.float) np.place(ret, wgood, self.func(wave[wgood])) return ret # option 2: a tuple elif isinstance(wave, tuple): return tuple([self.func(w) if (w >= self.blue_limit and w <= self.red_limit) else 0.0 for w in wave]) # option 3: a list elif isinstance(wave, list): return [self.func(w) if (w >= self.blue_limit and w <= self.red_limit) else 0.0 for w in wave] # option 4: a single value else: return self.func(wave) if (wave >= self.blue_limit and wave <= self.red_limit) else 0.0
def get_iris(catagory): data=np.loadtxt('iris.txt') np.place(data[:,0],data[:,0]!=catagory,-1) np.place(data[:,0],data[:,0]==catagory,1) return [np.concatenate((data[0:25],data[75:125])), np.concatenate((data[25:75],data[125:150]))]
def make_mask_map_4d(data, infile, outfile): """ Make mask map with 4d dimeions data: values for levels in infile. Shape = [4th dimension, regions] infile: input file to replace levels with values outfile: output file name """ from neurosynth.base.mask import Masker from neurosynth.base import imageutils from nibabel import nifti1 data = np.array(data) # Load image with masker masker = Masker(infile) img = imageutils.load_imgs(infile, masker) header = masker.get_header() shape = header.get_data_shape()[0:3] + (data.shape[0],) header.set_data_shape(shape) result = [] for t_dim, t_val in enumerate(data): result.append(img.copy()) for num, value in enumerate(t_val): np.place(result[t_dim], img == num + 1, [value]) result = np.hstack(result) header.set_data_dtype(result.dtype) # Avoids loss of precision img = nifti1.Nifti1Image(masker.unmask(result).squeeze(), None, header) img.to_filename(outfile)
def score_MCC(ground_truth, scores): ''' assuming the model output is the probability of being default, then this probability can be used for ranking. Then using the fraction of default in validation data to assign the proper threshold to the prediction ''' if isinstance(scores, pd.Series): scores = scores.values if isinstance(ground_truth, pd.Series): ground_truth = ground_truth.values tmp_ground_truth = np.copy(ground_truth) fault_frac = tmp_ground_truth.mean() #print 'score shape:', scores.shape, print 'mean of groud truth:', fault_frac thres_value = np.percentile(scores, 100.*(1-fault_frac), axis=0) print 'threshold for preds:', thres_value binary_scores = scores > thres_value binary_scores = binary_scores.astype(int) ## convert to sk-learn format np.place(binary_scores, binary_scores==0, -1) np.place(tmp_ground_truth, tmp_ground_truth==0, -1) return matthews_corrcoef(tmp_ground_truth, binary_scores)
def predict(self, z): zClass = [] within_dist = [] inputScaler = preprocessing.StandardScaler().fit(self.training_set) z = inputScaler.transform(z) for zz in z: zClass.append([self.pfn(zz)]) within = False normalized_zz = self.normalize(zz) for i in xrange(len(self.bests)): # pdb.set_trace() # within = within or (self.get_distance(self.normalized_best[i], normalized_zz) < self.best_distance[i]) # pdb.set_trace() within = ( within or ( abs(self.normalized_best[i] - normalized_zz) < abs(self.normalized_best[i] - self.best_distance[i]) ).all() ) within_dist.append([within]) # try: # within_dist.append([self.get_distance(self.normalized_best, zz)] < self.best_distance) # except: # within_dist.append([False]) # import pdb result = array(zClass) within_dist = array(within_dist) place(result, ~logical_or(result > self.limit, within_dist), [0.0]) # place(result, within_dist,[1.0]) # place(result, ~within_dist,[0.0]) # result = 1/(1+exp(-result*60+30)) ## sigmoid function # result = result * result * result * result * result * result return result
def add_noise_np(self, shingle_img, shingle_dim, rng): # assumes RGBA stain > shingle in both dim # assumes shingle is 'L' assert shingle_img is not None assert shingle_dim is not None assert rng is not None # Choose a random image and a random section inside of the image stain_img = self.images[rng.randint(len(self.images))] stain_w, stain_h = stain_img.size max_rand_x = stain_w - shingle_dim[1] max_rand_y = stain_h - shingle_dim[0] startx = rng.randint(max_rand_x) starty = rng.randint(max_rand_y) # Crop the image rand_cropped_stain = stain_img.crop((startx, starty, startx + shingle_dim[1], starty + shingle_dim[0])) # Invert both images and add them together shingle_inv = (255 - shingle_img) # .astype(np.uint16) stain_inv = (255 - np.asarray(rand_cropped_stain)) # .astype(np.uint16) final_img = shingle_inv + stain_inv np.place(final_img, final_img > 255, 255) assert final_img is not None # Invert image back to 255 final_img = 255 - final_img return final_img
def Estimate_sf_And_dist(locs, data, cellGraph, gmmDict, test_genes, tissue_mat, plot=False, unary_scale_factor=100, label_cost=10, algorithm='expansion'): ''' Relationship smooth factor with distance for estimating smooth factor. the function just like recal_dist_to_tissue() with tissue_mat, but getting best sf by using min(hamming_dist). Return Best_smooth_factor, related size of gmm labels' components and three distance. ''' result_df_new = pd.DataFrame() new_hamming = [] new_jaccard = [] new_hausdorff = [] con_list = [] for geneID in test_genes: #zeor_boundGenes: #de_counts: #if geneID not in result_df.index: count = data.loc[:, geneID].values gmm = gmmDict[geneID] Labels = gmm.predict(count.reshape(-1, 1)) zero_label = gmm.predict(np.min(count[count > 0]).reshape(-1, 1)) if sum(count == 0): np.place(Labels, count == 0, zero_label) con = count_component(locs, cellGraph, Labels) con_list.append(len(con)) hamming_dist_list = [] jaccard_dist_list = [] hausdorff_dist_list = [] test_df_list = pd.DataFrame() #figsize(10,8) for sf in np.arange(0, 110, 10): # 'Cldn9' need start from sf=0 temp_factor = sf newLabels = cut_graph_general(cellGraph, count, gmm, unary_scale_factor, temp_factor, label_cost, algorithm) #print(geneID) reverLabels = compute_p_reverLabels(locs, newLabels, gmm, count, cellGraph) p, node, com = compute_p_CSR(locs, reverLabels, gmm, count, cellGraph) # figsize(8,6) #print(sf,p) labels_array = np.array(reverLabels).reshape(1, -1) data_array = np.array((geneID, p, min(p), temp_factor, node), dtype=object).reshape(1, -1) t_array = np.hstack((data_array, labels_array)) c_labels = ['p_value', 'fdr', 'smooth_factor', 'nodes'] for i in np.arange(labels_array.shape[1]) + 1: temp_label = 'label_cell_' + str(i) c_labels.append(temp_label) test_df = pd.DataFrame(t_array[:, 1:], index=t_array[:, 0], columns=c_labels) dist_df, false_genes = calc_distance_df(locs, data, cellGraph, gmmDict, test_df, tissue_mat) if plot == True: plot_voronoi_boundary(geneID, locs, count, reverLabels, min(p)) print(sf, p, dist_df) hamming_dist_list.append(dist_df.iloc[:, 0].values) jaccard_dist_list.append(dist_df.iloc[:, 1].values) hausdorff_dist_list.append(dist_df.iloc[:, 2].values) test_df_list = pd.concat([test_df_list, test_df]) if len(jaccard_dist_list) > 0: min_inde = np.argmin( hamming_dist_list ) ## multiple min(p) select min(hamming_dist) amony new_hamming.append(hamming_dist_list[min_inde][0]) new_jaccard.append(jaccard_dist_list[min_inde][0]) new_hausdorff.append(hausdorff_dist_list[min_inde][0]) best_test_df = test_df_list.iloc[min_inde:min_inde + 1] result_df_new = pd.concat([result_df_new, best_test_df]) best_sf = result_df_new.smooth_factor.values dist_df_new = pd.DataFrame( [best_sf, con_list, new_hamming, new_jaccard, new_hausdorff]).T ### no norm dist dist_df_new.columns = [ 'smooth_factor', 'con_size', 'Hamming', 'Jaccard', 'Hausdorff' ] dist_df_new.index = result_df_new.index return result_df_new, dist_df_new
def compute_p_reverLabels(locs, newLabels, gmm, exp, cellGraph): ''' compute p values for each subgraph, and reverse labels of bad subgraph. return new reversal labels. ''' com_factor = 1 p_values = list() node_lists = list() reverLabel = newLabels.copy() a = exp[exp > 0] gmm_pred = gmm.predict(a.reshape(-1, 1)) zero_label = gmm.predict(np.min(a).reshape(-1, 1))[0] label_pred = gmm.predict(exp.reshape(-1, 1)) if len(np.where(exp == 0)[0]) > 0: np.place(label_pred, exp == 0, zero_label) gmm_pred = label_pred unique, counts = np.unique(gmm_pred, return_counts=True) con_components = count_component(locs, cellGraph, newLabels) for j in np.arange( len(con_components)): ## all subgraphs are normal,score all if len( con_components[j] ) <= 9: # len(exp)/2+10: ## too big pattern is not reverse labels node_list = con_components[j] com_size = len(node_list) temp_exp = exp[np.array(list(node_list))] gmm_pred_com = gmm.predict(temp_exp.reshape(-1, 1)) if sum(temp_exp == 0) > 0: # a=temp_exp[temp_exp>0] # zero_label = gmm.predict(np.min(a).reshape(-1,1))[0] np.place(gmm_pred_com, temp_exp == 0, zero_label) # check 0s unique_com, counts_com = np.unique(gmm_pred_com, return_counts=True) major_label = unique_com[np.where( counts_com == counts_com.max())[0][0]] label_count = counts[np.where(unique == major_label)[0]] count_in_com = counts_com.max() cover = exp.shape[0] / com_size pmf = poisson.pmf(count_in_com, com_size * (label_count / exp.shape[0]))[0] psf = poisson.sf(count_in_com, com_size * (label_count / exp.shape[0]))[0] prob_sf = (pmf + psf) * cover prob = (1 - poisson.cdf(count_in_com, com_size * (label_count / exp.shape[0]))[0]) * cover # if prob>=1: # prob=1 # p_values.append(prob) #print(prob_sf,prob,major_label,com_size,cover) # node_lists.append(np.array(list(node_list))) # for i in node_list: # plt.text(locs[i,0],locs[i,1],str(i)) # plot_voronoi_boundary(geneID,locs,exp,newLabels,prob) ## 1. find connected pattern of noise pattern if prob_sf > 0.1: test_node = node_list node_otherPattern = [] for i in test_node: if i in cellGraph[:, 0:2]: node = cellGraph[np.where(i == cellGraph[:, 0:2])[0], 0:2] node_array = node.reshape(1, -1)[0] other_node = [ int(val) for val in list( set(node_array) - set((test_node))) ] node_otherPattern.extend(other_node) # print(i, ' ',node_array,other_node) node_otherPattern = np.unique(node_otherPattern) # print(test_node,node_otherPattern) ## 2. judege nodes of connencted other pattern in one same pattern diff_con = [] for con in range(len(con_components)): if con != j: diff = set(node_otherPattern) - set( con_components[con]) if len(diff) == 0: diff_con.append(con) if len(diff_con) == 1: #reverLabel=newLabels.copy() rever = newLabels[list(con_components[diff_con[0]])][0] reverLabel[list(test_node)] = rever # plot_voronoi_boundary(geneID,locs,exp,reverLabel,prob) return reverLabel
def Min_Max(nt, nc, NumC, Bay, Height): it = 0 Movement = 0 relocation = 0 while it < NumC: p_l_c = np.where(Bay == Bay.min( keepdims=True)) #p_l_c = The position of lowest container ''' print(Bay) print(p_l_c) ''' p_r = int(p_l_c[0]) #the row position of p_l_c #print('p_r =', p_r) p_c = int(p_l_c[1]) #the column position of p_l_c if p_r == nt - int( Height[p_c] ): #if target container is on the top of a stack, directly retrieving it Bay[p_r][p_c] = NumC + 1 Height[p_c] = Height[p_c] - 1 ''' print('Height =', Height) ''' Movement += 1 np.place(Bay, Bay == NumC + 1, 0) ''' print('Round =',Movement,'\n',Bay) ''' np.place(Bay, Bay == 0, NumC + 1) ''' print('\n') ''' elif p_r > nt - Height[p_c]: r = nt - Height[p_c] ''' print('r = ', r) ''' while p_r > r: #while loop concept i = 0 Height_m = [ ] #create an empty list for column which is up to maximum height c_s_i = Bay.min(axis=0) - Bay[r][ p_c] #candidate stack including target container while nc > i: if Height[i] == nt: Height_m.append( i) #add column with maximum height to list i = i + 1 Height_m.append(p_c) # add the target column to list c_s = np.delete( c_s_i, Height_m, None ) #candidate stack after deleting target container and stack up to height limit if np.max(c_s) > 0: #find arg number arg_c = min(i for i in c_s if i > 0) else: arg_c = max(c_s) l_arg_c = [] for i in range( 0, nc ): # I ingore if there are many arg numbers, it will do the same things. It will influence the value of Height. if c_s_i[i] == arg_c: # find the location of arg_c l_arg_c.append(i) # add them to list r_arg_c = random.choice(l_arg_c) #random choose one of them Bay[nt - Height[r_arg_c] - 1][r_arg_c] = Bay[r][p_c] #relocation Bay[r][p_c] = NumC + 1 Height[p_c] = Height[p_c] - 1 Height[r_arg_c] = Height[r_arg_c] + 1 ''' print('\n') print('Height =', Height) ''' relocation += 1 Movement += 1 np.place(Bay, Bay == NumC + 1, 0) ''' print('Round =',Movement,'\n',Bay) ''' np.place(Bay, Bay == 0, NumC + 1) ''' print('relocation =', relocation) print('\n') ''' r = r + 100 ''' Bay[p_r][p_c] = NumC + 1 Height[p_c] = Height[p_c] - 1 #print('Height =', Height) Movement += 1 ''' #print('Height =', Height) np.place(Bay, Bay == NumC + 1, 0) ''' print('Round =',Movement,'\n',Bay) ''' np.place(Bay, Bay == 0, NumC + 1) break ''' print('\n') ''' it = it + 100 np.place(Bay, Bay == NumC + 1, 0) ''' print(Bay) ''' ''' np.place(Bay, Bay == 0, NumC + 1) ''' ''' print("Total movements =", Movement) print("Total relocation =", relocation) ''' #print('Height =', Height) #print('Bay =', Bay) return Height
def bay_test(Initial_Bay, NumC): Bay_test = Initial_Bay np.place(Initial_Bay, Initial_Bay == 0, NumC + 1) # strange syntax, transfer 0 to NumCon + 1 return Bay_test
x_values = np.linspace(x_min, x_max, x_max - x_min) # get the comparison values: yvalues_1 = Cls_1(x_values) yvalues_2 = Cls_2(x_values) # protect against zeroes: yvalues_1_temp = np.abs(yvalues_1) yvalues_2_temp = np.abs(yvalues_2) try: min2val_1 = np.amin(yvalues_1_temp[np.nonzero(yvalues_1_temp)]) min2val_2 = np.amin(yvalues_2_temp[np.nonzero(yvalues_2_temp)]) except: min2val_1 = cutoff min2val_2 = cutoff np.place(yvalues_1, yvalues_1 == 0.0, min2val_1) np.place(yvalues_2, yvalues_2 == 0.0, min2val_2) # computation of the percentual comparison: yvalues_comp_1 = (yvalues_1 - yvalues_2) / np.abs(yvalues_1) * 100.0 yvalues_comp_2 = (yvalues_1 - yvalues_2) / np.abs(yvalues_2) * 100.0 # protection against values too small: np.place(yvalues_comp_1, abs(yvalues_comp_1) < cutoff, [cutoff]) np.place(yvalues_comp_2, abs(yvalues_comp_2) < cutoff, [cutoff]) # get the Cls values: if do_abs: yvalues_comp_1 = np.abs(yvalues_comp_1) yvalues_comp_2 = np.abs(yvalues_comp_2) # get the plot bounds for comp 1:
def square(t, duty=0.5): """ Return a periodic square-wave waveform. The square wave has a period ``2*pi``, has value +1 from 0 to ``2*pi*duty`` and -1 from ``2*pi*duty`` to ``2*pi``. `duty` must be in the interval [0,1]. Note that this is not band-limited. It produces an infinite number of harmonics, which are aliased back and forth across the frequency spectrum. Parameters ---------- t : array_like The input time array. duty : array_like, optional Duty cycle. Default is 0.5 (50% duty cycle). If an array, causes wave shape to change over time, and must be the same length as t. Returns ------- y : ndarray Output array containing the square waveform. Examples -------- A 5 Hz waveform sampled at 500 Hz for 1 second: >>> from scipy import signal >>> import matplotlib.pyplot as plt >>> t = np.linspace(0, 1, 500, endpoint=False) >>> plt.plot(t, signal.square(2 * np.pi * 5 * t)) >>> plt.ylim(-2, 2) A pulse-width modulated sine wave: >>> plt.figure() >>> sig = np.sin(2 * np.pi * t) >>> pwm = signal.square(2 * np.pi * 30 * t, duty=(sig + 1)/2) >>> plt.subplot(2, 1, 1) >>> plt.plot(t, sig) >>> plt.subplot(2, 1, 2) >>> plt.plot(t, pwm) >>> plt.ylim(-1.5, 1.5) """ t, w = asarray(t), asarray(duty) w = asarray(w + (t - t)) t = asarray(t + (w - w)) if t.dtype.char in ['fFdD']: ytype = t.dtype.char else: ytype = 'd' y = zeros(t.shape, ytype) # width must be between 0 and 1 inclusive mask1 = (w > 1) | (w < 0) place(y, mask1, nan) # on the interval 0 to duty*2*pi function is 1 tmod = mod(t, 2 * pi) mask2 = (1 - mask1) & (tmod < w * 2 * pi) place(y, mask2, 1) # on the interval duty*2*pi to 2*pi function is # (pi*(w+1)-tmod) / (pi*(1-w)) mask3 = (1 - mask1) & (1 - mask2) place(y, mask3, -1) return y
def LA_N(Bay_test, N, NumC): lowest_con = 1 movement = 0 relocation = 0 NumC = 7 while lowest_con <= NumCon: row_lowest_con, column_lowest_con = Find_con_position( Bay_test, lowest_con) r = 1 N = min(2, NumC) #print('N =', N) #print('NumC =', NumC) while Bay_test[row_lowest_con - 1][column_lowest_con] <= NumCon and row_lowest_con != 0: Stack_N = Find_Stack_N(Bay_test, N, lowest_con) Stack_Height, Stack_not_N = not_in_Stack_N(Stack_N, Height) #print(int(min(Stack_Height))) while int(min(Stack_Height)) == NTier or len(Stack_N) == NTier: N -= 1 Stack_N = Find_Stack_N(Bay_test, N, lowest_con) Stack_Height, Stack_not_N = not_in_Stack_N(Stack_N, Height) #print('N =', N) #print('lowest container=', lowest_con) #print('Stack_N =', Stack_N) #print('Stack_Height =', Stack_Height) #print('Stack_not_N =', Stack_not_N) Top_Stack_N = Find_Top_Stack_N(Bay_test, Stack_N) #print('Top_Stack_N =', Top_Stack_N) Low_Stack = Find_Lowest_con_in_column(Bay_test, Stack_not_N) #print('Low_Stack =', Low_Stack) #print('r =', r) #print('len(Stack_N) =', len(Stack_N)) while r <= len(Stack_N): n = max(Top_Stack_N) #print('n = ',n) #print(lowest_con) row_n, column_n = Find_con_position(Bay_test, n) #print('row_n =', row_n, 'column_n =', column_n) if column_n == column_lowest_con: if min(Low_Stack) == NumCon + 1: for j in range(0, NCol): if Bay_test[NTier - 1][j] == NumCon + 1: row_low, column_low = NTier, j #print('row_low = ',row_low, 'column_low =', column_low) break elif max(Low_Stack) - n > 0: while min(Low_Stack) - n < 0: Low_Stack.remove(min(Low_Stack)) if min(Low_Stack) == NumCon + 1: for j in range(0, NCol): if Bay_test[NTier - 1][j] == NumCon + 1: row_low, column_low = NTier, j #print('row_low = ',row_low, 'column_low =', column_low) else: row, column_low = Find_con_position( Bay_test, min(Low_Stack)) row_low = NTier - Height[column_low] else: row, column_low = Find_con_position( Bay_test, max(Low_Stack)) row_low = NTier - Height[column_low] #print('row_low =', row_low, 'column_low =', column_low) break elif Bay_test[row_n][column_n] <= min( list(Bay_test[i][column_n] for i in range(0, NTier))): #print('min =', min(list(Bay_test[i][column_n] for i in range(0, NTier)))) for i in Stack_N: rounds = 0 for j in range(0, NTier - 1): if Bay_test[j][i] < Bay_test[j + 1][i]: rounds += 1 elif Bay_test[j][i] == NumCon + 1: rounds += 1 if Bay_test[i][NTier] != NumCon + 1: rounds += 1 if rounds == NTier: Stack_N.remove(i) if r == len(Stack_N): row, column_low = Find_con_position( Bay_test, min(Low_Stack)) row_low = NTier - Height[column_low] break else: r += 1 elif max(Low_Stack) > n: if min(Low_Stack) == NumCon + 1: for j in range(0, NCol): if Bay_test[NTier - 1][j] == NumCon + 1: row_low, column_low = NTier, j #print('row_low = ',row_low, 'column_low =', column_low) break else: while min(Low_Stack) - n < 0: Low_Stack.remove(min(Low_Stack)) row, column_low = Find_con_position( Bay_test, min(Low_Stack)) row_low = NTier - Height[column_low] #print(min(Low_Stack)) break elif r == len(Stack_N): row, column_low = Find_con_position( Bay_test, max(Low_Stack)) row_low = NTier - Height[column_low] break Top_Stack_N.remove(max(Top_Stack_N)) #print('Top_Stack_N =', Top_Stack_N) r += 1 Bay_test[row_low - 1][column_low] = Bay_test[row_n][column_n] Bay_test[row_n][column_n] = NumCon + 1 Height[column_low] += 1 Height[column_n] -= 1 np.place(Bay_test, Bay_test == NumCon + 1, 0) #print(Bay_test) #np.place(Bay_test, Bay_test == 0, NumCon + 1) movement += 1 relocation += 1 break if movement == 0: Bay_test[row_lowest_con][column_lowest_con] = NumCon + 1 Height[column_lowest_con] -= 1 np.place(Bay_test, Bay_test == NumCon + 1, 0) break #np.place(Bay_test, Bay_test == NumCon +1, 0) #print(Bay_test) np.place(Bay_test, Bay_test == 0, NumCon + 1) movement += 1 lowest_con += 1 NumC -= 1 return Bay_test
def richardson_lucy(image, psf, iterations=50, clip=True): """Richardson-Lucy deconvolution. Parameters ---------- image : ndarray Input degraded image (can be N dimensional). psf : ndarray The point spread function. iterations : int Number of iterations. This parameter plays the role of regularisation. clip : boolean, optional True by default. If true, pixel value of the result above 1 or under -1 are thresholded for skimage pipeline compatibility. Returns ------- im_deconv : ndarray The deconvolved image. Examples -------- >>> from skimage import color, data, restoration >>> camera = color.rgb2gray(data.camera()) >>> from scipy.signal import convolve2d >>> psf = np.ones((5, 5)) / 25 >>> camera = convolve2d(camera, psf, 'same') >>> camera += 0.1 * camera.std() * np.random.standard_normal(camera.shape) >>> deconvolved = restoration.richardson_lucy(camera, psf, 5) References ---------- .. [1] http://en.wikipedia.org/wiki/Richardson%E2%80%93Lucy_deconvolution """ # compute the times for direct convolution and the fft method. The fft is of # complexity O(N log(N)) for each dimension and the direct method does # straight arithmetic (and is O(n*k) to add n elements k times) direct_time = np.prod(image.shape + psf.shape) fft_time = np.sum([n * np.log(n) for n in image.shape + psf.shape]) # see whether the fourier transform convolution method or the direct # convolution method is faster (discussed in scikit-image PR #1792) time_ratio = 40.032 * fft_time / direct_time if time_ratio <= 1 or len(image.shape) > 2: convolve_method = fftconvolve else: convolve_method = convolve image = image.astype(np.float) psf = psf.astype(np.float) im_deconv = 0.5 * np.ones(image.shape) psf_mirror = psf[::-1, ::-1] for _ in range(iterations): eps = np.finfo(image.dtype).eps x = convolve_method(im_deconv, psf, 'same') np.place(x, x == 0, eps) relative_blur = image / x + eps im_deconv *= convolve_method(relative_blur, psf_mirror, 'same') if clip: im_deconv[im_deconv > 1] = 1 im_deconv[im_deconv < -1] = -1 return im_deconv
# =============================================================== # Buy or sell indicator array (1 = buy, 0 = sell) # Note: the "_bh" suffix indicates the corresponding arrays for a buy-and-hold AAPL strategy buy_or_sell = Y_test_pred.values buy_or_sell_bh = np.zeros_like(Y_test_pred) buy_or_sell_bh[0] = 1 # Convert prices array from type object to type float prices = open_prices[split_ind:].astype(float).values # The change in shares # positive = shares bought on day i (number of shares increases) # negative = shares sold on day i (number of shares decreases) shares_change = np.zeros_like(buy_or_sell).astype(int) np.place(shares_change, buy_or_sell == 1, shares) np.place(shares_change, buy_or_sell == 0, -shares) shares_change_bh = np.zeros_like(buy_or_sell_bh).astype(int) shares_change_bh[0] = shares # The change in cash # positive = gross proceeds from selling shares # negative = purchase amount for buying shares cash_change = np.zeros_like(buy_or_sell).astype(float) # Populate cash_change array for i in range(len(buy_or_sell)): # Buy if buy_or_sell[i] == 1: amount = shares * prices[i] + commission cash_change[i] = -amount
def maybe_upcast_putmask(result, mask, other): """ A safe version of putmask that potentially upcasts the result. The result is replaced with the first N elements of other, where N is the number of True values in mask. If the length of other is shorter than N, other will be repeated. Parameters ---------- result : ndarray The destination array. This will be mutated in-place if no upcasting is necessary. mask : boolean ndarray other : ndarray or scalar The source array or value Returns ------- result : ndarray changed : boolean Set to true if the result array was upcasted Examples -------- >>> result, _ = maybe_upcast_putmask(np.arange(1,6), np.array([False, True, False, True, True]), np.arange(21,23)) >>> result array([1, 21, 3, 22, 21]) """ if not isinstance(result, np.ndarray): raise ValueError("The result input must be a ndarray.") if mask.any(): # Two conversions for date-like dtypes that can't be done automatically # in np.place: # NaN -> NaT # integer or integer array -> date-like array if is_datetimelike(result.dtype): if is_scalar(other): if isna(other): other = result.dtype.type("nat") elif is_integer(other): other = np.array(other, dtype=result.dtype) elif is_integer_dtype(other): other = np.array(other, dtype=result.dtype) def changeit(): # try to directly set by expanding our array to full # length of the boolean try: om = other[mask] except (IndexError, TypeError): # IndexError occurs in test_upcast when we have a boolean # mask of the wrong shape # TypeError occurs in test_upcast when `other` is a bool pass else: om_at = om.astype(result.dtype) if (om == om_at).all(): new_result = result.values.copy() new_result[mask] = om_at result[:] = new_result return result, False # we are forced to change the dtype of the result as the input # isn't compatible r, _ = maybe_upcast(result, fill_value=other, copy=True) np.place(r, mask, other) return r, True # we want to decide whether place will work # if we have nans in the False portion of our mask then we need to # upcast (possibly), otherwise we DON't want to upcast (e.g. if we # have values, say integers, in the success portion then it's ok to not # upcast) new_dtype, _ = maybe_promote(result.dtype, other) if new_dtype != result.dtype: # we have a scalar or len 0 ndarray # and its nan and we are changing some values if is_scalar(other) or (isinstance(other, np.ndarray) and other.ndim < 1): if isna(other): return changeit() # we have an ndarray and the masking has nans in it else: if isna(other).any(): return changeit() try: np.place(result, mask, other) except TypeError: # e.g. int-dtype result and float-dtype other return changeit() return result, False
def maybe_upcast_putmask(result, mask, other): """ A safe version of putmask that potentially upcasts the result Parameters ---------- result : ndarray The destination array. This will be mutated in-place if no upcasting is necessary. mask : boolean ndarray other : ndarray or scalar The source array or value Returns ------- result : ndarray changed : boolean Set to true if the result array was upcasted """ if mask.any(): # Two conversions for date-like dtypes that can't be done automatically # in np.place: # NaN -> NaT # integer or integer array -> date-like array if is_datetimelike(result.dtype): if is_scalar(other): if isna(other): other = result.dtype.type('nat') elif is_integer(other): other = np.array(other, dtype=result.dtype) elif is_integer_dtype(other): other = np.array(other, dtype=result.dtype) def changeit(): # try to directly set by expanding our array to full # length of the boolean try: om = other[mask] om_at = om.astype(result.dtype) if (om == om_at).all(): new_result = result.values.copy() new_result[mask] = om_at result[:] = new_result return result, False except: pass # we are forced to change the dtype of the result as the input # isn't compatible r, _ = maybe_upcast(result, fill_value=other, copy=True) np.place(r, mask, other) return r, True # we want to decide whether place will work # if we have nans in the False portion of our mask then we need to # upcast (possibly), otherwise we DON't want to upcast (e.g. if we # have values, say integers, in the success portion then it's ok to not # upcast) new_dtype, _ = maybe_promote(result.dtype, other) if new_dtype != result.dtype: # we have a scalar or len 0 ndarray # and its nan and we are changing some values if (is_scalar(other) or (isinstance(other, np.ndarray) and other.ndim < 1)): if isna(other): return changeit() # we have an ndarray and the masking has nans in it else: if isna(other[mask]).any(): return changeit() try: np.place(result, mask, other) except: return changeit() return result, False
def main(): sim_config = simulation_configuration() sim_config.set_scenario_local_ref( h_t = 13.82e6, # m h_r = 20e3, # meters elevation = 60.0*np.pi/180, v_t = np.array([-2684.911, 1183.799, -671.829]), # m/s v_r = np.array([20, 20, 20]) # m/s ) #sim_config.jacobian_type = 'spherical' sim_config.receiver_antenna_gain = lambda p1,p2: 12.589 sim_config.rcs = lambda p1,p2: target_rcs.radar_cross_section(p1, 0, p2) sim_config.u_10 = 8.0 #sim_config.delay_chip = 1/gps_ca_chips_per_second # seconds delay_chip = sim_config.delay_chip number_of_delay_pixels = 128 - 50 number_of_doppler_pixels = 20 + 50 sim_config.doppler_increment_start = -50 sim_config.doppler_increment_end = 50 sim_config.doppler_resolution = (sim_config.doppler_increment_end - sim_config.doppler_increment_start)/number_of_doppler_pixels/3 sim_config.delay_increment_start = -0.5*delay_chip sim_config.delay_increment_end = 2*delay_chip #sim_config.delay_resolution = 0.01*delay_chip sim_config.delay_resolution = (sim_config.delay_increment_end - sim_config.delay_increment_start)/number_of_delay_pixels/3 sim_config.coherent_integration_time = 2e-2 # sec delay_increment_start = sim_config.delay_increment_start delay_increment_end = sim_config.delay_increment_end delay_resolution = sim_config.delay_resolution doppler_increment_start = sim_config.doppler_increment_start doppler_increment_end = sim_config.doppler_increment_end doppler_resolution = sim_config.doppler_resolution doppler_specular_point = eq_doppler_absolute_shift(np.array([0,0,0]), sim_config) rescaled_doppler_resolution = (doppler_increment_end - doppler_increment_start)/number_of_doppler_pixels rescaled_delay_resolution_chips = (delay_increment_end - delay_increment_start)/delay_chip/number_of_delay_pixels print("doppler res: {0}".format(rescaled_doppler_resolution)) print("delay res: {0}".format(rescaled_delay_resolution_chips)) # Surface mesh x_0 = 0 x_1 = 6e3 # meters n_x = 800 y_0 = -1e3 y_1 = 6e3 # meters n_y = 800 x_grid, y_grid = np.meshgrid( np.linspace(x_0, x_1, n_x), np.linspace(y_0, y_1, n_y) ) r = np.array([x_grid, y_grid, 0]) # Isolines and RCS z_grid_delay_chip = eq_delay_incremet(r, sim_config)/delay_chip doppler_specular_point = eq_doppler_absolute_shift(np.array([0,0,0]), sim_config) z_grid_doppler_increment = eq_doppler_absolute_shift(r, sim_config) - doppler_specular_point z_rcs = sim_config.rcs(r, sim_config) # Plot fig_rcs, ax_rcs = plt.subplots(1,figsize=(10, 4)) #contour_delay_chip = ax_rcs.contour( # x_grid, y_grid, z_grid_delay_chip, # np.arange(0, 2, 0.1), # cmap='winter', alpha = 0.3 # ) #contour_doppler = ax_rcs.contour( # x_grid, y_grid, z_grid_doppler_increment, # np.arange(-50, 50, 1), # cmap='jet', alpha = 0.3 # ) contour_rcs = ax_rcs.contourf(x_grid, y_grid, z_rcs, 55, cmap='jet', alpha = 0.8) ax_rcs.set_title('RCS') plt.xlabel('[km]') plt.ylabel('[km]') #fig_rcs.colorbar(contour_delay_chip, label='C/A chips') #fig_rcs.colorbar(contour_doppler, label='Hz') fig_rcs.colorbar(contour_rcs, label='Gain') target_delay_increment = 0.54 target_doppler_increment = 17.35 target_delay = 0.35 target_doppler = 12.5 target_iso_delay = ax_rcs.contour(x_grid, y_grid, z_grid_delay_chip, #[target_delay_increment-0.1], [target_delay - rescaled_delay_resolution_chips], colors='red', linewidths = 2.5, linestyles='dashed', ) target_iso_delay = ax_rcs.contour(x_grid, y_grid, z_grid_delay_chip, #[target_delay_increment+0.1], [target_delay + rescaled_delay_resolution_chips], colors='red', linewidths = 2.5, linestyles='dashed', ) target_iso_delay = ax_rcs.contour(x_grid, y_grid, z_grid_doppler_increment, #[target_doppler_increment-0.5], [target_doppler - rescaled_doppler_resolution], colors='red', linewidths = 2.5, linestyles='dashed', ) target_iso_delay = ax_rcs.contour(x_grid, y_grid, z_grid_doppler_increment, #[target_doppler_increment+0.5], [target_doppler + rescaled_doppler_resolution], colors='red', linewidths = 2.5, linestyles='dashed', ) ticks_y = ticker.FuncFormatter(lambda y, pos: '{0:g}'.format(y/1000)) ticks_x = ticker.FuncFormatter(lambda x, pos: '{0:g}'.format(x/1000)) ax_rcs.xaxis.set_major_formatter(ticks_x) ax_rcs.yaxis.set_major_formatter(ticks_y) # DDM ddm_sim = np.copy(simulate_ddm(sim_config)) sim_config.rcs = sea_rcs.radar_cross_section sim_config.rcs = lambda p1,p2: target_rcs.radar_cross_section(p1, 10, p2) sim_config.u_10 = 8.02 # m/s ddm_sim_1 = np.copy(simulate_ddm(sim_config)) ddm_diff = np.abs(ddm_sim - ddm_sim_1) fig_diff, ax_diff = plt.subplots(1,figsize=(10, 4)) plt.title('DDM diff simulation') plt.xlabel('C/A chips') plt.ylabel('Hz') contour_diff = ax_diff.imshow(ddm_diff, cmap='jet', extent=( delay_increment_start/delay_chip, delay_increment_end/delay_chip, doppler_increment_end, doppler_increment_start), aspect="auto" ) fig_diff.colorbar(contour_diff, label='Correlated Power [W]') fig_ddm, ax_ddm = plt.subplots(1,figsize=(10, 4)) plt.title('DDM original simulation') plt.xlabel('C/A chips') plt.ylabel('Hz') contour_sim = ax_ddm.imshow(ddm_sim, cmap='jet', extent=( delay_increment_start/delay_chip, delay_increment_end/delay_chip, doppler_increment_end, doppler_increment_start), aspect="auto" ) fig_ddm.colorbar(contour_sim, label='Correlated Power [W]') fig_ddm_1, ax_ddm_1 = plt.subplots(1,figsize=(10, 4)) plt.title('DDM original simulation 1') plt.xlabel('C/A chips') plt.ylabel('Hz') contour_sim_1 = ax_ddm_1.imshow(ddm_sim_1, cmap='jet', extent=( delay_increment_start/delay_chip, delay_increment_end/delay_chip, doppler_increment_end, doppler_increment_start), aspect="auto" ) fig_ddm_1.colorbar(contour_sim_1, label='Correlated Power [W]') # Image downscaling to desired resolution: # TODO: This is just an average of the pixels around the area # This is not valid, summation i srequired: # Di Simone > From a physical viewpoint, # such an approach should call for summation instead of averaging # https://stackoverflow.com/questions/48121916/numpy-resize-rescale-image fig_ddm_rescaled_diff, ax_ddm_rescaled_diff = plt.subplots(1,figsize=(10, 4)) plt.title('Simulation') plt.xlabel('C/A chips') plt.ylabel('Hz') #ddm_rescaled = rescale(ddm_diff, number_of_doppler_pixels, number_of_delay_pixels) ddm_sim_res = rescale(ddm_sim, number_of_doppler_pixels, number_of_delay_pixels) ddm_sim_1_res = rescale(ddm_sim_1, number_of_doppler_pixels, number_of_delay_pixels) ddm_diff_res = np.abs(ddm_sim_res - ddm_sim_1_res) contour_diff = ax_ddm_rescaled_diff.imshow(ddm_diff_res, cmap='jet', extent=( delay_increment_start/delay_chip, delay_increment_end/delay_chip, doppler_increment_end, doppler_increment_start), aspect='auto' ) fig_ddm_rescaled_diff.colorbar(contour_diff, label='Correlated Power [W]') fig_ddm_rescaled, ax_ddm_rescaled = plt.subplots(1,figsize=(10, 4)) plt.title('Simulation Rescaled') plt.xlabel('C/A chips') plt.ylabel('Hz') contour_rescaled = ax_ddm_rescaled.imshow(ddm_sim_res, cmap='jet', extent=( delay_increment_start/delay_chip, delay_increment_end/delay_chip, doppler_increment_end, doppler_increment_start), aspect='auto' ) fig_ddm_rescaled.colorbar(contour_rescaled, label='Correlated Power [W]') # DDM Noise T_noise_receiver = 225 k_b = 1.38e-23 # J/K y_noise = 1/sim_config.coherent_integration_time*k_b*T_noise_receiver print("expected SNR: {}".format(y_noise)) fig_snr, ax_snr = plt.subplots(1,figsize=(10, 4)) plt.title('SNR') plt.xlabel('C/A chips') plt.ylabel('Hz') ddm_snr = 10*np.log10(np.abs(ddm_diff_res)/y_noise) np.place(ddm_snr, ddm_snr < -30, np.nan) contour_snr = ax_snr.imshow(ddm_snr, cmap='jet', extent=( delay_increment_start/delay_chip, delay_increment_end/delay_chip, doppler_increment_end, doppler_increment_start), aspect='auto' ) fig_snr.colorbar(contour_snr, label='Correlated Power [W]') plt.show()
train_com = np.concatenate((cl1_train, cl2_train), axis=0) train_lab = np.concatenate((cl1_label, cl2_label), axis=0) [train_sff, train_labs] = shuffle(train_com, train_lab) fig = plt.figure() for i in range(16): plt.subplot(4, 4, i + 1) plt.tight_layout() plt.imshow(train_sff[i], cmap='gray', interpolation='none') plt.title("Digit: {}".format(train_labs[i])) plt.xticks([]) plt.yticks([]) fig np.place(train_labs, train_labs == cl1, [0]) np.place(train_labs, train_labs == cl2, [1]) ############################################################################################################ train_labs_cat = keras.utils.to_categorical(train_labs, 2) # train_sff = train_sff.astype('float32') train_sff /= 255 ftrain_sff = train_sff.reshape(train_labs_cat.shape[0], img_rows * img_cols) ################################################################################################################ model = Sequential() model.add(Dense(1024, input_dim=784, activation='sigmoid')) model.add(Dense(512, activation='sigmoid')) model.add(Dense(256, activation='sigmoid'))
def sawtooth(t, width=1): """ Return a periodic sawtooth or triangle waveform. The sawtooth waveform has a period ``2*pi``, rises from -1 to 1 on the interval 0 to ``width*2*pi``, then drops from 1 to -1 on the interval ``width*2*pi`` to ``2*pi``. `width` must be in the interval [0, 1]. Note that this is not band-limited. It produces an infinite number of harmonics, which are aliased back and forth across the frequency spectrum. Parameters ---------- t : array_like Time. width : array_like, optional Width of the rising ramp as a proportion of the total cycle. Default is 1, producing a rising ramp, while 0 produces a falling ramp. `width` = 0.5 produces a triangle wave. If an array, causes wave shape to change over time, and must be the same length as t. Returns ------- y : ndarray Output array containing the sawtooth waveform. Examples -------- A 5 Hz waveform sampled at 500 Hz for 1 second: >>> from scipy import signal >>> import matplotlib.pyplot as plt >>> t = np.linspace(0, 1, 500) >>> plt.plot(t, signal.sawtooth(2 * np.pi * 5 * t)) """ t, w = asarray(t), asarray(width) w = asarray(w + (t - t)) t = asarray(t + (w - w)) if t.dtype.char in ['fFdD']: ytype = t.dtype.char else: ytype = 'd' y = zeros(t.shape, ytype) # width must be between 0 and 1 inclusive mask1 = (w > 1) | (w < 0) place(y, mask1, nan) # take t modulo 2*pi tmod = mod(t, 2 * pi) # on the interval 0 to width*2*pi function is # tmod / (pi*w) - 1 mask2 = (1 - mask1) & (tmod < w * 2 * pi) tsub = extract(mask2, tmod) wsub = extract(mask2, w) place(y, mask2, tsub / (pi * wsub) - 1) # on the interval width*2*pi to 2*pi function is # (pi*(w+1)-tmod) / (pi*(1-w)) mask3 = (1 - mask1) & (1 - mask2) tsub = extract(mask3, tmod) wsub = extract(mask3, w) place(y, mask3, (pi * (wsub + 1) - tsub) / (pi * (1 - wsub))) return y
def active_to_full(self, active_joints, intial_position): full_joints = np.array(intial_position, copy=True, dtype=np.float) np.place(full_joints, self.active_joints_mask, active_joints) return full_joints
def num_roads(osmdata, nom, lineTbl, polyTbl, folder, cellsize, srs, rstTemplate): """ Select Roads and convert To Raster """ import datetime import os import numpy as np from osgeo import gdal from threading import Thread from gasp.fm.rst import rst_to_array from gasp.anls.exct import sel_by_attr from gasp.sql.anls.prox import splite_buffer from gasp.to.rst import shp_to_raster, array_to_raster from gasp.sql.mng.tbl import row_num time_a = datetime.datetime.now().replace(microsecond=0) NR = row_num(osmdata, lineTbl, where="roads IS NOT NULL", api='sqlite') time_b = datetime.datetime.now().replace(microsecond=0) if not NR: return None, {0: ('count_rows_roads', time_b - time_a)} timeGasto = {0: ('count_rows_roads', time_b - time_a)} # Get Roads Buffer LULC_CLS = '1221' if nom != "GLOBE_LAND_30" else '801' bfShps = [] def exportAndBuffer(): time_cc = datetime.datetime.now().replace(microsecond=0) roadFile = splite_buffer(osmdata, lineTbl, "bf_roads", "geometry", os.path.join(folder, 'bf_roads.gml'), whrClause="roads IS NOT NULL", outTblIsFile=True, dissolve=None) time_c = datetime.datetime.now().replace(microsecond=0) distRst = shp_to_raster(roadFile, None, cellsize, -1, os.path.join(folder, 'rst_roads.tif'), epsg=srs, rst_template=rstTemplate, api="gdal") time_d = datetime.datetime.now().replace(microsecond=0) bfShps.append(distRst) timeGasto[1] = ('buffer_roads', time_c - time_cc) timeGasto[2] = ('to_rst_roads', time_d - time_c) BUILDINGS = [] def exportBuild(): time_ee = datetime.datetime.now().replace(microsecond=0) NB = row_num(osmdata, polyTbl, where="building IS NOT NULL", api='sqlite') time_e = datetime.datetime.now().replace(microsecond=0) timeGasto[3] = ('check_builds', time_e - time_ee) if not NB: return bShp = sel_by_attr( osmdata, "SELECT geometry FROM {} WHERE building IS NOT NULL".format( polyTbl), os.path.join(folder, 'road_builds.shp'), api_gis='ogr') time_f = datetime.datetime.now().replace(microsecond=0) bRst = shp_to_raster(bShp, None, cellsize, -1, os.path.join(folder, 'road_builds.tif'), epsg=srs, rst_template=rstTemplate, api='gdal') time_g = datetime.datetime.now().replace(microsecond=0) BUILDINGS.append(bRst) timeGasto[4] = ('export_builds', time_f - time_e) timeGasto[5] = ('builds_to_rst', time_g - time_f) thrds = [ Thread(name="build-th", target=exportBuild), Thread(name='roads-th', target=exportAndBuffer) ] for t in thrds: t.start() for t in thrds: t.join() if not len(BUILDINGS): return {LULC_CLS: bfShps[0]} time_x = datetime.datetime.now().replace(microsecond=0) BUILD_ARRAY = rst_to_array(BUILDINGS[0], with_nodata=True) rst_array = rst_to_array(bfShps[0], with_nodata=True) np.place(rst_array, BUILD_ARRAY == 1, 0) newRaster = array_to_raster(rst_array, os.path.join(folder, 'fin_roads.tif'), rstTemplate, srs, gdal.GDT_Byte, noData=-1, gisApi='gdal') time_z = datetime.datetime.now().replace(microsecond=0) timeGasto[6] = ('sanitize_roads', time_z - time_x) return {int(LULC_CLS): newRaster}, timeGasto
arr : ndarray Array to put data into. mask : array_like Boolean mask array. Must have the same size as `arr`. !!! vals : 1-D sequence Values to put into `arr`. Only the first N elements are used, where N is the number of True values in `mask`. If `vals` is smaller than N, it will be repeated, and if elements of `arr` are to be masked, this sequence must be non-empty. See also copyto, put, take, extract """ arr = np.arange(6).reshape(2, 3) arr np.place(arr, arr>2, [10, 20]) #!!! inplace arr #array([[ 0, 1, 2], # [10, 20, 10]]) # also arr = np.arange(6).reshape(2, 3) arr np.putmask(arr, (arr>2), [10, 20]) #!!! the same arr # while np.copyto(arr, [10, 20], where=arr>2) # ValueError: could not broadcast input array from shape (2) into shape (2,3) np.place(arr, arr>1, [10, 20, 30]) arr
def update_globe_land_cover(original_globe_raster, osm_urban_atlas_raster, osm_globe_raster, epsg, updated_globe_raster, detailed_globe_raster): """ Update the original Glob Land 30 with the result of the conversion of OSM DATA to the Globe Land Cover nomenclature; Also updates he previous updated Glob Land 30 with the result of the conversion of osm data to the Urban Atlas Nomenclature """ import os import numpy as np from gasp.gt.fmrst import rst_to_array from gasp.gt.prop.rst import get_cellsize, get_nodata from gasp.gt.torst import obj_to_rst # ############################# # # Convert images to numpy array # # ############################# # np_globe_original = rst_to_array(original_globe_raster) np_globe_osm = rst_to_array(osm_globe_raster) np_ua_osm = rst_to_array(osm_urban_atlas_raster) # ################################## # # Check the dimension of both images # # ################################## # if np_globe_original.shape != np_globe_osm.shape: return ( 'The Globe Land 30 raster (original) do not have the same number' ' of columns/lines comparing with the Globe Land 30 derived ' 'from OSM data') elif np_globe_original.shape != np_ua_osm.shape: return ( 'The Globe Land 30 raster (original) do not have the same ' 'number of columns/lines comparing with the Urban Atlas raster ' 'derived from OSM data') elif np_globe_osm.shape != np_ua_osm.shape: return ( 'The Globe Land 30 derived from OSM data do not have the same ' 'number of columns/lines comparing with the Urban Atlas raster ' 'derived from OSM data') # ############## # # Check Cellsize # # ############## # cell_of_rsts = get_cellsize( [original_globe_raster, osm_globe_raster, osm_urban_atlas_raster], xy=True, gisApi='gdal') cell_globe_original = cell_of_rsts[original_globe_raster] cell_globe_osm = cell_of_rsts[osm_globe_raster] cell_ua_osm = cell_of_rsts[osm_urban_atlas_raster] if cell_globe_original != cell_globe_osm: return ( 'The cellsize of the Globe Land 30 raster (original) is not the ' 'same comparing with the Globe Land 30 derived from OSM data') elif cell_globe_original != cell_ua_osm: return ( 'The cellsize of the Globe Land 30 raster (original) is not the ' 'same comparing with the Urban Atlas raster derived from OSM data') elif cell_ua_osm != cell_globe_osm: return ( 'The cellsize of the Globe Land 30 derived from OSM data is not ' 'the same comparing with the Urban Atlas raster derived from ' 'OSM data') # ############################# # # Get the Value of Nodata Cells # # ############################# # nodata_glob_original = get_nodata(original_globe_raster, gisApi='gdal') nodata_glob_osm = get_nodata(osm_globe_raster, gisApi='gdal') nodata_ua_osm = get_nodata(osm_urban_atlas_raster, gisApi='gdal') # ######################################## # # Create a new map - Globe Land 30 Updated # # ######################################## # """ Create a new array with zeros... 1) The zeros will be replaced by the values in the Globe Land derived from OSM. 2) The zeros will be replaced by the values in the Original Globe Land at the cells with NULL data in the Globe Land derived from OSM. The meta array will identify values origins in the updated raster: 1 - Orinal Raster 2 - OSM Derived Raster """ update_array = np.zeros( (np_globe_original.shape[0], np_globe_original.shape[1])) update_meta_array = np.zeros( (np_globe_original.shape[0], np_globe_original.shape[1])) # 1) np.copyto(update_array, np_globe_osm, 'no', np_globe_osm != nodata_glob_osm) # 1) meta np.place(update_meta_array, update_array != 0, 2) # 2) meta np.place(update_meta_array, update_array == 0, 1) # 2) np.copyto(update_array, np_globe_original, 'no', update_array == 0) # 2) meta np.place(update_meta_array, update_array == nodata_glob_original, int(nodata_glob_original)) # noData to int np.place(update_array, update_array == nodata_glob_original, int(nodata_glob_original)) updated_meta = os.path.join( os.path.dirname(updated_globe_raster), '{n}_meta{e}'.format( n=os.path.splitext(os.path.basename(updated_globe_raster))[0], e=os.path.splitext(os.path.basename(updated_globe_raster))[1])) # Create Updated Globe Cover 30 obj_to_rst(update_array, updated_globe_raster, original_globe_raster, noData=int(nodata_glob_original)) # Create Updated Globe Cover 30 meta obj_to_rst(update_meta_array, updated_meta, original_globe_raster, noData=int(nodata_glob_original)) # ################################################# # # Create a new map - Globe Land 30 Detailed with UA # # ################################################# # np_update = rst_to_array(updated_globe_raster) detailed_array = np.zeros((np_update.shape[0], np_update.shape[1])) detailed_meta_array = np.zeros((np_update.shape[0], np_update.shape[1])) """ Replace 80 Globe Land for 11, 12, 13, 14 of Urban Atlas The meta array will identify values origins in the detailed raster: 1 - Updated Raster 2 - UA Derived Raster from OSM """ # Globe - Mantain some classes np.place(detailed_array, np_update == 30, 8) np.place(detailed_array, np_update == 30, 1) np.place(detailed_array, np_update == 40, 9) np.place(detailed_array, np_update == 40, 1) np.place(detailed_array, np_update == 50, 10) np.place(detailed_array, np_update == 50, 1) np.place(detailed_array, np_update == 10, 5) np.place(detailed_array, np_update == 10, 1) # Water bodies np.place(detailed_array, np_ua_osm == 50 or np_update == 60, 7) np.place(detailed_meta_array, np_ua_osm == 50 or np_update == 60, 1) # Urban - Where Urban Atlas IS NOT NULL np.place(detailed_array, np_ua_osm == 11, 1) np.place(detailed_meta_array, np_ua_osm == 11, 2) np.place(detailed_array, np_ua_osm == 12, 2) np.place(detailed_meta_array, np_ua_osm == 12, 2) np.place(detailed_array, np_ua_osm == 13, 3) np.place(detailed_meta_array, np_ua_osm == 13, 2) np.place(detailed_array, np_ua_osm == 14, 4) np.place(detailed_meta_array, np_ua_osm == 14, 2) # Urban Atlas - Class 30 to 6 np.place(detailed_array, np_ua_osm == 30, 6) np.place(detailed_meta_array, np_ua_osm == 30, 2) # Create Detailed Globe Cover 30 obj_to_rst(detailed_array, detailed_globe_raster, original_globe_raster, noData=0) # Create Detailed Globe Cover 30 meta detailed_meta = os.path.join( os.path.dirname(detailed_globe_raster), '{n}_meta{e}'.format( n=os.path.splitext(os.path.basename(detailed_meta))[0], e=os.path.splitext(os.path.basename(detailed_meta))[1])) obj_to_rst(detailed_meta_array, detailed_meta, original_globe_raster, noData=0)
def ntirogiannis2014(im): lib.debug_prefix.append('ng2014') debug_imwrite('input.png', im) im_h, _ = im.shape N, BG_prime = ng2014_normalize(im) O = otsu(N) debug_imwrite('O.png', O) letters = algorithm.all_letters(O) height_map = HeightMap(letters) ratio_sum = 0 for h in range(1, height_map.max_height() + 1): if len(height_map[h]) == 0: continue ratio_sum += height_map.ratio_pixels(h) / height_map.ratio_components( h) if ratio_sum > 1: break min_height = h if lib.debug: print('Accept components only >= height', h) OP = O.copy() for h in range(1, min_height): for letter in height_map[h]: sliced = letter.slice(OP) np.place(sliced, letter.raster(), 255) debug_imwrite('OP.png', OP) strokes = fast_stroke_width(OP) debug_imwrite('strokes.png', normalize_u8(strokes.clip(0, 10))) SW = int(round(strokes.sum() / np.count_nonzero(strokes))) if lib.debug: print('SW =', SW) S = skeleton(OP) debug_imwrite('S.png', S) S_inv = ~S # S_inv_32 = S_inv.astype(np.int32) # FG_count = np.count_nonzero(S_inv) FG_pos = im[S_inv.astype(bool)] FG_avg = FG_pos.mean() FG_std = FG_pos.std() # FG = (S_inv & im).astype(np.int32) # FG_avg = FG.sum() / float(FG_count) # FG_std = np.sqrt(((S_inv_32 & (FG - FG_avg)) ** 2).sum() / float(FG_count)) if lib.debug: print('FG:', FG_avg, FG_std) BG_avg = BG_prime.mean() BG_std = BG_prime.std() if lib.debug: print('BG:', BG_avg, BG_std) if FG_avg + FG_std != 0: C = -50 * np.log10((FG_avg + FG_std) / (BG_avg - BG_std)) k = -0.2 - 0.1 * C / 10 else: # This is the extreme case when the FG is 100% black, check the article explaination page before equation 5 C = -50 * np.log10((2.5) / (BG_avg - BG_std)) k = -0.2 - 0.1 * C / 10 if lib.debug: print('niblack:', C, k) local = niblack(N, window_size=(2 * SW) | 1, k=k) debug_imwrite('local.png', local) local_CCs = algorithm.all_letters(local) # NB: paper uses OP here, which results in neglecting all small components. O_inv = ~O O_inv_32 = O_inv.astype(np.int8, copy=False).astype(np.int32).astype(np.uint32, copy=False) label_map_O_inv = O_inv_32 & local_CCs[0].label_map CO_inv = np.zeros(im.shape, dtype=np.uint8) for cc in local_CCs: if np.count_nonzero(cc.slice(label_map_O_inv) == cc.label) / float( cc.area()) >= C / 100: CO_sliced = cc.slice(CO_inv) np.place(CO_sliced, cc.raster(), 255) CO = ~CO_inv debug_imwrite('CO.png', CO) CO_inv_dilated = cv2.dilate(CO_inv, rect33) FB = ~(CO_inv | ((~O) & CO_inv_dilated)) debug_imwrite('FB.png', FB) lib.debug_prefix.pop() return FB
def netCDF_filter_nan(data_file, phase_str, interim_path): ''' phase_str: train, val or test ''' data_dic = {'input_obs': None, 'input_ruitu': None, 'ground_truth': None} print('processing...:', data_file) ori_data = nc.Dataset(data_file) # 读取nc文件 ori_dimensions, ori_variables = ori_data.dimensions, ori_data.variables # 获取文件中的维度和变量 date_index, fortime_index, station_index = 1, 2, 3 # 根据三个维度,读取该变量在特定日期、特定站点的特定预报时刻的数值 var_obs = [] # var name list var_all = [] var_ruitu = [] for v in ori_variables: var_all.append(v) if v.find("_obs") != -1: var_obs.append(v) elif v.find('_M') != -1: var_ruitu.append(v) sta_id = ori_variables['station'][:].data print('sta_id:', sta_id) hour_index = ori_variables['foretimes'][:].data print('hour_index:', hour_index) day_index = ori_variables['date'][:].data print('day_index:', day_index) print(str(list(day_index)[-1]).split('.')[0]) # build a map for staion and its index station_dic = {} for i, s in enumerate(sta_id): station_dic[s] = i print(station_dic) NUMS = ori_dimensions['date'].size # 1188 for train input_ruitu_nan_list = [] input_obs_nan_list = [] #output_obs_nan_list=[] for i in range(NUMS - 1): input_ruitu_nan = ( ori_variables['t2m_M'][i, :, :].data == -9999.).all() input_obs_nan = ( ori_variables['t2m_obs'][i, :, :].data == -9999.).all() if input_ruitu_nan: input_ruitu_nan_list.append(i) if input_obs_nan: input_obs_nan_list.append(i) input_ruitu_nan_list_minus1 = [i - 1 for i in input_ruitu_nan_list] print('input_ruitu_nan_list_minus1:', input_ruitu_nan_list_minus1) print('input_obs_nan_list', input_obs_nan_list) deleted_obs_days = input_ruitu_nan_list_minus1 + input_obs_nan_list #bad_days print('deleted_obs_days:', (deleted_obs_days)) print('deleted_obs_days_nums:', len(deleted_obs_days)) input_obs_dic = dict.fromkeys(var_obs, None) input_ruitu_dic = dict.fromkeys(var_ruitu, None) ground_truth_dic = dict.fromkeys(var_obs, None) good_obs_days = [i for i in range(NUMS - 1) if i not in deleted_obs_days] print('The number of not seriously NaN days:', len(good_obs_days)) good_groundtruth_days = [i + 1 for i in good_obs_days] for v in var_obs: input_obs_dic[v] = ori_variables[v][good_obs_days, :, :].data ground_truth_dic[v] = ori_variables[v][ good_groundtruth_days, :, :].data for v in var_ruitu: input_ruitu_dic[v] = ori_variables[v][good_groundtruth_days, :, :].data for v in var_obs: np.place(input_obs_dic[v], input_obs_dic[v] == -9999., np.nan) # Fill missing value with mean value mean_ = np.nanmean(input_obs_dic[v]) # Calculate mean except np.nan where_are_NaNs = np.isnan(input_obs_dic[v]) input_obs_dic[v][where_are_NaNs] = mean_ np.place(ground_truth_dic[v], ground_truth_dic[v] == -9999., np.nan) mean_ = np.nanmean(ground_truth_dic[v]) # Calculate mean except np.nan where_are_NaNs = np.isnan(ground_truth_dic[v]) ground_truth_dic[v][where_are_NaNs] = mean_ data_dic['input_obs'] = input_obs_dic data_dic['ground_truth'] = ground_truth_dic for v in var_ruitu: np.place(input_ruitu_dic[v], input_ruitu_dic[v] == -9999., np.nan) mean_ = np.nanmean(input_ruitu_dic[v]) # Calculate mean except np.nan where_are_NaNs = np.isnan(input_ruitu_dic[v]) input_ruitu_dic[v][where_are_NaNs] = mean_ data_dic['input_ruitu'] = input_ruitu_dic save_pkl(data_dic, interim_path, '{}_non_NaN.dict'.format(phase_str)) return '{}_non_NaN.dict'.format(phase_str)
def generate(self, area, sigma=None, verbose=False): """ Generates simulated maps. Parameters ---------- area: float Area of generated maps, in deg^2 sigma: ndarray or None Map instrument noise, in Jy. If None, no instrument noise is added. verbose: bool Print informational messages as it runs. Returns ------- A tuple containing the input maps. If truthtable is set on initialization, also includes the truth table of positions and fluxes, where the positions are relative to the first map. Notes ----- Remember that the returned maps follow astropy.io.fits conventions, so the indexing into the maps is [y, x] """ if area <= 0.0: raise ValueError("Invalid (non-positive) area") if sigma is None: int_sigma = np.zeros(self._nbands, dtype=np.float32) elif type(sigma) == list: if len(sigma) != self._nbands: if len(sigma) == 1: int_sigma = sigma[0] * np.ones_like(self._wave) else: raise ValueError("Number of sigmas doesn't match number" " of wavelengths") else: int_sigma = np.asarray(sigma, dtype=np.float32) elif type(sigma) == np.ndarray: if len(sigma) != self._nbands: if len(sigma) == 1: int_sigma = sigma[0] * np.ones_like(self._wave) else: raise ValueError("Number of sigmas doesn't match number" " of wavelengths") else: int_sigma = sigma.astype(np.float32, copy=False) else: int_sigma = float(sigma) * np.ones_like(self._wave) if int_sigma.min() < 0: raise ValueError("Invalid (negative) instrument sigma") # Make the non-convolved images # The first step is to initialize the output maps. # Since we do the catalog in chunks (in typical applications # the catalog takes more memory than the maps), we must hold # all the maps in memory at once nextent = np.empty(self._nbands, dtype=np.int32) truearea = np.empty(self._nbands, dtype=np.float32) maps = [] for i in range(self._nbands): pixarea = (self._pixsize[i] / 3600.0)**2 nextent[i] = math.ceil(math.sqrt(area / pixarea)) truearea[i] = nextent[i]**2 * pixarea maps.append(np.zeros((nextent[i], nextent[i]), dtype=np.float32)) # Figure out how many sources to make truearea = truearea.mean() nsources_base = self._npersr * (math.pi / 180.0)**2 * truearea nsources = np.random.poisson(lam=nsources_base) if verbose: print("True area: {:0.2f} [deg^2]".format(truearea)) print("Number of sources to generate: {:d}".format(nsources)) # We do this in chunks if self._gensize == 0: # One big chunk nchunks = 1 chunks = np.array([nsources], dtype=np.int64) else: # Recall this is python 3 -- floating point division nchunks = math.ceil(nsources / self._gensize) chunks = self._gensize * np.ones(nchunks, dtype=np.int64) chunks[-1] = nsources - (nchunks - 1) * self._gensize assert chunks.sum() == nsources # Source generation loop nexgen = float(nextent[0]) if verbose: print("Generating sources") for i, nsrc in enumerate(chunks): if verbose and nchunks > 1: print(" Doing chunk {0:d} of {1:d}".format(i + 1, nchunks)) # Generate positions in base image, uniformly distributed # Note these are floating point xpos = nexgen * np.random.rand(nsrc) ypos = nexgen * np.random.rand(nsrc) # Get fluxes (in Jy) cat = self._gencat.generate(nsrc, wave=self._wave) fluxes = cat[-1].copy() # Set up truth table if needed if self._returntruth: truthtable = { 'x': xpos, 'y': ypos, 'z': cat[0], 'log10M': cat[1], 'sb': cat[2], 'log10sSFR': cat[3], 'log10Lir': cat[4], 'fluxdens': fluxes } # Try to be good about freeing up memory del cat # Add to first map without rescaling. # Note this has to happen in a for loop because multiple # sources can go in the -same- pixel cmap = maps[0] xf = np.floor(xpos) yf = np.floor(ypos) nx, ny = cmap.shape np.place(xf, xf > nx - 1, nx - 1) np.place(yf, yf > ny - 1, ny - 1) for cx, cy, cf in zip(xf, yf, fluxes[:, 0]): cmap[cy, cx] += cf # Note transpose # Other bands, with pixel scale adjustment for mapidx in range(1, self._nbands): posrescale = self._pixsize[0] / self._pixsize[mapidx] xf = np.floor(posrescale * xpos) yf = np.floor(posrescale * ypos) cmap = maps[mapidx] nx, ny = cmap.shape np.place(xf, xf > nx - 1, nx - 1) np.place(yf, yf > ny - 1, ny - 1) for cx, cy, cf in zip(xf, yf, fluxes[:, mapidx]): cmap[cy, cx] += cf # Note transpose if not self._returntruth: del fluxes, xpos, ypos, xf, yf else: del xf, yf # Now image details -- convolution, instrument noise for mapidx in range(self._nbands): if verbose: msg = "Preparing map for wavelength {0:5.1f} um "\ "extent: {1:d} x {2:d}" print( msg.format(self._wave[mapidx], maps[mapidx].shape[0], maps[mapidx].shape[1])) print(" Convolving") beam = self.get_gauss_beam(mapidx) maps[mapidx] = convolve(maps[mapidx], beam, boundary='wrap') if int_sigma[mapidx] > 0: if verbose: msg = " Adding instrument noise: {:0.4f} [Jy]" print(msg.format(int_sigma[mapidx])) maps[mapidx] += np.random.normal(scale=int_sigma[mapidx], size=maps[mapidx].shape) if self._returntruth: maps.append(truthtable) return maps
def dereverberate(wave, fs, params=None, estimate_execution_time=True, show_progress_bar=True): """ Estimates the impulse response in a room the recording took place :param wave: 1-D ndarray of wave samples :param fs: int - sampling frequency :param params: dict containing the algorithm parameters - keys: :param estimate_execution_time: should we print estimated execution time for each next frame :param show_progress_bar: should we print progress bar of estimation :returns: (h_stft_pow) 2-D ndarray power STFT of h_rir, (wave_dry) 1-D ndarray of the dry signal, (wave_wet) 1-D ndarray of the wet signal """ # estimating execution time loop_times = np.zeros(10) # =================== Parameters =================== if params is None: params = DEF_PARAMS # ==================== Windowing =================== win_len_ms = params["win_len"] win_ovlap_p = params["win_ovlap"] # ================ Times to samples ================ win_len = int(win_len_ms / 1000 * fs) win_ovlap = int(win_len * win_ovlap_p) window = np.hanning(win_len) # =================== Signal stft ================== nfft = first_larger_square(win_len) sig_stft = STFT(wave, window, win_ovlap, nfft) sig_stft = sig_stft[:, 0:nfft // 2 + 1] frame_count, frequency_count = sig_stft.shape # ==================== Constants =================== # length of the impulse response blocks = params["blocks"] # minimum gain of dry signal per frequency min_gain_dry = params["min_gain_dry"] # maximum impulse response estimate # max_h, blocks = read_impulse_response("deverb_test_samples/stalbans_a_mono.wav", fs, nfft, win_len_ms, win_ovlap_p) max_h = get_max_h_matrix('const', frequency_count, blocks) # bias used to keep magnitudes from getting stuck on a wrong minimum bias = params["bias"] # alpha and gamma - smoothing factors for impulse response magnitude and gain alpha = params["alpha"] gamma = params["gamma"] # ==================== Algorithm =================== # dry_stft and wet_stft are the estimated dry and reverberant signals in frequency-time domain dry_stft = np.zeros((frame_count, frequency_count), dtype=np.csingle) wet_stft = np.zeros((frame_count, frequency_count), dtype=np.csingle) # h_stft_pow is the estimated impulse response in frequency-time domain h_stft_pow = max_h / 2 # matrices with the information of currently estimated raw and dry signal (power spectra) raw_frames = np.ones((blocks, frequency_count)) dry_frames = np.zeros((blocks, frequency_count)) # c is a matrix to keep the raw estimated powers of the impulse response c = np.zeros((blocks, frequency_count)) # gain_dry and gain_wet are the frequency gains of the dry and wet signals gain_dry = np.ones(frequency_count) gain_wet = np.zeros(frequency_count) for i in range(frame_count): if estimate_execution_time: remaining = round(np.mean(loop_times) * (frame_count - i)) loop_times[1:] = loop_times[0:-1] loop_times[0] = time.time() print( "Processing frame {} of {}, estimated time left: {} ms".format( i + 1, frame_count, remaining)) frame = sig_stft[i, :] frame_power = np.power(np.abs(frame), 2) # estimate signals based on i-th frame for b in range(blocks): estimate = frame_power / raw_frames[b, :] np.place(estimate, estimate >= h_stft_pow[b, :], h_stft_pow[b, :] * bias + np.finfo(np.float32).eps) np.fmin(estimate, max_h[b, :], out=c[b, :]) h_stft_pow[b, :] = alpha * h_stft_pow[b, :] + (1 - alpha) * c[b, :] # calculating gains new_gain_dry = 1 - np.sum(dry_frames * h_stft_pow, axis=0) / frame_power np.place(new_gain_dry, new_gain_dry < min_gain_dry, min_gain_dry) gain_dry = gamma * gain_dry + (1 - gamma) * new_gain_dry new_gain_wet = 1 - gain_dry gain_wet = gamma * gain_wet + (1 - gamma) * new_gain_wet # calculatnig signals dry_stft[i, :] = gain_dry * frame wet_stft[i, :] = gain_wet * frame # shifting previous frames dry_frames[1:blocks, :] = dry_frames[0:blocks - 1, :] dry_frames[0, :] = np.power(np.abs(dry_stft[i, :]), 2) raw_frames[1:blocks, :] = raw_frames[0:blocks - 1, :] raw_frames[0, :] = frame_power if estimate_execution_time: loop_times[0] = round(1000 * (time.time() - loop_times[0])) if show_progress_bar: printProgressBar(i, frame_count, prefix='Progress', suffix='Complete', length=30) window = optimal_synth_window(window, win_ovlap) if TEST_SCOPE: t = (np.arange(frame_count) * (win_len_ms * (1 - win_ovlap_p))).astype(int) f = np.linspace(0, fs / 2, frequency_count).astype(int) txx, fxx = np.meshgrid(t, f) fig, axes = plt.subplots(3, 1, figsize=(10, 10)) axes[0].pcolormesh(txx, fxx, np.log10(np.power(np.abs(sig_stft.T), 2)), cmap=plt.get_cmap('plasma')) axes[0].set_title("Original signal") axes[1].pcolormesh(txx, fxx, np.log10(np.power(np.abs(dry_stft.T), 2)), cmap=plt.get_cmap('plasma')) axes[1].set_title("Dry signal") axes[2].pcolormesh(txx, fxx, np.log10(np.power(np.abs(wet_stft.T), 2)), cmap=plt.get_cmap('plasma')) axes[2].set_title("Reverberant signal") fig.show() wave_dry = reconstruct(dry_stft, window, win_ovlap) wave_wet = reconstruct(wet_stft, window, win_ovlap) return h_stft_pow, wave_dry, wave_wet
def calc_distance_df(locs, data_norm, cellGraph, gmmDict, fdr_df, tissue_mat_new, sort_tissue=False): hamming = list() jaccard = list() hausdorff = list() recal_genes = [] genelist = [] for geneID in fdr_df.index: exp = data_norm.loc[:, geneID].values gmm = gmmDict[geneID] target_seed_mat_list = list() temp_factor = int(fdr_df.loc[geneID].smooth_factor) newLabels = fdr_df.loc[geneID][4:].values.astype(int) node = fdr_df.loc[geneID, 'nodes'] p = fdr_df.loc[geneID, 'p_value'] num_isolate = count_isolate(locs, cellGraph, newLabels) num_noise = int((num_isolate[0] + num_isolate[1]) * 1 + (num_isolate[2] + num_isolate[3]) * 2 + (num_isolate[4] + num_isolate[5]) * 3) # print(num_noise,len(node[0]),(locs.shape[0]-num_noise),p) if len(p) == 1 and len(node[0]) == locs.shape[0]: recal_genes.append( geneID) ## no cuts genes, can't calculate hausdorff # elif len(p)==1 and min(p)>0.1 and len(node[0])==(locs.shape[0]-num_noise): ## all noise,like # genelist.append(geneID) # total_tissue_mat=np.sum(tissue_mat_new,axis=0) # total_target_mat=newLabels # hamming.append(compute_diff_vs_common_const10(total_tissue_mat , total_target_mat)) # jaccard.append(jaccard_dist(total_tissue_mat , total_target_mat)) # target_locs = locs[total_target_mat == 1] # tissue_locs = locs[total_tissue_mat == 1] # hausdorff.append(compute_hausdorff(target_locs,tissue_locs)) else: genelist.append(geneID) gmm_pred = gmm.predict(exp.reshape(-1, 1)) a = exp[exp > 0] zero_label = gmm.predict(np.min(a).reshape(-1, 1))[0] if sum(exp == 0) > 0: np.place(gmm_pred, exp == 0, zero_label) # print(major_label, unique_com, counts_com) # plot_voronoi_boundary(geneID, locs, exp, newLabels, min(p)) # get major label in min(p) segment target_seed_mat_list = list() node_list = node[np.argmin(p)] temp_vec = np.zeros(len(exp)) temp_vec[node_list] = 1 target_seed_mat_list.append(temp_vec) # for nn in node[np.argmin(p)]: # plt.text(locs[nn,0],locs[nn,1],str(nn)) # plot_voronoi_boundary(geneID, locs,exp,newLabels,min(p)) com_size = len(node_list) temp_exp = exp[np.array(list(node_list))] gmm_pred_com = gmm.predict(temp_exp.reshape(-1, 1)) if sum(temp_exp == 0) > 0: # a= temp_exp[temp_exp > 0] # zero_label = gmm.predict(np.min(a).reshape(-1,1))[0] np.place(gmm_pred_com, temp_exp == 0, zero_label) unique_com, counts_com = np.unique(gmm_pred_com, return_counts=True) major_label = unique_com[np.where( counts_com == counts_com.max())[0][0]] max_size = 0 size = [len(node[nn]) for nn in range(len(node))] p_sort = [p[i] for i in np.argsort(size)] node_sort = [node[i] for i in np.argsort(size)] #p_sort for p_index in np.arange( len(p)): #np.where(np.array(p) < 0.01)[0]: if p_sort[p_index] > min(p): #p[p_index] <= p_cutoff and temp_vec = np.zeros(len(exp)) node_list = node_sort[p_index] temp_exp = exp[np.array(list(node_list))] gmm_pred_com = gmm.predict(temp_exp.reshape(-1, 1)) if sum(temp_exp == 0) > 0: # a= temp_exp[temp_exp > 0] # zero_label = gmm.predict(np.min(a).reshape(-1,1))[0] np.place(gmm_pred_com, temp_exp == 0, zero_label) unique_com, counts_com = np.unique(gmm_pred_com, return_counts=True) temp_major_label = unique_com[np.where( counts_com == counts_com.max())[0][0]] # print(temp_major_label) if temp_major_label == major_label: ## get other pattern with major label #['Psap'] in Rep8 if len(node_list) + sum( np.sum(target_seed_mat_list, axis=0)) < (2 / 3) * len(exp) + 10: temp_vec[node_list] = 1 target_seed_mat_list.append(temp_vec) if len(node_list) > max_size: max_size = len(node_list) # print(max_size) else: if sum(target_seed_mat_list[0]) > max_size: max_size = sum(target_seed_mat_list[0]) else: if sum(target_seed_mat_list[0]) > max_size: max_size = sum( target_seed_mat_list[0] ) ## sum(np.sum(target_seed_mat_list,axis=0)) else: if sum(target_seed_mat_list[0]) > max_size: max_size = sum(target_seed_mat_list[0]) temp_list = list() if max_size > 5: for nn in target_seed_mat_list: if sum(nn == 1) > 5: temp_list.append(nn) else: for nn in target_seed_mat_list: if sum(nn == 1) == max_size: temp_list.append(nn) target_mat = np.array(temp_list) total_target_mat = np.sum(target_mat, axis=0) ## effect target mat: merge target mat as same label with min(p) if sum(total_target_mat) > len(exp) / 2 + 10: total_target_mat = abs( 1 - total_target_mat) ## 1-total_target_mat, for 'Hk3' # now find all tissue mat that with 50% of the tissue mat seg are in target temp_tissue_seg_list = list() best_overlap = list() best_overlap_val = 0 match_index = [] for ts_index in np.arange(len(tissue_mat_new)): # overlap = max(sum((tissue_mat_new[kk,:] + total_target_mat) == 2)/sum(tissue_mat_new[kk,:]), # sum((tissue_mat_new[kk,:] + total_target_mat) == 2)/sum(total_target_mat)) overlap = compute_inclusion_min(tissue_mat_new[ts_index], total_target_mat) # print(ts_index, overlap) if overlap > 0.45: match_index.append(ts_index) temp_tissue_seg_list.append(tissue_mat_new[ts_index]) if overlap >= best_overlap_val: #and best_overlap_val>=0.1 : # match_index.append(ts_index) best_overlap_val = overlap best_overlap = tissue_mat_new[ts_index].reshape(1, -1) if len(temp_tissue_seg_list) > 0: if sort_tissue == True: if len(temp_tissue_seg_list) == 1 and match_index[0] > 3: if match_index[0] == 4 or match_index[0] == 6: temp_tissue_seg_list.append(tissue_mat_new[5]) if match_index[0] == 5: temp_tissue_seg_list.append(tissue_mat_new[4]) temp_tissue_seg_list.append(tissue_mat_new[6]) match_tissue_mat = np.array(temp_tissue_seg_list) else: match_tissue_mat = np.array(temp_tissue_seg_list) else: match_tissue_mat = np.array(temp_tissue_seg_list) else: if best_overlap_val > 0: match_tissue_mat = np.array(best_overlap) ## 2. some points are missed by tissue mat, using min(hausdorff) to match tissue mat.['Postn'] in Rep8 else: ## all(overlap==0) dist_temp = [] temp_vec = np.zeros(locs.shape[0]) temp_vec[:] = 1 temp_vec[node[0]] = 0 ## mostly nodes u = locs[np.where(temp_vec == 1)] for ts_index in range(len(tissue_mat_new)): # ja_dist=jaccard_dist(temp_vec , tissue_mat_new[ts_index]) ## single noise not in any tissue mat. # ja_dist_temp.append(ja_dist) v = locs[np.where(tissue_mat_new[ts_index] == 1)] dist = compute_hausdorff(u, v) dist_temp.append(dist) temp_index = np.argmin(dist_temp) match_tissue_mat = tissue_mat_new[temp_index].reshape( 1, -1) total_tissue_mat = np.sum( match_tissue_mat, axis=0) ## get tissue_mat that matched target hamming.append( compute_diff_vs_common_const10(total_tissue_mat, total_target_mat)) jaccard.append(jaccard_dist(total_tissue_mat, total_target_mat)) target_locs = locs[total_target_mat == 1] tissue_locs = locs[total_tissue_mat == 1] hausdorff.append(compute_hausdorff(target_locs, tissue_locs)) # print(geneID, len(target_seed_mat_list), sum(total_target_mat), # len(temp_tissue_seg_list), sum(total_tissue_mat), jaccard_dist(total_tissue_mat , total_target_mat), # max(directed_hausdorff(tissue_locs, target_locs)[0], # directed_hausdorff(target_locs, tissue_locs)[0])) # plot_voronoi_boundary(geneID, locs, exp, total_target_mat, min(p)) #plot_voronoi_boundary(geneID, locs, exp, total_tissue_mat, min(p)) # print('------------------') data_array = np.array((hamming, jaccard, hausdorff), dtype=float).T c_labels = ['Hamming', 'Jaccard', 'Hausdorff'] dist_df = pd.DataFrame(data_array, index=genelist, columns=c_labels) # print out the target sum mat, and matched tissue sum mat to see whether they make sense return dist_df, recal_genes
def cross_jaccard(input_rasters, thresholds, verbose=False, logger=None): """ Calculate Jaccard coefficients between all the inpur rasters. This is a utility function that is intented to be used to compare fractions of the landscape. :param input_rasters list of input raster paths. :param thresholds vector of numeric tuples (x_min, x_max, y_min, y_max) values of thresholds. :param verbose: Boolean indicating how much information is printed out. :param logger: logger object to be used. :param ... additional arguments passed on to jaccard(). :return Pandas Dataframe with Jaccard coefficients between all rasters. """ # 1. Setup -------------------------------------------------------------- all_start = timer() if not logger: logging.basicConfig() llogger = logging.getLogger('cross_jaccard') llogger.setLevel(logging.DEBUG if verbose else logging.INFO) else: llogger = logger # Check the inputs assert len(input_rasters) > 1, "More than one input rasters are needed" assert len(thresholds) >= 1, "At least one tuple of thresholds is needed" # 2. Calculations -------------------------------------------------------- llogger.info(" [** COMPUTING JACCARD INDICES **]") all_jaccards = pd.DataFrame({ "feature1": [], "feature2": [], "threshold": [], "coef": [] }) n_rasters = len(input_rasters) # Generate counter information for all the computations. The results # matrix is always diagonally symmetrical. n_computations = int( (n_rasters * n_rasters - n_rasters) / 2 * len(thresholds)) no_computation = 1 for threshold in thresholds: if len(threshold) != 4: llogger.error("Threshold tuple needs 4 values") next for i in range(0, n_rasters): x_min, x_max, y_min, y_max = threshold raster1 = rasterio.open(input_rasters[i]) # To calculate the Jaccard index we are dealing with binary data # only. Avoid using masked arrays and replace NoData values with # zeros. raster1_nodata = raster1.nodata raster1_src = raster1.read(1) np.place(raster1_src, np.isclose(raster1_src, raster1_nodata), 0.0) for j in range(i + 1, n_rasters): raster2 = rasterio.open(input_rasters[j]) raster2_nodata = raster2.nodata raster2_src = raster2.read(1) np.place(raster2_src, np.isclose(raster2_src, raster2_nodata), 0.0) prefix = utils.get_iteration_prefix(no_computation, n_computations) llogger.info(("{} Calculating Jaccard ".format(prefix) + "index for [{0}, {1}] ".format(x_min, x_max) + "in {} ".format(input_rasters[i]) + "and, [{0}, {1}] ".format(y_min, y_max) + "in {}".format(input_rasters[j]))) coef = compute_jaccard(raster1_src, raster2_src, x_min=x_min, x_max=x_max, y_min=y_min, y_max=y_max) jaccards = pd.DataFrame({ "feature1": [input_rasters[i]], "feature2": [input_rasters[j]], "threshold": [threshold], "coef": [coef] }) all_jaccards = pd.concat([all_jaccards, jaccards]) no_computation += 1 all_jaccards.index = np.arange(0, len(all_jaccards.index), 1) all_end = timer() all_elapsed = round(all_end - all_start, 2) llogger.info(" [TIME] All processing took {} sec".format(all_elapsed)) return all_jaccards
filter_data = shuffle(np.loadtxt('spambase.data.csv', delimiter=',')) X_train, X_test, y_train, y_test = train_test_split(filter_data[:, np.arange(57)], filter_data[:, 57], test_size=0.50) indices = [np.nonzero(y_train==0)[0], np.nonzero(y_train)[0]] mean = np.transpose([np.mean(X_train[indices[0], :], axis=0), np.mean(X_train[indices[1], :], axis=0)]) std = np.transpose([np.std(X_train[indices[0], :], axis=0), np.std(X_train[indices[1], :], axis=0)]) ################################################################################################################################################################### # Experiment-1 : Classification with Gaussian Naive Bayes # # Compute prior probabilities for both classes and take their log.Find the feature indices which have zero standard deviation. # # If any such indices exist, Replace the zero standard deviation value with a small number # ################################################################################################################################################################### logP_class = np.log([1-np.mean(y_train), np.mean(y_train)]) zero_std = np.nonzero(std==0)[0] if (np.any(zero_std)): np.place(std, std==0, 0.0001) predict = [] for i in range(0, X_test.shape[0]): # Compute independent probabilities of features w.r.t. class P_X = np.divide(np.exp(-1*(np.divide(np.power(np.subtract(X_test[i, :].reshape(X_test.shape[1], 1), mean), 2), 2*np.power(std, 2)))), math.sqrt(2*np.pi)*std) # Compute class prediction for the test sample Class_X = np.argmax(logP_class+np.sum(np.nan_to_num(np.log(P_X)), axis=0)) predict.append(Class_X) print ("Experiment-1: Gaussian Naive Bayes") print ("Accuracy: ",metrics.accuracy_score(predict,y_test)) print ("Precision: ",metrics.precision_score(y_test, predict)) print ("Recall: ",metrics.recall_score(y_test, predict)) print ("Confusion Matrix:\n",metrics.confusion_matrix(y_test, predict)) ###################################################################################################################################################################
def JPEG_DCT(imgfile): # 3 channels x 8 bits, indexed-colors img = cv2.imread(imgfile, cv2.CV_LOAD_IMAGE_COLOR) wndName = "original indexed-color" cv2.namedWindow(wndName, cv2.WINDOW_AUTOSIZE) cv2.imshow(wndName, img) iHeight, iWidth = img.shape[:2] print "size:", iWidth, "x", iHeight # set size to multiply of 8 if (iWidth % 8) != 0: filler = img[:, iWidth - 1:, :] #print "width filler size", filler.shape for i in range(8 - (iWidth % 8)): img = np.append(img, filler, 1) if (iHeight % 8) != 0: filler = img[iHeight - 1:, :, :] #print "height filler size", filler.shape for i in range(8 - (iHeight % 8)): img = np.append(img, filler, 0) iHeight, iWidth = img.shape[:2] print "new size:", iWidth, "x", iHeight # convert to YCrCb, Y=luminance, Cr/Cb=red/blue-difference chrominance img = cv2.cvtColor(img, cv2.COLOR_BGR2YCR_CB) # array as storage for DCT + quant.result img2 = np.empty(shape=(iHeight, iWidth, 3)) # FORWARD ---------------- # do calc. for each 8x8 non-overlapping blocks cv2.imshow("dctimage", img2) for startY in range(0, iHeight, 8): for startX in range(0, iWidth, 8): for c in range(0, 3): block = img[startY:startY + 8, startX:startX + 8, c:c + 1].reshape(8, 8) # apply DCT for a block blockf = np.float32(block) # float conversion dst = cv2.dct(blockf) # dct # quantization of the DCT coefficients blockq = np.around(np.divide(dst, std_quant_tbl[c])) blockq = np.multiply(blockq, std_quant_tbl[c]) # store the result for y in range(8): for x in range(8): img2[startY + y, startX + x, c] = blockq[y, x] # INVERSE ---------------- for startY in range(0, iHeight, 8): for startX in range(0, iWidth, 8): for c in range(0, 3): block = img2[startY:startY + 8, startX:startX + 8, c:c + 1].reshape(8, 8) blockf = np.float32(block) # float conversion dst = cv2.idct(blockf) # inverse dct np.place(dst, dst > 255.0, 255.0) # saturation np.place(dst, dst < 0.0, 0.0) # grounding block = np.uint8(np.around(dst)) # store the results for y in range(8): for x in range(8): img[startY + y, startX + x, c] = block[y, x] print img.dtype # convert to BGR img = cv2.cvtColor(img, cv2.COLOR_YCR_CB2BGR) wndName1 = "DCT + quantization + inverseDCT" cv2.namedWindow(wndName1, cv2.WINDOW_AUTOSIZE) cv2.imshow(wndName1, img) cv2.waitKey(0) cv2.destroyWindow(wndName1) cv2.destroyWindow(wndName)
param_path = "../results/keep/fig031921/ber_3/params.json" params = settings.load_param(param_path) # n_sum = params["test_bits"] * params['SNR_AVERAGE'] * load_files n_sum = params['test_bits'] * params['trials'] snrs_db = np.linspace(params['graph_x_min'], params['graph_x_max'], params['graph_x_num']) fig, ax = graph.new_snr_ber_canvas(params['graph_x_min'], params['graph_x_max'], -4) ax.set_yticks([10**0, 10**-1, 10**-2, 10**-3, 10**-4]) previous_n_sum = params['previous_test_bits'] * params['trials'] pkl_path = "../results/keep/fig031921/ber_3/result.pkl" result = load_pkl_file(pkl_path) errors_sum = np.sum(result.non_cancell_error_array, axis=1) bers = errors_sum / previous_n_sum np.place(bers, bers == 0, None) ax.plot(snrs_db, bers, color='k', marker='x', linestyle=':', label="w/o canceller " + r"$(I=1)$", ms=10) errors_sum = np.sum(result.previous_errors, axis=1) bers = errors_sum / previous_n_sum np.place(bers, bers == 0, None) ax.plot(snrs_db, bers, color='k', marker='o', linestyle='--', label="Conventional " + r"$(I=1)$" + " [5]" , ms=10, markerfacecolor='None') pkl_path = "../results/keep/fig031921/ber_1/result.pkl" result = load_pkl_file(pkl_path) errors_sum = np.sum(result.errors, axis=1) bers = errors_sum / n_sum np.place(bers, bers == 0, None) ax.plot(snrs_db, bers, color='k', marker='^', linestyle='-', label="Proposed " + r"$(I=1)$", ms=10) # pkl_path = "../results/ofdm_fde_system_model/2021/01/21/22_24_29/result.pkl"
assert(magic_num==2049) raw_labels=f.read(num_labs) test_labs=[i for i in raw_labels] ncols=20 nrows=20 rand_indices=(10000*np.random.random([ncols*nrows])).astype("int") img_mat=np.zeros((img_height*nrows, img_width*ncols)) mat_rselector=np.matlib.repmat(np.floor(np.arange(0, nrows*img_height)/img_height), ncols*img_width, 1).transpose() mat_cselector=np.matlib.repmat(np.floor(np.arange(0, ncols*img_width)/img_width), nrows*img_height, 1) for i in range(0, nrows): labels=[] for j in range(0, ncols): np.place(img_mat, (mat_rselector==i) & (mat_cselector==j), train_imgs[rand_indices[i*ncols+j]]) labels.append(train_labs[rand_indices[i*ncols+j]]) print(labels) plt.imshow(img_mat, cmap="Greys") plt.show() num_classes=10 test_labs=keras.utils.to_categorical(test_labs) train_labs=keras.utils.to_categorical(train_labs) test_imgs=test_imgs.reshape(test_imgs.shape[0], img_height, img_width, 1) train_imgs=train_imgs.reshape(train_imgs.shape[0], img_height, img_width, 1) test_imgs=test_imgs.astype('float32') train_imgs=train_imgs.astype('float32') test_imgs/=255
def broadcast_to( value, index: Index, grouper: "Grouper" = None, ) -> Series: """Broastcast value to expected dimension, the result is a series with the given index Before calling this function, make sure that the index is already broadcasted based on the value. This means that index is always the wider then the value if it has one. Also the size for each group is larger then the length of the value if value doesn't have an index. Examples: >>> # 1. broadcast scalars/arrays(no index) >>> broadcast_to(1, Index(range(3))) >>> # Series: [1,1,1], index: range(3) >>> broadcast_to([1, 2], Index([0, 0, 1, 1])) >>> # ValueError >>> >>> # grouper: size() # [1, 1] >>> # grouper: result_index # ['a', 'b'] >>> broadcast_to([1, 2], Index([0, 0, 1, 1]), grouper) >>> # Series: [1, 2, 1, 2], index: [0, 0, 1, 1] >>> >>> # 2. broadcast a Series >>> broadcast(Series([1, 2]), index=[0, 0, 1, 1]) >>> # Series: [1, 1, 2, 2], index: [0, 0, 1, 1] >>> broadcast( >>> Series([1, 2], index=['a', 'b']), >>> index=[4, 4, 5, 5], >>> grouper, >>> ) >>> # Series: [1, 1, 2, 2], index: [4, 4, 5, 5] Args: value: Value to be broadcasted index: The index to broadcasted to grouper: The grouper of the original value Returns: The series with the given index """ if not is_list_like(value): return value if len(value) == 1 and not is_list_like(value[0]): return value[0] if not grouper: if len(value) == 0: return Series(value, index=index, dtype=object) # Series will raise the length problem return Series(value, index=index) gsizes = grouper.size() if gsizes.size == 0: return Series(value, index=index, dtype=object) # broadcast value to each group # length of each group is checked in _broadcast_base # A better way to distribute the value to each group? idx = np.concatenate( [grouper.groups[gdata] for gdata in grouper.result_index]) # make np.tile([[3, 4]], 2) to be [[3, 4], [3, 4]], # instead of [[3, 4, 3, 4]] repeats = (grouper.ngroups, ) + (1, ) * (np.ndim(value) - 1) # Can't do reindex for this case: # >>> df = tibble(a=[1, 1, 2, 2, 3, 3], _index=[1, 1, 2, 2, 3, 3]) # >>> df["b"] = [4, 5] # error out = Series(np.tile(value, repeats).tolist(), index=idx) if out.index.is_unique: return out.reindex(index) # recode index index_dict = out.index.to_frame().groupby(out.index).grouper.indices new_idx = Index(range(out.size)) new_index = np.ones(out.size, dtype=int) for ix, rowids in index_dict.items(): np.place(new_index, index == ix, rowids) out.index = new_idx out = out.reindex(new_index) new_idx = np.ones(out.size, dtype=int) for ix, rowids in index_dict.items(): new_idx[np.isin(new_index, rowids)] = ix out.index = new_idx return out
def voxel_samples3(image_path, patch_sizes): """ :param image_path: path to image generated by get_paths :param patch_sizes: a list or array of patch sizes desired for convolution. Must be odd > 1. :param total_sample_size: sampled patches for convolution, and corresponding labels :return: sampled patches for convolution, and corresponding labels """ # Load image from path Image = io.loadmat(image_path) # Extract total number of voxels Dims = np.prod(Image['scan'].shape) # Create 1 mask image Label = (Image['Tibia'] + 2 * Image['Femur']) np.place(Label, Label > 2, [2, 1]) if Image['isright'] == 0: Scan = np.fliplr(Image["scan"]) Label = np.fliplr(Label) else: Scan = Image["scan"] Index = np.array([]) Label2 = (Image['Tibia'] + Image['Femur']) B = morphology.distance_transform_cdt(np.absolute(Label2 - 1)) B = B.reshape(Dims) X = np.float32(range(np.max(B), 0, -1)) / range(1, np.max(B) + 1, 1) #print max(X) #print np.unique(B) #print min(X) No_samples = 36000 Index = np.array([]) samples = np.divide(X, sum(X)) * No_samples #print len(samples) for i in range(1, np.max(B) + 1): # print i Index = np.int32( np.append( Index, np.random.choice(np.squeeze(np.where(B == i)), (samples[i - 1]), replace=False))) Label = Label.reshape(Dims) # Find indexes of voxels to sample Gold_ind = np.random.choice(np.squeeze(np.where(Label == 1)), 12000, replace=False) Silv_ind = np.random.choice(np.squeeze(np.where(Label == 2)), 12000, replace=False) # Edge_ind = np.squeeze(np.where((B>0) & (B<=3))) # print Edge_ind.shape #Neg_Samp_ind = np.random.choice(np.squeeze(np.where(Label == 0)), 3*N_samp, replace=False) Index = np.hstack((Gold_ind, Silv_ind, Index)) np.random.shuffle(Index) Patch_List = [] # for each desired patch size, create patches for size in patch_sizes: PS = (size - 1) / 2 npad = ((PS, PS), (PS, PS), (PS, PS)) window_size = (size, size, size) image_pad = np.pad(Scan, pad_width=npad, mode='constant', constant_values=0) Patches = (view_as_windows(image_pad, window_size)).reshape( (Dims, size, size, size)) Patch_List.append( np.float32(Patches[Index, :, :, :].reshape(Index.shape[0], 1, size, size, size))) Inds = np.asarray(np.unravel_index(Index, Scan.shape, order='C')) Positions = Inds.reshape(3, 1, Inds.shape[1]).T Labels = Label[Index] Patch_List = tuple(Patch_List) return Patch_List, np.float32(Positions), np.int32(Labels)