def check_orbits(p1, t1, p2, t2, tmn, tmx, tol): n1 = t1 + p1 * np.arange(np.floor((tmn-t1)/p1), np.ceil((tmx-t1)/p1)) n1 = n1[(tmn <= n1) * (n1 <= tmx)] n2 = t2 + p2 * np.arange(np.floor((tmn-t2)/p2), np.ceil((tmx-t2)/p2)) n2 = n2[(tmn <= n2) * (n2 <= tmx)] delta = np.fabs(n1[:, None] - n2[None, :]) return max(len(n1), len(n2)) == np.sum(delta < tol)
def rand_checkers(n1=100, n2=100, n3=100, n4=100, sigma=0.1): """ Sample n1 and n2 points from a noisy checker""" nb1 = int(np.floor(n1 / 8)) nb2 = int(np.floor(n2 / 8)) nb3 = int(np.floor(n3 / 8)) nb4 = int(np.floor(n4 / 8)) xapp = np.reshape(np.zeros((nb1 + nb2 + nb3 + nb4) * 16), [(nb1 + nb2 + nb3 + nb4) * 8, 2]) yapp = np.ones((nb1 + nb2 + nb3 + nb4) * 8) idx = 0 nb = 2*nb1 for i in xrange(-2, 2): for j in xrange(-2, 2): yapp[idx:(idx + nb)] = [ fmod(i - j+100, 4)] * nb xapp[idx:(idx + nb), 0] = np.random.rand(nb) xapp[idx:(idx + nb), 0] += i + sigma * np.random.randn(nb) xapp[idx:(idx + nb), 1] = np.random.rand(nb) xapp[idx:(idx + nb), 1] += j + sigma * np.random.randn(nb) idx += nb ind = np.arange((nb1 + nb2 + nb3 + nb4) * 8) np.random.shuffle(ind) res = np.hstack([xapp, yapp[:, np.newaxis]]) return np.array(res[ind, :])
def shot( I, exp=0.1, flux=1e5, sensitivity=1.0, dark_c=None, io_noise=0.0, full_well=2 ** 10 - 1, el_per_ADU=1.0, offset=50.0 ): """\ I : intensity distribution flux : overall photon photons per seconds coming in exp : exposition time in sec io_noise : readout noise rms full_well : electron capacity of each pixel el_per_ADU : conversion effficiency of electrons to digitally counted units dark_curr : electrons per second per pixel on average """ I = np.asarray(I).astype(float) if I.sum() != 0.0: I = I / I.sum() photo_el = np.floor(sensitivity * np.random.poisson(exp * flux * I)) if dark_c is not None: therm_el = np.floor(np.random.poisson(dark_c * exp * np.ones_like(I))) else: therm_el = np.zeros_like(I) el = photo_el + therm_el el[el > full_well] = full_well out = offset + io_noise * np.random.standard_normal(I.shape) + el / el_per_ADU out[out < 0.0] = 0.0 return out.astype(int)
def build_X(bids, bot_or_human): X = bot_or_human X = X.drop("payment_account", 1) X = X.drop("address", 1) bids["day"] = np.floor((bids["time"] - startt) / one_day) bids["week"] = np.floor(bids["day"] / 7.0) # print bids.bidder_id[0] print "starting ips" X = ip(X, bids) print "starting bid order" X = bid_order(X, bids) print "starting dt" X = dt(X, bids) print "startin day" X = day(X, bids) print "starting n_bids" X = n_bids(X, bids) print "starting urls" X = urls(X, bids) # print 'starting bid order' # X = bid_order(X, bids) print "starting countries" X = user_countries_per_auction(X, bids) print "starting merch" X = merch(X, bids) return X
def __init__(self, filepath, qually, mode='444'): ''' ''' imOrig = cv2.imread(filepath,1) self.filepath = filepath self.mode = mode #Taxa de compressão e Redundancia self.CRate = 0; self.Redunc = 0 self.avgBits = 0 #Qualidade self.qually = qually #Dimensões da imagem original self.Mo, self.No, self.Do = imOrig.shape self.r, self.c = [8, 8] #DIMENSAO DOS BLOCOS #TRATA AS DIMENSOES DA IMAGEM (self.M, self.N, self.D), self.img = h.adjImg(imOrig) #NUMERO DE BLOCOS NA VERTICAL E HORIZONTAL self.nBlkRows = int(np.floor(self.M/self.r)) self.nBlkCols = int(np.floor(self.N/self.c)) #Gera Tabela de Qunatizaçao self.Z = h.genQntb(self.qually) #TRANSFORMA DE RGB PARA YCbCr self.Ymg = cv2.cvtColor(self.img, cv2.COLOR_BGR2YCR_CB) self.NumBits = 0 if self.Do == 2: self.NCHNL = 1 elif self.Do == 3: self.NCHNL = 3 # self.OUTCOMES = self._run_() self._run_()
def prepare_data(data_x, data_mask, data_y): ''' 将数据分为训练集,验证集和测试集 注意,因为要进行hstack, 行向量会变为列向量 ''' data_len = len(data_y) train_end = numpy.floor(data_len * 0.5) test_end = numpy.floor(data_len * 0.8) if data_x.ndim == 1: data_x.resize((data_x.shape[0],1)) if data_mask != [] and data_mask.ndim == 1: data_mask.resize((data_mask.shape[0],1)) if data_y.ndim == 1: data_y.resize((data_y.shape[0],1)) if data_mask == []: allData = numpy.concatenate((data_x,data_y), axis=1) else: allData = numpy.concatenate((data_x,data_mask,data_y), axis=1) train_data = allData[:train_end,...] test_data = allData[train_end:test_end,...] valid_data = allData[test_end:,...] return train_data, valid_data, test_data
def svd_example(): a = np.floor(np.random.rand(4, 4)*20-6) logger.info("Matrix A:\n %s", a) b = np.floor(np.random.rand(4, 1)*20-6) logger.info("Matrix B:\n %s", b) u, s, v_t = np.linalg.svd(a) # SVD decomposition of A logger.info("Matrix U:\n %s", u) logger.info("Matrix S:\n %s", s) logger.info("Matrix V(transpose:\n %s", u) logger.info("Computing inverse using linalg.pinv") # Computing the inverse using pinv inv_pinv = np.linalg.pinv(a) logger.info("pinv:\n %s", inv_pinv) # Computing inverse using matrix decomposition logger.info("Computing inverse using svd matrix decomposition") inv_svd = np.dot(np.dot(v_t.T, np.linalg.inv(np.diag(s))), u.T) logger.info("svd inverse:\n %s", inv_svd) logger.info("comparing the results from pinv and svd_inverse:\n %s", np.allclose(inv_pinv, inv_svd)) logger.info("Sol1: Solving x using pinv matrix... x=A^-1 x b") result_pinv_x = np.dot(inv_pinv, b) logger.info("Sol2: Solving x using svd_inverse matrix... x=A^-1 x b") result_svd_x = np.dot(inv_svd, b) if not np.allclose(result_pinv_x, result_svd_x): raise ValueError('Should have been True')
def spectrum(self, shape, surface_point, bound): """Returns the counts histogram (bins,counts) for """ wavelengths = [] key = shape.surface_identifier(surface_point) if not self.store.has_key(key): return None entries = self.store[key] if len(entries) == 0: return None for entry in entries: if entry[2] == bound: wavelengths.append(float(entry[1])) if len(wavelengths) is 0: return None wavelengths = np.array(wavelengths) min = wavelengths.min() max = wavelengths.max() if len(wavelengths) is 1: bins = np.arange(np.floor( wavelengths[0] - 1), np.ceil(wavelengths[0] + 2)) freq, bins = np.histogram(wavelengths, bins=bins) else: bins = np.arange(np.floor( wavelengths.min()-1), np.ceil(wavelengths.max()+2)) freq, bins = np.histogram(wavelengths, bins=bins) return Spectrum(bins[0:-1], freq)
def _find_tails(self, mag, rounding=True, half=5, full=10, flag=50): ''' Find how many of each of the tail pieces is necessary. Flag specifies the increment for a flag, barb for a full barb, and half for half a barb. Mag should be the magnitude of a vector (ie. >= 0). This returns a tuple of: (*number of flags*, *number of barbs*, *half_flag*, *empty_flag*) *half_flag* is a boolean whether half of a barb is needed, since there should only ever be one half on a given barb. *empty_flag* flag is an array of flags to easily tell if a barb is empty (too low to plot any barbs/flags. ''' #If rounding, round to the nearest multiple of half, the smallest #increment if rounding: mag = half * (mag / half + 0.5).astype(np.int) num_flags = np.floor(mag / flag).astype(np.int) mag = np.mod(mag, flag) num_barb = np.floor(mag / full).astype(np.int) mag = np.mod(mag, full) half_flag = mag >= half empty_flag = ~(half_flag | (num_flags > 0) | (num_barb > 0)) return num_flags, num_barb, half_flag, empty_flag
def _search_fine(sino, srad, step, init_cen, ratio, drop): """ Fine search for finding the rotation center. """ Nrow, Ncol = sino.shape centerfliplr = (Ncol + 1.0) / 2.0 - 1.0 # Use to shift the sinogram 2 to the raw CoR. shiftsino = np.int16(2 * (init_cen - centerfliplr)) _copy_sino = np.roll(np.fliplr(sino[1:]), shiftsino, axis=1) lefttake = 0 righttake = Ncol - 1 if init_cen <= centerfliplr: lefttake = np.ceil(srad + 1) righttake = np.floor(2 * init_cen - srad - 1) else: lefttake = np.ceil( init_cen - (Ncol - 1 - init_cen) + srad + 1) righttake = np.floor(Ncol - 1 - srad - 1) Ncol1 = righttake - lefttake + 1 mask = _create_mask(2 * Nrow - 1, Ncol1, 0.5 * ratio * Ncol, drop) numshift = np.int16((2 * srad + 1.0) / step) listshift = np.linspace(-srad, srad, num=numshift) listmetric = np.zeros(len(listshift), dtype='float32') num1 = 0 for i in listshift: _sino = ndimage.interpolation.shift( _copy_sino, (0, i), prefilter=False) sinojoin = np.vstack((sino, _sino)) listmetric[num1] = np.sum(np.abs(np.fft.fftshift( pyfftw.interfaces.numpy_fft.fft2( sinojoin[:, lefttake:righttake + 1]))) * mask) num1 = num1 + 1 minpos = np.argmin(listmetric) return init_cen + listshift[minpos] / 2.0
def connect_composition_III(self): synapse_list = [] Mt3v_list = self.non_columnar_neurons['Mt3v'] Mt3h_list = self.non_columnar_neurons['Mt3h'] for neuron in Mt3v_list: neuron.assign_pos(0., 0.) for neuron in Mt3h_list: neuron.assign_pos(0., 0.) rule3synapsesv = self.other_synapse_dict[self.other_synapse_dict['postname'] == 'Mt3v'] rule3synapsesh = self.other_synapse_dict[self.other_synapse_dict['postname'] == 'Mt3h'] dtnames = rule3synapsesv.dtype.names for cartridge in self.cartridges: synapse = Synapse(dict(zip(dtnames, [np.asscalar(p) for p in rule3synapsesv[0]]))) mtn = int(np.floor(cartridge.neurons['L2'].ypos / ((self.hexarray.Y[-1][-1]+1)/4))) synapse.link(cartridge.neurons['L2'], Mt3v_list[mtn]) synapse_list.append(synapse) synapse = Synapse(dict(zip(dtnames, [np.asscalar(p) for p in rule3synapsesh[0]]))) mtn = int(np.floor(cartridge.neurons['L2'].xpos / ((self.hexarray.X[-1][-1]+1)/4))) synapse.link(cartridge.neurons['L2'], Mt3h_list[mtn]) synapse_list.append(synapse) self.composition_rules.append({'synapses': synapse_list})
def qwtCanvasClip(canvas, canvasRect): x1 = np.ceil(canvasRect.left()) x2 = np.floor(canvasRect.right()) y1 = np.ceil(canvasRect.top()) y2 = np.floor(canvasRect.bottom()) r = QRect(x1, y1, x2-x1-1, y2-y1-1) return canvas.borderPath(r)
def getRowCol(self,lat,lon,returnFloat=False): """Return data row and column from given geographic coordinates (lat/lon decimal degrees). :param lat: Input latitude. :param lon: Input longitude. :param returnFloat: Boolean indicating whether floating point row/col coordinates should be returned. :returns: Tuple of row and column. """ ulx = self._geodict['xmin'] uly = self._geodict['ymax'] xdim = self._geodict['xdim'] ydim = self._geodict['ydim'] #check to see if we're in a scenario where the grid crosses the meridian if self._geodict['xmax'] < ulx and lon < ulx: lon += 360 col = (lon-ulx)/xdim row = (uly-lat)/ydim if returnFloat: return (row,col) return (np.floor(row).astype(int),np.floor(col).astype(int))
def _get_interv_graticule(self,pmin,pmax,dpar,mmin,mmax,dmer,verbose=True): def set_prec(d,n,nn=2): arcmin=False if d/n < 1.: d *= 60 arcmin = True nn = 1 x = d/n y = nn*x ex = np.floor(np.log10(y)) z = np.around(y/10**ex)*10**ex/nn if arcmin: z = 1./np.around(60./z) return z max_n_par = 18 max_n_mer = 36 n_par = (pmax-pmin)/dpar n_mer = (mmax-mmin)/dmer if n_par > max_n_par: dpar = set_prec((pmax-pmin)/dtor,max_n_par/2)*dtor if n_mer > max_n_mer: dmer = set_prec((mmax-mmin)/dtor,max_n_mer/2,nn=1)*dtor if dmer/dpar < 0.2 or dmer/dpar > 5.: dmer = dpar = max(dmer,dpar) vdeg = np.floor(np.around(dpar/dtor,10)) varcmin = (dpar/dtor-vdeg)*60. if verbose: print "The interval between parallels is %d deg %.2f'."%(vdeg,varcmin) vdeg = np.floor(np.around(dmer/dtor,10)) varcmin = (dmer/dtor-vdeg)*60. if verbose: print "The interval between meridians is %d deg %.2f'."%(vdeg,varcmin) return dpar,dmer
def interp(pic,flow): ys=np.arange(pic.shape[0]*pic.shape[1])/pic.shape[1] ud=(flow[:,:,0].reshape(-1)+ys)%pic.shape[0] xs=np.arange(pic.shape[0]*pic.shape[1])%pic.shape[1] lr=(flow[:,:,1].reshape(-1)+xs)%pic.shape[1] u=np.int32(np.floor(ud)) d=np.int32(np.ceil(ud))%pic.shape[0] udiffs=ud-u udiffs=np.dstack((udiffs,udiffs,udiffs)) l=np.int32(np.floor(lr)) r=np.int32(np.ceil(lr))%pic.shape[1] ldiffs=lr-l ldiffs=np.dstack((ldiffs,ldiffs,ldiffs)) ul=pic[u,l,:] ur=pic[u,r,:] dl=pic[d,l,:] dr=pic[d,r,:] udl=ul*(1-udiffs)+dl*udiffs udr=ur*(1-udiffs)+dr*udiffs ans=np.zeros(pic.shape) ans[ys,xs,:]=udl*(1-ldiffs)+udr*ldiffs return ans
def genice_lattice(atoms, box, matchfunc=None): global cell, celltype, waters, coord, density logger = logging.getLogger() filtered = [] if matchfunc is not None: for a in atoms: if matchfunc(a[0]): filtered.append(a) else: filtered = atoms dmin = shortest_distance(filtered) scale = 2.76 / dmin celltype = "triclinic" if (len(box) == 6): cell = np.array([[box[0],0,0],[box[1],box[2],0],[box[3],box[4],box[5]]]) else: cell = np.diag(box) volume = np.linalg.det(cell) icell = np.linalg.inv(cell) uniques = [] for name,x,y,z in filtered: rpos = np.dot([x,y,z], icell) rpos -= np.floor(rpos) #Do twice to reduce the floating point uncertainty. #(Hint: assume the case when x=-1e-33.) rpos -= np.floor(rpos) if is_unique(uniques, rpos): uniques.append(rpos) waters = uniques coord = "relative" # bondlen = 3 density = len(filtered)*18.0/(volume*scale**3*1e-24*6.022e23)
def allowable_ref_dividers(self, f_ref, require_integer_n, require_fractional_n): """ given a reference frequency and whether an integer-N solution is needed, return a qualified list of reference dividers that are within the limits of the PLL """ # first, we establish a minimum and maximum reference divider modulus.... # if we are forcing an integer-N solution, use those phase-detector frequency limits int_n_nref_max = (np.floor(f_ref / self.f_pfd_limits_integer_n[0])).astype(int) int_n_nref_min = (np.ceil(f_ref / self.f_pfd_limits_integer_n[1])).astype(int) frac_n_nref_max = (np.floor(f_ref / self.f_pfd_limits_fractional_n[0])).astype(int) frac_n_nref_min = (np.ceil(f_ref / self.f_pfd_limits_fractional_n[1])).astype(int) if require_integer_n: ref_divider_min = int_n_nref_min ref_divider_max = int_n_nref_max elif require_fractional_n: ref_divider_min = frac_n_nref_min ref_divider_max = frac_n_nref_max else: ref_divider_min = min(int_n_nref_min, frac_n_nref_min) ref_divider_max = max(int_n_nref_max, frac_n_nref_max) n_ref_allowed, n_ref_digital_codes = zip(*self.n_ref_data) # now, make a list of all the divider moduli in that range from min to max # making sure that the PLL can do each one allowable_divs = [] for n_ref_modulus in range(ref_divider_min, ref_divider_max+1): if n_ref_modulus in n_ref_allowed: allowable_divs.append(n_ref_modulus) return allowable_divs
def subsample(feature_list, trigger_list, clf, subsampling_rate = 0.75): """ clf: string -> 'perc' or 'nb' """ None_indices = [i for (i,trigger) in enumerate(trigger_list) if trigger == u'None'] All_other_indices = [i for (i,trigger) in enumerate(trigger_list) if trigger != u'None'] N = len(None_indices) N_pick = np.floor((1.0 - subsampling_rate) * N) #N_pick = len(All_other_indices) #now pick N_pick random 'None' samples among all of them. random_indices = np.floor(np.random.uniform(0, N , N_pick) ) subsample_of_None_indices = [None_indices[int(i)] for i in random_indices] # Identify indices of remaining samples after subsampling + randomise them. remaining_entries = subsample_of_None_indices + All_other_indices perm = np.random.permutation(len(remaining_entries)) remaining_entries = [remaining_entries[p] for p in perm] # Return the subsampled list of samples. if clf=='perc': subsampled_feature_list = [feature_list[i] for i in remaining_entries ] subsampled_trigger_list = [trigger_list[i] for i in remaining_entries ] return subsampled_feature_list, subsampled_trigger_list elif clf=='nb': subsampled_feature_list = feature_list.tocsr()[remaining_entries].tocoo() subsampled_trigger_list = np.asarray([trigger_list[i] for i in remaining_entries ]) return subsampled_feature_list, subsampled_trigger_list
def test_integer_inputs(data, method, center_data, fit_mean, with_errors, normalization): if method == 'scipy' and (fit_mean or with_errors): return t, y, dy = data t = np.floor(100 * t) t_int = t.astype(int) y = np.floor(100 * y) y_int = y.astype(int) dy = np.floor(100 * dy) dy_int = dy.astype('int32') frequency = 1E-2 * (0.8 + 0.01 * np.arange(40)) if not with_errors: dy = None dy_int = None kwds = dict(center_data=center_data, fit_mean=fit_mean, normalization=normalization) P_float = LombScargle(t, y, dy, **kwds).power(frequency,method=method) P_int = LombScargle(t_int, y_int, dy_int, **kwds).power(frequency, method=method) assert_allclose(P_float, P_int)
def PlotErrBoxPlot(x, y, delta, ax, showXTicks): if x.size < 1: return ids = np.floor((x)/delta).astype(np.int) data = [] for i in range(ids.min(), ids.max()+1): if (ids==i).any(): data.append(y[ids==i]) bp = plt.boxplot(data) # plt.plot(x,y,'.', color=c1, alpha=0.3) # set xticks if showXTicks: ticks = np.floor((np.arange(ids.min(), ids.max()+1)+0.5)*delta).astype(np.int) if np.unique(ticks).size < ticks.size: ticks = np.floor((np.arange(ids.min(), ids.max()+1)+0.5)*delta*10.)/10. xtickNames = plt.setp(ax, xticklabels=ticks) plt.setp(xtickNames, rotation=45) else: plt.setp(ax.get_xticklabels(), visible=False) for box in bp["boxes"]: box.set(color=c1) #box.set(facecolor=c1) for whisker in bp["whiskers"]: whisker.set(color=c1) for cap in bp["caps"]: cap.set(color=c1) for median in bp["medians"]: median.set(color=c2) for flier in bp["fliers"]: flier.set(color=c3, marker=".", alpha=0.15) #,s=6)
def plot_scatter_with_histograms(xvals, yvals, colour='k', oneToOneLine=True, xlabel=None, ylabel=None, title=None): gs = gridspec.GridSpec(5, 5) xmin = np.floor(min(xvals)) xmax = np.ceil(max(xvals)) ymin = np.floor(min(yvals)) ymax = np.ceil(max(yvals)) plt.subplot(gs[1:, 0:4]) plt.plot(xvals, yvals, 'o', color=colour) if xlabel is not None: plt.xlabel(xlabel) if ylabel is not None: plt.ylabel(ylabel) if oneToOneLine: oneToOneMax = max([max(xvals),max(yvals)]) plt.plot([0,oneToOneMax],[0,oneToOneMax],'b--') plt.xlim(xmin,xmax) plt.ylim(ymin,ymax) plt.subplot(gs[0, 0:4]) plt.hist(xvals, np.linspace(xmin,xmax,50)) plt.axis('off') plt.subplot(gs[1:,4]) plt.hist(yvals, np.linspace(ymin,ymax,50), orientation='horizontal') plt.axis('off') if title is not None: plt.suptitle(title)
def get_border_to_nucleus_properties(segment): pts = np.array(segment. isotropic_border_coords) res = min([segment.xres, segment.yres, segment.zres]) centroid = np.floor(np.mean(pts,0)) centroid_with_offset = np.floor(np.mean(segment.border_coords,0) + np.array([segment.bounding_box.xmin, segment.bounding_box.ymin, segment.bounding_box.zmin])) centroid_by_res = centroid_with_offset * np.array([segment.xres, segment.yres, segment.zres]) # if centroid_with_offset[2] == 26: # pdb.set_trace() dists = ((pts[:,0] - centroid[0])**2 + (pts[:,1] - centroid[1])**2 + (pts[:,2] - centroid[2])**2 ) **0.5 max_dist = dists.max() dists = dists/max_dist bins = np.arange(0, 1.05, 0.05) dist_hist = histogram(dists, bins = bins) [0] segment.add_feature("centroid_res", tuple(centroid_by_res)) segment.add_feature("border_to_nucleus_dist_hist", dist_hist) segment.add_feature("border_to_nucleus_dist_mean", np.mean(dists)) segment.add_feature("border_to_nucleus_dist_std", np.std(dists)) segment.add_feature("distance_to_border_scale_factor", max_dist) pts = np.array(segment.border_coords) segment.add_feature("centroid", tuple(centroid_with_offset.astype("int")))
def __init__(self, hipparcos, symbad, intervals): self.hipparcos = hipparcos self.symbad = symbad self.ra_min = min(hipparcos['RA_J2000']) self.ra_max = max(hipparcos['RA_J2000']) self.de_min = min(hipparcos['DE_J2000']) self.de_max = max(hipparcos['DE_J2000']) self.intervals = intervals self.ra_dif = self.ra_max - self.ra_min self.de_dif = self.de_max - self.de_min self.ra_sp = self.ra_dif/(self.intervals-1) self.de_sp = self.de_dif/(self.intervals-1) # set the position at the space grid for hipparcos catalog self.stars = pandas.DataFrame( numpy.zeros((len(self.hipparcos),2)), columns=['rec','dec']) self.stars['rec'] = numpy.floor( (self.hipparcos['RA_J2000'] - self.ra_min) / self.ra_sp) + 1 self.stars['dec'] = numpy.floor( (self.hipparcos['DE_J2000'] - self.de_min) / self.de_sp) + 1 # set the position at the space grid for symbad catalog self.symbad['rac'] = numpy.floor( (self.symbad['RA_J2000'] - self.ra_min) / self.ra_sp) + 1 self.symbad['dec'] = numpy.floor( (self.symbad['DE_J2000'] - self.de_min) / self.de_sp) + 1
def calc_slit_box_aps_1id(slit_box_corners, inclip=(1, 10, 1, 10)): """ Calculate the clip box based on given slip corners. Parameters ---------- slit_box_corners : np.ndarray Four corners of the slit box as a 4x2 matrix inclip : tuple, optional Extra inclipping to avoid clipping artifacts Returns ------- Tuple: Cliping indices as a tuple of four (clipFromTop, clipToBottom, clipFromLeft, clipToRight) """ return ( np.floor(slit_box_corners[:, 0].min()).astype( int) + inclip[0], # clip top row np.ceil(slit_box_corners[:, 0].max()).astype( int) - inclip[1], # clip bottom row np.floor(slit_box_corners[:, 1].min()).astype( int) + inclip[2], # clip left col np.ceil(slit_box_corners[:, 1].max()).astype( int) - inclip[3], # clip right col )
def solver_dt(t0, y0, f, dt, tk, method='expl_RK4', tuning='', v=0, outform='arr'): step=methods[method] #we calculate number of steps if np.abs(t0-tk/dt)-np.floor(np.abs(t0-tk/dt)) >= 0.5: nos = int(np.ceil(np.abs(t0-tk)/dt)) else: nos = int(np.floor(np.abs(t0-tk)/dt)) if v==2: print "Number of steps I`ll take is "+str(nos) #in solver_nos above code will be changed! soln = [[t0, y0]] for i in range(nos-1): tmp1, tmp2 = step(f, soln[i][1], soln[i][0], dt, tuning=tuning) soln.append([tmp1, tmp2]) if v>=1: print "solved. Took "+str(nos)+" steps to solve on interval ["+str(t0)+";"+str(tk)+"]" if v==2: print "solve method was "+method+". Additional tweaks for this solver were:"+tuning if v==2: print "output form was "+outform+"(arr- numpy array such that arr[i] = [t, y1, ..., yn], list - nested list list[i]=[t, array(y)])" if outform=='arr': return np.array([[i[0]]+ ([a for a in i[1]] if type(i[1])==np.ndarray else [i[1]]) for i in soln]) elif outform=='list': return soln
def multilook_attributes(atr_dict,lks_az,lks_rg): ##### atr = dict() for key, value in atr_dict.iteritems(): atr[key] = str(value) ##### calculate new data size length = int(atr['FILE_LENGTH']) width = int(atr['WIDTH']) length_mli = int(np.floor(length/lks_az)) width_mli = int(np.floor(width/lks_rg)) ##### Update attributes atr['FILE_LENGTH'] = str(length_mli) atr['WIDTH'] = str(width_mli) try: atr['Y_STEP'] = str(lks_az*float(atr['Y_STEP'])) atr['X_STEP'] = str(lks_rg*float(atr['X_STEP'])) except: pass try: atr['AZIMUTH_PIXEL_SIZE'] = str(lks_az*float(atr['AZIMUTH_PIXEL_SIZE'])) atr['RANGE_PIXEL_SIZE'] = str(lks_rg*float(atr['RANGE_PIXEL_SIZE'])) except: pass try: atr['ref_y'] = str(int(int(atr['ref_y'])/lks_az)) atr['ref_x'] = str(int(int(atr['ref_x'])/lks_rg)) except: pass try: atr['subset_y0'] = str(int(int(atr['subset_y0'])/lks_az)) atr['subset_y1'] = str(int(int(atr['subset_y1'])/lks_az)) atr['subset_x0'] = str(int(int(atr['subset_x0'])/lks_rg)) atr['subset_x1'] = str(int(int(atr['subset_x1'])/lks_rg)) except: pass return atr
def xzCrossingTime(self): """ Calculate times of crossing the xz-plane. This method calculates the times at which the xz-plane is crossed by the orbit. This is equivalent to finding the times where y=0. Returns ------- Time 1 : float First crossing time defined as having POSITIVE x position. Time 2 : float Second crossing time defined as having NEGATIVE x position. """ f = -self._w - arctan(tan(self._Omega)/cos(self._i)) E = 2.*arctan(sqrt((1-self.e)/(1.+self.e)) * tan(f/2.)) t1 = (E - self.e*sin(E))/self._n + self.tau p1 = self.xyzPos(t1) f += pi E = 2.*arctan(sqrt((1-self.e)/(1.+self.e)) * tan(f/2.)) t2 = (E - self.e*sin(E))/self._n + self.tau t1 -= self._per * numpy.floor(t1/self._per) t2 -= self._per * numpy.floor(t2/self._per) if p1[0] >= 0.0: # y position of p1 is > 0 return (t1, t2) else: return (t2, t1)
def errorScalingFactor(observable, beta): """ Look up the numerical factors to apply to the sky averaged parallax error in order to obtain error values for a given astrometric parameter, taking the Ecliptic latitude and the number of transits into account. Parameters ---------- observable - Name of astrometric observable (one of: alphaStar, delta, parallax, muAlphaStar, muDelta) beta - Values(s) of the Ecliptic latitude. Returns ------- Numerical factors to apply to the errors of the given observable. """ if isscalar(beta): index=int(floor(abs(sin(beta))*_numStepsSinBeta)) if index == _numStepsSinBeta: return _astrometricErrorFactors[observable][_numStepsSinBeta-1] else: return _astrometricErrorFactors[observable][index] else: indices = array(floor(abs(sin(beta))*_numStepsSinBeta), dtype=int) indices[(indices==_numStepsSinBeta)] = _numStepsSinBeta-1 return _astrometricErrorFactors[observable][indices]
def solve(self, pixel=None, solver='cvxopt.coneqp'): ''' Solves minimization problem using quadratic program formulation. ''' n, m = self.image.shape if not pixel: i, j = floor(n/2), floor(m/2) else: i, j = pixel r, p, q = self.psf_tensor.shape psf_bbox = (max(0, p/2-i), max(0, q/2-j), min(p, p/2-i+n), min(q, q/2-j+m)) img_bbox = (max(0, i-n/2), max(0, j-q/2), min(n, i+n/2), min(m, j+m/2)) self.A = self._compute_A(psf_bbox) self.normalization_factor = self._compute_normalization(psf_bbox) self.y = self._compute_y(img_bbox) Q = self._compute_Q() p = self._compute_p() G = self._compute_G() h = self._compute_h() result = cvxopt.solvers.qp(Q, p, G, h) self.result = result self.x = result['x'] return result['status']
def _drawGraticules(self,m,gd): par = np.arange(np.ceil(gd.ymin),np.floor(gd.ymax)+1,1.0) mer = np.arange(np.ceil(gd.xmin),np.floor(gd.xmax)+1,1.0) merdict = m.drawmeridians(mer,labels=[0,0,0,1],fontsize=10, linewidth=0.5,color='gray',zorder=GRATICULE_ZORDER) pardict = m.drawparallels(par,labels=[1,0,0,0],fontsize=10, linewidth=0.5,color='gray',zorder=GRATICULE_ZORDER) #loop over meridian and parallel dicts, change/increase font, draw ticks xticks = [] for merkey,mervalue in merdict.items(): merline,merlablist = mervalue merlabel = merlablist[0] merlabel.set_family('sans-serif') merlabel.set_fontsize(12.0) xticks.append(merline[0].get_xdata()[0]) yticks = [] for parkey,parvalue in pardict.items(): parline,parlablist = parvalue parlabel = parlablist[0] parlabel.set_family('sans-serif') parlabel.set_fontsize(12.0) yticks.append(parline[0].get_ydata()[0]) #plt.tick_params(axis='both',color='k',direction='in') plt.xticks(xticks,()) plt.yticks(yticks,()) m.ax.tick_params(direction='out')
def vert_cbar(self, resolution, log_scale, ax, label=None, label_fmt=None): r"""Display an image of the transfer function This function loads up matplotlib and displays the current transfer function. Parameters ---------- Examples -------- >>> tf = TransferFunction( (-10.0, -5.0) ) >>> tf.add_gaussian(-9.0, 0.01, 1.0) >>> tf.show() """ from matplotlib.ticker import FuncFormatter if label is None: label = "" alpha = self.alpha.y max_alpha = alpha.max() i_data = np.zeros((self.alpha.x.size, self.funcs[0].y.size, 3)) i_data[:, :, 0] = np.outer(self.funcs[0].y, np.ones(self.alpha.x.size)) i_data[:, :, 1] = np.outer(self.funcs[1].y, np.ones(self.alpha.x.size)) i_data[:, :, 2] = np.outer(self.funcs[2].y, np.ones(self.alpha.x.size)) ax.imshow(i_data, origin="lower", aspect="auto") ax.plot(alpha, np.arange(self.alpha.y.size), "w") # Set TF limits based on what is visible visible = np.argwhere(self.alpha.y > 1.0e-3 * self.alpha.y.max()) # Display colobar values xticks = ( np.arange(np.ceil(self.alpha.x[0]), np.floor(self.alpha.x[-1]) + 1, 1) - self.alpha.x[0] ) xticks *= (self.alpha.x.size - 1) / (self.alpha.x[-1] - self.alpha.x[0]) if len(xticks) > 5: xticks = xticks[:: len(xticks) // 5] # Add colorbar limits to the ticks (May not give ideal results) xticks = np.append(visible[0], xticks) xticks = np.append(visible[-1], xticks) # remove dupes xticks = list(set(xticks)) ax.yaxis.set_ticks(xticks) def x_format(x, pos): val = ( x * (self.alpha.x[-1] - self.alpha.x[0]) / (self.alpha.x.size - 1) + self.alpha.x[0] ) if log_scale: val = 10 ** val if label_fmt is None: if abs(val) < 1.0e-3 or abs(val) > 1.0e4: if not val == 0.0: e = np.floor(np.log10(abs(val))) return r"${:.2f}\times 10^{{ {:d} }}$".format( val / 10.0 ** e, int(e) ) else: return r"$0$" else: return f"{val:.1g}" else: return label_fmt % (val) ax.yaxis.set_major_formatter(FuncFormatter(x_format)) yticks = np.linspace(0, 1, 2, endpoint=True) * max_alpha ax.xaxis.set_ticks(yticks) def y_format(y, pos): s = f"{y:0.2f}" return s ax.xaxis.set_major_formatter(FuncFormatter(y_format)) ax.set_xlim(0.0, max_alpha) ax.get_xaxis().set_ticks([]) ax.set_ylim(visible[0], visible[-1]) ax.tick_params(axis="y", colors="white", size=10) ax.set_ylabel(label, color="white", size=10 * resolution / 512.0)
def main(eval_args): # ensures that weight initializations are all the same logging = utils.Logger(eval_args.local_rank, eval_args.save) # load a checkpoint logging.info('loading the model at:') logging.info(eval_args.checkpoint) checkpoint = torch.load(eval_args.checkpoint, map_location='cpu') args = checkpoint['args'] logging.info('loaded the model at epoch %d', checkpoint['epoch']) arch_instance = utils.get_arch_cells(args.arch_instance) model = AutoEncoder(args, None, arch_instance) model.load_state_dict(checkpoint['state_dict']) model = model.cuda() logging.info('args = %s', args) logging.info('num conv layers: %d', len(model.all_conv_layers)) logging.info('param size = %fM ', utils.count_parameters_in_M(model)) if eval_args.eval_mode == 'evaluate': # load train valid queue args.data = eval_args.data train_queue, valid_queue, num_classes, test_queue = datasets.get_loaders(args) if eval_args.eval_on_train: logging.info('Using the training data for eval.') valid_queue = train_queue if eval_args.eval_on_test: logging.info('Using the test data for eval.') valid_queue = test_queue # get number of bits num_output = utils.num_output(args.dataset, args) bpd_coeff = 1. / np.log(2.) / num_output valid_neg_log_p, valid_nelbo = test(valid_queue, model, num_samples=eval_args.num_iw_samples, args=args, logging=logging) logging.info('final valid nelbo %f', valid_nelbo) logging.info('final valid neg log p %f', valid_neg_log_p) logging.info('final valid nelbo in bpd %f', valid_nelbo * bpd_coeff) logging.info('final valid neg log p in bpd %f', valid_neg_log_p * bpd_coeff) else: bn_eval_mode = not eval_args.readjust_bn num_samples = 16 with torch.no_grad(): n = int(np.floor(np.sqrt(num_samples))) set_bn(model, bn_eval_mode, num_samples=36, t=eval_args.temp, iter=500) for ind in range(eval_args.repetition): # sampling is repeated. torch.cuda.synchronize() start = time() with autocast(): logits = model.sample(num_samples, eval_args.temp) output = model.decoder_output(logits) output_img = output.mean if isinstance(output, torch.distributions.bernoulli.Bernoulli) \ else output.sample() torch.cuda.synchronize() end = time() # save to file total_name = "{}/data_to_save_{}_{}.pickle".format(eval_args.save, eval_args.name_to_save, ind) with open(total_name, 'wb') as handle: pickle.dump(output_img.deatach().numpy(), handle, protocol=pickle.HIGHEST_PROTOCOL) output_tiled = utils.tile_image(output_img, n).cpu().numpy().transpose(1, 2, 0) logging.info('sampling time per batch: %0.3f sec', (end - start)) output_tiled = np.asarray(output_tiled * 255, dtype=np.uint8) output_tiled = np.squeeze(output_tiled) plt.imshow(output_tiled) plt.savefig("{}/generation_{}_{}".format(eval_args.save, eval_args.name_to_save, ind))
def load_twix_vd(fin, builder): twix_id, num_measurements = struct.unpack("II", fin.read(8)) # vd file can contain multiple measurements, but we only want the MRS # assume that the MRS is the last measurement measurement_index = num_measurements - 1 # measurement headers are each 152 bytes at start of file fin.seek(8 + 152 * measurement_index) meas_id, file_id, offset, length, patient_name, protocol_name = struct.unpack( "IIQQ64s64s", fin.read(152)) # offset points to where the actual data is in the file fin.seek(offset) # start with the header header_size = struct.unpack("I", fin.read(4))[0] header = fin.read(header_size - 4) header = header.decode('latin-1') builder.set_header_string(header) # read each scan until we hit the acq_end flag while True: # read the initial position, combined with DMA_length below that # tells us how to get to the start of the next scan initial_position = fin.tell() # the first four bytes contain some composite information, # read in an int and do bit shift magic to get the values temp = struct.unpack("I", fin.read(4))[0] DMA_length = temp & (2**26 - 1) pack_flag = (temp >> 25) & 1 PCI_rx = temp >> 26 meas_uid, scan_counter, time_stamp, pmu_time_stamp = struct.unpack( "IIII", fin.read(16)) system_type, ptab_pos_delay, ptab_pos_x, ptab_pos_y, ptab_pos_z, reserved = struct.unpack( "HHIIII", fin.read(20)) # more composite information eval_info_mask = struct.unpack("Q", fin.read(8))[0] acq_end = eval_info_mask & 1 rt_feedback = eval_info_mask >> 1 & 1 hp_feedback = eval_info_mask >> 2 & 1 sync_data = eval_info_mask >> 5 & 1 raw_data_correction = eval_info_mask >> 10 & 1 ref_phase_stab_scan = eval_info_mask >> 14 & 1 phase_stab_scan = eval_info_mask >> 15 & 1 sign_rev = eval_info_mask >> 17 & 1 phase_correction = eval_info_mask >> 21 & 1 pat_ref_scan = eval_info_mask >> 22 & 1 pat_ref_ima_scan = eval_info_mask >> 23 & 1 reflect = eval_info_mask >> 24 & 1 noise_adj_scan = eval_info_mask >> 25 & 1 # if acq_end is set, there is no more data if acq_end: break # there are some data frames that contain auxilliary data, we ignore those for now if rt_feedback or hp_feedback or phase_correction or noise_adj_scan or sync_data: fin.seek(initial_position + DMA_length) continue num_samples, num_channels = struct.unpack("HH", fin.read(4)) builder.set_num_channels(num_channels) loop_counters = struct.unpack("14H", fin.read(28)) cut_off_data, kspace_centre_column, coil_select, readout_offcentre = struct.unpack( "IHHI", fin.read(12)) time_since_rf, kspace_centre_line_num, kspace_centre_partition_num = struct.unpack( "IHH", fin.read(8)) slice_position = struct.unpack("7f", fin.read(28)) ice_program_params = struct.unpack("24H", fin.read(48)) reserved_params = struct.unpack("4H", fin.read(8)) fid_start_offset = ice_program_params[4] num_dummy_points = reserved_params[0] fid_start = fid_start_offset + num_dummy_points np = int(2**numpy.floor(numpy.log2(num_samples - fid_start))) builder.set_np(np) application_counter, application_mask, crc = struct.unpack( "HHI", fin.read(8)) # read the data for each channel in turn scan_data = numpy.zeros((num_channels, np), dtype='complex') for channel_index in range(num_channels): # start with the header dma_length, meas_uid, scan_counter, sequence_time, channel_id = struct.unpack( "III4xI4xH6x", fin.read(32)) # now the data itself, which consists of num_samples * 4 (bytes per float) * 2 (two floats per complex) raw_data = struct.unpack("<{}f".format(num_samples * 2), fin.read(num_samples * 4 * 2)) # we need to massage the list of floats into a numpy array of complex numbers data_iter = iter(raw_data) complex_iter = (complex(r, -i) for r, i in zip(data_iter, data_iter)) scan_data[channel_index, :] = numpy.fromiter( complex_iter, "complex64", num_samples)[fid_start:(fid_start + np)] builder.add_scan(loop_counters, scan_data) # move the file pointer to the start of the next scan fin.seek(initial_position + DMA_length)
def load_twix_vb(fin, builder): # first four bytes are the size of the header header_size = struct.unpack("I", fin.read(4))[0] # read the rest of the header minus the four bytes we already read header = fin.read(header_size - 4) # for some reason the last 24 bytes of the header contain some junk that is not a string header = header[:-24].decode('latin-1') builder.set_header_string(header) # the way that vb files are set up we just keep reading scans until the acq_end flag is set while True: # start by keeping track of where in the file this scan started # this will be used to jump to the start of the next scan start_position = fin.tell() # the first four bytes contain composite information temp = struct.unpack("I", fin.read(4))[0] # 25 LSBs contain DMA length (size of this scan) DMA_length = temp & (2**26 - 1) # next we have the "pack" flag bit and the rest is PCI_rx # not sure what either of these are for but break them out in case pack_flag = (temp >> 25) & 1 PCI_rx = temp >> 26 meas_uid, scan_counter, time_stamp, pmu_time_stamp = struct.unpack( "IIII", fin.read(16)) # next long int is actually a lot of bit flags # a lot of them don't seem to be relevant for spectroscopy eval_info_mask = struct.unpack("Q", fin.read(8))[0] acq_end = eval_info_mask & 1 rt_feedback = eval_info_mask >> 1 & 1 hp_feedback = eval_info_mask >> 2 & 1 sync_data = eval_info_mask >> 5 & 1 raw_data_correction = eval_info_mask >> 10 & 1 ref_phase_stab_scan = eval_info_mask >> 14 & 1 phase_stab_scan = eval_info_mask >> 15 & 1 sign_rev = eval_info_mask >> 17 & 1 phase_correction = eval_info_mask >> 21 & 1 pat_ref_scan = eval_info_mask >> 22 & 1 pat_ref_ima_scan = eval_info_mask >> 23 & 1 reflect = eval_info_mask >> 24 & 1 noise_adj_scan = eval_info_mask >> 25 & 1 if acq_end: break # if any of these flags are set then we should ignore the scan data if rt_feedback or hp_feedback or phase_correction or noise_adj_scan or sync_data: fin.seek(start_position + DMA_length) continue # now come the actual parameters of the scan num_samples, num_channels = struct.unpack("HH", fin.read(4)) builder.set_num_channels(num_channels) # the loop counters are a set of 14 shorts which are used as indices # for the parameters an acquisition might loop over, including # averaging repetitions, COSY echo time increments and CSI phase # encoding steps # we have no prior knowledge about which counters might loop in a given # scan so we have to read in all scans and then sort out the data shape loop_counters = struct.unpack("14H", fin.read(28)) cut_off_data, kspace_centre_column, coil_select, readout_offcentre = struct.unpack( "IHHI", fin.read(12)) time_since_rf, kspace_centre_line_num, kspace_centre_partition_num = struct.unpack( "IHH", fin.read(8)) ice_program_params = struct.unpack("4H", fin.read(8)) free_params = struct.unpack("4H", fin.read(8)) # there are some dummy points before the data starts num_dummy_points = free_params[0] # we want our np to be the largest power of two within the num_samples - num_dummy_points np = int(2**numpy.floor(numpy.log2(num_samples - num_dummy_points))) builder.set_np(np) slice_position = struct.unpack("7f", fin.read(28)) # construct a numpy ndarray to hold the data from all the channels in this scan scan_data = numpy.zeros((num_channels, np), dtype='complex') # loop over all the channels and extract data for channel_index in range(num_channels): channel_id, ptab_pos_neg = struct.unpack("Hh", fin.read(4)) raw_data = struct.unpack("<{}f".format(num_samples * 2), fin.read(num_samples * 4 * 2)) # turn the raw data into complex pairs data_iter = iter(raw_data) complex_iter = (complex(r, -i) for r, i in zip(data_iter, data_iter)) scan_data[channel_index, :] = numpy.fromiter( complex_iter, "complex64", num_samples)[num_dummy_points:(num_dummy_points + np)] # the vb format repeats all the header data for each channel in # turn, obviously this is redundant so we read all but the channel # index from the next header here fin.read(124) # pass the data from this scan to the builder builder.add_scan(loop_counters, scan_data) # go to the next scan and the top of the loop fin.seek(start_position + DMA_length)
######## geometry setup (moritz.huetten@pik) ######### boxWidth = float(boxWidth) accumrate = float(accumrate) ### CONSTANTS ### secpera = 31556926. ice_density = 910.0 # [kg m-3] yExtent = 2 * boxWidth # in km xExtent = 2 * 800 # in km # grid size: # of boxes ny = int(np.floor(yExtent / boxWidth / 2) * 2 + 1) # make it an odd number nx = int(np.floor(xExtent / boxWidth / 2) * 2 + 1) # make it an odd number # grid size: extent in km's, origin (0,0) in the center of the domain x = np.linspace(-xExtent / 2, xExtent / 2, nx) * 1000.0 y = np.linspace(-yExtent / 2, yExtent / 2, ny) * 1000.0 nxcenter = int(np.floor(0.5 * nx)) nycenter = int(np.floor(0.5 * ny)) thk = np.zeros((ny, nx)) topg = np.zeros((ny, nx)) ice_surface_temp = np.zeros((ny, nx)) precip = np.zeros((ny, nx))
def xmlToStructuredSong(xml_path, datasetToMusic21, datasetToMidiChords, datasetToChordComposition, datasetChords): ''' # Import xml file possible_durations = [4, 2, 1, 1/2, 1/4, 1/8, 1/16, 3, 3/2, 3/4, 3/8, 1/6, 1/12] # Define durations dictionary dur_dict = {} dur_dict[possible_durations[0]] = 'full' dur_dict[possible_durations[1]] = 'half' dur_dict[possible_durations[2]] = 'quarter' dur_dict[possible_durations[3]] = '8th' dur_dict[possible_durations[4]] = '16th' dur_dict[possible_durations[5]] = '32th' dur_dict[possible_durations[6]] = '64th' dur_dict[possible_durations[7]] = 'dot half' dur_dict[possible_durations[8]] = 'dot quarter' dur_dict[possible_durations[9]] = 'dot 8th' dur_dict[possible_durations[10]] = 'dot 16th' dur_dict[possible_durations[11]] = 'half note triplet' dur_dict[possible_durations[12]] = 'quarter note triplet' ''' # Import xml file possible_durations = [4, 2, 1, 1/2, 1/4, 1/8, 3, 3/2, 3/4, 1/6, 1/12] # Define durations dictionary dur_dict = {} dur_dict[possible_durations[0]] = 'full' dur_dict[possible_durations[1]] = 'half' dur_dict[possible_durations[2]] = 'quarter' dur_dict[possible_durations[3]] = '8th' dur_dict[possible_durations[4]] = '16th' dur_dict[possible_durations[5]] = '32th' dur_dict[possible_durations[6]] = 'dot half' dur_dict[possible_durations[7]] = 'dot quarter' dur_dict[possible_durations[8]] = 'dot 8th' dur_dict[possible_durations[9]] = 'half note triplet' dur_dict[possible_durations[10]] = 'quarter note triplet' # invert dict from Wjazz to Music21 chords Music21ToWjazz = {v: k for k, v in datasetToMusic21.items()} s = m21.converter.parse(xml_path) new_structured_song = {} new_structured_song['title'] = xml_path.split('/')[-1].split('.')[0] new_structured_song['tempo'] = s.metronomeMarkBoundaries()[0][2].number new_structured_song['beat duration [sec]'] = 60 / new_structured_song['tempo'] if not s.hasMeasures(): s = s.makeMeasures() #s.show('text') bar_num = 0 bars = [] beats = [] beat_pitch = [] beat_duration = [] beat_offset = [] chord = 'NC' for measure in s.getElementsByClass('Measure'): #measure.show('text') bar_num += 1 beat_num = 0 bar_duration = 0 for note in measure.notesAndRests: if 'Rest' in note.classSet: # detect rests pitch = 'R' distance = np.abs(np.array(possible_durations) - note.quarterLength) idx = distance.argmin() duration = dur_dict[possible_durations[idx]] offset = int(bar_duration * 96 / 4) # update beat arrays beat_pitch.append(pitch) beat_duration.append(duration) beat_offset.append(offset) elif 'ChordSymbol' in note.classSet: #chord m21chord = note.figure if m21chord in Music21ToWjazz.keys(): chord = Music21ToWjazz[m21chord] else: # add to WjazzToMusic21 datasetToMusic21[m21chord] = m21chord # derive chord composition and make it of 4 notes pitchNames = [str(p) for p in note.pitches] # The following bit is added # just for parameter modeling purposes if len(pitchNames) < 4: hd = m21.harmony.ChordStepModification('add', 7) note.addChordStepModification(hd, updatePitches=True) #chord = m21.chord.Chord(pitchNames) pitchNames = [str(p) for p in note.pitches] # midi conversion midiChord = [] for p in pitchNames: c = m21.pitch.Pitch(p) midiChord.append(c.midi) chord = m21chord NewChord = {} NewChord['Wjazz name'] = chord NewChord['music21 name'] = m21chord NewChord['chord composition'] = pitchNames NewChord['midi chord composition'] = midiChord NewChord['one-hot encoding'] = [] datasetChords.append(NewChord) # update dictionaries datasetToMidiChords[chord] = midiChord[:4] datasetToChordComposition[chord] = pitchNames[:4] # check for rests else: pitch = note.pitch.midi distance = np.abs(np.array(possible_durations) - note.quarterLength) idx = distance.argmin() duration = dur_dict[possible_durations[idx]] offset = int(bar_duration * 96 / 4) # update beat arrays beat_pitch.append(pitch) beat_duration.append(duration) beat_offset.append(offset) # update bar duration bar_duration += note.quarterLength # check if the beat is ended if np.floor(bar_duration) != beat_num: #print(np.floor(bar_duration), beat_num) while np.floor(bar_duration) - beat_num > 1: new_beat = {} new_beat['num beat'] = int(beat_num) + 1 new_beat['chord'] = chord new_beat['pitch'] = [] new_beat['duration'] = [] new_beat['offset'] = [] new_beat['scale'] = [] new_beat['bass'] = datasetToMidiChords[chord][0] new_beat['this beat duration [sec]'] = new_structured_song['beat duration [sec]'] beats.append(new_beat) beat_num += 1 if not chord: chord = 'NC' new_beat = {} new_beat['num beat'] = int(beat_num) + 1 new_beat['chord'] = chord new_beat['pitch'] = beat_pitch new_beat['duration'] = beat_duration new_beat['offset'] = beat_offset new_beat['scale'] = [] new_beat['bass'] = datasetToMidiChords[chord][0] new_beat['this beat duration [sec]'] = new_structured_song['beat duration [sec]'] beats.append(new_beat) if beat_num == 3: # append bar new_bar = {} new_bar['num bar'] = bar_num # over all song new_bar['beats'] = beats # beats 1,2,3,4 bars.append(new_bar) beats = [] beat_num = np.floor(bar_duration) beat_pitch = [] beat_duration = [] beat_offset = [] # compute chords array chord_array = [] for bar in bars: for beat in bar['beats']: chord_array.append(beat['chord']) # compute next chord last_chord = chord_array[0] next_chords = [] for i in range(len(chord_array)): if chord_array[i] != last_chord: next_chords.append(chord_array[i]) last_chord = chord_array[i] # compute array of next chords next_chords.append('NC') next_chord_array = [] next_chord_pointer = 0 last_chord = chord_array[0] for i in range(len(chord_array)): if chord_array[i] != last_chord: last_chord = chord_array[i] next_chord_pointer += 1 next_chord_array.append(next_chords[next_chord_pointer]) # compute next chord last_chord = bars[0]['beats'][0]['chord'] next_chords2 = [] for bar in bars: for beat in bar['beats']: if beat['chord'] != last_chord: next_chords2.append(beat['chord']) last_chord = beat['chord'] # add next chord to the beats last_chord = bars[0]['beats'][0]['chord'] next_chords2.append('NC') next_chord_pointer = 0 for bar in bars: for beat in bar['beats']: if beat['chord'] != last_chord: last_chord = beat['chord'] next_chord_pointer += 1 beat['next chord'] = next_chords2[next_chord_pointer] new_structured_song['bars'] = bars return new_structured_song, datasetToMusic21, datasetToMidiChords, datasetToChordComposition, datasetChords
def plot(newest_changes): filelist = os.listdir( '/home/maximilianklein/snapshot_data/{}/'.format(newest_changes)) site_linkss_file = [f for f in filelist if f.startswith('worldmap')][0] if newest_changes == 'newest-changes': date_range = site_linkss_file.split('worldmap-index-from-')[1].split( '.csv')[0].replace('-', ' ') print(date_range) csv_to_read = '/home/maximilianklein/snapshot_data/{}/{}'.format( newest_changes, site_linkss_file) df = pandas.DataFrame.from_csv(csv_to_read) major = df[df['total'] > 100] table_html = major.sort('Score', ascending=False).to_html(max_rows=10) # https://github.com/chdoig/pyladiesatx-bokeh-tutorial world_countries = wc.data.copy() country_xs = [world_countries[code]['lons'] for code in world_countries] country_ys = [world_countries[code]['lats'] for code in world_countries] country_names = [world_countries[code]['name'] for code in world_countries] def lookup_wigi(code): try: return df.ix[code]['Score'] except KeyError: return -1 index_vals = np.array([lookup_wigi(code) for code in world_countries]) def fmt(c): return int(np.nan_to_num(c)) colors = [ "#%02x%02x%02x" % (fmt(r), fmt(g), fmt(b)) for r, g, b in zip(np.floor(250 * (1 - index_vals)), np.floor(200 * (1 - index_vals)), np.floor(100 * index_vals)) ] source = ColumnDataSource(data=dict( name=country_names, wigi_index=[str(idx) for idx in index_vals])) # setup widgets TOOLS = "pan,wheel_zoom,box_zoom,reset,hover,save" title_suffix = 'Changes since {}'.format( date_range) if newest_changes == 'newest-changes' else 'All Time' p = figure(title="Gender by Country {}".format(title_suffix), tools=TOOLS) p.patches(country_xs, country_ys, fill_color=colors, source=source) hover = p.select(dict(type=HoverTool)) hover.point_policy = "follow_mouse" hover.tooltips = OrderedDict([ ("wigi", "@wigi_index"), ("Country", "@name"), ]) js_filename = "gender_by_country_{}.js".format(newest_changes) script_path = "./assets/js/" output_path = "./files/assets/js/" # generate javascript plot and corresponding script tag js, tag = autoload_static(p, CDN, script_path + js_filename) with open(output_path + js_filename, 'w') as js_file: js_file.write(js) return {'plot_tag': tag, 'table_html': table_html}
def detect_image_bboxes(self, image, true_boxes): # modified function to also plot ground truth bboxes along /w predictions start = timer() if self.model_image_size != (None, None): assert self.model_image_size[0]%32 == 0, 'Multiples of 32 required' assert self.model_image_size[1]%32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image(image, tuple(reversed(self.model_image_size))) else: new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), 'img')) font = ImageFont.truetype(font='font/FiraMono-Medium.otf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) print(label, (left, top), (right, bottom)) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle( [left + i, top + i, right - i, bottom - i], outline=self.colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw true_classes = true_boxes[:,-1] true_boxes = true_boxes[:,:4] for i, c in reversed(list(enumerate(true_classes))): try: predicted_class = self.class_names[c] except: print('Exception! class:', c) box = true_boxes[i] score = 1.0 # label = '{} {:.2f}'.format(predicted_class, score) label = 'Box' draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) print(label, (left, top), (right, bottom)) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle( [left + i, top + i, right - i, bottom - i], outline=(255,255,255)) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=(255,255,255)) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw end = timer() print(end - start) return image
def country(iso3, monthyear, proj="EPSG:3395", data_country=True, keep_data_raw=False, fcc_source="gfc", perc=50, gs_bucket=None): """Function formating the country data. This function downloads, computes and formats the country data. :param iso3: Country ISO 3166-1 alpha-3 code. :param proj: Projection definition (EPSG, PROJ.4, WKT) as in GDAL/OGR. Default to "EPSG:3395" (World Mercator). :param monthyear: Date (month and year) for WDPA data (e.g. "Aug2017"). :param data_country: Boolean for running data_country.sh to compute country landscape variables. Default to "True". :param keep_data_raw: Boolean to keep the data_raw folder. Default to "False". :param fcc_source: Source for forest-cover change data. Can be "gfc" (Global Forest Change 2015 Hansen data) or "roadless". Default to "gfc". :param perc: Tree cover percentage threshold to define forest (online used if fcc_source="gcf"). :param gs_bucket: Name of the google storage bucket to use. """ # Identify continent and country from iso3 print("Identify continent and country from iso3") # Geofabrik data file_geofab = pkg_resources.resource_filename("deforestprob", "data/ctry_geofab.csv") data_geofab = pd.read_csv(file_geofab, sep=";", header=0) # Country ctry_link_geofab = data_geofab.ctry_link[data_geofab.iso3 == iso3] ctry_link_geofab = ctry_link_geofab.iloc[0] # Continent continent = data_geofab.continent[data_geofab.iso3 == iso3] continent = continent.iloc[0].lower() # Create data_raw directory print("Create data_raw directory") make_dir("data_raw") # Download the zipfile from gadm.org print("Download data") url = "http://biogeo.ucdavis.edu/data/gadm2.8/shp/" + iso3 + "_adm_shp.zip" fname = "data_raw/" + iso3 + "_adm_shp.zip" urlretrieve(url, fname) # Extract files from zip print("Extract files from zip") destDir = "data_raw" f = ZipFile(fname) f.extractall(destDir) f.close() print("Files extracted") # Reproject cmd = "ogr2ogr -overwrite -s_srs EPSG:4326 -t_srs '" + proj + "' -f 'ESRI Shapefile' \ -lco ENCODING=UTF-8 data_raw/ctry_PROJ.shp data_raw/" + iso3 + "_adm0.shp" os.system(cmd) # Compute extent print("Compute extent") extent_latlong = extent_shp("data_raw/" + iso3 + "_adm0.shp") extent_proj = extent_shp("data_raw/ctry_PROJ.shp") # Region with buffer of 5km print("Region with buffer of 5km") xmin_reg = np.floor(extent_proj[0] - 5000) ymin_reg = np.floor(extent_proj[1] - 5000) xmax_reg = np.ceil(extent_proj[2] + 5000) ymax_reg = np.ceil(extent_proj[3] + 5000) extent_reg = (xmin_reg, ymin_reg, xmax_reg, ymax_reg) extent = " ".join(map(str, extent_reg)) # Tiles for SRTM data (see http://dwtkns.com/srtm/) print("Tiles for SRTM data") # SRTM tiles are 5x5 degrees # x: -180/+180 # y: +60/-60 xmin_latlong = np.floor(extent_latlong[0]) ymin_latlong = np.floor(extent_latlong[1]) xmax_latlong = np.ceil(extent_latlong[2]) ymax_latlong = np.ceil(extent_latlong[3]) # Compute SRTM tile numbers tile_left = np.int(np.ceil((xmin_latlong + 180.0) / 5.0)) tile_right = np.int(np.ceil((xmax_latlong + 180.0) / 5.0)) if (tile_right == tile_left): # Trick to make curl globbing work in data_country.sh tile_right = tile_left + 1 tile_top = np.int(np.ceil((-ymax_latlong + 60.0) / 5.0)) tile_bottom = np.int(np.ceil((-ymin_latlong + 60.0) / 5.0)) if (tile_bottom == tile_top): tile_bottom = tile_top + 1 # Format variables, zfill is for having 01 and not 1 tiles_long = str(tile_left).zfill(2) + "-" + str(tile_right).zfill(2) tiles_lat = str(tile_top).zfill(2) + "-" + str(tile_bottom).zfill(2) # Google EarthEngine task if (fcc_source == "gfc"): # Check data availability data_availability = ee_hansen.check(gs_bucket, iso3) # If not available, run GEE if data_availability is False: print("Run Google Earth Engine") task = ee_hansen.run_task(perc=perc, iso3=iso3, extent_latlong=extent_latlong, scale=30, proj=proj, gs_bucket=gs_bucket) print("GEE running on the following extent:") print(str(extent_latlong)) # Google EarthEngine task if (fcc_source == "roadless"): # Check data availability data_availability = ee_roadless.check(gs_bucket, iso3) # If not available, run GEE if data_availability is False: print("Run Google Earth Engine") task = ee_roadless.run_task(iso3=iso3, extent_latlong=extent_latlong, scale=30, proj=proj, gs_bucket=gs_bucket) print("GEE running on the following extent:") print(str(extent_latlong)) # Call data_country.sh if (data_country): script = pkg_resources.resource_filename("deforestprob", "shell/data_country.sh") args = [ "sh ", script, continent, ctry_link_geofab, iso3, "'" + proj + "'", "'" + extent + "'", tiles_long, tiles_lat, monthyear ] cmd = " ".join(args) os.system(cmd) # Forest computations if (fcc_source == "gfc"): # Download Google EarthEngine results print("Download Google Earth Engine results locally") ee_hansen.download(gs_bucket, iso3, path="data_raw") # Call forest_country.sh print("Forest computations") script = pkg_resources.resource_filename("deforestprob", "shell/forest_country.sh") args = ["sh ", script, "'" + proj + "'", "'" + extent + "'"] cmd = " ".join(args) os.system(cmd) # Forest computations if (fcc_source == "roadless"): # Download Google EarthEngine results print("Download Google Earth Engine results locally") ee_roadless.download(gs_bucket, iso3, path="data_raw") # Call forest_country.sh print("Forest computations") script = pkg_resources.resource_filename("deforestprob", "shell/forest_country.sh") args = ["sh ", script, "'" + proj + "'", "'" + extent + "'"] cmd = " ".join(args) os.system(cmd) # Delete data_raw if (keep_data_raw is False): for root, dirs, files in os.walk("data_raw", topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name))
def collect(self, idata, edata, rdata, params): #print " collecting statistics ...", #sys.stdout.flush() lstats = dict() species = len(params.target) # co-ordinate meshes lstats["XYx"] = self.XYx lstats["XYy"] = self.XYy lstats["YZy"] = self.YZy lstats["YZz"] = self.YZz lstats["XZx"] = self.XZx lstats["XZz"] = self.XZz # allocate storage for the other arrays lstats["iidXY"] = np.zeros((self.xbins, self.ybins)) lstats["iidXZ"] = np.zeros((self.xbins, self.zbins)) lstats["iidYZ"] = np.zeros((self.ybins, self.zbins)) lstats["evdXY"] = np.zeros((species+1, self.xbins, self.ybins)) lstats["evdXZ"] = np.zeros((species+1, self.xbins, self.zbins)) lstats["evdYZ"] = np.zeros((species+1, self.ybins, self.zbins)) lstats["rvdXY"] = np.zeros((species+1, self.xbins, self.ybins)) lstats["rvdXZ"] = np.zeros((species+1, self.xbins, self.zbins)) lstats["rvdYZ"] = np.zeros((species+1, self.ybins, self.zbins)) lstats["ridXY"] = np.zeros((species+1, self.xbins, self.ybins)) lstats["ridXZ"] = np.zeros((species+1, self.xbins, self.zbins)) lstats["ridYZ"] = np.zeros((species+1, self.ybins, self.zbins)) lstats["rddXYx"] = np.zeros((species+1, self.xbins, self.ybins)) lstats["rddXYy"] = np.zeros((species+1, self.xbins, self.ybins)) lstats["rddXZx"] = np.zeros((species+1, self.xbins, self.zbins)) lstats["rddXZz"] = np.zeros((species+1, self.xbins, self.zbins)) lstats["rddYZy"] = np.zeros((species+1, self.ybins, self.zbins)) lstats["rddYZz"] = np.zeros((species+1, self.ybins, self.zbins)) # how do implanted ions affect the statistics? if idata != None: for entry in idata: # extract stored data sid = entry[0] # species pos0 = entry[1] # final position # 2D projections of distributions binx = np.floor( (pos0[0]-self.xMin) / self.dx) biny = np.floor( (pos0[1]-self.yMin) / self.dy) binz = np.floor( (pos0[2]-self.zMin) / self.dz) if (binx >=0 and binx < self.xbins and biny >=0 and biny < self.ybins and binz >=0 and binz < self.zbins): lstats["iidXY"][binx, biny] += 1 lstats["iidXZ"][binx, binz] += 1 lstats["iidYZ"][biny, binz] += 1 # how do eroded atoms affect the statistics? if edata != None: for entry in edata: # extract stored data sid = entry[0] # species ID pos0 = entry[1] # initial position of sputtered atom # 2D projections of distributions binx = np.floor((pos0[0]-self.xMin) / self.dx) biny = np.floor((pos0[1]-self.yMin) / self.dy) binz = np.floor((pos0[2]-self.zMin) / self.dz) if (binx >=0 and binx < self.xbins and biny >=0 and biny < self.ybins and binz >=0 and binz < self.zbins): lstats["evdXY"][sid, binx, biny] += 1 lstats["evdXZ"][sid, binx, binz] += 1 lstats["evdYZ"][sid, biny, binz] += 1 # how do redistributed atoms affect the statistics? if rdata != None: for entry in rdata: # extract stored data sid = entry[0] # species ID pos0 = entry[1] # initial position pos1 = entry[2] # final position dpos = pos1-pos0 # displacement # 2D projections of distributions binx = np.floor((pos0[0]-self.xMin) / self.dx) biny = np.floor((pos0[1]-self.yMin) / self.dy) binz = np.floor((pos0[2]-self.zMin) / self.dz) if (binx >=0 and binx < self.xbins and biny >=0 and biny < self.ybins and binz >=0 and binz < self.zbins): lstats["rvdXY"][sid, binx, biny] += 1 lstats["rvdXZ"][sid, binx, binz] += 1 lstats["rvdYZ"][sid, biny, binz] += 1 lstats["rddXYx"][sid, binx, biny] += dpos[0] lstats["rddXYy"][sid, binx, biny] += dpos[1] lstats["rddXZx"][sid, binx, binz] += dpos[0] lstats["rddXZz"][sid, binx, binz] += dpos[2] lstats["rddYZy"][sid, biny, binz] += dpos[1] lstats["rddYZz"][sid, biny, binz] += dpos[2] # 2D projections of distributions binx = np.floor((pos1[0]-self.xMin) / self.dx) biny = np.floor((pos1[1]-self.yMin) / self.dy) binz = np.floor((pos1[2]-self.zMin) / self.dz) if (binx >=0 and binx < self.xbins and biny >=0 and biny < self.ybins and binz >=0 and binz < self.zbins): lstats["ridXY"][sid, binx, biny] += 1 lstats["ridXZ"][sid, binx, binz] += 1 lstats["ridYZ"][sid, biny, binz] += 1 # and update 2D projected distributions (also total of all species in position '0') for part1 in ["iid", "evd", "rvd", "rid"]: for part2 in ["XY", "XZ", "YZ"]: key = "%s%s" % (part1, part2) lstats[key][0] = np.sum(lstats[key][1:], axis=0) # and update 2D projected distributions (also total of all species in position '0') for part1 in ["rdd"]: for part2 in ["XYx", "XYy", "XZx", "XZz", "YZy", "YZz"]: key = "%s%s" % (part1, part2) lstats[key][0] = np.sum(lstats[key][1:], axis=0) return lstats
def _np_frame(data, window_length, hop_length): num_frames = 1 + int(np.floor((len(data) - window_length) // hop_length)) shape = (num_frames, window_length) strides = (data.strides[0] * hop_length, data.strides[0]) return np.lib.stride_tricks.as_strided(data, shape=shape, strides=strides)
# Import data data = pd.read_csv('01_data/data_stocks.csv') # Drop date variable data = data.drop(['DATE'], 1) # Dimensions of dataset n = data.shape[0] p = data.shape[1] # Make data a np.array data = data.values # Training and test data train_start = 0 train_end = int(np.floor(0.8*n)) test_start = train_end + 1 test_end = n data_train = data[np.arange(train_start, train_end), :] data_test = data[np.arange(test_start, test_end), :] # Scale data scaler = MinMaxScaler(feature_range=(-1, 1)) scaler.fit(data_train) data_train = scaler.transform(data_train) data_test = scaler.transform(data_test) # Build X and y X_train = data_train[:, 1:] y_train = data_train[:, 0] X_test = data_test[:, 1:]
from sklearn.externals import joblib from sklearn import preprocessing if __name__ == '__main__': np.random.seed(0) patient_data=pd.read_pickle("../../data/df/dataset.pickle").values patient_data=preprocessing.scale(patient_data) target=pd.read_pickle("../../data/df/target.pickle").values indices = np.random.permutation(len(target)) print "Total dataset size: "+str(patient_data.shape[0]) num_train=int(np.floor(patient_data.shape[0]*0.75)) print "Train set size: "+str(num_train) patient_data_train=patient_data[indices[0:num_train]] target_train=target[indices[0:num_train]] patient_data_test=patient_data[indices[num_train:]] target_test=target[indices[num_train:]] print "Test set size: "+str(target_test.shape[0]) #keep 95% of variance pca = decomposition.PCA(n_components=0.95) pca.fit(patient_data_train)
def get_boxes_grid(image_height, image_width): """ Return the boxes on image grid. calling this function when cfg.IS_MULTISCALE is True, otherwise, calling rdl_roidb.prepare_roidb(imdb) instead. """ # fixed a bug, change cfg.TRAIN.SCALES to cfg.TRAIN.SCALES_BASE # coz, here needs a ratio around 1.0, not the accutual size. # height and width of the feature map if cfg.NET_NAME == 'CaffeNet': height = np.floor((image_height * max(cfg.TRAIN.SCALES_BASE) - 1) / 4.0 + 1) height = np.floor((height - 1) / 2.0 + 1 + 0.5) height = np.floor((height - 1) / 2.0 + 1 + 0.5) width = np.floor((image_width * max(cfg.TRAIN.SCALES_BASE) - 1) / 4.0 + 1) width = np.floor((width - 1) / 2.0 + 1 + 0.5) width = np.floor((width - 1) / 2.0 + 1 + 0.5) elif cfg.NET_NAME == 'VGGnet': height = np.floor(image_height * max(cfg.TRAIN.SCALES_BASE) / 2.0 + 0.5) height = np.floor(height / 2.0 + 0.5) height = np.floor(height / 2.0 + 0.5) height = np.floor(height / 2.0 + 0.5) width = np.floor(image_width * max(cfg.TRAIN.SCALES_BASE) / 2.0 + 0.5) width = np.floor(width / 2.0 + 0.5) width = np.floor(width / 2.0 + 0.5) width = np.floor(width / 2.0 + 0.5) else: assert (1), 'The network architecture is not supported in utils.get_boxes_grid!' # compute the grid box centers h = np.arange(height) w = np.arange(width) y, x = np.meshgrid(h, w, indexing='ij') centers = np.dstack((x, y)) centers = np.reshape(centers, (-1, 2)) num = centers.shape[0] # compute width and height of grid box area = cfg.TRAIN.KERNEL_SIZE * cfg.TRAIN.KERNEL_SIZE aspect = cfg.TRAIN.ASPECTS # height / width num_aspect = len(aspect) widths = np.zeros((1, num_aspect), dtype=np.float32) heights = np.zeros((1, num_aspect), dtype=np.float32) for i in xrange(num_aspect): widths[0,i] = math.sqrt(area / aspect[i]) heights[0,i] = widths[0,i] * aspect[i] # construct grid boxes centers = np.repeat(centers, num_aspect, axis=0) widths = np.tile(widths, num).transpose() heights = np.tile(heights, num).transpose() x1 = np.reshape(centers[:,0], (-1, 1)) - widths * 0.5 x2 = np.reshape(centers[:,0], (-1, 1)) + widths * 0.5 y1 = np.reshape(centers[:,1], (-1, 1)) - heights * 0.5 y2 = np.reshape(centers[:,1], (-1, 1)) + heights * 0.5 boxes_grid = np.hstack((x1, y1, x2, y2)) / cfg.TRAIN.SPATIAL_SCALE return boxes_grid, centers[:,0], centers[:,1]
# Set random seed to make sure models are validated on the same validation images. # So you can compare the results of different models more intuitively. random.seed(16) val_indices=random.sample(range(0,len(val_input_names)),num_vals) # Do the training here for epoch in range(args.epoch_start_i, args.num_epochs): current_losses = [] cnt=0 # Equivalent to shuffling id_list = np.random.permutation(len(train_input_names)) num_iters = int(np.floor(len(id_list) / args.batch_size)) st = time.time() epoch_st=time.time() for i in range(num_iters): # st=time.time() input_image_batch = [] output_image_batch = [] # Collect a batch of images for j in range(args.batch_size): index = i*args.batch_size + j id = id_list[index] input_image = utils.load_image(train_input_names[id]) output_image = utils.load_image(train_output_names[id])
def preprocess_true_boxes(true_boxes, input_shape, anchors, num_classes): '''Preprocess true boxes to training input format Parameters ---------- true_boxes: array, shape=(m, T, 5) Absolute x_min, y_min, x_max, y_max, class_id relative to input_shape. input_shape: array-like, hw, multiples of 32 anchors: array, shape=(N, 2), wh num_classes: integer Returns ------- y_true: list of array, shape like yolo_outputs, xywh are reletive value ''' assert (true_boxes[..., 4]<num_classes).all(), 'class id must be less than num_classes' num_layers = len(anchors)//3 # default setting anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] if num_layers==3 else [[3,4,5], [0,1,2]] true_boxes = np.array(true_boxes, dtype='float32') input_shape = np.array(input_shape, dtype='int32') boxes_xy = (true_boxes[..., 0:2] + true_boxes[..., 2:4]) // 2 boxes_wh = true_boxes[..., 2:4] - true_boxes[..., 0:2] true_boxes[..., 0:2] = boxes_xy/input_shape[::-1] true_boxes[..., 2:4] = boxes_wh/input_shape[::-1] m = true_boxes.shape[0] grid_shapes = [input_shape//{0:32, 1:16, 2:8}[l] for l in range(num_layers)] y_true = [np.zeros((m,grid_shapes[l][0],grid_shapes[l][1],len(anchor_mask[l]),5+num_classes), dtype='float32') for l in range(num_layers)] # Expand dim to apply broadcasting. anchors = np.expand_dims(anchors, 0) anchor_maxes = anchors / 2. anchor_mins = -anchor_maxes valid_mask = boxes_wh[..., 0]>0 for b in range(m): # Discard zero rows. wh = boxes_wh[b, valid_mask[b]] if len(wh)==0: continue # Expand dim to apply broadcasting. wh = np.expand_dims(wh, -2) box_maxes = wh / 2. box_mins = -box_maxes intersect_mins = np.maximum(box_mins, anchor_mins) intersect_maxes = np.minimum(box_maxes, anchor_maxes) intersect_wh = np.maximum(intersect_maxes - intersect_mins, 0.) intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1] box_area = wh[..., 0] * wh[..., 1] anchor_area = anchors[..., 0] * anchors[..., 1] iou = intersect_area / (box_area + anchor_area - intersect_area) # Find best anchor for each true box best_anchor = np.argmax(iou, axis=-1) for t, n in enumerate(best_anchor): for l in range(num_layers): if n in anchor_mask[l]: i = np.floor(true_boxes[b,t,0]*grid_shapes[l][1]).astype('int32') j = np.floor(true_boxes[b,t,1]*grid_shapes[l][0]).astype('int32') k = anchor_mask[l].index(n) c = true_boxes[b,t, 4].astype('int32') y_true[l][b, j, i, k, 0:4] = true_boxes[b,t, 0:4] y_true[l][b, j, i, k, 4] = 1 y_true[l][b, j, i, k, 5+c] = 1 return y_true
#################### # SORT BY RICHNESS # #################### mock = mock[mock.argsort(order = ('rich', 'id'))[::-1]] cluster = cluster[cluster.argsort(order = ('rich', 'id'))[::-1]] ################### # BIN BY RICHNESS # ################### #set number of richness bins n_rich_bins = int(math.floor((opts.max_rich_bin - opts.min_rich_bin) / opts.rich_bin_size)) + 1 c_rich_bin_index = np.floor((cluster.rich - opts.min_rich_bin) / opts.rich_bin_size).astype('int') m_rich_bin_index = np.floor((mock.rich - opts.min_rich_bin) / opts.rich_bin_size).astype('int') x_rich_vals = (np.arange(n_rich_bins) + 0.5) * opts.rich_bin_size + opts.min_rich_bin ############ # BIN BY Z # ############ #set number of redshift bins n_z_bins = int(math.floor((opts.max_z_bin - opts.min_z_bin) / opts.z_bin_size)) + 1 c_z_bin_index = np.floor((cluster.z - opts.min_z_bin) / opts.z_bin_size).astype('int') m_z_bin_index = np.floor((mock.z - opts.min_z_bin) / opts.z_bin_size).astype('int') x_z_vals = (np.arange(n_z_bins) + 0.5) * opts.z_bin_size + opts.min_z_bin
def calc_xy_index_from_position(self, pos, lower_pos, max_index): ind = int(np.floor((pos - lower_pos) / self.resolution)) if 0 <= ind <= max_index: return ind else: return None
def create_grid_and_edges(data, drone_altitude, safety_distance): """ Returns a grid representation of a 2D configuration space along with Voronoi graph edges given obstacle data and the drone's altitude. """ # minimum and maximum north coordinates north_min = np.floor(np.min(data[:, 0] - data[:, 3])) north_max = np.ceil(np.max(data[:, 0] + data[:, 3])) # minimum and maximum east coordinates east_min = np.floor(np.min(data[:, 1] - data[:, 4])) east_max = np.ceil(np.max(data[:, 1] + data[:, 4])) # given the minimum and maximum coordinates we can # calculate the size of the grid. north_size = int(np.ceil((north_max - north_min))) east_size = int(np.ceil((east_max - east_min))) # Initialize an empty grid grid = np.zeros((north_size, east_size)) # Define a list to hold Voronoi points points = [] # Populate the grid with obstacles for i in range(data.shape[0]): north, east, alt, d_north, d_east, d_alt = data[i, :] if alt + d_alt + safety_distance > drone_altitude: obstacle = [ int(north - d_north - safety_distance - north_min), int(north + d_north + safety_distance - north_min), int(east - d_east - safety_distance - east_min), int(east + d_east + safety_distance - east_min), ] grid[obstacle[0]:obstacle[1] + 1, obstacle[2]:obstacle[3] + 1] = 1 # add center of obstacles to points list points.append([north - north_min, east - east_min]) # create a voronoi graph based on # location of obstacle centres graph = Voronoi(points) # check each edge from graph.ridge_vertices for collision edges = [] for v in graph.ridge_vertices: p1 = graph.vertices[v[0]].astype(int) p2 = graph.vertices[v[1]].astype(int) # test each pair p1 and p2 for collision using Bresenham # If the edge does not hit an obstacle add it to the list in_collision = False ridgeline = bresenham(p1[0], p1[1], p2[0], p2[1]) for b in ridgeline: # eliminate out of range points in the line if b[0] < 0 or b[0] >= grid.shape[0]: in_collision = True break if b[1] < 0 or b[1] >= grid.shape[1]: in_collision = True break # check if grid cell is an obstacle if grid[b[0], b[1]] == 1: in_collision = True break # keep ridge points not in collision if not in_collision: p1 = (p1[0], p1[1]) p2 = (p2[0], p2[1]) edges.append((p1, p2)) return grid, edges
def detect_all_signs(self, image, score_threshold): """ Will draw all the bounding boxes and return the list of classes, confidence scores in the boxes and the boxes themselves. """ start = timer() if self.model_image_size != (None, None): assert self.model_image_size[0] % 32 == 0, 'Multiples of 32 required' assert self.model_image_size[1] % 32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image(image, tuple(reversed(self.model_image_size))) else: new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') if self.gray_scale: image_data = rgb_2_gray(image_data) # print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) # print('Found {} boxes for {}'.format(len(out_boxes), 'img')) font = ImageFont.truetype(font='font/FiraMono-Medium.otf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 list_c, list_score, list_box = [], [], [] for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] if score < score_threshold: continue label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) # print(label, (left, top), (right, bottom)) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle( [left + i, top + i, right - i, bottom - i], outline=self.colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw list_c.append(c) list_score.append(score) list_box.append((left, top, right, bottom)) # x_min, y_min, x_max, y_max return image, list_c, list_score, list_box # (returns this if no sign is found)
def MergeDatCol_SigDat(SigDat_clean, DataColDat_Sub, LaneDict, StartVeh=4, EndVeh=14, RunNum=99): """ Merge Signal and Data col """ # Use pandas.merge_asof for merging t_Entry in data collection results to the nearest phase start time. # Similar to infimum ~ Greatest lower bound # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.merge_asof.html # A “backward” search selects the last row in the right DataFrame whose ‘on’ key is less than or equal to the left’s key. CombData_Ln_Dict = {} for Ln in LaneDict.keys(): CombData_Ln_Dict[Ln] = pd.merge_asof( DataColDat_Sub[DataColDat_Sub.Lane == Ln], SigDat_clean, left_on="t_Entry", right_on="G_st", direction="backward", ) # ***********************Reactivate Later**********************************# # assert(max(CombData_Ln_Dict[Ln].t_Entry -CombData_Ln_Dict[Ln].G_st) < 50) #Make sure you don't merge vehs to a diff cycle # Check the vehicles that eneter the intersection during Amber Indication CombData_Ln_Dict[Ln].loc[:, "VehinAmber"] = (CombData_Ln_Dict[Ln].t_Entry > CombData_Ln_Dict[Ln].G_end) CombData = pd.concat(CombData_Ln_Dict.values()) # #Debug: # Ln =1 # CombData_Ln_Dict[Ln].loc[:,"Debug"] = CombData_Ln_Dict[Ln].t_Entry -CombData_Ln_Dict[Ln].G_st # CombData = pd.concat(CombData_Ln_Dict.values()) # Get the vehicle numbers CombData.sort_values(["CycNum", "Lane"], inplace=True) # Get length of each group and then use arange CombData.loc[:, "VehNum"] = np.hstack( CombData.groupby( ["CycNum", "Lane"])["t_Entry"].apply(lambda x: np.arange(1, len(x) + 1)).values) CombData = CombData[[ "CycNum", "Lane", "LaneDesc", "tQueue", "t_Entry", "G_st", "G_end", "VehNum", "PhaseNum", ]] ################### Define conditions for Sat Flow ########################################### mask_SatFlow = (CombData.VehNum >= StartVeh) & ( CombData.VehNum <= EndVeh ) # Start vehicle is 4th (timeStamp for 4th vehicle is used for 5th vehicle) # mask_Headway = (CombData.VehNum >= StartVeh+1) & (CombData.VehNum <= EndVeh) # Start vehicle is 5th as we are directly getting the headway ## CombData_SatFlow = CombData[mask_SatFlow] CombData_Headway = CombData.copy() CombData_Headway.loc[:, "Headway"] = CombData_Headway.groupby( ["CycNum", "Lane"])["t_Entry"].diff() # mask2 = CombData_Headway.Headway<=5 # CombData_Headway = CombData_Headway[mask2] CombData_SatFlow = CombData_SatFlow.groupby(["CycNum", "LaneDesc"]).agg({ "t_Entry": ["min", "max"], "VehNum": ["min", "max"] }) CombData_SatFlow.columns = [ "_".join(col).strip() for col in CombData_SatFlow.columns.values ] CombData_SatFlow.loc[:, "AvgHeadway"] = ( CombData_SatFlow.t_Entry_max - CombData_SatFlow.t_Entry_min) / ( CombData_SatFlow.VehNum_max - CombData_SatFlow.VehNum_min) CombData_SatFlow.reset_index(drop=False, inplace=True) CombDataSum = CombData_SatFlow.groupby(["LaneDesc" ])["AvgHeadway"].describe() CombDataSum.loc[:, "SatFlow"] = np.floor(3600 / CombDataSum["mean"]) CombDataSum.loc[:, "RunNum"] = RunNum return (CombDataSum, CombData_SatFlow, CombData_Headway)
def probability_v_rank(counts, out = None): ''' Plots the probability (normalized frequency) versus rank for an arbitrary 1D vector of counts ''' counts = np.sort(counts) probs = counts / np.sum(counts) ranks = range(1, len(probs) + 1) ranks.reverse() # Probability fig = plt.figure(figsize=(6,4), dpi = 300) ax = plt.gca() ax.scatter(ranks, probs, alpha = 0.5) ax.set_yscale('log') ax.set_xscale('log') ax.axis([1, np.power(10, np.ceil(np.log10(max(ranks)))), \ np.power(10, np.floor(np.log10(min(probs)))), \ np.power(10, np.ceil(np.log10(max(probs))))]) plt.xlabel('Rank') plt.ylabel('Normalized frequency') plt.title('Transform templates from {}'.format(collection.name)) plt.grid(True) if out: fig.savefig(out + ' prob_rank.png') np.savetxt(out + ' probs.txt', sorted(probs, reverse = True)) # plt.show() # Count fig = plt.figure(figsize=(6,4), dpi = 300) ax = plt.gca() ax.scatter(ranks, counts, alpha = 0.5) ax.set_yscale('log') ax.set_xscale('log') ax.axis([1, np.power(10, np.ceil(np.log10(max(ranks)))), \ np.power(10, np.floor(np.log10(min(counts)))), \ np.power(10, np.ceil(np.log10(max(counts))))]) plt.xlabel('Rank') plt.ylabel('Counts') plt.title('Transform templates from {}'.format(collection.name)) plt.grid(True) if out: fig.savefig(out + ' count_rank.png') np.savetxt(out + ' counts.txt', sorted(counts, reverse = True)) # plt.show() # Coverage missing = np.ones_like(probs) missing[0] = 0.0 for i in range(1, len(probs)): missing[i] = missing[i-1] + probs[i] missing = np.ones_like(missing) - missing fig = plt.figure(figsize=(6,4), dpi = 300) ax = plt.gca() ax.scatter(ranks, missing, alpha = 0.5) ax.set_xscale('log') ax.axis([1, np.power(10, np.ceil(np.log10(max(ranks)))), \ 0, 1]) plt.xlabel('Rank threshold for inclusion') plt.ylabel('Estimated minimum coverage') plt.title('Transform templates from {}'.format(collection.name)) plt.grid(True) if out: fig.savefig(out + ' missing_rank.png') # plt.show() return
def i4_sobol(dim_num, seed): """ Parameters: Input, integer DIM_NUM, the number of spatial dimensions. DIM_NUM must satisfy 1 <= DIM_NUM <= 40. Input/output, integer SEED, the "seed" for the sequence. This is essentially the index in the sequence of the quasirandom value to be generated. On output, SEED has been set to the appropriate next value, usually simply SEED+1. If SEED is less than 0 on input, it is treated as though it were 0. An input value of 0 requests the first (0-th) element of the sequence. Output, real QUASI(DIM_NUM), the next quasirandom vector. """ global atmost global dim_max global dim_num_save global initialized global lastq global log_max global maxcol global poly global recipd global seed_save global v if 'initialized' not in list(globals().keys()): initialized = 0 dim_num_save = -1 if not initialized or dim_num != dim_num_save: initialized = 1 dim_max = 40 dim_num_save = -1 log_max = 30 seed_save = -1 # Initialize (part of) V. v = np.zeros((dim_max, log_max)) v[0:40, 0] = np.transpose([ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ]) v[2:40, 1] = np.transpose([ 1, 3, 1, 3, 1, 3, 3, 1, 3, 1, 3, 1, 3, 1, 1, 3, 1, 3, 1, 3, 1, 3, 3, 1, 3, 1, 3, 1, 3, 1, 1, 3, 1, 3, 1, 3, 1, 3 ]) v[3:40, 2] = np.transpose([ 7, 5, 1, 3, 3, 7, 5, 5, 7, 7, 1, 3, 3, 7, 5, 1, 1, 5, 3, 3, 1, 7, 5, 1, 3, 3, 7, 5, 1, 1, 5, 7, 7, 5, 1, 3, 3 ]) v[5:40, 3] = np.transpose([ 1, 7, 9, 13, 11, 1, 3, 7, 9, 5, 13, 13, 11, 3, 15, 5, 3, 15, 7, 9, 13, 9, 1, 11, 7, 5, 15, 1, 15, 11, 5, 3, 1, 7, 9 ]) v[7:40, 4] = np.transpose([ 9, 3, 27, 15, 29, 21, 23, 19, 11, 25, 7, 13, 17, 1, 25, 29, 3, 31, 11, 5, 23, 27, 19, 21, 5, 1, 17, 13, 7, 15, 9, 31, 9 ]) v[13:40, 5] = np.transpose([ 37, 33, 7, 5, 11, 39, 63, 27, 17, 15, 23, 29, 3, 21, 13, 31, 25, 9, 49, 33, 19, 29, 11, 19, 27, 15, 25 ]) v[19:40, 6] = np.transpose([ 13, 33, 115, 41, 79, 17, 29, 119, 75, 73, 105, 7, 59, 65, 21, 3, 113, 61, 89, 45, 107 ]) v[37:40, 7] = np.transpose([7, 23, 39]) # Set POLY. poly = [ 1, 3, 7, 11, 13, 19, 25, 37, 59, 47, 61, 55, 41, 67, 97, 91, 109, 103, 115, 131, 193, 137, 145, 143, 241, 157, 185, 167, 229, 171, 213, 191, 253, 203, 211, 239, 247, 285, 369, 299 ] atmost = 2**log_max - 1 # Find the number of bits in ATMOST. maxcol = i4_bit_hi1(atmost) # Initialize row 1 of V. v[0, 0:maxcol] = 1 # Things to do only if the dimension changed. if dim_num != dim_num_save: # Check parameters. if dim_num < 1 or dim_max < dim_num: print('I4_SOBOL - Fatal error!') print(' The spatial dimension DIM_NUM should satisfy:') print(' 1 <= DIM_NUM <= %d' % dim_max) print(' But this input value is DIM_NUM = %d' % dim_num) return dim_num_save = dim_num # Initialize the remaining rows of V. for i in range(2, dim_num + 1): # The bits of the integer POLY(I) gives the form of polynomial I. # Find the degree of polynomial I from binary encoding. j = poly[i - 1] m = 0 j //= 2 while j > 0: j //= 2 m += 1 # Expand this bit pattern to separate components of the logical array INCLUD. j = poly[i - 1] includ = np.zeros(m) for k in range(m, 0, -1): j2 = j // 2 includ[k - 1] = (j != 2 * j2) j = j2 # Calculate the remaining elements of row I as explained # in Bratley and Fox, section 2. for j in range(m + 1, maxcol + 1): newv = v[i - 1, j - m - 1] l = 1 for k in range(1, m + 1): l *= 2 if includ[k - 1]: newv = np.bitwise_xor(int(newv), int(l * v[i - 1, j - k - 1])) v[i - 1, j - 1] = newv # Multiply columns of V by appropriate power of 2. l = 1 for j in range(maxcol - 1, 0, -1): l *= 2 v[0:dim_num, j - 1] = v[0:dim_num, j - 1] * l # RECIPD is 1/(common denominator of the elements in V). recipd = 1.0 / (2 * l) lastq = np.zeros(dim_num) seed = int(np.floor(seed)) if seed < 0: seed = 0 l = 1 if seed == 0: lastq = np.zeros(dim_num) elif seed == seed_save + 1: # Find the position of the right-hand zero in SEED. l = i4_bit_lo0(seed) elif seed <= seed_save: seed_save = 0 lastq = np.zeros(dim_num) for seed_temp in range(int(seed_save), int(seed)): l = i4_bit_lo0(seed_temp) for i in range(1, dim_num + 1): lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]), int(v[i - 1, l - 1])) l = i4_bit_lo0(seed) elif seed_save + 1 < seed: for seed_temp in range(int(seed_save + 1), int(seed)): l = i4_bit_lo0(seed_temp) for i in range(1, dim_num + 1): lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]), int(v[i - 1, l - 1])) l = i4_bit_lo0(seed) # Check that the user is not calling too many times! if maxcol < l: print('I4_SOBOL - Fatal error!') print(' Too many calls!') print(' MAXCOL = %d\n' % maxcol) print(' L = %d\n' % l) return # Calculate the new components of QUASI. quasi = np.zeros(dim_num) for i in range(1, dim_num + 1): quasi[i - 1] = lastq[i - 1] * recipd lastq[i - 1] = np.bitwise_xor(int(lastq[i - 1]), int(v[i - 1, l - 1])) seed_save = seed seed += 1 return [quasi, seed]