def coarser(x, y, r=0.1): # getting the point density for a coarser subset (seeds) # of points that are close to grid cell centers (xc, yc) cc = np.arange(r / 4, 1, r / 2) xc, yc = np.meshgrid(cc, cc) xc, yc = xc.flatten(), yc.flatten() # kdtrees for the points and the cell centers grid = kdtree(np.transpose((xc, yc))) tree = kdtree(np.transpose((x, y))) # those points in tree that are close to grid seed = grid.query_ball_tree(tree, r=r / 2) seeds = [x for x in seed if x != []] # as in density lsts = tree.query_ball_tree(tree, r=r) m = len(lsts) dens = np.zeros(m) # but now a loop only over seeds for l in seeds: i = l[0] dens[i] = len(lsts[i]) disk = np.pi * r**2 return (xc, yc, dens / disk)
def geometric_graph(positions, radius): tree = kdtree(positions) edges = list(tree.query_pairs(radius)) g = Graph(n=len(positions), edges=edges) return g
def time_kdtree(pt, k, leafsize, n_jobs): t0 = time() tr = kdtree(pt, leafsize = leafsize) t1 = time() tr.query(pt, k = k, n_jobs = n_jobs) t2 = time() return (t1 - t0, t2 - t1)
def _buildTree(self, simDataRa, simDataDec, leafsize=100): """Build KD tree on simDataRA/Dec and set radius (via setRad) for matching. simDataRA, simDataDec = RA and Dec values (in radians). leafsize = the number of Ra/Dec pointings in each leaf node.""" if np.any(np.abs(simDataRa) > np.pi*2.0) or np.any(np.abs(simDataDec) > np.pi*2.0): raise ValueError('Expecting RA and Dec values to be in radians.') x, y, z = self._treexyz(simDataRa, simDataDec) data = list(zip(x, y, z)) if np.size(data) > 0: try: self.opsimtree = kdtree(data, leafsize=leafsize, balanced_tree=False, compact_nodes=False) except TypeError: self.opsimtree = kdtree(data, leafsize=leafsize) else: raise ValueError('SimDataRA and Dec should have length greater than 0.')
def low_pass_smoothing(vertices, iterations, lambda1, mu): vertices_smoothed = np.zeros(np.shape(vertices)) for n in range(iterations): for i in range(vertices.shape[0]): vertices_tree = kdtree(vertices) kdtree.visualize(vertices_tree) _, idx = vertices_tree.query(vertices[i], k=7) q = np.zeros((1, 3)) # set the intial sum of nearest points m = len( idx ) # number of nearest points (actually minus 1 because this including the point itself) for j in range(m): q += vertices[idx[j]] if n % 2 == 0: vertices_smoothed[i] = vertices[i] + ( q - m * vertices[i]) * lambda1 / (m - 1) else: vertices_smoothed[i] = vertices[i] + ( q - m * vertices[i]) * mu / (m - 1) vertices = vertices_smoothed return vertices
def _buildTree(self, simDataRa, simDataDec, leafsize=100): """Build KD tree on simDataRA/Dec and set radius (via setRad) for matching. simDataRA, simDataDec = RA and Dec values (in radians). leafsize = the number of Ra/Dec pointings in each leaf node.""" if np.any(np.abs(simDataRa) > np.pi*2.0) or np.any(np.abs(simDataDec) > np.pi*2.0): raise ValueError('Expecting RA and Dec values to be in radians.') x, y, z = self._treexyz(simDataRa, simDataDec) data = zip(x, y, z) if np.size(data) > 0: try: self.opsimtree = kdtree(data, leafsize=leafsize, balanced_tree=False, compact_nodes=False) except TypeError: self.opsimtree = kdtree(data, leafsize=leafsize) else: raise ValueError('SimDataRA and Dec should have length greater than 0.')
def tree(xp, yp, r = 0.01): tree = kdtree(np.transpose((xp, yp))) lsts = tree.query_ball_tree(tree, r = r) l = 0 for i in range(len(xp)): l += len(lsts[i]) return l
def tree(n, r = 0.01): xp = np.random.random(n) yp = np.random.random(n) tree = kdtree(np.transpose((xp, yp))) lsts = tree.query_ball_tree(tree, r = r) l = 0 for i in range(len(xp)): l += len(lsts[i]) return l
def density(x, y, r=0.1): # getting the point density for each point disk = np.pi * r**2 tree = kdtree(np.transpose((x, y))) lsts = tree.query_ball_tree(tree, r=r) m = len(lsts) dens = np.zeros(m) for i in range(m): dens[i] = len(lsts[i]) return dens / disk
def test_tree_time(random_postions=True): def tree_xyz(lat, lon): """ convert positions on a sphere to 3d coordinates """ x = np.cos(lat) * np.cos(lon) y = np.cos(lat) * np.sin(lon) z = np.sin(lat) return x, y, z npts = 3e6 np.random.seed(42) if random_postions: # random points on a sphere. lat = np.arccos(2.*np.random.uniform(low=0, high=1, size=npts) - 1.) - np.pi/2. lon = np.random.uniform(low=0, high=2*np.pi, size=npts) else: nside = 16 lat, lon = hp.pix2ang(nside, np.arange(hp.nside2npix(nside))) npix = lat.size lat = np.repeat(lat, np.ceil(npts/npix)) lon = np.repeat(lon, np.ceil(npts/npix)) lat = lat[0:npts] lon = lon[0:npts] x, y, z = tree_xyz(lat, lon) # Postiions to search on nside = 128 position_lat, position_lon = hp.pix2ang(nside, np.arange(hp.nside2npix(nside))) position_lat = np.pi/2. - position_lat pos_x, pos_y, pos_z = tree_xyz(position_lat, position_lon) leafsize = 100 # Trying out solution from: https://github.com/scipy/scipy/issues/6108#issuecomment-217117065 tree = kdtree(zip(x, y, z), leafsize=leafsize, balanced_tree=False, compact_nodes=False) # Set a search radius radius = 2. x0, y0, z0 = (1, 0, 0) x1, y1, z1 = tree_xyz(np.radians(radius), 0) search_rad = np.sqrt((x1-x0)**2+(y1-y0)**2+(z1-z0)**2) found_points = 0 for px, py, pz in zip(pos_x, pos_y, pos_z): indices = tree.query_ball_point((px, py, pz), search_rad) found_points += len(indices)
def grid_point_cloud(x, y, z, width=1): """ Aggregates a point cloud (x, y, z) to a grid with grid cell spacing width. Returns: (mean, std, xbounds, ybounds) """ from scipy.spatial import cKDTree as kdtree xmin, xmax = x.min(), x.max() ymin, ymax = y.min(), y.max() xb = np.arange(xmin, xmax + width, width) yb = np.arange(ymin, ymax + width, width) xr = xb[:-1] + width / 2.0 yr = yb[:-1] + width / 2.0 xc, yc = np.meshgrid(xr, yr) shape = xc.shape xc = xc.ravel() yc = yc.ravel() tree = kdtree(np.transpose((x, y))) grid = kdtree(np.transpose((xc, yc))) lsts = grid.query_ball_tree(tree, r=width / np.sqrt(2)) n = len(lsts) m = np.zeros(n) s = np.zeros(n) for i in range(n): if len(lsts[i]) < 3: m[i] = np.nan s[i] = np.nan else: j = lsts[i] m[i] = z[j].mean() s[i] = z[j].std() m.shape = shape s.shape = shape return (m, s, xb, yb)
def thinning(x, y, z, r): from scipy.spatial import cKDTree as kdtree b = np.ones(len(x), dtype='bool') tree = kdtree(np.transpose((x, y))) lsts = tree.query_ball_tree(tree, r=r) for l in lsts: if len(l) > 1: la = np.array(l) dz = np.abs(z[la] - np.median(z[la])) b[la] = False b[la[np.argmin(dz)]] = True print('keeping %.3f %%' % (100. * len(b[b]) / len(b))) return (x[b], y[b], z[b])
def buildTree(simDataRa, simDataDec, leafsize=100): """Build KD tree on simDataRA/Dec and set radius (via setRad) for matching. simDataRA, simDataDec = RA and Dec values (in radians). leafsize = the number of Ra/Dec pointings in each leaf node.""" if np.any(np.abs(simDataRa) > np.pi*2.0) or np.any(np.abs(simDataDec) > np.pi*2.0): raise ValueError('Expecting RA and Dec values to be in radians.') x, y, z = treexyz(simDataRa, simDataDec) data = zip(x,y,z) if np.size(data) > 0: starTree = kdtree(data, leafsize=leafsize) else: raise ValueError('SimDataRA and Dec should have length greater than 0.') return starTree
def buildTree(simDataRa, simDataDec, leafsize=100): """Build KD tree on simDataRA/Dec and set radius (via setRad) for matching. simDataRA, simDataDec = RA and Dec values (in radians). leafsize = the number of Ra/Dec pointings in each leaf node.""" if np.any(np.abs(simDataRa) > np.pi * 2.0) or np.any( np.abs(simDataDec) > np.pi * 2.0): raise ValueError('Expecting RA and Dec values to be in radians.') x, y, z = treexyz(simDataRa, simDataDec) data = list(zip(x, y, z)) if np.size(data) > 0: starTree = kdtree(data, leafsize=leafsize) else: raise ValueError( 'SimDataRA and Dec should have length greater than 0.') return starTree
def main(n, r=0.1): x = (np.random.random(n) - 0.5) * 4 y = (np.random.random(n) - 0.5) * 3 z = np.exp(-x * x - y * y) tree = kdtree(np.transpose((x, y))) slp = np.zeros(n) for i in range(n): nb = tree.query_ball_point((x[i], y[i]), r) pts = np.transpose((x[nb], y[nb], z[nb])) nx, ny, nz = FitPlane(pts) slp[i] = np.sqrt(nx * nx + ny * ny) / nz slp = np.arctan(slp) * 180 / np.pi pl.title('Numerical') pl.scatter(x, y, c=slp) pl.colorbar() pl.axes().set_aspect('equal') pl.show() # theoretical results rp = np.sqrt(x * x + y * y) sp = 2 * rp * np.exp(-rp * rp) sp = np.arctan(sp) * 180 / np.pi pl.title('Theory') pl.scatter(x, y, c=sp) pl.colorbar() pl.axes().set_aspect('equal') pl.show() pl.title('absolute difference') pl.scatter(x, y, c=slp - sp, cmap=pl.cm.seismic, vmin=-25, vmax=25) pl.colorbar() pl.axes().set_aspect('equal') pl.show() pl.title('relative difference') pl.scatter(x, y, c=100. * (slp - sp) / sp, cmap=pl.cm.seismic, vmin=-25, vmax=25) pl.colorbar() pl.axes().set_aspect('equal') pl.show()
def get_interpolation_matrix( self, npts=20, epsilon=None ): tree_2d = kdtree(self.topography[:, :2]) xy = Utils.ndgrid(self.mesh_3d.vectorCCx, self.mesh_3d.vectorCCy) distance, inds = tree_2d.query(xy, k=npts) if epsilon is None: epsilon = np.min([self.mesh_3d.hx.min(), self.mesh_3d.hy.min()]) w = 1. / (distance + epsilon)**2 w = Utils.sdiag(1./np.sum(w, axis=1)) * (w) I = Utils.mkvc( np.arange(inds.shape[0]).reshape([-1, 1]).repeat(npts, axis=1) ) J = Utils.mkvc(inds) self._P = sp.coo_matrix( (Utils.mkvc(w), (I, J)), shape=(inds.shape[0], self.topography.shape[0]) ) mesh_1d = Mesh.TensorMesh([np.r_[self.hz[:-1], 1e20]]) z = self.P*self.topography[:, 2] self._actinds = Utils.surface2ind_topo(self.mesh_3d, np.c_[xy, z]) Z = np.empty(self.mesh_3d.vnC, dtype=float, order='F') Z = self.mesh_3d.gridCC[:, 2].reshape( (self.mesh_3d.nCx*self.mesh_3d.nCy, self.mesh_3d.nCz), order='F' ) ACTIND = self._actinds.reshape( (self.mesh_3d.nCx*self.mesh_3d.nCy, self.mesh_3d.nCz), order='F' ) self._Pz = [] # This part can be cythonized or parallelized for i_xy in range(self.mesh_3d.nCx*self.mesh_3d.nCy): actind_temp = ACTIND[i_xy, :] z_temp = -(Z[i_xy, :] - z[i_xy]) self._Pz.append(mesh_1d.getInterpolationMat(z_temp[actind_temp]))
def __init__(self, init_num, size, stp): self.num = init_num self.size = size self.stp = stp self.one = 1.0 / size self.rad = 0.04 edge = 0.2 self.xy = edge + random((init_num, 2)) * (1.0 - 2 * edge) self.v = zeros((init_num, 2), 'float') self.dx = zeros((init_num, 2), 'float') self.random_velocity(self.stp) self.tree = kdtree(self.xy) self.slowdown = 0.99999
def remove_close(points, radius): """ given an (n,m) set of points where n = 2 or n =3 return a list of points where no point is closer than radius para::points: numpy array(m,2) or (m,3), coordinates of the points para::radius: the least distance between the points return: the unique points and its index """ tree = kdtree(points) consumed = np.zeros(len(points), dtype=np.bool) unique = np.zeros(len(points), dtype=np.bool) for k in range(len(points)): if consumed[k]: continue neighbors = tree.query_ball_point(points[k], r=radius) consumed[neighbors] = True unique[k] = True index = np.array([k for k in range(len(points))]) return points[unique], index[unique]
def _buildTree(self, ra, dec, leafsize=100): """Build KD tree on simDataRA/Dec and set radius (via setRad) for matching. simDataRA, simDataDec = RA and Dec values (in radians). leafsize = the number of Ra/Dec pointings in each leaf node.""" if np.any(np.abs(ra) > np.pi*2.0) or np.any(np.abs(dec) > np.pi*2.0): raise ValueError('Expecting RA and Dec values to be in radians.') x, y, z = self._treexyz(ra, dec) data = list(zip(x, y, z)) if np.size(data) > 0: self.hptree = kdtree(data, leafsize=leafsize) # try: # XXX--this is crashing the kernel? Not sure why, need to make sure other way speed-tests OK # self.hptree = kdtree(data, leafsize=leafsize, balanced_tree=False, compact_nodes=False) # except TypeError: # self.hptree = kdtree(data, leafsize=leafsize) else: raise ValueError('RA and Dec should have length greater than 0.')
def postprocess(self): """ This MUST be called, once, after the XML has been loaded. It: - ensures relation-members know about their parent object (incl. the plantref if parent is plant); - calculates the total area of each relation; - calculates the centroid of each relation; - performs spatial containment queries to label solar panels as members of a plant (i.e. plantref) if they're geographically inside them. """ # Find all objects that are relations and also plants. Then push down the metadata through their children (only for the temporary data), and also calculate the area for the parents. rels_postprocessed = 0 self.plantoutlines = [] # {lat, lon, outlinepath, plantref} a list of ways that are either plants themselves, or non-inner members of plant relations; i.e. potential geo containers for panels for curitem in self.objs: if curitem['tag_power']=='plant' and curitem['objtype']=='way': self.plantoutlines.append({'lat': curitem['lat'], 'lon': curitem['lon'], 'outlinepath': self.waydata[curitem['id']]['outlinepath'], 'plantref': curitem['plantref']}) if curitem['tag_power'] in ['plant', 'generator'] and curitem['objtype']=='relation': if curitem['tag_power']=='plant': plantitem = curitem plantref = ('relation', curitem['id']) else: plantitem = None plantref = None curitem['calc_area'] = 0 self._recurse_relation_info(curitem, plantitem, plantref) rels_postprocessed += 1 print("Postprocessed %i power=* relations" % rels_postprocessed) print("Plant outlines for geo containment search: %i" % len(self.plantoutlines)) plantoutlines_kdtree = kdtree([[item['lat'], item['lon']] for item in self.plantoutlines]) # Now, for every generator object that DOESN'T have a plantref, we find its nearest-neighbour potential-containers and check for containment for curitem in self.objs: if curitem['tag_power']=='generator' and not curitem.get('plantref', None): for distance, arrayposition in zip(*plantoutlines_kdtree.query([curitem['lat'], curitem['lon']], 3, distance_upper_bound=1)): if distance != np.inf: if self.plantoutlines[arrayposition]['outlinepath'].contains_point([curitem['lat'], curitem['lon']]): curitem['plantref'] = self.plantoutlines[arrayposition]['plantref'] #print(" spatially inferred generator %s/%s belongs to plant %s" % (curitem['objtype'], curitem['id'], curitem['plantref'])) if False: # This should NOT NORMALLY be activated. It inserts "guesstimate" power capacities for small-scale solar PV for curitem in self.objs: if curitem['tag_power']=='generator' and curitem.get('calc_capacity', 0)==0 and not curitem.get('plantref', None): curitem['calc_capacity'] = guess_kilowattage(curitem)
def __init__(self, nactions, input_ranges, nelemns=[], npoints=0, k=1, alpha=0.3, lm=0.95): if not (nelemns == False) ^ (npoints == False): raise ValueError('Plese indicate either: [nelemns] Xor [npoints]') if nelemns: self.cl = self.ndlinspace(input_ranges, nelemns) else: self.cl = self.CreateRandomSpace(input_ranges, npoints) self.lbounds = [] self.ubounds = [] self.k = k self.shape = self.cl.shape self.nactions = nactions self.Q = zeros((self.shape[0], nactions)) # self.Q = uniform(-100,0,(self.shape[0],nactions))+0.0 self.e = zeros((self.shape[0], nactions)) + 0.0 # self.ac = zeros((self.shape[0]))+0.0 #classifiers activation self.ac = [] self.knn = [] self.alpha = alpha self.lm = lm # good 0.95 self.last_state = zeros((1, self.shape[1])) + 0.0 for r in input_ranges: self.lbounds.append(r[0]) self.ubounds.append(r[1]) self.lbounds = array(self.lbounds) self.ubounds = array(self.ubounds) self.cl = array(self.RescaleInputs(self.cl)) self.knntree = kdtree(self.cl)
def hp_kd_tree(nside=set_default_nside(), leafsize=100): """ Generate a KD-tree of healpixel locations Parameters ---------- nside : int A valid healpix nside leafsize : int (100) Leafsize of the kdtree Returns ------- tree : scipy kdtree """ hpid = np.arange(hp.nside2npix(nside)) ra, dec = _hpid2RaDec(nside, hpid) x, y, z = treexyz(ra, dec) tree = kdtree(list(zip(x, y, z)), leafsize=leafsize, balanced_tree=False, compact_nodes=False) return tree
def main(fn, m=5000, r=1): pts = np.load(fn) x = pts[:m, 0] y = pts[:m, 1] z = pts[:m, 2] n = len(z) tree = kdtree(np.transpose((x, y))) slp = np.zeros(n) for i in range(n): nb = tree.query_ball_point((x[i], y[i]), r) pts = np.transpose((x[nb], y[nb], z[nb])) nx, ny, nz = FitPlane(pts) slp[i] = np.sqrt(nx * nx + ny * ny) / nz slp = np.arctan(slp) * 180 / np.pi pl.figure(1, (10.24, 7.68)) pl.title('Pozo slope') pl.scatter(x, y, c=slp, s=3, vmin=0, vmax=90) pl.colorbar() pl.axes().set_aspect('equal') pl.savefig('%s.png' % sys.argv[0][:-3])
def step(self, separation, cohesion, alignment): self.xy += self.v nearby = self.get_nearby() separation = self.stp * separation alignment = self.stp * alignment cohesion = self.stp * cohesion self.dx[:, :] = 0 for i, near in enumerate(nearby): near = list(set(near).difference(set([i]))) if not near: continue self.separation(i, near, separation) self.alignment(i, near, alignment) self.cohesion(i, near, cohesion) self.v += self.dx / 3.0 self.v *= self.slowdown self.tree = kdtree(self.xy)
data = np.load('phot.npz') phot = data['phot'].copy() data.close() umjd = np.unique(phot['mjd']) phot.sort(order='mjd') left = np.searchsorted(phot['mjd'], umjd) right = np.searchsorted(phot['mjd'], umjd, side='right') nstars = right - left goodMJD = umjd[np.where(nstars == nstars.max())] # create a kdtree for all the ra,dec points x, y, z = treexyz(phot['RA'], phot['dec']) tree = kdtree(zip(x, y, z), leafsize=100) starIDs = np.zeros(phot.size, dtype=int) - 666 # Array to say if the star has matched multiple times multiFlag = np.zeros(phot.size, dtype=int) radius = 300. / 3600. # return everything within 300 arcsec? x0, y0, z0 = (1, 0, 0) x1, y1, z1 = treexyz(np.radians(radius), 0) rad = np.sqrt((x1 - x0)**2 + (y1 - y0)**2 + (z1 - z0)**2) # Match everything to one of the frames with lots of stars good = np.where(phot['mjd'] == goodMJD) onID = 1 for i, blah in enumerate(phot[good]): indices = np.array(tree.query_ball_point((x[i], y[i], z[i]), rad))
def __init__(self,points): self.points = points self.tree = kdtree(points)
def time_bin_magseries_with_errs(times, mags, errs, binsize=540.0): ''' This bins the given mag timeseries in time using the binsize given. binsize is in seconds. ''' # check if the input arrays are ok if not(times.shape and mags.shape and errs.shape and len(times) > 10 and len(mags) > 10): LOGERROR("input time/mag arrays don't have enough elements") return # find all the finite values of the magnitudes and times finiteind = np.isfinite(mags) & np.isfinite(times) & np.isfinite(errs) finite_times = times[finiteind] finite_mags = mags[finiteind] finite_errs = errs[finiteind] # convert binsize in seconds to JD units binsizejd = binsize/(86400.0) nbins = int(np.ceil((np.nanmax(finite_times) - np.nanmin(finite_times))/binsizejd) + 1) minjd = np.nanmin(finite_times) jdbins = [(minjd + x*binsizejd) for x in range(nbins)] # make a KD-tree on the JDs so we can do fast distance calculations. we # need to add a bogus y coord to make this a problem that KD-trees can # solve. time_coords = np.array([[x,1.0] for x in finite_times]) jdtree = kdtree(time_coords) binned_finite_timeseries_indices = [] collected_binned_mags = {} for jd in jdbins: # find all bin indices close to within binsizejd of this point # using the kdtree query. we use the p-norm = 1 (I think this # means straight-up pairwise distance? FIXME: check this) bin_indices = jdtree.query_ball_point(np.array([jd,1.0]), binsizejd/2.0, p=1.0) # if the bin_indices have already been collected, then we're # done with this bin, move to the next one. if they haven't, # then this is the start of a new bin. if (bin_indices not in binned_finite_timeseries_indices and len(bin_indices)) > 0: binned_finite_timeseries_indices.append(bin_indices) # convert to ndarrays binned_finite_timeseries_indices = [np.array(x) for x in binned_finite_timeseries_indices] collected_binned_mags['jdbins_indices'] = binned_finite_timeseries_indices collected_binned_mags['jdbins'] = jdbins collected_binned_mags['nbins'] = len(binned_finite_timeseries_indices) # collect the finite_times binned_jd = np.array([np.median(finite_times[x]) for x in binned_finite_timeseries_indices]) collected_binned_mags['binnedtimes'] = binned_jd collected_binned_mags['binsize'] = binsize # median bin the magnitudes according to the calculated indices collected_binned_mags['binnedmags'] = ( np.array([np.median(finite_mags[x]) for x in binned_finite_timeseries_indices]) ) # FIXME: calculate the error in the median-binned magnitude correctly # for now, just take the median of the errors in this bin collected_binned_mags['binnederrs'] = ( np.array([np.median(finite_errs[x]) for x in binned_finite_timeseries_indices]) ) return collected_binned_mags
venus_avoid = np.radians(5.) # rad # Sun twilight limit sun_alt_limit = np.radians(-12.) # rad # Telescope pointing altitude limit. alt_limit = np.radians(20.) nside = 32 map_size = hp.nside2npix(nside) dec, ra = hp.pix2ang(nside, np.arange(map_size)) dec = np.pi/2. - dec leafsize = 100 x, y, z = treexyz(ra, dec) healTree = kdtree(list(zip(x, y, z)), leafsize=leafsize) # Need to build a kdtree to quick search for healpixes within a radius mjd_start = 59580.033829 survey_length = 10 # days time_step = 10. # minitues mjds = np.arange(mjd_start, mjd_start + survey_length+time_step/60./24., time_step/60./24.) names = ['u', 'g', 'r', 'i', 'z', 'y', 'airmass', 'mask'] types = [float]*7 types.append(int) results = np.zeros((mjds.size, map_size), dtype=list(zip(names, types)))
venus_avoid = np.radians(5.) # rad # Sun twilight limit sun_alt_limit = np.radians(-12.) # rad # Telescope pointing altitude limit. alt_limit = np.radians(20.) nside = 32 map_size = hp.nside2npix(nside) dec, ra = hp.pix2ang(nside, np.arange(map_size)) dec = np.pi / 2. - dec leafsize = 100 x, y, z = treexyz(ra, dec) healTree = kdtree(list(zip(x, y, z)), leafsize=leafsize) # Need to build a kdtree to quick search for healpixes within a radius mjd_start = 59580.033829 survey_length = 10 # days time_step = 10. # minitues mjds = np.arange(mjd_start, mjd_start + survey_length + time_step / 60. / 24., time_step / 60. / 24.) names = ['u', 'g', 'r', 'i', 'z', 'y', 'airmass', 'mask'] types = [float] * 7 types.append(int) results = np.zeros((mjds.size, map_size), dtype=list(zip(names, types))) results['mask'] = 1
venus_avoid = np.radians(5.) # rad # Sun twilight limit sun_alt_limit = np.radians(-12.) # rad # Telescope pointing altitude limit. alt_limit = np.radians(20.) nside = 32 map_size = hp.nside2npix(nside) dec,ra = hp.pix2ang(nside, np.arange(map_size)) dec = np.pi/2. - dec leafsize = 100 x,y,z = treexyz(ra, dec) healTree = kdtree(zip(x,y,z),leafsize=leafsize) # Need to build a kdtree to quick search for healpixes within a radius mjd_start = 59580.033829 survey_length = 10 # days time_step = 10. # minitues mjds = np.arange(mjd_start, mjd_start + survey_length+time_step/60./24., time_step/60./24.) names=['u','g','r','i','z','y','airmass', 'mask'] types = [float]*7 types.append(int) results = np.zeros((mjds.size,map_size), dtype=zip(names,types) )
def phase_bin_magseries(phases, mags, binsize=0.005, minbinelems=7): ''' This bins a magnitude timeseries in phase using the binsize (in phase) provided. minbinelems is the minimum number of elements in each bin. ''' # check if the input arrays are ok if not (phases.shape and mags.shape and len(phases) > 10 and len(mags) > 10): LOGERROR("input time/mag arrays don't have enough elements") return # find all the finite values of the magnitudes and phases finiteind = np.isfinite(mags) & np.isfinite(phases) finite_phases = phases[finiteind] finite_mags = mags[finiteind] nbins = int( np.ceil((np.nanmax(finite_phases) - np.nanmin(finite_phases)) / binsize) + 1) minphase = np.nanmin(finite_phases) phasebins = [(minphase + x * binsize) for x in range(nbins)] # make a KD-tree on the PHASEs so we can do fast distance calculations. we # need to add a bogus y coord to make this a problem that KD-trees can # solve. time_coords = np.array([[x, 1.0] for x in finite_phases]) phasetree = kdtree(time_coords) binned_finite_phaseseries_indices = [] collected_binned_mags = {} for phase in phasebins: # find all bin indices close to within binsize of this point using the # kdtree query. we use the p-norm = 1 for pairwise Euclidean distance. bin_indices = phasetree.query_ball_point(np.array([phase, 1.0]), binsize / 2.0, p=1.0) # if the bin_indices have already been collected, then we're # done with this bin, move to the next one. if they haven't, # then this is the start of a new bin. if (bin_indices not in binned_finite_phaseseries_indices and len(bin_indices) >= minbinelems): binned_finite_phaseseries_indices.append(bin_indices) # convert to ndarrays binned_finite_phaseseries_indices = [ np.array(x) for x in binned_finite_phaseseries_indices ] collected_binned_mags['phasebins_indices'] = ( binned_finite_phaseseries_indices) collected_binned_mags['phasebins'] = phasebins collected_binned_mags['nbins'] = len(binned_finite_phaseseries_indices) # collect the finite_phases binned_phase = np.array([ np.median(finite_phases[x]) for x in binned_finite_phaseseries_indices ]) collected_binned_mags['binnedphases'] = binned_phase collected_binned_mags['binsize'] = binsize # median bin the magnitudes according to the calculated indices collected_binned_mags['binnedmags'] = (np.array([ np.median(finite_mags[x]) for x in binned_finite_phaseseries_indices ])) return collected_binned_mags
data = np.load('phot.npz') phot = data['phot'].copy() data.close() umjd = np.unique(phot['mjd']) phot.sort(order='mjd') left = np.searchsorted(phot['mjd'], umjd) right = np.searchsorted(phot['mjd'], umjd, side='right') nstars = right-left goodMJD = umjd[np.where(nstars == nstars.max())] # create a kdtree for all the ra,dec points x, y, z = treexyz(phot['RA'], phot['dec']) tree = kdtree(zip(x,y,z), leafsize=100) starIDs = np.zeros(phot.size, dtype=int)-666 # Array to say if the star has matched multiple times multiFlag = np.zeros(phot.size, dtype=int) radius = 300./3600. # return everything within 300 arcsec? x0, y0, z0 = (1, 0, 0) x1, y1, z1 = treexyz(np.radians(radius), 0) rad = np.sqrt((x1-x0)**2+(y1-y0)**2+(z1-z0)**2) # Match everything to one of the frames with lots of stars good = np.where(phot['mjd'] == goodMJD) onID = 1 for i, blah in enumerate(phot[good]): indices = np.array(tree.query_ball_point((x[i], y[i], z[i]), rad))
def time_bin_magseries_with_errs(times, mags, errs, binsize=540.0, minbinelems=7): '''This bins the given mag timeseries in time using the binsize given. binsize is in seconds. minbinelems is the number of minimum elements in a bin. ''' # check if the input arrays are ok if not (times.shape and mags.shape and errs.shape and len(times) > 9 and len(mags) > 9): LOGERROR("input time/mag arrays don't have enough elements") return # find all the finite values of the magnitudes and times finiteind = np.isfinite(mags) & np.isfinite(times) & np.isfinite(errs) finite_times = times[finiteind] finite_mags = mags[finiteind] finite_errs = errs[finiteind] # convert binsize in seconds to JD units binsizejd = binsize / (86400.0) nbins = int( np.ceil((np.nanmax(finite_times) - np.nanmin(finite_times)) / binsizejd) + 1) minjd = np.nanmin(finite_times) jdbins = [(minjd + x * binsizejd) for x in range(nbins)] # make a KD-tree on the JDs so we can do fast distance calculations. we # need to add a bogus y coord to make this a problem that KD-trees can # solve. time_coords = np.array([[x, 1.0] for x in finite_times]) jdtree = kdtree(time_coords) binned_finite_timeseries_indices = [] collected_binned_mags = {} for jd in jdbins: # find all bin indices close to within binsize of this point using the # kdtree query. we use the p-norm = 1 for pairwise Euclidean distance. bin_indices = jdtree.query_ball_point(np.array([jd, 1.0]), binsizejd / 2.0, p=1.0) # if the bin_indices have already been collected, then we're # done with this bin, move to the next one. if they haven't, # then this is the start of a new bin. if (bin_indices not in binned_finite_timeseries_indices and len(bin_indices) >= minbinelems): binned_finite_timeseries_indices.append(bin_indices) # convert to ndarrays binned_finite_timeseries_indices = [ np.array(x) for x in binned_finite_timeseries_indices ] collected_binned_mags['jdbins_indices'] = binned_finite_timeseries_indices collected_binned_mags['jdbins'] = np.array(jdbins) collected_binned_mags['nbins'] = len(binned_finite_timeseries_indices) # collect the finite_times binned_jd = np.array( [np.median(finite_times[x]) for x in binned_finite_timeseries_indices]) collected_binned_mags['binnedtimes'] = binned_jd collected_binned_mags['binsize'] = binsize # median bin the magnitudes according to the calculated indices collected_binned_mags['binnedmags'] = (np.array( [np.median(finite_mags[x]) for x in binned_finite_timeseries_indices])) # FIXME: calculate the error in the median-binned magnitude correctly # for now, just take the median of the errors in this bin collected_binned_mags['binnederrs'] = (np.array( [np.median(finite_errs[x]) for x in binned_finite_timeseries_indices])) return collected_binned_mags
for voxel_width in np.linspace(0.05, 0.25, 5): print('voxel width:', voxel_width) # voxel bounds vxb = np.linspace(0, shape[2], int(shape[2] / voxel_width) + 1) vyb = np.linspace(0, shape[1], int(shape[1] / voxel_width) + 1) vzb = np.linspace(0, shape[0], int(shape[0] / voxel_width) + 1) # voxel center coordinates vyc, vzc, vxc = np.meshgrid((vyb[1:] + vyb[:-1]) / 2, (vzb[1:] + vzb[:-1]) / 2, (vxb[1:] + vxb[:-1]) / 2) vxl = np.c_[vxc.ravel(), vyc.ravel(), vzc.ravel()] # KD-Tree tree = kdtree(pts) # it would be more honest to take a ball neighborhood, # but I'll just take the 20 nn for speed and simplicity # problem: nn might not form a sphere dd, ii = tree.query(vxl, k=20) wi = gaussian_weights(dd, sigma) vi = var[ii] # Gaussian kernel interpolation var_gki = np.sum(vi * wi, axis=1) / np.sum(wi, axis=1) var_gki.shape = vzc.shape # digital surface model dsm = vzb[np.nanargmin(var_gki, axis=0)] + voxel_width / 2
print('reading points from %s ..' % fn_las) fp = lp.file.File(fn_las) pts = np.c_[fp.x, fp.y, fp.z] fp.close() pmin = pts.min(axis = 0) pts -= pmin print('reading points from %s ..' % fn_las_ext) fp = lp.file.File(fn_las_ext) ext = np.c_[fp.x, fp.y, fp.z] - pmin fp.close() print('building kdtree ..') tr = kdtree(pts) print('query ..') d, k = tr.query(ext, n_jobs = -1) print(d.max(), d.mean(), d.min()) print('create mask ..') mask = np.ones(pts.shape[0], dtype = 'bool') mask[k] = False print('reading waveforms ..') idx, pts = wf.Read(fn_las, fn_wdp, mask = mask) print('export to LAS file ..') wf.ExportLAS('fwf-' + fn_las_ext, pts[:,3], pts)
def phase_bin_magseries_with_errs(phases, mags, errs, binsize=0.005): ''' This bins a magnitude timeseries in phase using the binsize (in phase) provided. ''' # check if the input arrays are ok if not(phases.shape and mags.shape and len(phases) > 10 and len(mags) > 10): LOGERROR("input time/mag arrays don't have enough elements") return # find all the finite values of the magnitudes and phases finiteind = np.isfinite(mags) & np.isfinite(phases) & np.isfinite(errs) finite_phases = phases[finiteind] finite_mags = mags[finiteind] finite_errs = errs[finiteind] nbins = int(np.ceil((np.nanmax(finite_phases) - np.nanmin(finite_phases))/binsize) + 1) minphase = np.nanmin(finite_phases) phasebins = [(minphase + x*binsize) for x in range(nbins)] # make a KD-tree on the PHASEs so we can do fast distance calculations. we # need to add a bogus y coord to make this a problem that KD-trees can # solve. time_coords = np.array([[x,1.0] for x in finite_phases]) phasetree = kdtree(time_coords) binned_finite_phaseseries_indices = [] collected_binned_mags = {} for phase in phasebins: # find all bin indices close to within binsize of this point # using the kdtree query. we use the p-norm = 1 (I think this # means straight-up pairwise distance? FIXME: check this) bin_indices = phasetree.query_ball_point(np.array([phase,1.0]), binsize/2.0, p=1.0) # if the bin_indices have already been collected, then we're # done with this bin, move to the next one. if they haven't, # then this is the start of a new bin. if (bin_indices not in binned_finite_phaseseries_indices and len(bin_indices)) > 0: binned_finite_phaseseries_indices.append(bin_indices) # convert to ndarrays binned_finite_phaseseries_indices = [np.array(x) for x in binned_finite_phaseseries_indices] collected_binned_mags['phasebins_indices'] = ( binned_finite_phaseseries_indices ) collected_binned_mags['phasebins'] = phasebins collected_binned_mags['nbins'] = len(binned_finite_phaseseries_indices) # collect the finite_phases binned_phase = np.array([np.median(finite_phases[x]) for x in binned_finite_phaseseries_indices]) collected_binned_mags['binnedphases'] = binned_phase collected_binned_mags['binsize'] = binsize # median bin the magnitudes according to the calculated indices collected_binned_mags['binnedmags'] = ( np.array([np.median(finite_mags[x]) for x in binned_finite_phaseseries_indices]) ) collected_binned_mags['binnederrs'] = ( np.array([np.median(finite_errs[x]) for x in binned_finite_phaseseries_indices]) ) return collected_binned_mags