def coarser(x, y, r=0.1):
# getting the point density for a coarser subset (seeds)
# of points that are close to grid cell centers (xc, yc)
cc = np.arange(r / 4, 1, r / 2)
xc, yc = np.meshgrid(cc, cc)
xc, yc = xc.flatten(), yc.flatten()
# kdtrees for the points and the cell centers
grid = kdtree(np.transpose((xc, yc)))
tree = kdtree(np.transpose((x, y)))
# those points in tree that are close to grid
seed = grid.query_ball_tree(tree, r=r / 2)
seeds = [x for x in seed if x != []]
# as in density
lsts = tree.query_ball_tree(tree, r=r)
m = len(lsts)
dens = np.zeros(m)
# but now a loop only over seeds
for l in seeds:
i = l[0]
dens[i] = len(lsts[i])
disk = np.pi * r**2
return (xc, yc, dens / disk)
def geometric_graph(positions, radius):
tree = kdtree(positions)
edges = list(tree.query_pairs(radius))
g = Graph(n=len(positions), edges=edges)
return g
def time_kdtree(pt, k, leafsize, n_jobs):
t0 = time()
tr = kdtree(pt, leafsize = leafsize)
t1 = time()
tr.query(pt, k = k, n_jobs = n_jobs)
t2 = time()
return (t1 - t0, t2 - t1)
def _buildTree(self, simDataRa, simDataDec, leafsize=100):
"""Build KD tree on simDataRA/Dec and set radius (via setRad) for matching.
simDataRA, simDataDec = RA and Dec values (in radians).
leafsize = the number of Ra/Dec pointings in each leaf node."""
if np.any(np.abs(simDataRa) > np.pi*2.0) or np.any(np.abs(simDataDec) > np.pi*2.0):
raise ValueError('Expecting RA and Dec values to be in radians.')
x, y, z = self._treexyz(simDataRa, simDataDec)
data = list(zip(x, y, z))
if np.size(data) > 0:
try:
self.opsimtree = kdtree(data, leafsize=leafsize, balanced_tree=False, compact_nodes=False)
except TypeError:
self.opsimtree = kdtree(data, leafsize=leafsize)
else:
raise ValueError('SimDataRA and Dec should have length greater than 0.')
def low_pass_smoothing(vertices, iterations, lambda1, mu):
vertices_smoothed = np.zeros(np.shape(vertices))
for n in range(iterations):
for i in range(vertices.shape[0]):
vertices_tree = kdtree(vertices)
kdtree.visualize(vertices_tree)
_, idx = vertices_tree.query(vertices[i], k=7)
q = np.zeros((1, 3)) # set the intial sum of nearest points
m = len(
idx
) # number of nearest points (actually minus 1 because this including the point itself)
for j in range(m):
q += vertices[idx[j]]
if n % 2 == 0:
vertices_smoothed[i] = vertices[i] + (
q - m * vertices[i]) * lambda1 / (m - 1)
else:
vertices_smoothed[i] = vertices[i] + (
q - m * vertices[i]) * mu / (m - 1)
vertices = vertices_smoothed
return vertices
def _buildTree(self, simDataRa, simDataDec, leafsize=100):
"""Build KD tree on simDataRA/Dec and set radius (via setRad) for matching.
simDataRA, simDataDec = RA and Dec values (in radians).
leafsize = the number of Ra/Dec pointings in each leaf node."""
if np.any(np.abs(simDataRa) > np.pi*2.0) or np.any(np.abs(simDataDec) > np.pi*2.0):
raise ValueError('Expecting RA and Dec values to be in radians.')
x, y, z = self._treexyz(simDataRa, simDataDec)
data = zip(x, y, z)
if np.size(data) > 0:
try:
self.opsimtree = kdtree(data, leafsize=leafsize, balanced_tree=False, compact_nodes=False)
except TypeError:
self.opsimtree = kdtree(data, leafsize=leafsize)
else:
raise ValueError('SimDataRA and Dec should have length greater than 0.')
def tree(xp, yp, r = 0.01):
tree = kdtree(np.transpose((xp, yp)))
lsts = tree.query_ball_tree(tree, r = r)
l = 0
for i in range(len(xp)):
l += len(lsts[i])
return l
def tree(n, r = 0.01):
xp = np.random.random(n)
yp = np.random.random(n)
tree = kdtree(np.transpose((xp, yp)))
lsts = tree.query_ball_tree(tree, r = r)
l = 0
for i in range(len(xp)):
l += len(lsts[i])
return l
def density(x, y, r=0.1):
# getting the point density for each point
disk = np.pi * r**2
tree = kdtree(np.transpose((x, y)))
lsts = tree.query_ball_tree(tree, r=r)
m = len(lsts)
dens = np.zeros(m)
for i in range(m):
dens[i] = len(lsts[i])
return dens / disk
def test_tree_time(random_postions=True):
def tree_xyz(lat, lon):
"""
convert positions on a sphere to 3d coordinates
"""
x = np.cos(lat) * np.cos(lon)
y = np.cos(lat) * np.sin(lon)
z = np.sin(lat)
return x, y, z
npts = 3e6
np.random.seed(42)
if random_postions:
# random points on a sphere.
lat = np.arccos(2.*np.random.uniform(low=0, high=1, size=npts) - 1.) - np.pi/2.
lon = np.random.uniform(low=0, high=2*np.pi, size=npts)
else:
nside = 16
lat, lon = hp.pix2ang(nside, np.arange(hp.nside2npix(nside)))
npix = lat.size
lat = np.repeat(lat, np.ceil(npts/npix))
lon = np.repeat(lon, np.ceil(npts/npix))
lat = lat[0:npts]
lon = lon[0:npts]
x, y, z = tree_xyz(lat, lon)
# Postiions to search on
nside = 128
position_lat, position_lon = hp.pix2ang(nside, np.arange(hp.nside2npix(nside)))
position_lat = np.pi/2. - position_lat
pos_x, pos_y, pos_z = tree_xyz(position_lat, position_lon)
leafsize = 100
# Trying out solution from: https://github.com/scipy/scipy/issues/6108#issuecomment-217117065
tree = kdtree(zip(x, y, z), leafsize=leafsize, balanced_tree=False, compact_nodes=False)
# Set a search radius
radius = 2.
x0, y0, z0 = (1, 0, 0)
x1, y1, z1 = tree_xyz(np.radians(radius), 0)
search_rad = np.sqrt((x1-x0)**2+(y1-y0)**2+(z1-z0)**2)
found_points = 0
for px, py, pz in zip(pos_x, pos_y, pos_z):
indices = tree.query_ball_point((px, py, pz), search_rad)
found_points += len(indices)
def grid_point_cloud(x, y, z, width=1):
"""
Aggregates a point cloud (x, y, z) to a
grid with grid cell spacing width.
Returns:
(mean, std, xbounds, ybounds)
"""
from scipy.spatial import cKDTree as kdtree
xmin, xmax = x.min(), x.max()
ymin, ymax = y.min(), y.max()
xb = np.arange(xmin, xmax + width, width)
yb = np.arange(ymin, ymax + width, width)
xr = xb[:-1] + width / 2.0
yr = yb[:-1] + width / 2.0
xc, yc = np.meshgrid(xr, yr)
shape = xc.shape
xc = xc.ravel()
yc = yc.ravel()
tree = kdtree(np.transpose((x, y)))
grid = kdtree(np.transpose((xc, yc)))
lsts = grid.query_ball_tree(tree, r=width / np.sqrt(2))
n = len(lsts)
m = np.zeros(n)
s = np.zeros(n)
for i in range(n):
if len(lsts[i]) < 3:
m[i] = np.nan
s[i] = np.nan
else:
j = lsts[i]
m[i] = z[j].mean()
s[i] = z[j].std()
m.shape = shape
s.shape = shape
return (m, s, xb, yb)
def thinning(x, y, z, r):
from scipy.spatial import cKDTree as kdtree
b = np.ones(len(x), dtype='bool')
tree = kdtree(np.transpose((x, y)))
lsts = tree.query_ball_tree(tree, r=r)
for l in lsts:
if len(l) > 1:
la = np.array(l)
dz = np.abs(z[la] - np.median(z[la]))
b[la] = False
b[la[np.argmin(dz)]] = True
print('keeping %.3f %%' % (100. * len(b[b]) / len(b)))
return (x[b], y[b], z[b])
def buildTree(simDataRa, simDataDec,
leafsize=100):
"""Build KD tree on simDataRA/Dec and set radius (via setRad) for matching.
simDataRA, simDataDec = RA and Dec values (in radians).
leafsize = the number of Ra/Dec pointings in each leaf node."""
if np.any(np.abs(simDataRa) > np.pi*2.0) or np.any(np.abs(simDataDec) > np.pi*2.0):
raise ValueError('Expecting RA and Dec values to be in radians.')
x, y, z = treexyz(simDataRa, simDataDec)
data = zip(x,y,z)
if np.size(data) > 0:
starTree = kdtree(data, leafsize=leafsize)
else:
raise ValueError('SimDataRA and Dec should have length greater than 0.')
return starTree
def buildTree(simDataRa, simDataDec, leafsize=100):
"""Build KD tree on simDataRA/Dec and set radius (via setRad) for matching.
simDataRA, simDataDec = RA and Dec values (in radians).
leafsize = the number of Ra/Dec pointings in each leaf node."""
if np.any(np.abs(simDataRa) > np.pi * 2.0) or np.any(
np.abs(simDataDec) > np.pi * 2.0):
raise ValueError('Expecting RA and Dec values to be in radians.')
x, y, z = treexyz(simDataRa, simDataDec)
data = list(zip(x, y, z))
if np.size(data) > 0:
starTree = kdtree(data, leafsize=leafsize)
else:
raise ValueError(
'SimDataRA and Dec should have length greater than 0.')
return starTree
def main(n, r=0.1):
x = (np.random.random(n) - 0.5) * 4
y = (np.random.random(n) - 0.5) * 3
z = np.exp(-x * x - y * y)
tree = kdtree(np.transpose((x, y)))
slp = np.zeros(n)
for i in range(n):
nb = tree.query_ball_point((x[i], y[i]), r)
pts = np.transpose((x[nb], y[nb], z[nb]))
nx, ny, nz = FitPlane(pts)
slp[i] = np.sqrt(nx * nx + ny * ny) / nz
slp = np.arctan(slp) * 180 / np.pi
pl.title('Numerical')
pl.scatter(x, y, c=slp)
pl.colorbar()
pl.axes().set_aspect('equal')
pl.show()
# theoretical results
rp = np.sqrt(x * x + y * y)
sp = 2 * rp * np.exp(-rp * rp)
sp = np.arctan(sp) * 180 / np.pi
pl.title('Theory')
pl.scatter(x, y, c=sp)
pl.colorbar()
pl.axes().set_aspect('equal')
pl.show()
pl.title('absolute difference')
pl.scatter(x, y, c=slp - sp, cmap=pl.cm.seismic, vmin=-25, vmax=25)
pl.colorbar()
pl.axes().set_aspect('equal')
pl.show()
pl.title('relative difference')
pl.scatter(x,
y,
c=100. * (slp - sp) / sp,
cmap=pl.cm.seismic,
vmin=-25,
vmax=25)
pl.colorbar()
pl.axes().set_aspect('equal')
pl.show()
def get_interpolation_matrix(
self,
npts=20,
epsilon=None
):
tree_2d = kdtree(self.topography[:, :2])
xy = Utils.ndgrid(self.mesh_3d.vectorCCx, self.mesh_3d.vectorCCy)
distance, inds = tree_2d.query(xy, k=npts)
if epsilon is None:
epsilon = np.min([self.mesh_3d.hx.min(), self.mesh_3d.hy.min()])
w = 1. / (distance + epsilon)**2
w = Utils.sdiag(1./np.sum(w, axis=1)) * (w)
I = Utils.mkvc(
np.arange(inds.shape[0]).reshape([-1, 1]).repeat(npts, axis=1)
)
J = Utils.mkvc(inds)
self._P = sp.coo_matrix(
(Utils.mkvc(w), (I, J)),
shape=(inds.shape[0], self.topography.shape[0])
)
mesh_1d = Mesh.TensorMesh([np.r_[self.hz[:-1], 1e20]])
z = self.P*self.topography[:, 2]
self._actinds = Utils.surface2ind_topo(self.mesh_3d, np.c_[xy, z])
Z = np.empty(self.mesh_3d.vnC, dtype=float, order='F')
Z = self.mesh_3d.gridCC[:, 2].reshape(
(self.mesh_3d.nCx*self.mesh_3d.nCy, self.mesh_3d.nCz), order='F'
)
ACTIND = self._actinds.reshape(
(self.mesh_3d.nCx*self.mesh_3d.nCy, self.mesh_3d.nCz), order='F'
)
self._Pz = []
# This part can be cythonized or parallelized
for i_xy in range(self.mesh_3d.nCx*self.mesh_3d.nCy):
actind_temp = ACTIND[i_xy, :]
z_temp = -(Z[i_xy, :] - z[i_xy])
self._Pz.append(mesh_1d.getInterpolationMat(z_temp[actind_temp]))
def __init__(self, init_num, size, stp):
self.num = init_num
self.size = size
self.stp = stp
self.one = 1.0 / size
self.rad = 0.04
edge = 0.2
self.xy = edge + random((init_num, 2)) * (1.0 - 2 * edge)
self.v = zeros((init_num, 2), 'float')
self.dx = zeros((init_num, 2), 'float')
self.random_velocity(self.stp)
self.tree = kdtree(self.xy)
self.slowdown = 0.99999
def get_interpolation_matrix(
self,
npts=20,
epsilon=None
):
tree_2d = kdtree(self.topography[:, :2])
xy = Utils.ndgrid(self.mesh_3d.vectorCCx, self.mesh_3d.vectorCCy)
distance, inds = tree_2d.query(xy, k=npts)
if epsilon is None:
epsilon = np.min([self.mesh_3d.hx.min(), self.mesh_3d.hy.min()])
w = 1. / (distance + epsilon)**2
w = Utils.sdiag(1./np.sum(w, axis=1)) * (w)
I = Utils.mkvc(
np.arange(inds.shape[0]).reshape([-1, 1]).repeat(npts, axis=1)
)
J = Utils.mkvc(inds)
self._P = sp.coo_matrix(
(Utils.mkvc(w), (I, J)),
shape=(inds.shape[0], self.topography.shape[0])
)
mesh_1d = Mesh.TensorMesh([np.r_[self.hz[:-1], 1e20]])
z = self.P*self.topography[:, 2]
self._actinds = Utils.surface2ind_topo(self.mesh_3d, np.c_[xy, z])
Z = np.empty(self.mesh_3d.vnC, dtype=float, order='F')
Z = self.mesh_3d.gridCC[:, 2].reshape(
(self.mesh_3d.nCx*self.mesh_3d.nCy, self.mesh_3d.nCz), order='F'
)
ACTIND = self._actinds.reshape(
(self.mesh_3d.nCx*self.mesh_3d.nCy, self.mesh_3d.nCz), order='F'
)
self._Pz = []
# This part can be cythonized or parallelized
for i_xy in range(self.mesh_3d.nCx*self.mesh_3d.nCy):
actind_temp = ACTIND[i_xy, :]
z_temp = -(Z[i_xy, :] - z[i_xy])
self._Pz.append(mesh_1d.getInterpolationMat(z_temp[actind_temp]))
def remove_close(points, radius):
"""
given an (n,m) set of points where n = 2 or n =3 return a list of points where no point is closer than radius
para::points: numpy array(m,2) or (m,3), coordinates of the points
para::radius: the least distance between the points
return: the unique points and its index
"""
tree = kdtree(points)
consumed = np.zeros(len(points), dtype=np.bool)
unique = np.zeros(len(points), dtype=np.bool)
for k in range(len(points)):
if consumed[k]:
continue
neighbors = tree.query_ball_point(points[k], r=radius)
consumed[neighbors] = True
unique[k] = True
index = np.array([k for k in range(len(points))])
return points[unique], index[unique]
def _buildTree(self, ra, dec, leafsize=100):
"""Build KD tree on simDataRA/Dec and set radius (via setRad) for matching.
simDataRA, simDataDec = RA and Dec values (in radians).
leafsize = the number of Ra/Dec pointings in each leaf node."""
if np.any(np.abs(ra) > np.pi*2.0) or np.any(np.abs(dec) > np.pi*2.0):
raise ValueError('Expecting RA and Dec values to be in radians.')
x, y, z = self._treexyz(ra, dec)
data = list(zip(x, y, z))
if np.size(data) > 0:
self.hptree = kdtree(data, leafsize=leafsize)
# try:
# XXX--this is crashing the kernel? Not sure why, need to make sure other way speed-tests OK
# self.hptree = kdtree(data, leafsize=leafsize, balanced_tree=False, compact_nodes=False)
# except TypeError:
# self.hptree = kdtree(data, leafsize=leafsize)
else:
raise ValueError('RA and Dec should have length greater than 0.')
def postprocess(self):
"""
This MUST be called, once, after the XML has been loaded.
It:
- ensures relation-members know about their parent object (incl. the plantref if parent is plant);
- calculates the total area of each relation;
- calculates the centroid of each relation;
- performs spatial containment queries to label solar panels as members of a plant (i.e. plantref) if they're geographically inside them.
"""
# Find all objects that are relations and also plants. Then push down the metadata through their children (only for the temporary data), and also calculate the area for the parents.
rels_postprocessed = 0
self.plantoutlines = [] # {lat, lon, outlinepath, plantref} a list of ways that are either plants themselves, or non-inner members of plant relations; i.e. potential geo containers for panels
for curitem in self.objs:
if curitem['tag_power']=='plant' and curitem['objtype']=='way':
self.plantoutlines.append({'lat': curitem['lat'], 'lon': curitem['lon'], 'outlinepath': self.waydata[curitem['id']]['outlinepath'], 'plantref': curitem['plantref']})
if curitem['tag_power'] in ['plant', 'generator'] and curitem['objtype']=='relation':
if curitem['tag_power']=='plant':
plantitem = curitem
plantref = ('relation', curitem['id'])
else:
plantitem = None
plantref = None
curitem['calc_area'] = 0
self._recurse_relation_info(curitem, plantitem, plantref)
rels_postprocessed += 1
print("Postprocessed %i power=* relations" % rels_postprocessed)
print("Plant outlines for geo containment search: %i" % len(self.plantoutlines))
plantoutlines_kdtree = kdtree([[item['lat'], item['lon']] for item in self.plantoutlines])
# Now, for every generator object that DOESN'T have a plantref, we find its nearest-neighbour potential-containers and check for containment
for curitem in self.objs:
if curitem['tag_power']=='generator' and not curitem.get('plantref', None):
for distance, arrayposition in zip(*plantoutlines_kdtree.query([curitem['lat'], curitem['lon']], 3, distance_upper_bound=1)):
if distance != np.inf:
if self.plantoutlines[arrayposition]['outlinepath'].contains_point([curitem['lat'], curitem['lon']]):
curitem['plantref'] = self.plantoutlines[arrayposition]['plantref']
#print(" spatially inferred generator %s/%s belongs to plant %s" % (curitem['objtype'], curitem['id'], curitem['plantref']))
if False: # This should NOT NORMALLY be activated. It inserts "guesstimate" power capacities for small-scale solar PV
for curitem in self.objs:
if curitem['tag_power']=='generator' and curitem.get('calc_capacity', 0)==0 and not curitem.get('plantref', None):
curitem['calc_capacity'] = guess_kilowattage(curitem)
def __init__(self,
nactions,
input_ranges,
nelemns=[],
npoints=0,
k=1,
alpha=0.3,
lm=0.95):
if not (nelemns == False) ^ (npoints == False):
raise ValueError('Plese indicate either: [nelemns] Xor [npoints]')
if nelemns:
self.cl = self.ndlinspace(input_ranges, nelemns)
else:
self.cl = self.CreateRandomSpace(input_ranges, npoints)
self.lbounds = []
self.ubounds = []
self.k = k
self.shape = self.cl.shape
self.nactions = nactions
self.Q = zeros((self.shape[0], nactions))
# self.Q = uniform(-100,0,(self.shape[0],nactions))+0.0
self.e = zeros((self.shape[0], nactions)) + 0.0
# self.ac = zeros((self.shape[0]))+0.0 #classifiers activation
self.ac = []
self.knn = []
self.alpha = alpha
self.lm = lm # good 0.95
self.last_state = zeros((1, self.shape[1])) + 0.0
for r in input_ranges:
self.lbounds.append(r[0])
self.ubounds.append(r[1])
self.lbounds = array(self.lbounds)
self.ubounds = array(self.ubounds)
self.cl = array(self.RescaleInputs(self.cl))
self.knntree = kdtree(self.cl)
def hp_kd_tree(nside=set_default_nside(), leafsize=100):
"""
Generate a KD-tree of healpixel locations
Parameters
----------
nside : int
A valid healpix nside
leafsize : int (100)
Leafsize of the kdtree
Returns
-------
tree : scipy kdtree
"""
hpid = np.arange(hp.nside2npix(nside))
ra, dec = _hpid2RaDec(nside, hpid)
x, y, z = treexyz(ra, dec)
tree = kdtree(list(zip(x, y, z)), leafsize=leafsize, balanced_tree=False, compact_nodes=False)
return tree
def main(fn, m=5000, r=1):
pts = np.load(fn)
x = pts[:m, 0]
y = pts[:m, 1]
z = pts[:m, 2]
n = len(z)
tree = kdtree(np.transpose((x, y)))
slp = np.zeros(n)
for i in range(n):
nb = tree.query_ball_point((x[i], y[i]), r)
pts = np.transpose((x[nb], y[nb], z[nb]))
nx, ny, nz = FitPlane(pts)
slp[i] = np.sqrt(nx * nx + ny * ny) / nz
slp = np.arctan(slp) * 180 / np.pi
pl.figure(1, (10.24, 7.68))
pl.title('Pozo slope')
pl.scatter(x, y, c=slp, s=3, vmin=0, vmax=90)
pl.colorbar()
pl.axes().set_aspect('equal')
pl.savefig('%s.png' % sys.argv[0][:-3])
def step(self, separation, cohesion, alignment):
self.xy += self.v
nearby = self.get_nearby()
separation = self.stp * separation
alignment = self.stp * alignment
cohesion = self.stp * cohesion
self.dx[:, :] = 0
for i, near in enumerate(nearby):
near = list(set(near).difference(set([i])))
if not near:
continue
self.separation(i, near, separation)
self.alignment(i, near, alignment)
self.cohesion(i, near, cohesion)
self.v += self.dx / 3.0
self.v *= self.slowdown
self.tree = kdtree(self.xy)
data = np.load('phot.npz')
phot = data['phot'].copy()
data.close()
umjd = np.unique(phot['mjd'])
phot.sort(order='mjd')
left = np.searchsorted(phot['mjd'], umjd)
right = np.searchsorted(phot['mjd'], umjd, side='right')
nstars = right - left
goodMJD = umjd[np.where(nstars == nstars.max())]
# create a kdtree for all the ra,dec points
x, y, z = treexyz(phot['RA'], phot['dec'])
tree = kdtree(zip(x, y, z), leafsize=100)
starIDs = np.zeros(phot.size, dtype=int) - 666
# Array to say if the star has matched multiple times
multiFlag = np.zeros(phot.size, dtype=int)
radius = 300. / 3600. # return everything within 300 arcsec?
x0, y0, z0 = (1, 0, 0)
x1, y1, z1 = treexyz(np.radians(radius), 0)
rad = np.sqrt((x1 - x0)**2 + (y1 - y0)**2 + (z1 - z0)**2)
# Match everything to one of the frames with lots of stars
good = np.where(phot['mjd'] == goodMJD)
onID = 1
for i, blah in enumerate(phot[good]):
indices = np.array(tree.query_ball_point((x[i], y[i], z[i]), rad))
def __init__(self,points):
self.points = points
self.tree = kdtree(points)
def time_bin_magseries_with_errs(times, mags, errs, binsize=540.0):
'''
This bins the given mag timeseries in time using the binsize given. binsize
is in seconds.
'''
# check if the input arrays are ok
if not(times.shape and mags.shape and errs.shape and
len(times) > 10 and len(mags) > 10):
LOGERROR("input time/mag arrays don't have enough elements")
return
# find all the finite values of the magnitudes and times
finiteind = np.isfinite(mags) & np.isfinite(times) & np.isfinite(errs)
finite_times = times[finiteind]
finite_mags = mags[finiteind]
finite_errs = errs[finiteind]
# convert binsize in seconds to JD units
binsizejd = binsize/(86400.0)
nbins = int(np.ceil((np.nanmax(finite_times) -
np.nanmin(finite_times))/binsizejd) + 1)
minjd = np.nanmin(finite_times)
jdbins = [(minjd + x*binsizejd) for x in range(nbins)]
# make a KD-tree on the JDs so we can do fast distance calculations. we
# need to add a bogus y coord to make this a problem that KD-trees can
# solve.
time_coords = np.array([[x,1.0] for x in finite_times])
jdtree = kdtree(time_coords)
binned_finite_timeseries_indices = []
collected_binned_mags = {}
for jd in jdbins:
# find all bin indices close to within binsizejd of this point
# using the kdtree query. we use the p-norm = 1 (I think this
# means straight-up pairwise distance? FIXME: check this)
bin_indices = jdtree.query_ball_point(np.array([jd,1.0]),
binsizejd/2.0, p=1.0)
# if the bin_indices have already been collected, then we're
# done with this bin, move to the next one. if they haven't,
# then this is the start of a new bin.
if (bin_indices not in binned_finite_timeseries_indices and
len(bin_indices)) > 0:
binned_finite_timeseries_indices.append(bin_indices)
# convert to ndarrays
binned_finite_timeseries_indices = [np.array(x) for x in
binned_finite_timeseries_indices]
collected_binned_mags['jdbins_indices'] = binned_finite_timeseries_indices
collected_binned_mags['jdbins'] = jdbins
collected_binned_mags['nbins'] = len(binned_finite_timeseries_indices)
# collect the finite_times
binned_jd = np.array([np.median(finite_times[x])
for x in binned_finite_timeseries_indices])
collected_binned_mags['binnedtimes'] = binned_jd
collected_binned_mags['binsize'] = binsize
# median bin the magnitudes according to the calculated indices
collected_binned_mags['binnedmags'] = (
np.array([np.median(finite_mags[x])
for x in binned_finite_timeseries_indices])
)
# FIXME: calculate the error in the median-binned magnitude correctly
# for now, just take the median of the errors in this bin
collected_binned_mags['binnederrs'] = (
np.array([np.median(finite_errs[x])
for x in binned_finite_timeseries_indices])
)
return collected_binned_mags
venus_avoid = np.radians(5.) # rad
# Sun twilight limit
sun_alt_limit = np.radians(-12.) # rad
# Telescope pointing altitude limit.
alt_limit = np.radians(20.)
nside = 32
map_size = hp.nside2npix(nside)
dec, ra = hp.pix2ang(nside, np.arange(map_size))
dec = np.pi/2. - dec
leafsize = 100
x, y, z = treexyz(ra, dec)
healTree = kdtree(list(zip(x, y, z)), leafsize=leafsize)
# Need to build a kdtree to quick search for healpixes within a radius
mjd_start = 59580.033829
survey_length = 10 # days
time_step = 10. # minitues
mjds = np.arange(mjd_start,
mjd_start + survey_length+time_step/60./24.,
time_step/60./24.)
names = ['u', 'g', 'r', 'i', 'z', 'y', 'airmass', 'mask']
types = [float]*7
types.append(int)
results = np.zeros((mjds.size, map_size), dtype=list(zip(names, types)))
venus_avoid = np.radians(5.) # rad
# Sun twilight limit
sun_alt_limit = np.radians(-12.) # rad
# Telescope pointing altitude limit.
alt_limit = np.radians(20.)
nside = 32
map_size = hp.nside2npix(nside)
dec, ra = hp.pix2ang(nside, np.arange(map_size))
dec = np.pi / 2. - dec
leafsize = 100
x, y, z = treexyz(ra, dec)
healTree = kdtree(list(zip(x, y, z)), leafsize=leafsize)
# Need to build a kdtree to quick search for healpixes within a radius
mjd_start = 59580.033829
survey_length = 10 # days
time_step = 10. # minitues
mjds = np.arange(mjd_start, mjd_start + survey_length + time_step / 60. / 24.,
time_step / 60. / 24.)
names = ['u', 'g', 'r', 'i', 'z', 'y', 'airmass', 'mask']
types = [float] * 7
types.append(int)
results = np.zeros((mjds.size, map_size), dtype=list(zip(names, types)))
results['mask'] = 1
venus_avoid = np.radians(5.) # rad
# Sun twilight limit
sun_alt_limit = np.radians(-12.) # rad
# Telescope pointing altitude limit.
alt_limit = np.radians(20.)
nside = 32
map_size = hp.nside2npix(nside)
dec,ra = hp.pix2ang(nside, np.arange(map_size))
dec = np.pi/2. - dec
leafsize = 100
x,y,z = treexyz(ra, dec)
healTree = kdtree(zip(x,y,z),leafsize=leafsize)
# Need to build a kdtree to quick search for healpixes within a radius
mjd_start = 59580.033829
survey_length = 10 # days
time_step = 10. # minitues
mjds = np.arange(mjd_start,
mjd_start + survey_length+time_step/60./24.,
time_step/60./24.)
names=['u','g','r','i','z','y','airmass', 'mask']
types = [float]*7
types.append(int)
results = np.zeros((mjds.size,map_size), dtype=zip(names,types) )
def phase_bin_magseries(phases, mags, binsize=0.005, minbinelems=7):
'''
This bins a magnitude timeseries in phase using the binsize (in phase)
provided.
minbinelems is the minimum number of elements in each bin.
'''
# check if the input arrays are ok
if not (phases.shape and mags.shape and len(phases) > 10
and len(mags) > 10):
LOGERROR("input time/mag arrays don't have enough elements")
return
# find all the finite values of the magnitudes and phases
finiteind = np.isfinite(mags) & np.isfinite(phases)
finite_phases = phases[finiteind]
finite_mags = mags[finiteind]
nbins = int(
np.ceil((np.nanmax(finite_phases) - np.nanmin(finite_phases)) /
binsize) + 1)
minphase = np.nanmin(finite_phases)
phasebins = [(minphase + x * binsize) for x in range(nbins)]
# make a KD-tree on the PHASEs so we can do fast distance calculations. we
# need to add a bogus y coord to make this a problem that KD-trees can
# solve.
time_coords = np.array([[x, 1.0] for x in finite_phases])
phasetree = kdtree(time_coords)
binned_finite_phaseseries_indices = []
collected_binned_mags = {}
for phase in phasebins:
# find all bin indices close to within binsize of this point using the
# kdtree query. we use the p-norm = 1 for pairwise Euclidean distance.
bin_indices = phasetree.query_ball_point(np.array([phase, 1.0]),
binsize / 2.0,
p=1.0)
# if the bin_indices have already been collected, then we're
# done with this bin, move to the next one. if they haven't,
# then this is the start of a new bin.
if (bin_indices not in binned_finite_phaseseries_indices
and len(bin_indices) >= minbinelems):
binned_finite_phaseseries_indices.append(bin_indices)
# convert to ndarrays
binned_finite_phaseseries_indices = [
np.array(x) for x in binned_finite_phaseseries_indices
]
collected_binned_mags['phasebins_indices'] = (
binned_finite_phaseseries_indices)
collected_binned_mags['phasebins'] = phasebins
collected_binned_mags['nbins'] = len(binned_finite_phaseseries_indices)
# collect the finite_phases
binned_phase = np.array([
np.median(finite_phases[x]) for x in binned_finite_phaseseries_indices
])
collected_binned_mags['binnedphases'] = binned_phase
collected_binned_mags['binsize'] = binsize
# median bin the magnitudes according to the calculated indices
collected_binned_mags['binnedmags'] = (np.array([
np.median(finite_mags[x]) for x in binned_finite_phaseseries_indices
]))
return collected_binned_mags
data = np.load('phot.npz')
phot = data['phot'].copy()
data.close()
umjd = np.unique(phot['mjd'])
phot.sort(order='mjd')
left = np.searchsorted(phot['mjd'], umjd)
right = np.searchsorted(phot['mjd'], umjd, side='right')
nstars = right-left
goodMJD = umjd[np.where(nstars == nstars.max())]
# create a kdtree for all the ra,dec points
x, y, z = treexyz(phot['RA'], phot['dec'])
tree = kdtree(zip(x,y,z), leafsize=100)
starIDs = np.zeros(phot.size, dtype=int)-666
# Array to say if the star has matched multiple times
multiFlag = np.zeros(phot.size, dtype=int)
radius = 300./3600. # return everything within 300 arcsec?
x0, y0, z0 = (1, 0, 0)
x1, y1, z1 = treexyz(np.radians(radius), 0)
rad = np.sqrt((x1-x0)**2+(y1-y0)**2+(z1-z0)**2)
# Match everything to one of the frames with lots of stars
good = np.where(phot['mjd'] == goodMJD)
onID = 1
for i, blah in enumerate(phot[good]):
indices = np.array(tree.query_ball_point((x[i], y[i], z[i]), rad))
def time_bin_magseries_with_errs(times,
mags,
errs,
binsize=540.0,
minbinelems=7):
'''This bins the given mag timeseries in time using the binsize given.
binsize is in seconds.
minbinelems is the number of minimum elements in a bin.
'''
# check if the input arrays are ok
if not (times.shape and mags.shape and errs.shape and len(times) > 9
and len(mags) > 9):
LOGERROR("input time/mag arrays don't have enough elements")
return
# find all the finite values of the magnitudes and times
finiteind = np.isfinite(mags) & np.isfinite(times) & np.isfinite(errs)
finite_times = times[finiteind]
finite_mags = mags[finiteind]
finite_errs = errs[finiteind]
# convert binsize in seconds to JD units
binsizejd = binsize / (86400.0)
nbins = int(
np.ceil((np.nanmax(finite_times) - np.nanmin(finite_times)) /
binsizejd) + 1)
minjd = np.nanmin(finite_times)
jdbins = [(minjd + x * binsizejd) for x in range(nbins)]
# make a KD-tree on the JDs so we can do fast distance calculations. we
# need to add a bogus y coord to make this a problem that KD-trees can
# solve.
time_coords = np.array([[x, 1.0] for x in finite_times])
jdtree = kdtree(time_coords)
binned_finite_timeseries_indices = []
collected_binned_mags = {}
for jd in jdbins:
# find all bin indices close to within binsize of this point using the
# kdtree query. we use the p-norm = 1 for pairwise Euclidean distance.
bin_indices = jdtree.query_ball_point(np.array([jd, 1.0]),
binsizejd / 2.0,
p=1.0)
# if the bin_indices have already been collected, then we're
# done with this bin, move to the next one. if they haven't,
# then this is the start of a new bin.
if (bin_indices not in binned_finite_timeseries_indices
and len(bin_indices) >= minbinelems):
binned_finite_timeseries_indices.append(bin_indices)
# convert to ndarrays
binned_finite_timeseries_indices = [
np.array(x) for x in binned_finite_timeseries_indices
]
collected_binned_mags['jdbins_indices'] = binned_finite_timeseries_indices
collected_binned_mags['jdbins'] = np.array(jdbins)
collected_binned_mags['nbins'] = len(binned_finite_timeseries_indices)
# collect the finite_times
binned_jd = np.array(
[np.median(finite_times[x]) for x in binned_finite_timeseries_indices])
collected_binned_mags['binnedtimes'] = binned_jd
collected_binned_mags['binsize'] = binsize
# median bin the magnitudes according to the calculated indices
collected_binned_mags['binnedmags'] = (np.array(
[np.median(finite_mags[x]) for x in binned_finite_timeseries_indices]))
# FIXME: calculate the error in the median-binned magnitude correctly
# for now, just take the median of the errors in this bin
collected_binned_mags['binnederrs'] = (np.array(
[np.median(finite_errs[x]) for x in binned_finite_timeseries_indices]))
return collected_binned_mags
for voxel_width in np.linspace(0.05, 0.25, 5):
print('voxel width:', voxel_width)
# voxel bounds
vxb = np.linspace(0, shape[2], int(shape[2] / voxel_width) + 1)
vyb = np.linspace(0, shape[1], int(shape[1] / voxel_width) + 1)
vzb = np.linspace(0, shape[0], int(shape[0] / voxel_width) + 1)
# voxel center coordinates
vyc, vzc, vxc = np.meshgrid((vyb[1:] + vyb[:-1]) / 2,
(vzb[1:] + vzb[:-1]) / 2,
(vxb[1:] + vxb[:-1]) / 2)
vxl = np.c_[vxc.ravel(), vyc.ravel(), vzc.ravel()]
# KD-Tree
tree = kdtree(pts)
# it would be more honest to take a ball neighborhood,
# but I'll just take the 20 nn for speed and simplicity
# problem: nn might not form a sphere
dd, ii = tree.query(vxl, k=20)
wi = gaussian_weights(dd, sigma)
vi = var[ii]
# Gaussian kernel interpolation
var_gki = np.sum(vi * wi, axis=1) / np.sum(wi, axis=1)
var_gki.shape = vzc.shape
# digital surface model
dsm = vzb[np.nanargmin(var_gki, axis=0)] + voxel_width / 2
print('reading points from %s ..' % fn_las)
fp = lp.file.File(fn_las)
pts = np.c_[fp.x, fp.y, fp.z]
fp.close()
pmin = pts.min(axis = 0)
pts -= pmin
print('reading points from %s ..' % fn_las_ext)
fp = lp.file.File(fn_las_ext)
ext = np.c_[fp.x, fp.y, fp.z] - pmin
fp.close()
print('building kdtree ..')
tr = kdtree(pts)
print('query ..')
d, k = tr.query(ext, n_jobs = -1)
print(d.max(), d.mean(), d.min())
print('create mask ..')
mask = np.ones(pts.shape[0], dtype = 'bool')
mask[k] = False
print('reading waveforms ..')
idx, pts = wf.Read(fn_las, fn_wdp, mask = mask)
print('export to LAS file ..')
wf.ExportLAS('fwf-' + fn_las_ext, pts[:,3], pts)
def phase_bin_magseries_with_errs(phases, mags, errs, binsize=0.005):
'''
This bins a magnitude timeseries in phase using the binsize (in phase)
provided.
'''
# check if the input arrays are ok
if not(phases.shape and mags.shape and len(phases) > 10 and len(mags) > 10):
LOGERROR("input time/mag arrays don't have enough elements")
return
# find all the finite values of the magnitudes and phases
finiteind = np.isfinite(mags) & np.isfinite(phases) & np.isfinite(errs)
finite_phases = phases[finiteind]
finite_mags = mags[finiteind]
finite_errs = errs[finiteind]
nbins = int(np.ceil((np.nanmax(finite_phases) -
np.nanmin(finite_phases))/binsize) + 1)
minphase = np.nanmin(finite_phases)
phasebins = [(minphase + x*binsize) for x in range(nbins)]
# make a KD-tree on the PHASEs so we can do fast distance calculations. we
# need to add a bogus y coord to make this a problem that KD-trees can
# solve.
time_coords = np.array([[x,1.0] for x in finite_phases])
phasetree = kdtree(time_coords)
binned_finite_phaseseries_indices = []
collected_binned_mags = {}
for phase in phasebins:
# find all bin indices close to within binsize of this point
# using the kdtree query. we use the p-norm = 1 (I think this
# means straight-up pairwise distance? FIXME: check this)
bin_indices = phasetree.query_ball_point(np.array([phase,1.0]),
binsize/2.0, p=1.0)
# if the bin_indices have already been collected, then we're
# done with this bin, move to the next one. if they haven't,
# then this is the start of a new bin.
if (bin_indices not in binned_finite_phaseseries_indices and
len(bin_indices)) > 0:
binned_finite_phaseseries_indices.append(bin_indices)
# convert to ndarrays
binned_finite_phaseseries_indices = [np.array(x) for x in
binned_finite_phaseseries_indices]
collected_binned_mags['phasebins_indices'] = (
binned_finite_phaseseries_indices
)
collected_binned_mags['phasebins'] = phasebins
collected_binned_mags['nbins'] = len(binned_finite_phaseseries_indices)
# collect the finite_phases
binned_phase = np.array([np.median(finite_phases[x])
for x in binned_finite_phaseseries_indices])
collected_binned_mags['binnedphases'] = binned_phase
collected_binned_mags['binsize'] = binsize
# median bin the magnitudes according to the calculated indices
collected_binned_mags['binnedmags'] = (
np.array([np.median(finite_mags[x])
for x in binned_finite_phaseseries_indices])
)
collected_binned_mags['binnederrs'] = (
np.array([np.median(finite_errs[x])
for x in binned_finite_phaseseries_indices])
)
return collected_binned_mags
