def ztree(self,
zkey=None,
scale=None,
minthresh=10,
maxthresh=20,
np=None):
if scale is None:
scale = fc.scale(self['locations'].min(axis=0),
self['locations'].ptp(axis=0))
zkey = sharedmem.empty(self.numpoints, dtype=fc.fckeytype)
with sharedmem.MapReduce(np=np) as pool:
chunksize = 1024 * 1024
def work(i):
X, Y, Z = self['locations'][i:i + chunksize].T
fc.encode(X, Y, Z, scale=scale, out=zkey[i:i + chunksize])
pool.map(work, range(0, len(zkey), chunksize))
# use sharemem.argsort, because it is faster
arg = sharedmem.argsort(zkey, np=np)
return zt.Tree(zkey=zkey,
scale=scale,
arg=arg,
minthresh=minthresh,
maxthresh=maxthresh)
def ztree(self, zkey=None, scale=None, minthresh=10, maxthresh=20, np=None):
if scale is None:
scale = fc.scale(self['locations'].min(axis=0), self['locations'].ptp(axis=0))
zkey = sharedmem.empty(self.numpoints, dtype=fc.fckeytype)
with sharedmem.MapReduce(np=np) as pool:
chunksize = 1024 * 1024
def work(i):
X, Y, Z = self['locations'][i:i+chunksize].T
fc.encode(X, Y, Z, scale=scale, out=zkey[i:i+chunksize])
pool.map(work, range(0, len(zkey), chunksize))
# use sharemem.argsort, because it is faster
arg = sharedmem.argsort(zkey, np=np)
return zt.Tree(zkey=zkey, scale=scale, arg=arg, minthresh=minthresh, maxthresh=maxthresh)
def paint(pos, color, luminosity, sml, camera, CCD, tree=None,
return_tree_and_sml=False, normalize=True, np=None, direct_write=False,
cumulative=True):
""" pos = (x, y, z)
color can be None or 1 or array
luminosity can be None or 1 or array
sml can be None or 1, or array
camera is Camera
CCD is array of camera.shape, 2.
CCD[..., 0] is the color channel (sum of color * luminosity)
CCD[..., 1] is the luminosity channel (sum of luminosity)
if color is None, CCD.shape == camera.shape
if color is not None,
CCD.shape == camera.shape, 2
CCD[..., 0] is color
CCD[..., 1] is brightness
if normalize is False, do not do CCD[..., 0] will be
the weighted sum of color.
if normalize is True, CCD[..., 0] will be the weighted average of color
if direct_write is true, each process will directly write to CCD (CCD
must be on sharedmem)
if cumulative is False, original content in CCD will be disregarded.
if cumulative is True, original content in CCD will be preserved (+=)
"""
CCDlimit = 20 * 1024 * 1024 # 20M pixel per small CCD
camera.shape = (CCD.shape[0], CCD.shape[1])
nCCD = int((CCD.shape[0] * CCD.shape[1] / CCDlimit) ** 0.5)
if np is None: np = sharedmem.cpu_count()
if nCCD <= np ** 0.5:
nCCD = int(np ** 0.5 + 1)
cams = camera.divide(nCCD, nCCD)
cams = cams.reshape(-1, 3)
if tree is None:
scale = fc.scale([x.min() for x in pos], [x.ptp() for x in pos])
zkey = sharedmem.empty(len(pos[0]), dtype=fc.fckeytype)
with sharedmem.MapReduce(np=np) as pool:
chunksize = 1024 * 1024
def work(i):
sl = slice(i, i+chunksize)
x, y, z = pos
fc.encode(x[sl], y[sl], z[sl], scale=scale, out=zkey[i:i+chunksize])
pool.map(work, range(0, len(zkey), chunksize))
arg = sharedmem.argsort(zkey)
tree = zt.Tree(zkey=zkey, scale=scale, arg=arg, minthresh=8, maxthresh=20)
if sml is None:
sml = sharedmem.empty(len(zkey), 'f4')
with sharedmem.MapReduce(np=np) as pool:
chunksize = 1024 * 64
def work(i):
setupsml(tree, [x[i:i+chunksize] for x in pos],
out=sml[i:i+chunksize])
pool.map(work, range(0, len(zkey), chunksize))
def writeCCD(i, sparse, myCCD):
cam, ox, oy = cams[i]
#print i, sparse, len(cams)
if sparse:
index, C, L = myCCD
x = index[0] + ox
y = index[1] + oy
p = CCD.flat
if color is not None:
ind = numpy.ravel_multi_index((x, y, 0), CCD.shape)
if cumulative:
p[ind] += C
else:
p[ind] = C
ind = numpy.ravel_multi_index((x, y, 1), CCD.shape)
if cumulative:
p[ind] += L
else:
p[ind] = L
else:
ind = numpy.ravel_multi_index((x, y), CCD.shape)
if cumulative:
p[ind] += L
else:
p[ind] = L
else:
if color is not None:
if cumulative:
CCD[ox:ox + cam.shape[0], oy:oy+cam.shape[1], :] += myCCD
else:
CCD[ox:ox + cam.shape[0], oy:oy+cam.shape[1], :] = myCCD
else:
if cumulative:
CCD[ox:ox + cam.shape[0], oy:oy+cam.shape[1]] += myCCD[..., 1]
else:
CCD[ox:ox + cam.shape[0], oy:oy+cam.shape[1]] = myCCD[..., 1]
with sharedmem.MapReduce(np=np) as pool:
def work(i):
cam, ox, oy = cams[i]
myCCD = numpy.zeros(cam.shape, dtype=('f8', 2))
cam.paint(pos[0], pos[1], pos[2],
sml, color, luminosity, out=myCCD, tree=tree)
mask = (myCCD[..., 1] != 0)
if mask.sum() < 0.1 * myCCD[..., 1].size:
index = mask.nonzero()
C = myCCD[..., 0][mask]
L = myCCD[..., 1][mask]
sparse, myCCD = True, (index, C, L)
else:
sparse, myCCD = False, myCCD
if not direct_write:
return i, sparse, myCCD
else:
writeCCD(i, sparse, myCCD)
return 0, 0, 0
def reduce(i, sparse, myCCD):
if not direct_write:
writeCCD(i, sparse, myCCD)
pool.map(work, range(len(cams)), reduce=reduce)
if color is not None and normalize:
CCD[..., 0] /= CCD[..., 1]
if return_tree_and_sml:
return CCD, tree, sml
else:
tree = None
sml = None
return CCD
def paint(pos,
color,
luminosity,
sml,
camera,
CCD,
tree=None,
return_tree_and_sml=False,
normalize=True,
np=None,
direct_write=False,
cumulative=True):
""" pos = (x, y, z)
color can be None or 1 or array
luminosity can be None or 1 or array
sml can be None or 1, or array
camera is Camera
CCD is array of camera.shape, 2.
CCD[..., 0] is the color channel (sum of color * luminosity)
CCD[..., 1] is the luminosity channel (sum of luminosity)
if color is None, CCD.shape == camera.shape
if color is not None,
CCD.shape == camera.shape, 2
CCD[..., 0] is color
CCD[..., 1] is brightness
if normalize is False, do not do CCD[..., 0] will be
the weighted sum of color.
if normalize is True, CCD[..., 0] will be the weighted average of color
if direct_write is true, each process will directly write to CCD (CCD
must be on sharedmem)
if cumulative is False, original content in CCD will be disregarded.
if cumulative is True, original content in CCD will be preserved (+=)
"""
CCDlimit = 20 * 1024 * 1024 # 20M pixel per small CCD
camera.shape = (CCD.shape[0], CCD.shape[1])
nCCD = int((CCD.shape[0] * CCD.shape[1] / CCDlimit)**0.5)
if np is None: np = sharedmem.cpu_count()
if nCCD <= np**0.5:
nCCD = int(np**0.5 + 1)
cams = camera.divide(nCCD, nCCD)
cams = cams.reshape(-1, 3)
if tree is None:
scale = fc.scale([x.min() for x in pos], [x.ptp() for x in pos])
zkey = sharedmem.empty(len(pos[0]), dtype=fc.fckeytype)
with sharedmem.MapReduce(np=np) as pool:
chunksize = 1024 * 1024
def work(i):
sl = slice(i, i + chunksize)
x, y, z = pos
fc.encode(x[sl],
y[sl],
z[sl],
scale=scale,
out=zkey[i:i + chunksize])
pool.map(work, range(0, len(zkey), chunksize))
arg = numpy.argsort(zkey)
tree = zt.Tree(zkey=zkey,
scale=scale,
arg=arg,
minthresh=8,
maxthresh=20)
if sml is None:
sml = sharedmem.empty(len(zkey), 'f4')
with sharedmem.MapReduce(np=np) as pool:
chunksize = 1024 * 64
def work(i):
setupsml(tree, [x[i:i + chunksize] for x in pos],
out=sml[i:i + chunksize])
pool.map(work, range(0, len(zkey), chunksize))
def writeCCD(i, sparse, myCCD):
cam, ox, oy = cams[i]
#print i, sparse, len(cams)
if sparse:
index, C, L = myCCD
x = index[0] + ox
y = index[1] + oy
p = CCD.flat
if color is not None:
ind = numpy.ravel_multi_index((x, y, 0), CCD.shape)
if cumulative:
p[ind] += C
else:
p[ind] = C
ind = numpy.ravel_multi_index((x, y, 1), CCD.shape)
if cumulative:
p[ind] += L
else:
p[ind] = L
else:
ind = numpy.ravel_multi_index((x, y), CCD.shape)
if cumulative:
p[ind] += L
else:
p[ind] = L
else:
if color is not None:
if cumulative:
CCD[ox:ox + cam.shape[0], oy:oy + cam.shape[1], :] += myCCD
else:
CCD[ox:ox + cam.shape[0], oy:oy + cam.shape[1], :] = myCCD
else:
if cumulative:
CCD[ox:ox + cam.shape[0],
oy:oy + cam.shape[1]] += myCCD[..., 1]
else:
CCD[ox:ox + cam.shape[0],
oy:oy + cam.shape[1]] = myCCD[..., 1]
with sharedmem.MapReduce(np=np) as pool:
def work(i):
cam, ox, oy = cams[i]
myCCD = numpy.zeros(cam.shape, dtype=('f8', 2))
cam.paint(pos[0],
pos[1],
pos[2],
sml,
color,
luminosity,
out=myCCD,
tree=tree)
mask = (myCCD[..., 1] != 0)
if mask.sum() < 0.1 * myCCD[..., 1].size:
index = mask.nonzero()
C = myCCD[..., 0][mask]
L = myCCD[..., 1][mask]
sparse, myCCD = True, (index, C, L)
else:
sparse, myCCD = False, myCCD
if not direct_write:
return i, sparse, myCCD
else:
writeCCD(i, sparse, myCCD)
return 0, 0, 0
def reduce(i, sparse, myCCD):
if not direct_write:
writeCCD(i, sparse, myCCD)
pool.map(work, range(len(cams)), reduce=reduce)
if color is not None and normalize:
CCD[..., 0] /= CCD[..., 1]
if return_tree_and_sml:
return CCD, tree, sml
else:
tree = None
sml = None
return CCD