def list_bricks(ns):
t0 = time()
if ns.filelist is not None:
d = dict([(parse_filename(fn.strip()), fn.strip())
for fn in open(ns.filelist, 'r').readlines()])
else:
d = dict(iter_tractor(ns.src))
if ns.verbose:
print('enumerated %d bricks in %g seconds' % (
len(d), time() - t0))
#- Load list of bricknames to use
if ns.bricklist is not None:
bricklist = np.loadtxt(ns.bricklist, dtype='S8')
# TODO: skip unknown bricks?
d = dict([(brickname.decode(), d[brickname])
for brickname in bricklist])
t0 = time()
bricks = sorted(d.items())
return bricks
def list_bricks(ns):
t0 = time()
if ns.filelist is not None:
d = dict([(parse_filename(fn.strip()), fn.strip())
for fn in open(ns.filelist, 'r').readlines()])
else:
d = dict(iter_tractor(ns.src))
if ns.verbose:
print('enumerated %d bricks in %g seconds' % (
len(d), time() - t0))
#- Load list of bricknames to use
if ns.bricklist is not None:
bricklist = np.loadtxt(ns.bricklist, dtype='S8')
# TODO: skip unknown bricks?
d = dict([(brickname.decode(), d[brickname])
for brickname in bricklist])
t0 = time()
bricks = sorted(d.items())
return bricks
def list_bricks(ns):
t0 = time()
if ns.filelist is not None:
d = dict([(parse_filename(fn.strip()), fn.strip())
for fn in open(ns.filelist, 'r').readlines()])
else:
d = dict(iter_tractor(ns.src))
if ns.verbose:
print('enumerated %d bricks in %g seconds' % (len(d), time() - t0))
if ns.bricksdesc is not None:
bricksdesc = fitsio.read(ns.bricksdesc, 1, upper=True)
# ADM convert from bytes_ to str_ type if fitsio version < 1.
if bricksdesc["BRICKNAME"].dtype.type == np.bytes_:
bricksdesc = dict([(item['BRICKNAME'].decode(), item)
for item in bricksdesc])
else:
bricksdesc = dict([(item['BRICKNAME'], item)
for item in bricksdesc])
else:
bricksdesc = None
#- Load list of bricknames to use
if ns.bricklist is not None:
bricklist = np.loadtxt(ns.bricklist, dtype='S8')
# TODO: skip unknown bricks?
d = dict([(brickname.decode(), d[brickname])
for brickname in bricklist])
t0 = time()
with sharedmem.MapReduce(np=ns.numproc) as pool:
chunksize = 1024
keys = list(d.keys())
def work(i):
return [(brickname, d[brickname],
read_region(brickname, d[brickname], bricksdesc))
for brickname in keys[i:i + chunksize]]
bricks = sum(pool.map(work, range(0, len(keys), chunksize)), [])
if ns.verbose:
print('read regions of %d bricks in %g seconds' %
(len(bricks), time() - t0))
return bricks
def list_bricks(ns):
t0 = time()
if ns.filelist is not None:
d = dict([(parse_filename(fn.strip()), fn.strip())
for fn in open(ns.filelist, 'r').readlines()])
else:
d = dict(iter_tractor(ns.src))
if ns.verbose:
print('enumerated %d bricks in %g seconds' % (
len(d), time() - t0))
if ns.bricksdesc is not None:
bricksdesc = fitsio.read(ns.bricksdesc, 1, upper=True)
bricksdesc = dict([(item['BRICKNAME'].decode(), item) for item in bricksdesc])
else:
bricksdesc = None
#- Load list of bricknames to use
if ns.bricklist is not None:
bricklist = np.loadtxt(ns.bricklist, dtype='S8')
# TODO: skip unknown bricks?
d = dict([(brickname.decode(), d[brickname])
for brickname in bricklist])
t0 = time()
with sharedmem.MapReduce(np=ns.numproc) as pool:
chunksize = 1024
keys = list(d.keys())
def work(i):
return [
(brickname, d[brickname], read_region(brickname, d[brickname], bricksdesc))
for brickname in keys[i:i+chunksize]
]
bricks = sum(pool.map(work, range(0, len(keys), chunksize)), [])
if ns.verbose:
print('read regions of %d bricks in %g seconds' % (
len(bricks), time() - t0))
return bricks
