def list_bricks(ns): t0 = time() if ns.filelist is not None: d = dict([(parse_filename(fn.strip()), fn.strip()) for fn in open(ns.filelist, 'r').readlines()]) else: d = dict(iter_tractor(ns.src)) if ns.verbose: print('enumerated %d bricks in %g seconds' % ( len(d), time() - t0)) #- Load list of bricknames to use if ns.bricklist is not None: bricklist = np.loadtxt(ns.bricklist, dtype='S8') # TODO: skip unknown bricks? d = dict([(brickname.decode(), d[brickname]) for brickname in bricklist]) t0 = time() bricks = sorted(d.items()) return bricks
def list_bricks(ns): t0 = time() if ns.filelist is not None: d = dict([(parse_filename(fn.strip()), fn.strip()) for fn in open(ns.filelist, 'r').readlines()]) else: d = dict(iter_tractor(ns.src)) if ns.verbose: print('enumerated %d bricks in %g seconds' % (len(d), time() - t0)) if ns.bricksdesc is not None: bricksdesc = fitsio.read(ns.bricksdesc, 1, upper=True) # ADM convert from bytes_ to str_ type if fitsio version < 1. if bricksdesc["BRICKNAME"].dtype.type == np.bytes_: bricksdesc = dict([(item['BRICKNAME'].decode(), item) for item in bricksdesc]) else: bricksdesc = dict([(item['BRICKNAME'], item) for item in bricksdesc]) else: bricksdesc = None #- Load list of bricknames to use if ns.bricklist is not None: bricklist = np.loadtxt(ns.bricklist, dtype='S8') # TODO: skip unknown bricks? d = dict([(brickname.decode(), d[brickname]) for brickname in bricklist]) t0 = time() with sharedmem.MapReduce(np=ns.numproc) as pool: chunksize = 1024 keys = list(d.keys()) def work(i): return [(brickname, d[brickname], read_region(brickname, d[brickname], bricksdesc)) for brickname in keys[i:i + chunksize]] bricks = sum(pool.map(work, range(0, len(keys), chunksize)), []) if ns.verbose: print('read regions of %d bricks in %g seconds' % (len(bricks), time() - t0)) return bricks
def list_bricks(ns): t0 = time() if ns.filelist is not None: d = dict([(parse_filename(fn.strip()), fn.strip()) for fn in open(ns.filelist, 'r').readlines()]) else: d = dict(iter_tractor(ns.src)) if ns.verbose: print('enumerated %d bricks in %g seconds' % ( len(d), time() - t0)) if ns.bricksdesc is not None: bricksdesc = fitsio.read(ns.bricksdesc, 1, upper=True) bricksdesc = dict([(item['BRICKNAME'].decode(), item) for item in bricksdesc]) else: bricksdesc = None #- Load list of bricknames to use if ns.bricklist is not None: bricklist = np.loadtxt(ns.bricklist, dtype='S8') # TODO: skip unknown bricks? d = dict([(brickname.decode(), d[brickname]) for brickname in bricklist]) t0 = time() with sharedmem.MapReduce(np=ns.numproc) as pool: chunksize = 1024 keys = list(d.keys()) def work(i): return [ (brickname, d[brickname], read_region(brickname, d[brickname], bricksdesc)) for brickname in keys[i:i+chunksize] ] bricks = sum(pool.map(work, range(0, len(keys), chunksize)), []) if ns.verbose: print('read regions of %d bricks in %g seconds' % ( len(bricks), time() - t0)) return bricks