def __init__(self, tabfile):
self.tabfile = tabfile
first = Snapshot(tabfile % 0,
'cmugadget.SubHaloTab',
idtype='u8',
floattype='f4')
self.Nhalo = int(first.C['NtotSubgroups'])
Depot.__init__(self, subdtype, first.C['Nfiles'])
def __init__(self, tabfile):
self.tabfile = tabfile
self.posvelfile = self.tabfile.replace('_tab_', '_posvel_')
self.idsfile = self.tabfile.replace('_tab_', '_ids_')
first = Snapshot(tabfile % 0,
'cmugadget.SubHaloTab',
idtype='u8',
floattype='f4')
Depot.__init__(self, pdtype, first.C['Nfiles'])
def work(i):
snap = Snapshot(argv[1] % i, 'cmugadget', template=first)
print 'in', i
sfr = snap[0, 'sfr']
id = snap[0, 'id']
mask = sfr > 0
print 'done', i, mask.sum(), sfr.max(), sfr.min()
return mask.sum()
def work(i):
snap = Snapshot(argv[1] % i, 'cmugadget', template=first)
print 'i', i
sfr = snap[0, 'sfr']
id = snap[0, 'id']
print 'i', i, 'read'
mask = sfr > 0
idall[offset[i]:offset[i]+size[i]] = id[mask]
sfrall[offset[i]:offset[i]+size[i]] = sfr[mask]
print 'i', i, 'assign'
print 'i', i, mask.sum()
def prefetcher(fid):
print 'pref', fid
try:
snap = Snapshot(snapname % fid, 'cmugadget')
snap[0, 'pos']
snap[0, 'ie']
snap[0, 'ye']
snap[0, 'sml']
snap[0, 'mass']
except:
pass
return
def work(i):
snap = Snapshot(argv[1] % i, 'cmugadget', template=first)
numpy.random.seed(seedtable[i])
# print 'in', i
with pool.critical:
pos = snap[1, 'pos']
r = numpy.random.uniform(size=len(pos))
mask = r < 100.**3 / first.C['Ntot'][1]
pos = pos[mask]
# print 'done', i, mask.sum(), sfr.max(), sfr.min()
with pool.critical:
numpy.savetxt(stdout, pos, fmt='%0.7g')
stdout.flush()
def fetch(self, id):
f = Snapshot(self.tabfile % id,
'cmugadget.GroupTab',
idtype='u8',
floattype='f4')
rt = numpy.empty(f.C['N'][0], dtype=groupdtype)
rt['pos'] = f[0, 'pos']
rt['vel'] = f[0, 'vel']
rt['mass'] = f[0, 'mass']
rt['len'] = f[0, 'length']
rt['massbytype'] = f[0, 'massbytype']
rt['lenbytype'] = f[0, 'lenbytype']
rt['nhalo'] = 0
return rt
def read(self,
fids=None,
use_gas=True,
use_bh=True,
use_star=True,
use_halo=False,
use_disk=False,
use_bulge=False,
numthreads=None):
if fids is not None:
snapnames = [self.snapname % i for i in fids]
elif '%d' in self.snapname:
snapnames = [self.snapname % i for i in range(self.C['Nfiles'])]
else:
snapnames = [self.snapname]
snapshots = [Snapshot(snapname, self.format) for snapname in snapnames]
if use_gas:
self.gas.take_snapshots(snapshots,
ptype=self.ptype['gas'],
nthreads=numthreads,
cut=self.cut)
if use_halo:
self.halo.take_snapshots(snapshots,
ptype=self.ptype['halo'],
nthreads=numthreads,
cut=self.cut)
if use_disk:
self.disk.take_snapshots(snapshots,
ptype=self.ptype['disk'],
nthreads=numthreads,
cut=self.cut)
if use_bulge:
self.bulge.take_snapshots(snapshots,
ptype=self.ptype['bulge'],
nthreads=numthreads,
cut=self.cut)
if use_bh:
self.bh.take_snapshots(snapshots,
ptype=self.ptype['bh'],
nthreads=numthreads,
cut=self.cut)
if use_star:
self.star.take_snapshots(snapshots,
ptype=self.ptype['star'],
nthreads=numthreads,
cut=self.cut)
self.invalidate()
def fetch(self, id):
f = Snapshot(self.tabfile % id,
'cmugadget.SubHaloTab',
idtype='u8',
floattype='f4')
rt = numpy.empty(f.C['N'][1], dtype=subdtype)
rt['pos'] = f[1, 'halopos']
rt['vel'] = f[1, 'halovel']
rt['len'] = f[1, 'halolen']
rt['mass'] = f[1, 'halomass']
rt['vdisp'] = f[1, 'haloveldisp']
rt['vcirc'] = f[1, 'halovmax']
rt['rcirc'] = f[1, 'halovmaxrad']
# i don't understand this; we do not want halo parent
# it's always zero
rt['parent'] = f[1, 'haloparent']
rt['lenbytype'] = 0
rt['massbytype'] = 0
return rt
def __init__(self, snapid, ROOT='../'):
SnapDirReadOnly.__init__(self, snapid, ROOT)
snapid = self.snapid
self.snapname = ROOT + '/snapdir/snapdir_%03d/snapshot_%03d.%%d' %(snapid, snapid)
first = Snapshot(self.snapname % 0, 'cmugadget')
self.first = first
self.C = first.C
C = first.C
self.cosmology = Cosmology(h=C['h'], M=C['OmegaM'], L=C['OmegaL'])
self.U = self.cosmology.U
self.schema = first.schema
self.collectivedir = ROOT + \
'/isolatedir/isolate_%03d/%%03d' % snapid
self.tabfile = ROOT + \
'/groups_and_subgroups/groups_%03d/subhalo_tab_%03d.%%d' %(snapid, snapid)
if not os.path.isfile(self.tabfile % 0):
self.tabfile = ROOT + \
'/groups_and_subgroups/no_collective_halos/groups_%03d/subhalo_tab_%03d.%%d' \
%(snapid, snapid)
self.grouptabfile = self.tabfile.replace('subhalo', 'group')
from sys import argv
from gaepsi.snapshot import Snapshot
import numpy
import os.path
import sharedmem
idsfile = argv[1]
sfgasdir = argv[2]
first = Snapshot(idsfile % 0,
'cmugadget.GroupIDs',
idtype='u8',
floattype='f4')
sfgasid = numpy.fromfile(sfgasdir + '/sfgasid', dtype='u8')
sfgassfr = numpy.fromfile(sfgasdir + '/sfgassfr', dtype='f4')
matchedfile = file(sfgasdir + '/matchedsfr', 'w')
matchedidfile = file(sfgasdir + '/matchedid', 'w')
with sharedmem.Pool(use_threads=False) as pool:
def work(i):
snap = Snapshot(idsfile % i,
'cmugadget.GroupIDs',
idtype='u8',
floattype='f4')
id = snap[0, 'id']
result = numpy.zeros(len(id), dtype='f4')
lookedid = numpy.zeros(len(id), dtype='u8')
ind = sfgasid.searchsorted(id)
ind.clip(0, len(sfgasid) - 1, out=ind)
def splitMain(args):
snap = args.snap
for type in range(6):
try:
os.makedirs(snap.subhalodir + '/%d' % type)
except OSError:
pass
file(snap.subhalodir + '/header.txt', mode='w').write(str(snap.C))
print snap.tabfile
depot_sub = SubHaloDepot(snap.tabfile)
depot_group = GroupDepot(snap.grouptabfile)
if not args.check:
wrong_file_or_die(snap.groupfile,
depot_group.Ngroups * groupdtype.itemsize)
depot_par = SubHaloParDepot(snap.tabfile)
F = {}
for type in range(6):
F[type] = {}
for field in pdtype.fields:
if type != 4 and \
(field == 'SEDindex' or field == 'recfrac'):
continue
F[type][field] = snap.open(type, field, mode='w')
halodump = file(snap.subhalofile, 'w')
groupdump = file(snap.groupfile, 'w')
realgroupid = 0
for i in range(len(depot_sub)):
tab = Snapshot(snap.tabfile % i,
'cmugadget.SubHaloTab',
idtype='u8',
floattype='f4')
group = depot_group.consume(tab.C['N'][0])
for groupid in range(tab.C['N'][0]):
if snap.hascollective(realgroupid, group[groupid]):
fakenhalo = tab[0, 'nhalo'][groupid]
depot_subcol = SubHaloDepot(
snap.collectivetabfile(realgroupid))
realhalo = depot_subcol.consume()
nhalo = len(realhalo)
# skip the pars from ids file (it's of wrong order)
if not args.check:
depot_parcol = SubHaloParDepot(
snap.collectivetabfile(realgroupid))
pars = depot_parcol.consume()
junk = depot_par.consume(group['len'][groupid])
assert len(pars) == group['len'][groupid]
else:
nhalo = tab[0, 'nhalo'][groupid]
if not args.check:
pars = depot_par.consume(group['len'][groupid])
realhalo = depot_sub.consume(nhalo)
halo = numpy.empty(nhalo + 1, dtype=realhalo.dtype)
halo[:-1] = realhalo
# last halo is contamination
halo[-1]['mass'] = numpy.nan
halo[-1]['pos'] = numpy.nan
halo[:]['groupid'] = realgroupid
halo[-1]['len'] = group['len'][groupid] - halo['len'][:-1].sum()
assert (halo['len'] >= 0).all()
assert (halo['len'][0:-2] >= halo['len'][1:-1]).all()
parstart = numpy.concatenate(([0], halo['len'].cumsum()))
parend = parstart[1:]
#make sure group and tab are of the same file
assert group[groupid]['len'] == tab[0, 'length'][groupid]
if not args.check:
for haloid in range(nhalo + 1):
parinhalo = pars[parstart[haloid]:parend[haloid]]
parinhalo.sort(order='type')
start = parinhalo['type'].searchsorted(range(6),
side='left')
end = parinhalo['type'].searchsorted(range(6),
side='right')
for type in range(6):
thistype = parinhalo[start[type]:end[type]]
halo[haloid]['massbytype'][type] = thistype[
'mass'].sum(dtype='f8')
halo[haloid]['lenbytype'][type] = len(thistype)
for field in F[type]:
thistype[field].tofile(F[type][field])
group[groupid]['nhalo'] = nhalo
print i, 'group', groupid, 'nhalo', \
group['nhalo'][groupid], \
'npar', group['len'][groupid], halo[-1]['len'], \
'collective', snap.hascollective(realgroupid, group[groupid])
if not args.check:
halo.tofile(halodump)
halodump.flush()
assert numpy.all(
group[groupid]['lenbytype'] == halo['lenbytype'].sum(
axis=0))
if not numpy.allclose(group[groupid]['massbytype'],
halo['massbytype'].sum(axis=0)):
print group[groupid]['massbytype'], halo['massbytype'].sum(
axis=0)
realgroupid += 1
if not args.check:
group.tofile(groupdump)
groupdump.flush()
for type in range(6):
for field in pdtype.fields:
F[type][field].flush()
assert depot_sub.Nhalo == depot_sub.total_read
def use(self,
snapname,
format,
components={},
bhcomponents={
'bhmass': 'f4',
'bhmdot': 'f4',
'id': 'u8'
},
starcomponents={
'sft': 'f4',
'mass': 'f4'
},
gas=0,
halo=1,
disk=2,
bulge=3,
star=4,
bh=5,
cut=None,
gascomponents=None,
periodic=True):
if gascomponents is not None:
self.components = gascomponents
else:
self.components = components
self.components['mass'] = 'f4'
self.components['sml'] = 'f4'
self.snapname = snapname
self.format = format
self.F['gas'] = Field(components=self.components)
self.F['bh'] = Field(components=bhcomponents)
self.F['star'] = Field(components=starcomponents)
self.F['halo'] = Field(components=self.components)
self.F['disk'] = Field(components=self.components)
self.F['bulge'] = Field(components=self.components)
self.ptype = {
"gas": gas,
"halo": halo,
"disk": disk,
"bulge": bulge,
"star": star,
"bh": bh,
}
try:
snapname = self.snapname % 0
except TypeError:
snapname = self.snapname
snap = Snapshot(snapname, self.format)
self.gas.init_from_snapshot(snap)
self.bh.init_from_snapshot(snap)
self.star.init_from_snapshot(snap)
self.halo.init_from_snapshot(snap)
self.disk.init_from_snapshot(snap)
self.bulge.init_from_snapshot(snap)
self.C = snap.C
if cut is not None:
self.cut.take(cut)
else:
try:
boxsize = snap.C['boxsize']
self.cut.take(
Cut(xcut=[0, boxsize],
ycut=[0, boxsize],
zcut=[0, boxsize]))
except:
pass
self.boxsize = ones(3) * snap.C['boxsize']
self.redshift = snap.C['redshift']
self.invalidate()
self.periodic = periodic
#! python
from gaepsi.snapshot import Snapshot
from sys import argv
template = Snapshot(argv[2], argv[1])
for filename in argv[2:]:
try:
snap = Snapshot(filename, argv[1], template=template)
snap.check()
print snap.file, 'ok'
except IOError as e:
print filename, e
def open(template, fid, create=False):
if '%d' in template:
snap = Snapshot(template % fid, args.format, create=create)
else:
snap = Snapshot(template, args.format, create=create)
return snap
def getsnap(snapname):
try:
return Snapshot(snapname, self.format, template=self._template)
except IOError as e:
warnings.warn('file %s skipped for %s' % (snapname, str(e)))
return None
from gaepsi.snapshot import Snapshot
from sys import argv
print len(argv[1:])
sfr = 0
for fn in argv[1:]:
snap = Snapshot(fn, 'cmugadget')
assert snap.C['Nfiles'] == len(argv[1:])
sfr += snap[0, 'sfr'].sum(dtype='f8')
print sfr
def main():
camera = Camera(int(length), int(width))
camera.fade = False
camera.lookat(pos=[
cub.newboxsize[0] * 0.5,
cub.newboxsize[1] * 0.5,
cub.newboxsize[2]],
target=[
cub.newboxsize[0] * 0.5,
cub.newboxsize[1] * 0.5,
cub.newboxsize[2] * 0.5],
up=[0, 1, 0])
camera.ortho(near=0, far=cub.newboxsize[2],
extent=[-cub.newboxsize[0] * 0.5,
cub.newboxsize[0] * 0.5,
-cub.newboxsize[1] * 0.5,
cub.newboxsize[1] * 0.5])
if not os.path.exists(ccdbuffer):
CCD = numpy.memmap(ccdbuffer, mode='w+', dtype=('f4', 2), shape=camera.shape)
CCD[...] = 0
def prefetcher(fid):
print 'pref', fid
try:
snap = Snapshot(snapname % fid, 'cmugadget')
snap[0, 'pos']
snap[0, 'ie']
snap[0, 'ye']
snap[0, 'sml']
snap[0, 'mass']
except:
pass
return
chunksize = 64
def prefetch(fid):
loadnext = [
sharedmem.background(
lambda fidnext: prefetcher(fidnext),
fidnext)
for fidnext in range(fid, fid + chunksize) if fidnext < Nfile ]
return loadnext
loadnext = prefetch(0)
for fid in range(0, Nfile, chunksize):
print fid, 'start'
for thread in loadnext:
thread.wait()
del loadnext
nextsnap = [Snapshot(snapname % f, 'cmugadget')
for f in range(fid, fid + chunksize) if f < Nfile]
loadnext = prefetch(fid + chunksize)
makegigapan(nextsnap, camera, CCD)
del nextsnap
print fid, 'done'
CCD[..., 0] /= CCD[..., 1]
else:
CCD = numpy.memmap(ccdbuffer, mode='r', dtype=('f4', 2))
CCD = CCD.reshape(-1, 2)
print 'CCD done'
C, L = CCD[..., 0], CCD[..., 1]
C = C.reshape(-1)
L = L.reshape(-1)
with sharedmem.MapReduce() as pool:
chunksize = 1024 * 1024 * 20
def work(i):
sl = slice(i, i + chunksize)
l = nl_(L[sl])
return l.vmax, l.vmin
vmax, vmin = numpy.array(pool.map(work, range(0, len(L), chunksize))).T
vmax = numpy.max(vmax) - 0.2
vmin = numpy.min(vmin)
print vmax, vmin
im = numpy.memmap(imagename,
mode='w+', dtype=('u1', 3), shape=L.shape)
with sharedmem.MapReduce() as pool:
chunksize = 1024 * 1024 * 128
def work(i):
sl = slice(i, i + chunksize)
im[sl] = image(nl_(C[sl], vmin=3.0, vmax=8.0), nl_(L[sl],
vmax=vmax, vmin=vmin) ** 0.8, cmap=cm.RdBu_r)[..., :3]
pool.map(work, range(0, len(L), chunksize))
im.flush()
from gaepsi.render import *
from matplotlib import cm
from sys import argv
from gaepsi.snapshot import Snapshot
from gaepsi.gaplot import create_cosmology
import os.path
from gaepsi.compiledbase.geometry import Cubenoid
from sys import argv
execfile(argv[1])
# center = [ 62995.8671875, 40180.12109375, 52267.77734375]
# pos of most massive halo from halo catalog
# unfold from here.
#snapname = '/physics/yfeng1/e5/run/000/output/snapdir_1366/snapshot_1366.%d'
snap = Snapshot(snapname % 0, 'cmugadget')
cosmology = create_cosmology(snap.C)
Nfile = snap.C['Nfiles']
boxsize = snap.C['boxsize']
cub = Cubenoid(M, [0, 0, 0], [boxsize, boxsize, boxsize],
center=0, neworigin=0)
ratio = cub.newboxsize[0] / cub.newboxsize[1]
width = int(width * dpi)
length = int(ratio * width)
print 'real length is', 1.0 * length / dpi, 'inch'
print 'kpc per inch is', cub.newboxsize[0] / length * dpi
print 'kpc per dot is', cub.newboxsize[0] / length
print 'pixel size is wxl', width, length
print 'total pixels = ', width * length / 1024 / 1024., 'M'
def opensnap(self, fid):
return Snapshot(self.snapname %fid, 'cmugadget', template=self.first)
from gaepsi.snapshot import Snapshot
from gaepsi.tools.analyze import HaloCatalog
import numpy
import time
import sharedmem
import os.path
from sys import argv
first = Snapshot(argv[1] % 0, 'cmugadget', idtype='u8')
class Timer:
def __enter__(self):
self.start = time.clock()
return self
def __exit__(self, *args):
self.end = time.clock()
self.interval = self.end - self.start
Ntot = first.C['Ntot']
Nout = first.C['Nfiles']
for i in range(Nout):
output = Snapshot(argv[2] % i, 'cmugadget', template=first, create=True)
output.C['N'] = Ntot * (i + 1) // Nout - Ntot * i // Nout
output.save()
for ptype in range(6):
buffer = {}
j = 0
for blk in first.schema:
ses_import.end()
# when opt.range == None, there is only one file in the snapshot
if opt.range != None:
snapfiles = [opt.snapname % i for i in range(opt.range[0], opt.range[1])]
else:
if opt.rangefile != None:
from numpy import loadtxt
snapids = loadtxt(opt.rangefile)
snapfiles = [opt.snapname % i for i in snapids]
else:
snapfiles = [opt.snapname]
#decide the boxsize
if comm.rank == 0:
snap = Snapshot(snapfiles[0], opt.format)
boxsize = snap.C['boxsize']
del snap
else:
boxsize = None
opt.boxsize = comm.bcast(boxsize)
del boxsize
# create the stripes and layers
stripe = Stripe(comm=comm,
imagesize=opt.geometry,
xcut=opt.xcut,
ycut=opt.ycut,
zcut=opt.zcut)
if opt.gas != None:
from gaepsi.snapshot import Snapshot
import sharedmem
import numpy
from sys import argv
import os.path
first = Snapshot(argv[1] % 0, 'cmugadget')
def work(i):
snap = Snapshot(argv[1] % i, 'cmugadget', template=first)
print 'in', i
sfr = snap[0, 'sfr']
id = snap[0, 'id']
mask = sfr > 0
print 'done', i, mask.sum(), sfr.max(), sfr.min()
return mask.sum()
with sharedmem.Pool() as pool:
size = pool.map(work, range(first.C['Nfiles']), ordered=True)
print 'phase 1 done'
size = numpy.array(size, 'i8')
total = size.sum()
offset = numpy.concatenate(([0], size.cumsum()))
print 'allocation total pars', total
idall = sharedmem.empty(total, dtype=first.schema['id'].dtype)
sfrall = sharedmem.empty(total, dtype='f4')
def work(i):
from gaepsi.snapshot import Snapshot
from gaepsi.tools.analyze import HaloCatalog
import numpy
import time
import sharedmem
import os.path
from sys import argv
firstcat = Snapshot(argv[1] % 0,
'cmugadget.GroupTab',
idtype='u8',
floattype='f4')
Ncat = (firstcat[0, 'length'] > 8500000).nonzero()[0].max()
class Timer:
def __enter__(self):
self.start = time.clock()
return self
def __exit__(self, *args):
self.end = time.clock()
self.interval = self.end - self.start
TheBigGuys = range(Ncat)
def outputname(haloid, j):
outputname = '%03d/' % haloid + os.path.basename(argv[2])
return outputname % j
def use(self,
snapname,
format,
periodic=False,
origin=[0, 0, 0.],
boxsize=None,
mapfile=None,
**kwargs):
""" kwargs will be passed to reader,
only particles within origin and boxsize are loaded.
"""
self.snapname = snapname
if format == 'hdf5':
self.format = 'hdf5'
elif format == 'bigfile':
self.format = 'bigfile'
else:
self.format = Reader(format, **kwargs)
try:
snapname = self.snapname % 0
except TypeError:
snapname = self.snapname
snap = Snapshot(snapname, self.format)
self.C = snap.C
self._template = snap
self._schema = snap.schema
self.origin[...] = numpy.ones(3) * origin
if boxsize is not None:
self.need_cut = True
else:
self.need_cut = False
if mapfile is not None:
self.map = MeshIndex.fromfile(mapfile)
else:
self.map = None
if boxsize is None and 'boxsize' in self.C:
boxsize = numpy.ones(3) * self.C['boxsize']
if boxsize is not None:
self.boxsize[...] = boxsize
else:
self.boxsize[...] = 1.0
self.periodic = periodic
self.cosmology = create_cosmology(self.C)
try:
self.redshift = self.C['redshift']
except:
self.redshift = 0.0
self.schema('gas', 0, ['sml', 'mass', 'id', 'ye', 'ie', 'entropy'])
self.schema('bh', 5, ['bhmass', 'bhmdot', 'id'])
self.schema('dark', 1, ['mass', 'id'])
self.schema('star', 4, ['mass', 'sft', 'id'])
