def __init__(self, tabfilename, format, count=10, **kwargs):
"""
tabfilename is like groups_019/group_tab_019.%d.
"""
g = Snapshot(tabfilename % 0, format + '.GroupTab',
**kwargs)
if count < 0 or count > g.C['Ntot'][0]:
count = g.C['Ntot'][0]
i = 0
# decide number of files to open
nread = 0
tabs = []
while nread < count:
g = Snapshot(tabfilename % i, format + '.GroupTab',
**kwargs)
nread += g.C['N'][0]
i = i + 1
tabs.append(g)
#print 'will read', len(tabs), 'files'
Field.__init__(self, numpoints=count, components={'offset':'i8',
'length':'i8', 'massbytype':('f8', 6), 'mass':'f8', 'pos':('f8', 3),
'vel':('f8', 3)})
if len(tabs) > 0:
self.take_snapshots(tabs, ptype=0)
del tabs
# fix the offset which may overflow for large halos
self['offset'][1:] = self['length'].cumsum()[:-1]
nread = 0
nshallread = self['length'].sum()
i = 0
idslen = numpy.zeros(g.C['Nfiles'], dtype='i8')
while nread < nshallread:
idslen[i] = numpy.fromfile(tabfilename.replace('_tab_', '_ids_')
% i, dtype='i4', count=3)[2]
nread += idslen[i]
i = i + 1
idsoffset = numpy.concatenate(([0], idslen.cumsum()))
ids = sharedmem.empty(idslen.sum(), dtype=g.C['idtype'])
#print 'reading', i, 'id files'
with sharedmem.Pool() as pool:
def work(i):
more = numpy.memmap(tabfilename.replace('_tab_', '_ids_')
% i, dtype=g.C['idtype'], mode='r', offset=28)
ids[idsoffset[i]:idsoffset[i] + idslen[i]] = more
pool.map(work, range(i))
self.ids = packarray(ids, self['length'])
for i in range(self.numpoints):
self.ids[i].sort()
def __init__(self, tabfilename, format, count=10, **kwargs):
"""
tabfilename is like groups_019/group_tab_019.%d.
"""
g = Snapshot(tabfilename % 0, format + '.GroupTab',
**kwargs)
if count < 0 or count > g.C['Ntot'][0]:
count = g.C['Ntot'][0]
i = 0
# decide number of files to open
nread = 0
tabs = []
while nread < count:
g = Snapshot(tabfilename % i, format + '.GroupTab',
**kwargs)
nread += g.C['N'][0]
i = i + 1
tabs.append(g)
#print 'will read', len(tabs), 'files'
Field.__init__(self, numpoints=count, components={'offset':'i8',
'length':'i8', 'massbytype':('f8', 6), 'mass':'f8', 'pos':('f8', 3),
'vel':('f8', 3)})
if len(tabs) > 0:
self.take_snapshots(tabs, ptype=0)
del tabs
# fix the offset which may overflow for large halos
self['offset'][1:] = self['length'].cumsum()[:-1]
nread = 0
nshallread = self['length'].sum()
i = 0
idslen = numpy.zeros(g.C['Nfiles'], dtype='i8')
while nread < nshallread:
idslen[i] = numpy.fromfile(tabfilename.replace('_tab_', '_ids_')
% i, dtype='i4', count=3)[2]
nread += idslen[i]
i = i + 1
idsoffset = numpy.concatenate(([0], idslen.cumsum()))
ids = sharedmem.empty(idslen.sum(), dtype=g.C['idtype'])
#print 'reading', i, 'id files'
with sharedmem.Pool() as pool:
def work(i):
more = numpy.memmap(tabfilename.replace('_tab_', '_ids_')
% i, dtype=g.C['idtype'], mode='r', offset=28)
ids[idsoffset[i]:idsoffset[i] + idslen[i]] = more
pool.map(work, range(i))
self.ids = packarray(ids, self['length'])
for i in range(self.numpoints):
self.ids[i].sort()
def readbh(self):
rawbh = numpy.load(ROOT + '/bhcorr/bh_%03d.npz' % self.snapid)
bh = Field(numpoints=len(rawbh['bhmass']))
assert self.C['Ntot'][5] == len(bh)
for component in rawbh.files:
bh[component] = rawbh[component].copy()
return bh
def schema(self,
ftype,
types,
components=None,
tree=True,
locations='pos'):
""" give particle types a name, and associate
it with components.
dtype is the base dtype of the locations.
components is a list of components in the Field
and also the blocks to read from snapshot files.
if components is None, all blocks defined
in the reader will be read in.
"""
schemed = {}
if components is None:
components = self._schema
if locations not in components:
components += [locations]
for comp in components:
if isinstance(comp, tuple):
schemed[comp[0]] = comp[1]
elif comp in self._schema:
schemed[comp] = self._schema[comp].dtype
self.F[ftype] = Field(components=schemed, locations=locations)
self.P[ftype] = _ensurelist(types)
if not tree: self.T[ftype] = False
def crop_snapshot(center, size, map, format, snapname, output):
cut = Cut(center=center, size=size)
mesh = Meshmap(map)
gas = Field(
components={
'mass': 'f4',
'rho': 'f4',
'ie': 'f4',
'xHI': 'f4',
'ye': 'f4',
'id': 'u8',
'sfr': 'f4',
'met': 'f4',
'sml': 'f4'
})
bh = Field(components={
'mass': 'f4',
'bhmdot': 'f4',
'bhmass': 'f4',
'id': 'u8'
})
star = Field(components={
'mass': 'f4',
'sft': 'f4',
'met': 'f4',
'id': 'u8'
})
fids = mesh.cut2fid(cut)
print fids
snapshots = [Snapshot(snapname % fid, format) for fid in fids]
gas.take_snapshots(snapshots, 0, cut=cut)
star.take_snapshots(snapshots, 4, cut=cut)
bh.take_snapshots(snapshots, 5, cut=cut)
left = center - size * 0.5
gas['locations'][:, :] -= left[newaxis, :]
star['locations'][:, :] -= left[newaxis, :]
bh['locations'][:, :] -= left[newaxis, :]
print bh['id']
out = Snapshot(output, format, create=True)
for f in snapshots[0].header.dtype.names:
out.header[f] = snapshots[0].header[f]
out.header['unused'][0:3] = center
out.header['boxsize'] = size[0]
gas.dump_snapshots([out], 0)
star.dump_snapshots([out], 4)
bh.dump_snapshots([out], 5)
from gaepsi.field import Field
from numpy import array
field = Field(numpoints=1, components={'mass':'f4', 'vel':('f4', 3), 'sml':'f4'}, boxsize=20)
field['locations'][0,:] = array([10.0,10.0,10.0])
field['vel'][0,:] = array([10.0,20.0,30.0])
field['sml'][0] = 10.0
field['mass'][0] = 1.0
# steps is the number of iterations to scan over all snapshot files.
# in each step each core loads in one file and an alltoall communication
# distributes relevant particles to the core hosting the stripe
steps = max([len(snaplist) for snaplist in snaplist_all])
#snaplist on this core
snaplist = snaplist_all[comm.rank]
# start the main loop
for step in range(steps):
# the fields may be None if the core doesn't read in a file in this step
# it happens at the end of the iterations, when NCPU doesn't divide the
# number of snapshot files given by opt.range
ses_step = timer.session("step %d" % step)
ses_reading = timer.session('reading fields')
bhfield = Field(boxsize=opt.boxsize, components={'bhmass': 'f4'})
starfield = Field(boxsize=opt.boxsize, components={'sft': 'f4'})
gascomps = []
if opt.sfr != None:
gascomps += ['sfr']
if opt.temp != None:
gascomps += ['ie', 'ye']
if opt.gas != None:
gascomps += ['mass', 'sml']
gasfield = Field(boxsize=opt.boxsize,
components={
'mass': 'f4',
'sml': 'f4',
'ie': 'f4',
'ye': 'f4',
'sfr': 'f4'
def use(self,
snapname,
format,
components={},
bhcomponents={
'bhmass': 'f4',
'bhmdot': 'f4',
'id': 'u8'
},
starcomponents={
'sft': 'f4',
'mass': 'f4'
},
gas=0,
halo=1,
disk=2,
bulge=3,
star=4,
bh=5,
cut=None,
gascomponents=None,
periodic=True):
if gascomponents is not None:
self.components = gascomponents
else:
self.components = components
self.components['mass'] = 'f4'
self.components['sml'] = 'f4'
self.snapname = snapname
self.format = format
self.F['gas'] = Field(components=self.components)
self.F['bh'] = Field(components=bhcomponents)
self.F['star'] = Field(components=starcomponents)
self.F['halo'] = Field(components=self.components)
self.F['disk'] = Field(components=self.components)
self.F['bulge'] = Field(components=self.components)
self.ptype = {
"gas": gas,
"halo": halo,
"disk": disk,
"bulge": bulge,
"star": star,
"bh": bh,
}
try:
snapname = self.snapname % 0
except TypeError:
snapname = self.snapname
snap = Snapshot(snapname, self.format)
self.gas.init_from_snapshot(snap)
self.bh.init_from_snapshot(snap)
self.star.init_from_snapshot(snap)
self.halo.init_from_snapshot(snap)
self.disk.init_from_snapshot(snap)
self.bulge.init_from_snapshot(snap)
self.C = snap.C
if cut is not None:
self.cut.take(cut)
else:
try:
boxsize = snap.C['boxsize']
self.cut.take(
Cut(xcut=[0, boxsize],
ycut=[0, boxsize],
zcut=[0, boxsize]))
except:
pass
self.boxsize = ones(3) * snap.C['boxsize']
self.redshift = snap.C['redshift']
self.invalidate()
self.periodic = periodic
def crop_snapshot(center, size, map, format, snapname, output):
cut = Cut(center = center, size=size)
mesh = Meshmap(map)
gas = Field(components={'mass':'f4', 'rho':'f4', 'ie':'f4', 'xHI':'f4', 'ye':'f4', 'id':'u8', 'sfr':'f4', 'met':'f4', 'sml':'f4'})
bh = Field(components={'mass':'f4', 'bhmdot':'f4', 'bhmass':'f4', 'id':'u8'})
star = Field(components={'mass':'f4', 'sft':'f4', 'met':'f4', 'id':'u8'})
fids = mesh.cut2fid(cut)
print fids
snapshots = [Snapshot(snapname % fid, format) for fid in fids]
gas.take_snapshots(snapshots, 0, cut=cut)
star.take_snapshots(snapshots, 4, cut=cut)
bh.take_snapshots(snapshots, 5, cut=cut)
left = center - size * 0.5
gas['locations'][:, :] -= left[newaxis, :]
star['locations'][:, :] -= left[newaxis, :]
bh['locations'][:, :] -= left[newaxis, :]
print bh['id']
out = Snapshot(output, format, create=True)
for f in snapshots[0].header.dtype.names:
out.header[f] = snapshots[0].header[f]
out.header['unused'][0:3] = center
out.header['boxsize'] = size[0]
gas.dump_snapshots([out], 0)
star.dump_snapshots([out], 4)
bh.dump_snapshots([out], 5)
# steps is the number of iterations to scan over all snapshot files.
# in each step each core loads in one file and an alltoall communication
# distributes relevant particles to the core hosting the stripe
steps = max([len(snaplist) for snaplist in snaplist_all])
# snaplist on this core
snaplist = snaplist_all[comm.rank]
# start the main loop
for step in range(steps):
# the fields may be None if the core doesn't read in a file in this step
# it happens at the end of the iterations, when NCPU doesn't divide the
# number of snapshot files given by opt.range
ses_step = timer.session("step %d" % step)
ses_reading = timer.session("reading fields")
bhfield = Field(boxsize=opt.boxsize, components={"bhmass": "f4"})
starfield = Field(boxsize=opt.boxsize, components={"sft": "f4"})
gascomps = []
if opt.sfr != None:
gascomps += ["sfr"]
if opt.temp != None:
gascomps += ["ie", "ye"]
if opt.gas != None:
gascomps += ["mass", "sml"]
gasfield = Field(boxsize=opt.boxsize, components={"mass": "f4", "sml": "f4", "ie": "f4", "ye": "f4", "sfr": "f4"})
if opt.temp == None:
del gasfield["ye"]
del gasfield["ie"]
if opt.sfr == None:
del gasfield["sfr"]
if step < len(snaplist):