def work(j):
input = Snapshot(inputname(j), 'cmugadget', template=first)
id = input[None, 'id']
t1 = 0
t2 = 0
t3 = 0
for i in TheBigGuys:
masks = []
output = Snapshot(outputname(i, j),
'cmugadget',
create=True,
template=first)
for ptype in range(6):
with Timer() as timer:
id = input[ptype, 'id']
mask = cat.mask(id, groupid=i)
masks.append(mask)
output.C['N'][ptype] = mask.sum()
t1 += timer.interval
for blk in input.schema:
if not (None, blk) in input: continue
with Timer() as timer:
junk = input[None, blk]
for ptype in range(6):
if not (ptype, blk) in input: continue
if output.C['N'][ptype] == 0: continue
output[ptype, blk][:] = input[ptype, blk][masks[ptype]]
t2 += timer.interval
with Timer() as timer:
output.save()
t3 += timer.interval
return j, t1, t2, t3
def work(j):
input = Snapshot(inputname(j), 'cmugadget', template=first)
id = input[None, 'id']
t1 = 0
t2 = 0
t3 = 0
for i in TheBigGuys:
masks = []
output = Snapshot(outputname(i, j), 'cmugadget', create=True, template=first)
for ptype in range(6):
with Timer() as timer:
id = input[ptype, 'id']
mask = cat.mask(id, groupid=i)
masks.append(mask)
output.C['N'][ptype] = mask.sum()
t1 += timer.interval
for blk in input.schema:
if not (None, blk) in input: continue
with Timer() as timer:
junk = input[None, blk]
for ptype in range(6):
if not (ptype, blk) in input: continue
if output.C['N'][ptype] == 0: continue
output[ptype, blk][:] = input[ptype, blk][masks[ptype]]
t2 += timer.interval
with Timer() as timer:
output.save()
t3 += timer.interval
return j, t1, t2, t3
with Timer() as timer:
junk = input[None, blk]
for ptype in range(6):
if not (ptype, blk) in input: continue
if output.C['N'][ptype] == 0: continue
output[ptype, blk][:] = input[ptype, blk][masks[ptype]]
t2 += timer.interval
with Timer() as timer:
output.save()
t3 += timer.interval
return j, t1, t2, t3
def reduce(j, t1, t2, t3):
print j, Nfiles, 'times', t1, t2, t3
with sharedmem.Pool() as pool:
pool.map(work, range(Nfiles), reduce=reduce)
for i in TheBigGuys:
Ntot = numpy.zeros(6, 'i8')
for j in range(Nfiles):
output = Snapshot(outputname(i, j), 'cmugadget', template=first)
Ntot += output.C['N']
print i, Ntot
for j in range(Nfiles):
output = Snapshot(outputname(i, j), 'cmugadget', template=first)
output.C['Ntot'][:] = Ntot
output.save('header')
class Timer:
def __enter__(self):
self.start = time.clock()
return self
def __exit__(self, *args):
self.end = time.clock()
self.interval = self.end - self.start
Ntot = first.C['Ntot']
Nout = first.C['Nfiles']
for i in range(Nout):
output = Snapshot(argv[2] % i, 'cmugadget', template=first, create=True)
output.C['N'] = Ntot * (i + 1) // Nout - Ntot * i // Nout
output.save()
for ptype in range(6):
buffer = {}
j = 0
for blk in first.schema:
buffer[blk] = numpy.empty(0, first.schema[blk].dtype)
for i in range(Nout):
output = Snapshot(argv[2] % i,
'cmugadget',
template=first,
readonly=False)
while output.C['N'][ptype] > len(buffer['pos']):
print 'need to read ', j, output.C['N'][ptype], len(buffer['pos'])
newinput = Snapshot(argv[1] % j, 'cmugadget', template=first)
with Timer() as timer:
junk = input[None, blk]
for ptype in range(6):
if not (ptype, blk) in input: continue
if output.C['N'][ptype] == 0: continue
output[ptype, blk][:] = input[ptype, blk][masks[ptype]]
t2 += timer.interval
with Timer() as timer:
output.save()
t3 += timer.interval
return j, t1, t2, t3
def reduce(j, t1, t2, t3):
print j, Nfiles, 'times', t1, t2, t3
with sharedmem.Pool() as pool:
pool.map(work, range(Nfiles),
reduce=reduce)
for i in TheBigGuys:
Ntot = numpy.zeros(6, 'i8')
for j in range(Nfiles):
output = Snapshot(outputname(i, j), 'cmugadget', template=first)
Ntot += output.C['N']
print i, Ntot
for j in range(Nfiles):
output = Snapshot(outputname(i, j), 'cmugadget', template=first)
output.C['Ntot'][:] = Ntot
output.save('header')
first = Snapshot(argv[1] % 0, 'cmugadget', idtype='u8')
class Timer:
def __enter__(self):
self.start = time.clock()
return self
def __exit__(self, *args):
self.end = time.clock()
self.interval = self.end - self.start
Ntot = first.C['Ntot']
Nout = first.C['Nfiles']
for i in range(Nout):
output = Snapshot(argv[2] % i, 'cmugadget', template=first,
create=True)
output.C['N'] = Ntot * (i + 1) // Nout - Ntot * i // Nout
output.save()
for ptype in range(6):
buffer = {}
j = 0
for blk in first.schema:
buffer[blk] = numpy.empty(0, first.schema[blk].dtype)
for i in range(Nout):
output = Snapshot(argv[2] % i, 'cmugadget', template=first,
readonly=False)
while output.C['N'][ptype] > len(buffer['pos']):
print 'need to read ', j, output.C['N'][ptype], len(buffer['pos'])
newinput = Snapshot(argv[1] % j, 'cmugadget', template=first)
j = j + 1
for blk in newinput.schema:
