def work(j): input = Snapshot(inputname(j), 'cmugadget', template=first) id = input[None, 'id'] t1 = 0 t2 = 0 t3 = 0 for i in TheBigGuys: masks = [] output = Snapshot(outputname(i, j), 'cmugadget', create=True, template=first) for ptype in range(6): with Timer() as timer: id = input[ptype, 'id'] mask = cat.mask(id, groupid=i) masks.append(mask) output.C['N'][ptype] = mask.sum() t1 += timer.interval for blk in input.schema: if not (None, blk) in input: continue with Timer() as timer: junk = input[None, blk] for ptype in range(6): if not (ptype, blk) in input: continue if output.C['N'][ptype] == 0: continue output[ptype, blk][:] = input[ptype, blk][masks[ptype]] t2 += timer.interval with Timer() as timer: output.save() t3 += timer.interval return j, t1, t2, t3
with Timer() as timer: junk = input[None, blk] for ptype in range(6): if not (ptype, blk) in input: continue if output.C['N'][ptype] == 0: continue output[ptype, blk][:] = input[ptype, blk][masks[ptype]] t2 += timer.interval with Timer() as timer: output.save() t3 += timer.interval return j, t1, t2, t3 def reduce(j, t1, t2, t3): print j, Nfiles, 'times', t1, t2, t3 with sharedmem.Pool() as pool: pool.map(work, range(Nfiles), reduce=reduce) for i in TheBigGuys: Ntot = numpy.zeros(6, 'i8') for j in range(Nfiles): output = Snapshot(outputname(i, j), 'cmugadget', template=first) Ntot += output.C['N'] print i, Ntot for j in range(Nfiles): output = Snapshot(outputname(i, j), 'cmugadget', template=first) output.C['Ntot'][:] = Ntot output.save('header')
class Timer: def __enter__(self): self.start = time.clock() return self def __exit__(self, *args): self.end = time.clock() self.interval = self.end - self.start Ntot = first.C['Ntot'] Nout = first.C['Nfiles'] for i in range(Nout): output = Snapshot(argv[2] % i, 'cmugadget', template=first, create=True) output.C['N'] = Ntot * (i + 1) // Nout - Ntot * i // Nout output.save() for ptype in range(6): buffer = {} j = 0 for blk in first.schema: buffer[blk] = numpy.empty(0, first.schema[blk].dtype) for i in range(Nout): output = Snapshot(argv[2] % i, 'cmugadget', template=first, readonly=False) while output.C['N'][ptype] > len(buffer['pos']): print 'need to read ', j, output.C['N'][ptype], len(buffer['pos']) newinput = Snapshot(argv[1] % j, 'cmugadget', template=first)
first = Snapshot(argv[1] % 0, 'cmugadget', idtype='u8') class Timer: def __enter__(self): self.start = time.clock() return self def __exit__(self, *args): self.end = time.clock() self.interval = self.end - self.start Ntot = first.C['Ntot'] Nout = first.C['Nfiles'] for i in range(Nout): output = Snapshot(argv[2] % i, 'cmugadget', template=first, create=True) output.C['N'] = Ntot * (i + 1) // Nout - Ntot * i // Nout output.save() for ptype in range(6): buffer = {} j = 0 for blk in first.schema: buffer[blk] = numpy.empty(0, first.schema[blk].dtype) for i in range(Nout): output = Snapshot(argv[2] % i, 'cmugadget', template=first, readonly=False) while output.C['N'][ptype] > len(buffer['pos']): print 'need to read ', j, output.C['N'][ptype], len(buffer['pos']) newinput = Snapshot(argv[1] % j, 'cmugadget', template=first) j = j + 1 for blk in newinput.schema: