def _stitch(self, phase):
fname = self.zpfile(phase)
flist = [fname]
#nproc = len(glob.glob(os.path.join(self.path,'id*')))
if self.mpi:
flist += glob.glob(
os.path.join(
self.path, 'id*/%s-id*.%s.%s.%s' %
(self.id, self.step, phase, self.ext)))
nf = len(flist)
if not test_pickle(fname):
if nf > 1:
# for i in range(1,nproc):
# flist.append(os.path.join(self.path,'id%d/%s-id%d.%s.%s.%s' % (i, self.id, i, self.step, phase, self.ext)))
flist.sort()
print(("Read-Merge-Write for ", phase))
data = pd.DataFrame()
for f in flist:
df = pd.read_csv(f, comment='#', index_col=0)
data = data.add(df, fill_value=0)
data.to_pickle(fname + '.p')
else:
print(("Read-Write for ", phase))
df = pd.read_csv(fname, comment='#', index_col=0)
df.to_pickle(fname + '.p')
def _clean(self,phase,forced=False):
fname=self.zpfile(phase)
if test_pickle(fname) or forced:
if self.mpi: flist = glob.glob(os.path.join(self.path,'id*/%s-id*.%s.%s.%s' % (self.id, self.step, phase, self.ext)))
if len(flist) != 0:
print(("Clean for ",phase))
for f in flist: os.remove(f)