def __Split(self, split, reduction, sfiles, tfiles, flow, stdout, stdin,
jobmult, suffix, prefix, src_suffix):
'''Split jobs for pscons'''
if self.jobs < split[1]:
jobs = self.jobs * jobmult
w = int(1 + float(split[1]) / jobs) # length of one chunk
else:
jobs = split[1]
w = 1
jobmult = 1
par_sfiles = copy.copy(sfiles)
par_targets = {}
for tfile in tfiles:
par_targets[tfile] = []
prev_par_tfiles = []
bigjobs = split[1] - jobs * (w - 1)
cflow = flow
for i in range(jobs):
if i < bigjobs:
chunk = w
skip = i * w
else:
chunk = w - 1
skip = bigjobs * w + (i - bigjobs) * chunk
for j in split[2]:
if 0 == j and stdin and \
flow.rfind('$SOURCE') < 0 and \
flow.rfind('${SOURCES[0]}') < 0:
# For the first input (if stdin), use windowing
# to avoid creation of a chunk file
par_sfiles[j] = sfiles[j]
cflow = '''
window n%d=%d f%d=%d squeeze=n icpu=%d ncpu=%d |
''' % (split[0], chunk, split[0], skip, i % self.jobs,
self.jobs) + flow
else:
source = sfiles[j] + '_' + str(i)
par_sfiles[j] = source
self.Flow(source,
sfiles[j],
'''
window n%d=%d f%d=%d squeeze=n icpu=%d ncpu=%d
''' % (split[0], chunk, split[0], skip,
i % self.jobs, self.jobs),
noderotate=stdin)
par_tfiles = []
for j in range(len(tfiles)):
tfile = tfiles[j]
par_tfile = tfile + '__' + str(i)
par_tfiles.append(par_tfile)
par_targets[tfile].append(par_tfile)
par_sfiles0 = copy.copy(par_sfiles)
if i >= self.jobs:
par_sfiles0.append(tfile + '__' + str(i % self.jobs))
# operation on one chunk
self.Flow(par_tfiles, par_sfiles0, cflow, stdout, stdin, 1, suffix,
prefix, src_suffix)
# Reduce parallel TARGETS down to original TARGETS:
for tfile in tfiles:
self.Flow(tfile,
par_targets[tfile],
'%s ${SOURCES[1:%d]}' % (reduction, jobs),
local=1)
def __Split(self,split,reduction,
sfiles,tfiles,flow,stdout,stdin,jobmult,suffix,prefix,src_suffix):
'''Split jobs for pscons'''
if self.jobs < split[1]:
jobs = self.jobs*jobmult
w = int(1+float(split[1])/jobs) # length of one chunk
else:
jobs = split[1]
w = 1
jobmult = 1
par_sfiles = copy.copy(sfiles)
par_targets = {}
for tfile in tfiles:
par_targets[tfile] = []
prev_par_tfiles = []
bigjobs = split[1] - jobs*(w-1)
cflow = flow
for i in range(jobs):
if i < bigjobs:
chunk=w
skip=i*w
else:
chunk=w-1
skip=bigjobs*w+(i-bigjobs)*chunk
for j in split[2]:
if 0 == j and stdin and \
flow.rfind('$SOURCE') < 0 and \
flow.rfind('${SOURCES[0]}') < 0:
# For the first input (if stdin), use windowing
# to avoid creation of a chunk file
par_sfiles[j] = sfiles[j]
cflow = '''
window n%d=%d f%d=%d squeeze=n icpu=%d ncpu=%d |
''' % (split[0],chunk,split[0],skip,
i%self.jobs,self.jobs) + flow
else:
source = sfiles[j] + '_' + str(i)
par_sfiles[j] = source
self.Flow(source,sfiles[j],
'''
window n%d=%d f%d=%d squeeze=n icpu=%d ncpu=%d
''' % (split[0],chunk,split[0],skip,
i%self.jobs,self.jobs),
noderotate=stdin)
par_tfiles = []
for j in range(len(tfiles)):
tfile = tfiles[j]
par_tfile = tfile + '__' + str(i)
par_tfiles.append(par_tfile)
par_targets[tfile].append(par_tfile)
par_sfiles0 = copy.copy(par_sfiles)
if i >= self.jobs:
par_sfiles0.append (tfile + '__' + str(i % self.jobs))
# operation on one chunk
self.Flow(par_tfiles,par_sfiles0,cflow,
stdout,stdin,1,
suffix,prefix,src_suffix)
# Reduce parallel TARGETS down to original TARGETS:
for tfile in tfiles:
self.Flow(tfile,par_targets[tfile],
'%s ${SOURCES[1:%d]}' % (reduction,jobs),
local=1)
