def Flow(self,target,source,flow,stdout=1,stdin=1,rsfflow=1,
suffix=sfsuffix,prefix=sfprefix,src_suffix=sfsuffix,
split=[],np=1,reduce='cat',jobmult=1,local=0,noderotate=1,
workdir=None,wall=''):
if not flow:
return None
if type(target) is types.ListType:
tfiles = target
else:
tfiles = string.split(target)
if source:
if type(source) is types.ListType:
sfiles = source
else:
sfiles = string.split(source)
else:
sfiles = []
if self.hostname[-15:] == 'tacc.utexas.edu':
mpirun = '%s tacc_affinity' % self.mpirun
else:
mpirun = '%s -np %s' % (self.mpirun,np)
if split:
if len(split) < 2:
split.append(1)
if len(split) < 3:
split.append(range(len(sfiles)))
if reduce.find('axis=') < 0:
reduction = '%s axis=%d' % (reduce,split[0])
else:
reduction = reduce
if split[1] == 'omp' or split[1] == 'mpi':
splitpar = 'split=%d ' % split[0]
if reduce == 'add':
splitpar += ' join=0'
else:
join = re.search('cat\s+axis=(\d)',reduce)
if join:
splitpar += ' join=%s' % join.group(1)
flow = '|'.join(map(lambda x: ' '.join([split[1],splitpar,x]),flow.split('|')))
for k in split[2]:
# par=${SOURCES[k]} -> _par=${SOURCES[k]}
flow = re.sub(r'(\S+=\${SOURCES\[%d\]})' % k,'_\\1',flow)
elif self.jobs > 1 and rsfflow and sfiles:
# Split the flow into parallel flows
self.__Split(split,reduction,
sfiles,tfiles,flow,stdout,stdin,
jobmult,suffix,prefix,src_suffix)
return
sources = []
if sfiles:
for file in sfiles:
if ('.' not in file):
file = file + src_suffix
sources.append(file)
else:
stdin=0
# May need to do it remotely
if local:
node = 'localhost'
else: # get it from the rotating list
node = self.nodes[self.ip]
if noderotate:
self.ip = self.ip + 1
if self.ip == len(self.nodes):
self.ip = 0
if node != 'localhost':
if self.raddenv:
remote = ' '
else:
remote = '%s %s ' % (WhereIs('env'),self.environ)
else:
remote = ''
command = rsf.flow.Flow(sources,flow,self.bindir,rsfflow,
self.checkpar,self.coms,prefix,self.progsuffix,
remote,stdout,stdin,self.timer,mpirun,workdir,
self.batch,np,wall)
# May need to do it remotely
if remote:
command = re.sub('"','\\"',command)
if self.raddenv:
# runonnode cans to use hosts.txt to avoid pscond reruns
#command = string.join([self.runonnode,self.hosts,'\"',self.raddenv,
# '; cd ',self.cwd,';',command,'\"'])
command = string.join([self.runonnode,'\"',self.raddenv,
'; cd ',self.cwd,';',command,'\"'])
else:
# runonnode cans to use hosts.txt to avoid pscond reruns
#command = string.join([self.runonnode,self.hosts,
# '\"cd ',self.cwd,';',command,'\"'])
command = string.join([self.runonnode,
'\"cd ',self.cwd,';',command,'\"'])
targets = []
for file in tfiles:
if (not re.search(suffix + '$',file)) and ('.' not in file):
file = file + suffix
targets.append(file)
if workdir:
command = re.sub(r'\$(SOURCE|TARGET)',r'${\1.abspath}',command)
command = re.sub(r'\$\{(SOURCES|TARGETS)(\[[^\]]+\])\}',r'${\1\2.abspath}',command)
flow = self.Command(targets,sources,command)
if workdir:
Clean(flow,workdir)
if suffix == sfsuffix:
binaries = map(lambda x, self=self: self.path + x + '@',
filter(lambda x, suffix=suffix:
x[-len(suffix):] == suffix,targets))
if binaries:
Clean(flow,binaries)
self.Default(flow)
return flow
def Flow(self,
target,
source,
flow,
stdout=1,
stdin=1,
rsfflow=1,
suffix=sfsuffix,
prefix=sfprefix,
src_suffix=sfsuffix,
split=[],
np=1,
reduce='cat',
jobmult=1,
local=0,
noderotate=1,
workdir=None,
wall=''):
if not flow:
return None
if type(target) is list:
tfiles = target
else:
tfiles = target.split()
if source:
if type(source) is list:
sfiles = source
else:
sfiles = source.split()
else:
sfiles = []
if self.hostname[-15:] == 'tacc.utexas.edu':
mpirun = '%s tacc_affinity' % self.mpirun
else:
mpirun = '%s -np %s' % (self.mpirun, np)
if split:
if len(split) < 2:
split.append(1)
if len(split) < 3:
split.append(list(range(len(sfiles))))
if reduce.find('axis=') < 0:
reduction = '%s axis=%d' % (reduce, split[0])
else:
reduction = reduce
if split[1] == 'omp' or split[1] == 'mpi':
splitpar = 'split=%d ' % split[0]
if reduce == 'add':
splitpar += ' join=0'
else:
join = re.search('cat\s+axis=(\d)', reduce)
if join:
splitpar += ' join=%s' % join.group(1)
flow = '|'.join([
' '.join([split[1], splitpar, x]) for x in flow.split('|')
])
for k in split[2]:
# par=${SOURCES[k]} -> _par=${SOURCES[k]}
flow = re.sub(r'(\S+=\${SOURCES\[%d\]})' % k, '_\\1', flow)
elif self.jobs > 1 and rsfflow and sfiles:
# Split the flow into parallel flows
self.__Split(split, reduction, sfiles, tfiles, flow, stdout,
stdin, jobmult, suffix, prefix, src_suffix)
return
sources = []
if sfiles:
for file in sfiles:
if ('.' not in file):
file = file + src_suffix
sources.append(file)
else:
stdin = 0
# May need to do it remotely
if local:
node = 'localhost'
else: # get it from the rotating list
node = self.nodes[self.ip]
if noderotate:
self.ip = self.ip + 1
if self.ip == len(self.nodes):
self.ip = 0
if node != 'localhost':
remote = '%s $( %s $)' % (WhereIs('env'), self.environ)
else:
remote = ''
command = rsf.flow.Flow(sources, flow, self.bindir, rsfflow,
self.checkpar, self.coms, prefix,
self.progsuffix, remote, stdout, stdin,
self.timer, mpirun, workdir, self.batch, np,
wall)
# May need to do it remotely
if remote:
command = re.sub('"', '\\"', command)
command = ' '.join(
['$( ssh', node, '$) \"cd ', self.cwd, ';', command, '\"'])
targets = []
for file in tfiles:
if (not re.search(suffix + '$', file)) and ('.' not in file):
file = file + suffix
targets.append(file)
if suffix == sfsuffix and re.search('/', targets[0]):
subdir = os.path.dirname(os.path.join(self.path, targets[0]))
rsf.path.mkdir(subdir)
command = command + ' --out=%s' % os.path.join(
self.path, '${TARGET}@')
if workdir:
command = re.sub(r'\$(SOURCE|TARGET)', r'${\1.abspath}', command)
command = re.sub(r'\$\{(SOURCES|TARGETS)(\[[^\]]+\])\}',
r'${\1\2.abspath}', command)
flow = self.Command(targets, sources, command)
if workdir:
Clean(flow, workdir)
if suffix == sfsuffix:
binaries = list(
map(lambda x, self=self: self.path + x + '@',
list(
filter(lambda x, suffix=suffix: x[-len(suffix):] ==
suffix,
targets))))
if binaries:
Clean(flow, binaries)
self.Default(flow)
return flow
def Flow(self,
target,
source,
flow,
stdout=1,
stdin=1,
rsfflow=1,
suffix=sfsuffix,
prefix=sfprefix,
src_suffix=sfsuffix,
split=[],
np=1,
reduce='cat',
local=0,
noderotate=1):
if not flow:
return None
if isinstance(target, basestring):
tfiles = target.split()
else:
tfiles = list(target)
if source:
if isinstance(source, basestring):
sfiles = source.split()
else:
sfiles = list(source)
else:
sfiles = []
mpirun = '%s -np %s' % (self.mpirun, np)
if split:
if len(split) < 2:
split.append(1)
if len(split) < 3:
split.append(range(len(sfiles)))
if reduce.find('axis=') < 0:
reduction = '%s axis=%d' % (reduce, split[0])
else:
reduction = reduce
if split[1] == 'omp' or split[1] == 'mpi':
splitpar = 'split=%d ' % split[0]
if reduce == 'add':
splitpar += ' join=0'
else:
join = re.search('cat\s+axis=(\d)', reduce)
if join:
splitpar += ' join=%s' % join.group(1)
flow = '|'.join([
' '.join([split[1], splitpar, x]) for x in flow.split('|')
])
elif self.jobs > 1 and rsfflow and sfiles:
# Split the flow into parallel flows
self.__Split(split, reduction, sfiles, tfiles, flow, stdout,
stdin, suffix, prefix, src_suffix)
return
sources = []
if sfiles:
for file in sfiles:
if ('.' not in file):
file = file + src_suffix
sources.append(file)
else:
stdin = 0
# May need to do it remotely
if local:
node = 'localhost'
else: # get it from the rotating list
if noderotate:
numtries = 0
while (self.taskonnode[self.ip]
and numtries < 2 * len(self.nodes)):
time.sleep(.01)
self.ip = (self.ip + 1) % len(
self.nodes) # increment and modulo
numtries += 1
if (numtries > len(self.nodes)):
print("having problems finding idle node")
myip = self.ip # local variable remembers the ip being used
self.taskonnode[myip] = True
node = self.nodes[ip]
self.ip = (self.ip + 1) % len(
self.nodes) # increment and modulo
if node != 'localhost':
if self.raddenv:
remote = ' '
else:
remote = '%s %s ' % (WhereIs('env'), self.environ)
else:
remote = ''
command = rsf.flow.Flow(sources, flow, self.bindir, rsfflow,
self.checkpar, self.coms, prefix,
self.progsuffix, remote, stdout, stdin,
self.timer, mpirun)
# May need to do it remotely
if remote:
command = re.sub('"', '\\"', command)
if self.raddenv:
command = ' '.join([
WhereIs('ssh'), node, '\"', self.raddenv, '; cd ',
self.cwd, ';', command, '\"'
])
else:
command = ' '.join([
WhereIs('ssh'), node, '\"cd ', self.cwd, ';', command, '\"'
])
targets = []
for file in tfiles:
if (not re.search(suffix + '$', file)) and ('.' not in file):
file = file + suffix
targets.append(file)
flow = self.Command(targets, sources, command)
# free up the node for reuse
if not local:
if noderotate:
self.taskonnode[myip] = FALSE
if suffix == sfsuffix:
binaries = list(
map(lambda x, self=self: self.path + x + '@',
list(
filter(lambda x, suffix=suffix: x[-len(suffix):] ==
suffix,
targets))))
if binaries:
Clean(flow, binaries)
self.Default(flow)
return flow
