def Flow(self,target,source,flow,stdout=1,stdin=1,rsfflow=1, suffix=sfsuffix,prefix=sfprefix,src_suffix=sfsuffix, split=[],np=1,reduce='cat',jobmult=1,local=0,noderotate=1, workdir=None,wall=''): if not flow: return None if type(target) is types.ListType: tfiles = target else: tfiles = string.split(target) if source: if type(source) is types.ListType: sfiles = source else: sfiles = string.split(source) else: sfiles = [] if self.hostname[-15:] == 'tacc.utexas.edu': mpirun = '%s tacc_affinity' % self.mpirun else: mpirun = '%s -np %s' % (self.mpirun,np) if split: if len(split) < 2: split.append(1) if len(split) < 3: split.append(range(len(sfiles))) if reduce.find('axis=') < 0: reduction = '%s axis=%d' % (reduce,split[0]) else: reduction = reduce if split[1] == 'omp' or split[1] == 'mpi': splitpar = 'split=%d ' % split[0] if reduce == 'add': splitpar += ' join=0' else: join = re.search('cat\s+axis=(\d)',reduce) if join: splitpar += ' join=%s' % join.group(1) flow = '|'.join(map(lambda x: ' '.join([split[1],splitpar,x]),flow.split('|'))) for k in split[2]: # par=${SOURCES[k]} -> _par=${SOURCES[k]} flow = re.sub(r'(\S+=\${SOURCES\[%d\]})' % k,'_\\1',flow) elif self.jobs > 1 and rsfflow and sfiles: # Split the flow into parallel flows self.__Split(split,reduction, sfiles,tfiles,flow,stdout,stdin, jobmult,suffix,prefix,src_suffix) return sources = [] if sfiles: for file in sfiles: if ('.' not in file): file = file + src_suffix sources.append(file) else: stdin=0 # May need to do it remotely if local: node = 'localhost' else: # get it from the rotating list node = self.nodes[self.ip] if noderotate: self.ip = self.ip + 1 if self.ip == len(self.nodes): self.ip = 0 if node != 'localhost': if self.raddenv: remote = ' ' else: remote = '%s %s ' % (WhereIs('env'),self.environ) else: remote = '' command = rsf.flow.Flow(sources,flow,self.bindir,rsfflow, self.checkpar,self.coms,prefix,self.progsuffix, remote,stdout,stdin,self.timer,mpirun,workdir, self.batch,np,wall) # May need to do it remotely if remote: command = re.sub('"','\\"',command) if self.raddenv: # runonnode cans to use hosts.txt to avoid pscond reruns #command = string.join([self.runonnode,self.hosts,'\"',self.raddenv, # '; cd ',self.cwd,';',command,'\"']) command = string.join([self.runonnode,'\"',self.raddenv, '; cd ',self.cwd,';',command,'\"']) else: # runonnode cans to use hosts.txt to avoid pscond reruns #command = string.join([self.runonnode,self.hosts, # '\"cd ',self.cwd,';',command,'\"']) command = string.join([self.runonnode, '\"cd ',self.cwd,';',command,'\"']) targets = [] for file in tfiles: if (not re.search(suffix + '$',file)) and ('.' not in file): file = file + suffix targets.append(file) if workdir: command = re.sub(r'\$(SOURCE|TARGET)',r'${\1.abspath}',command) command = re.sub(r'\$\{(SOURCES|TARGETS)(\[[^\]]+\])\}',r'${\1\2.abspath}',command) flow = self.Command(targets,sources,command) if workdir: Clean(flow,workdir) if suffix == sfsuffix: binaries = map(lambda x, self=self: self.path + x + '@', filter(lambda x, suffix=suffix: x[-len(suffix):] == suffix,targets)) if binaries: Clean(flow,binaries) self.Default(flow) return flow
def Flow(self, target, source, flow, stdout=1, stdin=1, rsfflow=1, suffix=sfsuffix, prefix=sfprefix, src_suffix=sfsuffix, split=[], np=1, reduce='cat', jobmult=1, local=0, noderotate=1, workdir=None, wall=''): if not flow: return None if type(target) is list: tfiles = target else: tfiles = target.split() if source: if type(source) is list: sfiles = source else: sfiles = source.split() else: sfiles = [] if self.hostname[-15:] == 'tacc.utexas.edu': mpirun = '%s tacc_affinity' % self.mpirun else: mpirun = '%s -np %s' % (self.mpirun, np) if split: if len(split) < 2: split.append(1) if len(split) < 3: split.append(list(range(len(sfiles)))) if reduce.find('axis=') < 0: reduction = '%s axis=%d' % (reduce, split[0]) else: reduction = reduce if split[1] == 'omp' or split[1] == 'mpi': splitpar = 'split=%d ' % split[0] if reduce == 'add': splitpar += ' join=0' else: join = re.search('cat\s+axis=(\d)', reduce) if join: splitpar += ' join=%s' % join.group(1) flow = '|'.join([ ' '.join([split[1], splitpar, x]) for x in flow.split('|') ]) for k in split[2]: # par=${SOURCES[k]} -> _par=${SOURCES[k]} flow = re.sub(r'(\S+=\${SOURCES\[%d\]})' % k, '_\\1', flow) elif self.jobs > 1 and rsfflow and sfiles: # Split the flow into parallel flows self.__Split(split, reduction, sfiles, tfiles, flow, stdout, stdin, jobmult, suffix, prefix, src_suffix) return sources = [] if sfiles: for file in sfiles: if ('.' not in file): file = file + src_suffix sources.append(file) else: stdin = 0 # May need to do it remotely if local: node = 'localhost' else: # get it from the rotating list node = self.nodes[self.ip] if noderotate: self.ip = self.ip + 1 if self.ip == len(self.nodes): self.ip = 0 if node != 'localhost': remote = '%s $( %s $)' % (WhereIs('env'), self.environ) else: remote = '' command = rsf.flow.Flow(sources, flow, self.bindir, rsfflow, self.checkpar, self.coms, prefix, self.progsuffix, remote, stdout, stdin, self.timer, mpirun, workdir, self.batch, np, wall) # May need to do it remotely if remote: command = re.sub('"', '\\"', command) command = ' '.join( ['$( ssh', node, '$) \"cd ', self.cwd, ';', command, '\"']) targets = [] for file in tfiles: if (not re.search(suffix + '$', file)) and ('.' not in file): file = file + suffix targets.append(file) if suffix == sfsuffix and re.search('/', targets[0]): subdir = os.path.dirname(os.path.join(self.path, targets[0])) rsf.path.mkdir(subdir) command = command + ' --out=%s' % os.path.join( self.path, '${TARGET}@') if workdir: command = re.sub(r'\$(SOURCE|TARGET)', r'${\1.abspath}', command) command = re.sub(r'\$\{(SOURCES|TARGETS)(\[[^\]]+\])\}', r'${\1\2.abspath}', command) flow = self.Command(targets, sources, command) if workdir: Clean(flow, workdir) if suffix == sfsuffix: binaries = list( map(lambda x, self=self: self.path + x + '@', list( filter(lambda x, suffix=suffix: x[-len(suffix):] == suffix, targets)))) if binaries: Clean(flow, binaries) self.Default(flow) return flow
def Flow(self, target, source, flow, stdout=1, stdin=1, rsfflow=1, suffix=sfsuffix, prefix=sfprefix, src_suffix=sfsuffix, split=[], np=1, reduce='cat', local=0, noderotate=1): if not flow: return None if isinstance(target, basestring): tfiles = target.split() else: tfiles = list(target) if source: if isinstance(source, basestring): sfiles = source.split() else: sfiles = list(source) else: sfiles = [] mpirun = '%s -np %s' % (self.mpirun, np) if split: if len(split) < 2: split.append(1) if len(split) < 3: split.append(range(len(sfiles))) if reduce.find('axis=') < 0: reduction = '%s axis=%d' % (reduce, split[0]) else: reduction = reduce if split[1] == 'omp' or split[1] == 'mpi': splitpar = 'split=%d ' % split[0] if reduce == 'add': splitpar += ' join=0' else: join = re.search('cat\s+axis=(\d)', reduce) if join: splitpar += ' join=%s' % join.group(1) flow = '|'.join([ ' '.join([split[1], splitpar, x]) for x in flow.split('|') ]) elif self.jobs > 1 and rsfflow and sfiles: # Split the flow into parallel flows self.__Split(split, reduction, sfiles, tfiles, flow, stdout, stdin, suffix, prefix, src_suffix) return sources = [] if sfiles: for file in sfiles: if ('.' not in file): file = file + src_suffix sources.append(file) else: stdin = 0 # May need to do it remotely if local: node = 'localhost' else: # get it from the rotating list if noderotate: numtries = 0 while (self.taskonnode[self.ip] and numtries < 2 * len(self.nodes)): time.sleep(.01) self.ip = (self.ip + 1) % len( self.nodes) # increment and modulo numtries += 1 if (numtries > len(self.nodes)): print("having problems finding idle node") myip = self.ip # local variable remembers the ip being used self.taskonnode[myip] = True node = self.nodes[ip] self.ip = (self.ip + 1) % len( self.nodes) # increment and modulo if node != 'localhost': if self.raddenv: remote = ' ' else: remote = '%s %s ' % (WhereIs('env'), self.environ) else: remote = '' command = rsf.flow.Flow(sources, flow, self.bindir, rsfflow, self.checkpar, self.coms, prefix, self.progsuffix, remote, stdout, stdin, self.timer, mpirun) # May need to do it remotely if remote: command = re.sub('"', '\\"', command) if self.raddenv: command = ' '.join([ WhereIs('ssh'), node, '\"', self.raddenv, '; cd ', self.cwd, ';', command, '\"' ]) else: command = ' '.join([ WhereIs('ssh'), node, '\"cd ', self.cwd, ';', command, '\"' ]) targets = [] for file in tfiles: if (not re.search(suffix + '$', file)) and ('.' not in file): file = file + suffix targets.append(file) flow = self.Command(targets, sources, command) # free up the node for reuse if not local: if noderotate: self.taskonnode[myip] = FALSE if suffix == sfsuffix: binaries = list( map(lambda x, self=self: self.path + x + '@', list( filter(lambda x, suffix=suffix: x[-len(suffix):] == suffix, targets)))) if binaries: Clean(flow, binaries) self.Default(flow) return flow