def validateFile(validationCommands,fileList): valid = [] if not isinstance(fileList,list): fileList = list(fileList) for f in fileList: ext = os.path.splitext(f)[1] if ext in validationCommand: command += validationCommand[ext] + " %s" % f try: cF.callProcess(command) valid.append(True) except subprocess.CalledProcessError: valid.append(False) else: valid.append(False) return valid
def generate(self): # Parent config dict is none configDicts = [None] if self.cCluster.jobIdxParent >= self.cCluster.nJobs-1: raise ValueError("jobIdxParent=%i not feasible, since you generate %i jobs from idx 0 till %i !" % (self.cCluster.jobIdxParent,self.cCluster.nJobs, self.cCluster.nJobs-1) ) for jobIdx in range(0,self.cCluster.nJobs): print("Generating MPI Job: SimpleExecutable =================") print("-> JobIndex: %i" % jobIdx + (" (not files because parent job!)" if jobIdx <= self.cCluster.jobIdxParent else "") ) # first make a new self.config self.config = self.makeInterpolationConfig(); # setting the jobIdx self.config["Job"]["jobIdx"] = str(jobIdx) self.config["Job"]["submitCommand"] = " ".join([self.config["Cluster"]["submitCommand"] , (configDicts[-1].Cluster.submitArgsChainJob if jobIdx != 0 else "") , self.config["Job"]["submitArgs"]]) # final interpolation of all automatic generated options and conversion to self.configDict # and checking of feasible values before template writting # from now self.cCluster , self.cJob and are available (reference to self.configDict) self.convertValues() # only generate files for the job for jobIndices which are greater then jobIdxParent # jobIdxParent is the parent job, which we base our nJobs on if jobIdx > self.cCluster.jobIdxParent : self.checkConfig(interact=self.cCluster.interact) self.printOptions() # make job script dir (if exists, try to remove it, if no -> abort) cf.makeDirectory( self.cJob.scriptDir, name="Job script dir", defaultMakeEmpty=False, interact= self.cCluster.interact) self.writeJobScriptArgs( os.path.join(self.cJob.scriptDir, "submitScriptArgs.txt" ) ) # write all templates self.writeTemplates(self, configDicts); # check if we submit the job if self.cCluster.submitJobs: print("Trying to submit job with command: \n%s" % self.cJob.submitCommand) cf.callProcess(self.cJob.submitCommand) #end if # save conficDict for next jobIdx (possibly) configDicts.append(self.configDict) print("==========================================================") # Write total submit file to first folder config0 = configDicts[1]; filePath = os.path.join(config0.Job.scriptDir,"submitAll.sh") f = open(filePath,"w+") commands = []; for c in configDicts: if c: commands.append(c.Job.submitCommand) f.write("\n".join(commands)) cf.makeExecutable(filePath);
def generate(self): configDicts = [None] if self.cCluster.jobIdxParent >= self.cCluster.nJobs-1: raise CE.MyValueError("jobIdxParent=%i not feasible, since you generate %i jobs from idx 0 till %i !" % (self.cCluster.jobIdxParent,self.cCluster.nJobs, self.cCluster.nJobs-1) ) for jobIdx in range(0,self.cCluster.nJobs): CE.printHeader("Generating MPI Job: Tool Pipeline ========================") CE.printKeyMessage("JobIndex","%i" % jobIdx + (" (no generating, because parent job!)" if jobIdx <= self.cCluster.jobIdxParent else "") ) # first make a new self.config self.config = self.makeInterpolationConfig(); # setting the jobIdx self.config["Job"]["jobIdx"] = str(jobIdx) self.config["Job"]["submitCommand"] = " ".join([self.config["Cluster"]["submitCommand"] , (configDicts[-1].Cluster.submitArgsChainJob if jobIdx > self.cCluster.jobIdxParent+1 else "") , self.config["Job"]["submitArgs"]]) if configDicts[-1]: self.config["Pipeline-PreProcess"]["validationInfoFile"] = configDicts[-1]["Pipeline-PostProcess"]["validationInfoFile"] else: self.config["Pipeline-PreProcess"]["validationInfoFile"] = "" # final interpolation of all automatic generated options and conversion to self.configDict # and checking of feasible values before template writting # from now self.cCluster , self.cJob and self.cRigidBodySim are available (reference to self.configDict) self.convertValues() # only make jobs which are greater then parent! if jobIdx > self.cCluster.jobIdxParent: self.checkConfig(interact=self.cCluster.interact) self.printOptions() # make job script dir (if exists, try to remove it, if no -> abort) cF.makeDirectory( self.cJob.scriptDir, name="Job script dir", defaultMakeEmpty=False, interact= self.cCluster.interact) self.writeJobScriptArgs( os.path.join(self.cJob.scriptDir, "configureScriptArgs.txt" ) ) # write all templates self.writeTemplates(self, configDicts); # check if we submit command # overwrite submit command by appending all submit args if self.cCluster.submitJobs: print("Trying to submit job with command: \n%s" % self.cJob.submitCommand) cF.callProcess(self.cJob.submitCommand) # save conficDict for next jobIdx (possibly) # TODO save config dict in job script folder (json file) configDicts.append(self.configDict) CE.printHeader("==========================================================") # Write total submit file to first folder config0 = configDicts[self.cCluster.jobIdxParent+2]; filePath = os.path.join(config0.Job.scriptDir,"submitAll.sh") f = open(filePath,"w+") commands = []; for c in configDicts[self.cCluster.jobIdxParent+2:]: if c: commands.append(c.Job.submitCommand) f.write("\n".join(commands)) cF.makeExecutable(filePath);