def toTemplateString(self):
tempstr="job_begin\n";
if self.name is not None:
tempstr+="\tname "+self.name+"\n"
else:
raise PipelineError("[PipelineClusterJob] Attempted to produce template string with no defined name!")
if self.memory is not None:
tempstr+="\tmemory "+self.memory+"\n"
if self.queue is not None:
tempstr+="\tqueue "+self.queue+"\n"
if self.module is not None:
tempstr+="\tmodule "+self.module+"\n"
if self.directory is not None:
tempstr+="\tdirectory "+self.directory+"\n"
if self.status is not None:
tempstr+="\tstatus "+self.status+"\n"
# print("type of self.cmd: %s" % (self.cmd.__class__.__name__))
# print("# of commands: %d" % len(self.cmd))
if len(self.cmd) <= 0:
raise PipelineError("[PipelineClusterJob] Attempted to produce template string with no defined commands!")
elif len(self.cmd) == 1:
#TODO make these paths come from settings
tempstr+="\tcmd %s \"%s\" \n" %(BiotoolsSettings.getValue("HANDLER_SCRIPT"),self.cmd[0])
else:
tempstr+="\tcmd_begin\n"
for cmd in self.cmd:
#TODO find out how multi-line commands work, & whether you can run these with the runwithenv commands
tempstr+="\t\t"+cmd+"\n"
tempstr+="\tcmd_end\n"
tempstr+="job_end\n"
if len(self.order_after) > 0:
for prior in self.order_after:
tempstr+="order "+self.getFullName()+" after "+prior+"\n"
return tempstr
def replaceVars(instr,subjob,grouplbl,cumsuffix,prefix,prefix2=None):
#TODO: fill in stub
errors=[]
if instr is None:
errors.append("instr is None")
if subjob is None:
errors.append("subjob is None")
if grouplbl is None:
errors.append("grouplbl is None")
if cumsuffix is None:
errors.append("cumsuffix is None")
if prefix is None:
errors.append("prefix is None")
if subjob.parent.isCrossJob and (prefix2 is None):
errors.append("prefix2 is None for crossjob")
if len(errors) != 0:
raise PipelineError("[Pipeline.PipelineUtil.replaceVars] " + (", ".join(errors)))
retstr=instr
for var in subjob.parent.vars:
search=var
replace=subjob.parent.vars[var]
search=re.sub(r'\$','\\\$',search)
#print ("replacing %s with %s" %(search,replace))
retstr=re.sub(search,replace,retstr)
#replace ADJPREFIX with $PREFIX$CUMSUFFIX - totally for convenience,
#can still use $CUMSUFFIX directly, for input files that need it
if not subjob.parent.clearsuffixes:
retstr=re.sub('\$ADJPREFIX','$PREFIX$CUMSUFFIX',retstr)
else:
retstr=re.sub('\$ADJPREFIX','$PREFIX',retstr)
retstr=re.sub('\$CUMSUFFIX',cumsuffix,retstr)
retstr=re.sub('\$SUFFIX',subjob.parent.suffix,retstr);
for key in BiotoolsSettings.getKeyList():
search='\$'+key
replace=BiotoolsSettings.getValue(key)
#TODO: this is for testing
if sys.platform == 'win32':
replace=re.sub("\\\\","\\\\\\\\",replace)
#print("replacing %s with %s" % (search,replace))
retstr=re.sub(search,replace,retstr);
retstr=re.sub('\$GROUPLBL',grouplbl,retstr);
retstr=re.sub('\$CUMSUFFIX',cumsuffix,retstr);
retstr=re.sub('\$PREFIX',prefix,retstr);
if subjob.parent.isCrossJob:
retstr=re.sub('\$PREFIX2',prefix2,retstr);
return retstr
def toSJMStrings(self,splitOpts,baseName,grouplbl):
sjm_strings={}
#get name of string content should be added to:
nodeQueue=[]
cumsuffixQueue=[]
#set starting nodes
for item in self.getSourceNodes():
nodeQueue.append(item)
cumsuffixQueue.append("")
while len(nodeQueue) > 0:
node=nodeQueue.pop(0)
cumsuffix=cumsuffixQueue.pop(0)
#if job is comparing pairs of samples,
if node.template.isCrossJob:
raise PipelineError("[PipelineTemplate.AnalysisPipeline.toSJMStrings] CrossJob Translation not yet implemented")
#handle selected pairs
#TODO, what to do with no selection (missing optionfile)? Fail or do ALL pairwise?
else:
#otherwise translate template once per file
for sample in self.samples.keys():
Sample=self.samples[sample]
stringName=self.getFileNameForString(splitOpts,baseName, node, Sample)
if not (sjm_strings.has_key(stringName)):
sjm_strings[stringName]=""
parentNode=self.getParentOfNode(node)
#TODO get template string & append it to sjm_strings[stringName]
sjm_strings[stringName]+=node.template.toString(grouplbl,cumsuffix,Sample.ID)
#TODO add any extra link-related job dependencies manually
if not (splitOpts['step']):
#TODO link across templates, link back to parents
derp=""
if parentNode is not None:
print("%s <<< %s | %s <- %s | % s : %s" % (stringName, node.template.name,node.subname,parentNode.template.name,parentNode.subname, Sample.ID))
else:
print("%s <<< %s | %s : %s" % (stringName, node.template.name,node.subname,Sample.ID))
for item in self.getTargetsOfNode(node):
nodeQueue.append(item)
if node.template.clearsuffixes:
cumsuffixQueue.append("")
else:
cumsuffixQueue.append(cumsuffix+node.template.suffix)
#TODO add log_dir line to strings
logdir=BiotoolsSettings.getValue("CURDIR")+os.sep+"sjm_logs"
if os.path.exists(logdir):
if not (os.path.isdir(logdir)):
raise PipelineError("[PipelineTemplate.AnalysisPipeline.toSJMStrings] log directory path: %s already exists and is not a directory" % logdir)
else:
os.mkdir(logdir)
if not (os.path.isdir(logdir)):
raise PipelineError("[PipelineTemplate.AnalysisPipeline.toSJMStrings] failed to create log directory")
for sjm in sjm_strings.keys():
sjm_strings[sjm]=sjm_strings.get(sjm)+"log_dir %s" % logdir
return sjm_strings
def parseSubJob(subjobopt,clusterjob):
#subjobvars={};
commasplit=subjobopt.split(",");
for commaItem in commasplit:
eqsplit=commaItem.split("=")
# print("split on equal sign: %s" % eqsplit)
if (len(eqsplit)!=2):
raise PipelineError("[PipelineTemplateGenerator.parseVars] invalid argument syntax! should have 2 elements separated by '=', have: %d" % len(eqsplit));
attrib_name=str(eqsplit[0].strip())
attrib_val=str(eqsplit[1])
# print("attrib_name:")
# print(attrib_name)
# print("attrib_val:")
# print(attrib_val)
if attrib_name == "order_after":
# print("found order_after!!!");
# print("parsing: " + attrib_val);
if ':' in attrib_val:
arr=attrib_val.split(":");
# print("split order after: " + arr);
clusterjob.order_after.append(arr);
else:
# print("order after: " + attrib_val);
clusterjob.order_after.append(attrib_val);
elif attrib_name == "cmd":
# print("found cmd!!!");
# print("split cmd: " + attrib_val);
clusterjob.cmd.append(attrib_val);
else:
# print("found " + attrib_name + " !!!")
setattr(clusterjob, attrib_name, attrib_val)
if clusterjob.module is None:
clusterjob.module=BiotoolsSettings.getValue("MODULEFILE")
if clusterjob.directory is None:
clusterjob.directory=BiotoolsSettings.getValue("CURDIR")
if clusterjob.queue is None:
clusterjob.queue=BiotoolsSettings.getValue("JOBQUEUE")
def templateDir():
#return os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))),"jobtemplates")#get path to this script, get directory name, and go up one level, then append template dir name
return BiotoolsSettings.getValue("SJM_TEMPLATE_DIR")