def runPipelineWithConfig(taskName, name, pipeline, conf, queue):
"""
This is for internal use only
"""
templateDir = os.path.join(conf("dirs.clovr_pipelines_template_dir"), pipeline.TEMPLATE_NAME)
templateConfig = os.path.join(templateDir, "pipeline_tmpl.config")
templateLayout = os.path.join(templateDir, "pipeline.layout")
foutName = os.path.join("/tmp", str(time.time()))
fout = open(foutName, "w")
for line in handleIncludes(open(templateConfig)):
fout.write(config.replaceStr(line, conf) + "\n")
fout.close()
cmd = ["run_pipeline.pl", "--config=" + foutName, "--templatelayout=" + templateLayout, "--taskname=" + taskName]
if queue:
cmd.append("--queue=" + queue)
cmd = " ".join(cmd)
res = []
exitCode = runSingleProgram(cmd, res.append, None)
##
# If we got a weird exit code or more than one line was print or nothing was printed
# then something bad happened
if exitCode != 0 or len(res) > 1 or not res:
raise ProgramRunError(cmd, exitCode)
##
# This should be the pipeline ID
return createPipeline(taskName, name, res[0].strip(), pipeline, conf)
def resumePipeline(pipeline, queue=None):
cmd = ["resume_pipeline.pl", "--pipeline_id=" + pipeline.pid, "--taskname=" + pipeline.taskName]
if queue:
cmd.append("--queue=" + queue)
cmd = " ".join(cmd)
res = []
exitCode = runSingleProgram(cmd, res.append, None)
##
# If we got a weird exit code or more than one line was print or nothing was printed
# then something bad happened
if exitCode != 0 or len(res) > 1 or not res:
raise ProgramRunError(cmd, exitCode)
return pipeline
def resumePipeline(pipeline, queue=None):
cmd = [
'resume_pipeline.pl', '--pipeline_id=' + pipeline.pid,
'--taskname=' + pipeline.taskName
]
if queue:
cmd.append('--queue=' + queue)
cmd = ' '.join(cmd)
res = []
exitCode = runSingleProgram(cmd, res.append, None)
##
# If we got a weird exit code or more than one line was print or nothing was printed
# then something bad happened
if exitCode != 0 or len(res) > 1 or not res:
raise ProgramRunError(cmd, exitCode)
return pipeline
def runPipelineWithConfig(taskName, name, pipeline, conf, queue):
"""
This is for internal use only
"""
templateDir = os.path.join(conf('dirs.clovr_pipelines_template_dir'),
pipeline.TEMPLATE_NAME)
templateConfig = os.path.join(templateDir, 'pipeline_tmpl.config')
templateLayout = os.path.join(templateDir, 'pipeline.layout')
foutName = os.path.join('/tmp', str(time.time()))
fout = open(foutName, 'w')
for line in handleIncludes(open(templateConfig)):
fout.write(config.replaceStr(line, conf) + '\n')
fout.close()
cmd = [
'run_pipeline.pl', '--config=' + foutName,
'--templatelayout=' + templateLayout, '--taskname=' + taskName
]
if queue:
cmd.append('--queue=' + queue)
cmd = ' '.join(cmd)
res = []
exitCode = runSingleProgram(cmd, res.append, None)
##
# If we got a weird exit code or more than one line was print or nothing was printed
# then something bad happened
if exitCode != 0 or len(res) > 1 or not res:
raise ProgramRunError(cmd, exitCode)
##
# This should be the pipeline ID
return createPipeline(taskName, name, res[0].strip(), pipeline, conf)
def runSingleProgramEx(conf, cmd, stdoutf, stderrf):
cmd = replaceStr(cmd, conf)
exitCode = runSingleProgram(cmd, stdoutf, stderrf)
if exitCode != 0:
raise ProgramRunError(cmd, exitCode)
def runSingleProgramEx(conf, cmd, stdoutf, stderrf):
cmd = replaceStr(cmd, conf)
exitCode = runSingleProgram(cmd, stdoutf, stderrf)
if exitCode != 0:
raise ProgramRunError(cmd, exitCode)
def runSystemEx(cmd):
"""This just ignores all stdout"""
code = runSingleProgram(cmd, None, errorPrintS)
if code != 0:
raise ProgramRunError(cmd, code)
def runSystemEx(cmd):
"""This just ignores all stdout"""
code = runSingleProgram(cmd, None, errorPrintS)
if code != 0:
raise ProgramRunError(cmd, code)