def createJob(workflow, cmssw_ver, arch):
workflow_args = FixWFArgs(cmssw_ver, arch, workflow, GetMatrixOptions(cmssw_ver, arch))
cmd = format("rm -rf %(workflow)s %(workflow)s_*; mkdir %(workflow)s; cd %(workflow)s; PATH=%(das_utils)s:$PATH runTheMatrix.py --maxSteps=0 -l %(workflow)s %(workflow_args)s",workflow=workflow,workflow_args=workflow_args, das_utils=CMS_BOT_DIR+"/das-utils")
print("Running ",cmd)
e, o = run_cmd(cmd)
if e: print("ERROR:%s:%s" % (workflow, o))
try:
workflow_dir = glob.glob(format("%(workflow)s/%(workflow)s_*", workflow=workflow))[0]
run_cmd(format("mv %(workflow)s/runall-report-step123-.log %(workflow_dir)s/workflow.log; touch %(workflow_dir)s/cmdLog; mv %(workflow_dir)s .; rm -rf %(workflow)s", workflow=workflow, workflow_dir=workflow_dir))
print("Commands for workflow %s generated" % workflow)
except Exception as e:
print("ERROR: Creating workflow job:",workflow,str(e))
run_cmd("rm -rf %s %s_*" % (workflow,workflow))
def runStep1Only(basedir, workflow, args=''):
args = FixWFArgs (os.environ["CMSSW_VERSION"],os.environ["SCRAM_ARCH"],workflow,args)
workdir = os.path.join(basedir, workflow)
matrixCmd = 'runTheMatrix.py --maxSteps=0 -l ' + workflow +' '+args
try:
if not os.path.isdir(workdir):
os.makedirs(workdir)
except Exception as e:
print("runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : can't create thread folder: " + str(e))
try:
ret = doCmd(matrixCmd, False, workdir)
except Exception as e:
print("runPyRelVal> ERROR during test PyReleaseValidation steps, workflow "+str(workflow)+" : caught exception: " + str(e))
return
def runThreadMatrix(basedir, workflow, args='', logger=None, force=False):
if (not force) and logger and logger.relvalAlreadyDone(workflow):
print "Message>> Not ruuning workflow ", workflow, " as it is already ran"
return
args = FixWFArgs(os.environ["CMSSW_VERSION"], os.environ["SCRAM_ARCH"],
workflow, args)
workdir = os.path.join(basedir, workflow)
matrixCmd = 'runTheMatrix.py -l ' + workflow + ' ' + args
try:
if not os.path.isdir(workdir):
os.makedirs(workdir)
except Exception, e:
print "runPyRelVal> ERROR during test PyReleaseValidation, workflow " + str(
workflow) + " : can't create thread folder: " + str(e)
def runThreadMatrix(basedir, workflow, args='', logger=None, wf_err={}):
args = FixWFArgs(os.environ["CMSSW_VERSION"], os.environ["SCRAM_ARCH"],
workflow, args)
workdir = os.path.join(basedir, workflow)
matrixCmd = 'runTheMatrix.py -l ' + workflow + ' ' + args
try:
if not os.path.isdir(workdir):
os.makedirs(workdir)
except Exception as e:
print("runPyRelVal> ERROR during test PyReleaseValidation, workflow " +
str(workflow) + " : can't create thread folder: " + str(e))
wftime = time.time()
try:
ret = doCmd(matrixCmd, False, workdir)
except Exception as e:
print("runPyRelVal> ERROR during test PyReleaseValidation, workflow " +
str(workflow) + " : caught exception: " + str(e))
wftime = time.time() - wftime
outfolders = [
file for file in os.listdir(workdir)
if re.match("^" + str(workflow) + "_", file)
]
if len(outfolders) == 0: return
outfolder = os.path.join(basedir, outfolders[0])
wfdir = os.path.join(workdir, outfolders[0])
ret = doCmd("rm -rf " + outfolder + "; mkdir -p " + outfolder)
ret = doCmd(
"find . -mindepth 1 -maxdepth 1 -name '*.xml' -o -name '*.log' -o -name '*.py' -o -name '*.json' -o -name 'cmdLog' -type f | xargs -i mv '{}' "
+ outfolder + "/", False, wfdir)
logRE = re.compile('^(.*/[0-9]+(\.[0-9]+|)_([^/]+))/step1_dasquery.log$')
for logFile in glob.glob(outfolder + "/step1_dasquery.log"):
m = logRE.match(logFile)
if not m: continue
ret = doCmd("cp " + logFile + " " + m.group(1) + "/step1_" +
m.group(3) + ".log")
ret = doCmd("mv " + os.path.join(workdir, "runall-report-step*.log") +
" " + os.path.join(outfolder, "workflow.log"))
ret = doCmd("echo " + str(wftime) + " > " +
os.path.join(outfolder, "time.log"))
ret = doCmd("hostname -s > " + os.path.join(outfolder, "hostname"))
if wf_err: json.dump(wf_err, open("%s/known_error.json" % outfolder, "w"))
if logger: logger.updateRelValMatrixPartialLogs(basedir, outfolders[0])
shutil.rmtree(workdir)
return