def processScheduledQARequest(requestId, ibName, arch, buildpath=DEFAULT_BUILDPATH, cmspath=DEFAULT_CMSPATH, tests=DEFAULT_TESTS): cycle = getCycleNameByReleaseName(ibName) os.environ["SCRAM_ARCH"]=arch os.environ["CMSINTBLD_CMS_PATH"]=cmspath tagCollectorAPI.setRequestBuilding(requestId, ibName,machine=socket.gethostname(), pid=os.getpid()) startTest(ibName, cycle, buildpath, tests, arch) tagCollectorAPI.finishRequest(requestId)
def setStatusBuilding(self, rel):
if self.doDeBuG: print 'setting status to building for', self.id, rel
tc.setRequestBuilding(request_id=self.id,
release_name=rel,
machine=socket.gethostname(),
pid=os.getpid())
sys.stdout.flush()
sys.stderr.flush()
def processScheduledQARequest(requestId, ibName, arch, buildpath=DEFAULT_BUILDPATH, cmspath=DEFAULT_CMSPATH, tests=DEFAULT_TESTS):
cycle = getCycleNameByReleaseName(ibName)
os.environ["SCRAM_ARCH"]=arch
os.environ["CMSINTBLD_CMS_PATH"]=cmspath
tagCollectorAPI.setRequestBuilding(requestId, ibName,machine=socket.gethostname(), pid=os.getpid())
qaStampDir= os.path.join("/afs/cern.ch/cms/sw/ReleaseCandidates",arch,"www/qaTest")
qaStampFile = os.path.join(qaStampDir,ibName)
if (not os.path.exists(qaStampFile)):
os.system("mkdir -p %s; touch %s" % (qaStampDir,qaStampFile))
startTest(ibName, cycle, buildpath, tests, arch)
tagCollectorAPI.finishRequest(requestId)
def test_build_and_fail_ib(self):
ibData = tagCollectorAPI.getIBPendingRequests()
self.assertTrue(ibData)
tagCollectorAPI.setRequestBuilding(
ibData["id"], ibData["release_name"], machine=socket.gethostname(), pid=os.getpid()
)
ibBuildingRequests = tagCollectorAPI.getBuildingIBRequests()
self.assertTrue(ibBuildingRequests)
self.assertEquals(self.release_name, ibBuildingRequests[0]["release_name"])
self.assertEquals(self.architecture_name, ibBuildingRequests[0]["architecture_name"])
self.assertEquals("IB", ibBuildingRequests[0]["type"])
isFinished = tagCollectorAPI.failRequest(ibBuildingRequests[0]["id"])
self.assertTrue(isFinished)
def test_build_and_finish_qa(self):
qaData = tagCollectorAPI.getQAPendingRequests()
self.assertTrue(qaData)
tagCollectorAPI.setRequestBuilding(
qaData["id"], qaData["release_name"], machine=socket.gethostname(), pid=os.getpid()
)
qaBuildingRequests = tagCollectorAPI.getBuildingQARequests()
self.assertTrue(qaBuildingRequests)
self.assertEquals(self.release_name, qaBuildingRequests[0]["release_name"])
self.assertEquals(self.architecture_name, qaBuildingRequests[0]["architecture_name"])
self.assertEquals("QA", qaBuildingRequests[0]["type"])
isFinished = tagCollectorAPI.finishRequest(qaBuildingRequests[0]["id"])
self.assertTrue(isFinished)
def processScheduledQARequest(requestId,
ibName,
arch,
buildpath=DEFAULT_BUILDPATH,
cmspath=DEFAULT_CMSPATH,
tests=DEFAULT_TESTS):
cycle = getCycleNameByReleaseName(ibName)
os.environ["SCRAM_ARCH"] = arch
os.environ["CMSINTBLD_CMS_PATH"] = cmspath
tagCollectorAPI.setRequestBuilding(requestId,
ibName,
machine=socket.gethostname(),
pid=os.getpid())
startTest(ibName, cycle, buildpath, tests, arch)
tagCollectorAPI.finishRequest(requestId)
def test_build_and_fail_ib(self):
ibData = tagCollectorAPI.getIBPendingRequests()
self.assertTrue(ibData)
tagCollectorAPI.setRequestBuilding(ibData['id'],
ibData['release_name'],
machine=socket.gethostname(),
pid=os.getpid())
ibBuildingRequests = tagCollectorAPI.getBuildingIBRequests()
self.assertTrue(ibBuildingRequests)
self.assertEquals(self.release_name,
ibBuildingRequests[0]['release_name'])
self.assertEquals(self.architecture_name,
ibBuildingRequests[0]['architecture_name'])
self.assertEquals('IB', ibBuildingRequests[0]['type'])
isFinished = tagCollectorAPI.failRequest(ibBuildingRequests[0]['id'])
self.assertTrue(isFinished)
def test_build_and_finish_qa(self):
qaData = tagCollectorAPI.getQAPendingRequests()
self.assertTrue(qaData)
tagCollectorAPI.setRequestBuilding(qaData['id'],
qaData['release_name'],
machine=socket.gethostname(),
pid=os.getpid())
qaBuildingRequests = tagCollectorAPI.getBuildingQARequests()
self.assertTrue(qaBuildingRequests)
self.assertEquals(self.release_name,
qaBuildingRequests[0]['release_name'])
self.assertEquals(self.architecture_name,
qaBuildingRequests[0]['architecture_name'])
self.assertEquals('QA', qaBuildingRequests[0]['type'])
isFinished = tagCollectorAPI.finishRequest(qaBuildingRequests[0]['id'])
self.assertTrue(isFinished)
def process():
# Get the first task from the list
# Check if we know what to do
# Mark it as started
# Start doing it
parser = OptionParser(usage="%prog process [options]")
parser.add_option("--match-arch", metavar="REGEX", dest="matchArch", help="Limit architectures to those matching REGEX", default=".*")
parser.add_option("--match-release", metavar="REGEX", dest="matchRelease", help="Limit releases to those matching REGEX", default=".*")
parser.add_option("--work-dir", "--top-dir", metavar="PATH", dest="workdir", help="Work dir where processing happens", default=None)
parser.add_option("--jobs", "-j", type="int", metavar="N", dest="jobs", help="Number of parallel building threads", default=1)
parser.add_option("--builders", type="int", metavar="N", dest="builders", help="Number of packages built in parallel", default=1)
parser.add_option("--debug", metavar="PATH", dest="debug", help="Print out what's happening", action="store_true", default=False)
parser.add_option("--dry-run", "-n", metavar="BOOL", dest="dryRun", help="Do not execute", action="store_true", default=False)
parser.add_option("--testbed", metavar="BOOL", dest="useTestBed", help="Use the testbed tag collector to ", action="store_true", default=False)
parser.add_option("--max-load", type="int", metavar="LOAD", dest="maxLoad", help="Do not execute if average last 15 minutes load > LOAD", default=8)
opts, args = parser.parse_args()
if not opts.workdir:
print "Please specify a workdir"
sys.exit(1)
if exists("/etc/iss.nologin"):
print "/etc/iss.nologin found. Not doing anything and waiting for machine out of maintainance mode."
sys.exit(1)
opts.workdir = abspath(opts.workdir)
thisPath=dirname(__file__)
getstatusoutput(format(
"%(here)s/syncLogs.py %(workdir)s",
here=thisPath,
workdir=opts.workdir))
lockPath = join(opts.workdir, "cms", ".cmsLock")
lock = Lock(lockPath, True, 60*60*12)
if not lock:
if opts.debug:
print "Lock found in %s" % lockPath
sys.exit(1)
lock.__del__()
if overloaded(opts.maxLoad):
print "Current load exceeds maximum allowed of %s." % opts.maxLoad
sys.exit(1)
options = {"release_pattern": opts.matchRelease,
"architecture_pattern": opts.matchArch}
if opts.useTestBed:
options["tcBaseURL"] = TESTBED_URL
tasks = tagCollectorAPI.listPendingTasks(**options)
print tasks
if not len(tasks):
if opts.debug:
print "Nothing to be done which matches release %s and architecture %s" % (opts.matchArch, opts.matchRelease)
sys.exit(1)
# Default payload options.
payload = {"debug": False}
task_id, architecture_name, release_name, payloadNew = tasks[0]
payload.update(payloadNew)
if not payload.has_key("build-task"):
print "Request task %s is not a valid build task" % task_id
sys.exit(1)
buildTask = payload["build-task"]
if not buildTask in ["build-package"]:
print "Unknown task for request %s: %s" % (task_id, buildTask)
sys.exit(1)
if opts.dryRun:
print "Dry run. Not building"
sys.exit(1)
options = {"request_id": task_id,
"release_name": release_name,
"machine": socket.gethostname(),
"pid": os.getpid()}
if opts.useTestBed:
options["tcBaseURL"] = TESTBED_URL
options["results_url"] = "http://cmssdt.cern.ch/SDT/tc-ib-logs/%s/log.%s.html" % (socket.gethostname(), task_id)
ok = tagCollectorAPI.setRequestBuilding(**options)
if not ok:
print "Could not change request %s state to building" % task_id
sys.exit(1)
# Build the package.
# We gracefully handle any exception (broken pipe, ctrl-c, SIGKILL)
# by failing the request if they happen. We also always cat
# the log for this build in a global log file.
log = ""
getstatusoutput(format(
"echo 'Log not sync-ed yet' > %(workdir)s/log.%(task_id)s;\n"
"%(here)s/syncLogs.py %(workdir)s",
task_id=task_id,
here=thisPath,
workdir=opts.workdir))
try:
print "Building..."
error, log = getstatusoutput(format("set -e ;\n"
"mkdir -p %(workdir)s/%(task_id)s ;\n"
"export CMS_PATH=%(workdir)s/cms ;\n"
"cd %(workdir)s ;\n"
"( export CVSROOT=:pserver:[email protected]/local/reps/CMSSW ;\n"
" export CVS_PASSFILE=%(workdir)s/.cvspass ;\n"
" echo '/1 :pserver:[email protected]:/cvs_server/repositories/CMSSW %(cvspass)s' > $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]:2401/cvs/CMSSW %(cvspass)s' >> $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]:2401/cvs_server/repositories/CMSSW %(cvspass)s' >> $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]/local/reps/CMSSW %(cvspass)s' >> $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]/local/reps/CMSSW %(cvspass)s' >> $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]:2401/local/reps/CMSSW %(cvspass)s' >> $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]:2401/local/reps/CMSSW %(cvspass)s' >> $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]:2401/local/reps/CMSSW %(cvspass)s' >> $CVS_PASSFILE;\n"
" git clone https://github.com/cms-sw/cmsdist.git %(task_id)s/CMSDIST;\n"
" pushd %(task_id)s/CMSDIST; git checkout %(cmsdistTag)s; popd;\n"
" PKGTOOLS_TAG=\"`echo %(pkgtoolsTag)s | sed -e's/\\(V[0-9]*-[0-9]*\\).*/\\1-XX/'`\";\n"
" git clone https://github.com/cms-sw/pkgtools.git %(task_id)s/PKGTOOLS\n"
" pushd %(task_id)s/PKGTOOLS; git checkout $PKGTOOLS_TAG; popd;\n"
" echo \"### RPM cms dummy `date +%%s`\n%%prep\n%%build\n%%install\n\" > %(task_id)s/CMSDIST/dummy.spec ;\n"
" set -x ;\n"
" rm -rf %(workdir)s/cms %(workdir)s/b ;\n"
" perl -p -i -e 's/### RPM cms cmssw.*/### RPM cms cmssw %(base_release_name)s/' %(task_id)s/CMSDIST/cmssw.spec ;\n"
" perl -p -i -e 's/### RPM cms cmssw-patch.*/### RPM cms cmssw-patch %(real_release_name)s/' %(task_id)s/CMSDIST/cmssw-patch.spec ;\n"
" %(workdir)s/%(task_id)s/PKGTOOLS/cmsBuild %(debug)s --new-scheduler --cmsdist %(workdir)s/%(task_id)s/CMSDIST %(ignoreErrors)s --builders %(builders)s -j %(jobs)s --repository %(repository)s --architecture %(architecture)s --work-dir %(workdir)s/cms build %(package)s ;\n"
" %(workdir)s/%(task_id)s/PKGTOOLS/cmsBuild %(debug)s --new-scheduler --cmsdist %(workdir)s/%(task_id)s/CMSDIST --repository %(repository)s --upload-tmp-repository %(tmpRepository)s %(syncBack)s --architecture %(architecture)s --work-dir %(workdir)s/cms upload %(package)s ;\n"
" set +x ;\n"
" echo AUTOIB SUCCESS) 2>&1 | tee %(workdir)s/log.%(task_id)s",
workdir=opts.workdir,
cvspass=CMSSW_CVSPASS,
debug=payload["debug"] == True and "--debug" or "",
cmsdistTag=sanitize(payload["CMSDIST"]),
pkgtoolsTag=sanitize(payload["PKGTOOLS"]),
architecture=sanitize(architecture_name),
release_name=sanitize(release_name),
base_release_name=re.sub("_[^_]*patch[0-9]*$", "", sanitize(payload["real_release_name"])),
real_release_name=sanitize(payload["real_release_name"]),
package=sanitize(payload["package"]),
repository=sanitize(payload["repository"]),
syncBack=payload["syncBack"] == True and "--sync-back" or "",
ignoreErrors=payload["ignoreErrors"] == True and "-k" or "",
tmpRepository=sanitize(payload["tmpRepository"]),
task_id=task_id,
jobs=opts.jobs,
builders=opts.builders))
getstatusoutput(format("echo 'Task %(task_id)s completed successfully.' >> %(workdir)s/log.%(task_id)s",
workdir=opts.workdir,
task_id=task_id))
except Exception, e:
log = open(format("%(workdir)s/log.%(task_id)s", workdir=opts.workdir, task_id=task_id)).read()
log += "\nInterrupted externally."
log += str(e)
getstatusoutput(format("echo 'Interrupted externally' >> %(workdir)s/log.%(task_id)s",
workdir=opts.workdir,
task_id=task_id))
def setStatusBuilding(self, rel):
if self.doDeBuG: print 'setting status to building for', self.id, rel
tc.setRequestBuilding(request_id=self.id, release_name=rel, machine=socket.gethostname(), pid=os.getpid())
sys.stdout.flush()
sys.stderr.flush()
def process():
# Get the first task from the list
# Check if we know what to do
# Mark it as started
# Start doing it
parser = OptionParser(usage="%prog process [options]")
parser.add_option("--match-arch",
metavar="REGEX",
dest="matchArch",
help="Limit architectures to those matching REGEX",
default=".*")
parser.add_option("--match-release",
metavar="REGEX",
dest="matchRelease",
help="Limit releases to those matching REGEX",
default=".*")
parser.add_option("--work-dir",
"--top-dir",
metavar="PATH",
dest="workdir",
help="Work dir where processing happens",
default=None)
parser.add_option("--jobs",
"-j",
type="int",
metavar="N",
dest="jobs",
help="Number of parallel building threads",
default=1)
parser.add_option("--builders",
type="int",
metavar="N",
dest="builders",
help="Number of packages built in parallel",
default=1)
parser.add_option("--debug",
metavar="PATH",
dest="debug",
help="Print out what's happening",
action="store_true",
default=False)
parser.add_option("--dry-run",
"-n",
metavar="BOOL",
dest="dryRun",
help="Do not execute",
action="store_true",
default=False)
parser.add_option("--testbed",
metavar="BOOL",
dest="useTestBed",
help="Use the testbed tag collector to ",
action="store_true",
default=False)
parser.add_option(
"--max-load",
type="int",
metavar="LOAD",
dest="maxLoad",
help="Do not execute if average last 15 minutes load > LOAD",
default=8)
opts, args = parser.parse_args()
if not opts.workdir:
print "Please specify a workdir"
sys.exit(1)
if exists("/etc/iss.nologin"):
print "/etc/iss.nologin found. Not doing anything and waiting for machine out of maintainance mode."
sys.exit(1)
opts.workdir = abspath(opts.workdir)
thisPath = dirname(__file__)
getstatusoutput(
format("%(here)s/syncLogs.py %(workdir)s",
here=thisPath,
workdir=opts.workdir))
lockPath = join(opts.workdir, "cms", ".cmsLock")
lock = Lock(lockPath, True, 60 * 60 * 12)
if not lock:
if opts.debug:
print "Lock found in %s" % lockPath
sys.exit(1)
lock.__del__()
if overloaded(opts.maxLoad):
print "Current load exceeds maximum allowed of %s." % opts.maxLoad
sys.exit(1)
options = {
"release_pattern": opts.matchRelease,
"architecture_pattern": opts.matchArch
}
if opts.useTestBed:
options["tcBaseURL"] = TESTBED_URL
tasks = tagCollectorAPI.listPendingTasks(**options)
print tasks
if not len(tasks):
if opts.debug:
print "Nothing to be done which matches release %s and architecture %s" % (
opts.matchArch, opts.matchRelease)
sys.exit(1)
# Default payload options.
payload = {"debug": False}
task_id, architecture_name, release_name, payloadNew = tasks[0]
payload.update(payloadNew)
if not payload.has_key("build-task"):
print "Request task %s is not a valid build task" % task_id
sys.exit(1)
buildTask = payload["build-task"]
if not buildTask in ["build-package"]:
print "Unknown task for request %s: %s" % (task_id, buildTask)
sys.exit(1)
if opts.dryRun:
print "Dry run. Not building"
sys.exit(1)
options = {
"request_id": task_id,
"release_name": release_name,
"machine": socket.gethostname(),
"pid": os.getpid()
}
if opts.useTestBed:
options["tcBaseURL"] = TESTBED_URL
options[
"results_url"] = "http://cmssdt.cern.ch/SDT/tc-ib-logs/%s/log.%s.html" % (
socket.gethostname(), task_id)
ok = tagCollectorAPI.setRequestBuilding(**options)
if not ok:
print "Could not change request %s state to building" % task_id
sys.exit(1)
# Build the package.
# We gracefully handle any exception (broken pipe, ctrl-c, SIGKILL)
# by failing the request if they happen. We also always cat
# the log for this build in a global log file.
log = ""
getstatusoutput(
format(
"echo 'Log not sync-ed yet' > %(workdir)s/log.%(task_id)s;\n"
"%(here)s/syncLogs.py %(workdir)s",
task_id=task_id,
here=thisPath,
workdir=opts.workdir))
try:
print "Building..."
error, log = getstatusoutput(
format(
"set -e ;\n"
"mkdir -p %(workdir)s/%(task_id)s ;\n"
"export CMS_PATH=%(workdir)s/cms ;\n"
"cd %(workdir)s ;\n"
"( export CVSROOT=:pserver:[email protected]/local/reps/CMSSW ;\n"
" export CVS_PASSFILE=%(workdir)s/.cvspass ;\n"
" echo '/1 :pserver:[email protected]:/cvs_server/repositories/CMSSW %(cvspass)s' > $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]:2401/cvs/CMSSW %(cvspass)s' >> $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]:2401/cvs_server/repositories/CMSSW %(cvspass)s' >> $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]/local/reps/CMSSW %(cvspass)s' >> $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]/local/reps/CMSSW %(cvspass)s' >> $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]:2401/local/reps/CMSSW %(cvspass)s' >> $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]:2401/local/reps/CMSSW %(cvspass)s' >> $CVS_PASSFILE ;\n"
" echo '/1 :pserver:[email protected]:2401/local/reps/CMSSW %(cvspass)s' >> $CVS_PASSFILE;\n"
" git clone https://github.com/cms-sw/cmsdist.git %(task_id)s/CMSDIST;\n"
" pushd %(task_id)s/CMSDIST; git checkout %(cmsdistTag)s; popd;\n"
" PKGTOOLS_TAG=\"`echo %(pkgtoolsTag)s | sed -e's/\\(V[0-9]*-[0-9]*\\).*/\\1-XX/'`\";\n"
" git clone https://github.com/cms-sw/pkgtools.git %(task_id)s/PKGTOOLS\n"
" pushd %(task_id)s/PKGTOOLS; git checkout $PKGTOOLS_TAG; popd;\n"
" echo \"### RPM cms dummy `date +%%s`\n%%prep\n%%build\n%%install\n\" > %(task_id)s/CMSDIST/dummy.spec ;\n"
" set -x ;\n"
" rm -rf %(workdir)s/cms %(workdir)s/b ;\n"
" perl -p -i -e 's/### RPM cms cmssw.*/### RPM cms cmssw %(base_release_name)s/' %(task_id)s/CMSDIST/cmssw.spec ;\n"
" perl -p -i -e 's/### RPM cms cmssw-patch.*/### RPM cms cmssw-patch %(real_release_name)s/' %(task_id)s/CMSDIST/cmssw-patch.spec ;\n"
" %(workdir)s/%(task_id)s/PKGTOOLS/cmsBuild %(debug)s --new-scheduler --cmsdist %(workdir)s/%(task_id)s/CMSDIST %(ignoreErrors)s --builders %(builders)s -j %(jobs)s --repository %(repository)s --architecture %(architecture)s --work-dir %(workdir)s/cms build %(package)s ;\n"
" %(workdir)s/%(task_id)s/PKGTOOLS/cmsBuild %(debug)s --new-scheduler --cmsdist %(workdir)s/%(task_id)s/CMSDIST --repository %(repository)s --upload-tmp-repository %(tmpRepository)s %(syncBack)s --architecture %(architecture)s --work-dir %(workdir)s/cms upload %(package)s ;\n"
" set +x ;\n"
" echo AUTOIB SUCCESS) 2>&1 | tee %(workdir)s/log.%(task_id)s",
workdir=opts.workdir,
cvspass=CMSSW_CVSPASS,
debug=payload["debug"] == True and "--debug" or "",
cmsdistTag=sanitize(payload["CMSDIST"]),
pkgtoolsTag=sanitize(payload["PKGTOOLS"]),
architecture=sanitize(architecture_name),
release_name=sanitize(release_name),
base_release_name=re.sub(
"_[^_]*patch[0-9]*$", "",
sanitize(payload["real_release_name"])),
real_release_name=sanitize(payload["real_release_name"]),
package=sanitize(payload["package"]),
repository=sanitize(payload["repository"]),
syncBack=payload["syncBack"] == True and "--sync-back" or "",
ignoreErrors=payload["ignoreErrors"] == True and "-k" or "",
tmpRepository=sanitize(payload["tmpRepository"]),
task_id=task_id,
jobs=opts.jobs,
builders=opts.builders))
getstatusoutput(
format(
"echo 'Task %(task_id)s completed successfully.' >> %(workdir)s/log.%(task_id)s",
workdir=opts.workdir,
task_id=task_id))
except Exception, e:
log = open(
format("%(workdir)s/log.%(task_id)s",
workdir=opts.workdir,
task_id=task_id)).read()
log += "\nInterrupted externally."
log += str(e)
getstatusoutput(
format(
"echo 'Interrupted externally' >> %(workdir)s/log.%(task_id)s",
workdir=opts.workdir,
task_id=task_id))