def start(self):
check_beginning_time = time.time()
print 'Starting IB requests state check'
building_requests = tagCollectorAPI.getBuildingIBRequests()
now = datetime.datetime.today()
for building_request in building_requests:
btime = time.strptime(building_request['building_timestamp'],
'%Y-%m-%d %H:%M:%S')
build_start_time = datetime.datetime(*btime[0:6])
delta = (now - build_start_time)
total_building_time = delta.days * 24 * 60 * 60 + delta.seconds
if total_building_time >= MAX_BUILDING_TIME_IN_SECONDS:
isSetToFailed = tagCollectorAPI.failRequest(
building_request['id'], 0, 0, 0, 0, '')
while not isSetToFailed and (time.time() - check_beginning_time
) > MAX_CHECK_RUNNING_TIME:
isSetToFailed = tagCollectorAPI.failRequest(
data['id'], 0, 0, 0, 0, '')
if not isSetToFailed:
time.sleep(STATUS_SETTING_RETRY_TIME)
print "Cancelled build request: \n%s\nReason: %s\n" % (
str(building_request),
'The building time is more than %s hours' %
str(MAX_BUILDING_TIME_IN_SECONDS / 60 / 60),
)
self.sendMail = True
def start(self):
check_beginning_time = time.time()
print 'Starting IB requests state check'
building_requests = tagCollectorAPI.getBuildingIBRequests()
now = datetime.datetime.today()
for building_request in building_requests:
btime = time.strptime(building_request['building_timestamp'], '%Y-%m-%d %H:%M:%S')
build_start_time = datetime.datetime(*btime[0:6])
delta = (now - build_start_time)
total_building_time = delta.days*24*60*60 + delta.seconds
if total_building_time >= MAX_BUILDING_TIME_IN_SECONDS:
isSetToFailed = tagCollectorAPI.failRequest(building_request['id'], 0,0,0,0,'')
while not isSetToFailed and (time.time()-check_beginning_time) > MAX_CHECK_RUNNING_TIME:
isSetToFailed = tagCollectorAPI.failRequest(data['id'], 0,0,0,0,'')
if not isSetToFailed:
time.sleep(STATUS_SETTING_RETRY_TIME)
print "Cancelled build request: \n%s\nReason: %s\n" % (str(building_request), 'The building time is more than %s hours' % str(MAX_BUILDING_TIME_IN_SECONDS / 60 / 60),)
self.sendMail=True
def test_build_and_fail_ib(self):
ibData = tagCollectorAPI.getIBPendingRequests()
self.assertTrue(ibData)
tagCollectorAPI.setRequestBuilding(
ibData["id"], ibData["release_name"], machine=socket.gethostname(), pid=os.getpid()
)
ibBuildingRequests = tagCollectorAPI.getBuildingIBRequests()
self.assertTrue(ibBuildingRequests)
self.assertEquals(self.release_name, ibBuildingRequests[0]["release_name"])
self.assertEquals(self.architecture_name, ibBuildingRequests[0]["architecture_name"])
self.assertEquals("IB", ibBuildingRequests[0]["type"])
isFinished = tagCollectorAPI.failRequest(ibBuildingRequests[0]["id"])
self.assertTrue(isFinished)
def test_build_and_fail_ib(self):
ibData = tagCollectorAPI.getIBPendingRequests()
self.assertTrue(ibData)
tagCollectorAPI.setRequestBuilding(ibData['id'],
ibData['release_name'],
machine=socket.gethostname(),
pid=os.getpid())
ibBuildingRequests = tagCollectorAPI.getBuildingIBRequests()
self.assertTrue(ibBuildingRequests)
self.assertEquals(self.release_name,
ibBuildingRequests[0]['release_name'])
self.assertEquals(self.architecture_name,
ibBuildingRequests[0]['architecture_name'])
self.assertEquals('IB', ibBuildingRequests[0]['type'])
isFinished = tagCollectorAPI.failRequest(ibBuildingRequests[0]['id'])
self.assertTrue(isFinished)
# map over the params ...
self.id = int(ret['id'])
self.ref = ret['release_name']
latestIB = False
if self.ref[-2:] == '_X': latestIB = True
if latestIB:
try:
self.ref = self.getLatestIB(self.ref)
except Exception, e:
print "ERROR when trying to find latest IB, aborting"
print str(e)
tc.commentRequest(
self.id, "ERROR: Could not find latest IB for " + self.ref)
self.setStatusBuilding(self.ref)
tc.failRequest(self.id)
informRequestor(
[ret['email']],
"Sorry, can not process your custom IB request " +
str(ret['id']) + " due to missing IB " + self.ref +
" for SCRAM_ARCH " + plat, plat)
return False
self.tagList = ret['tags']
self._stamp = str(self.id)
self.getUserInfo(ret)
if self.doDeBuG:
print "found request id ", self.id, " for:"
print "\tIB :", self.ref
print "\tuser :", self.user
log = open(format("%(workdir)s/log.%(task_id)s", workdir=opts.workdir, task_id=task_id)).read()
log += "\nInterrupted externally."
log += str(e)
getstatusoutput(format("echo 'Interrupted externally' >> %(workdir)s/log.%(task_id)s",
workdir=opts.workdir,
task_id=task_id))
error, saveLog = getstatusoutput(format("set -e ;\n"
"echo '#### Log file for %(task_id)s' >> %(workdir)s/log ;\n"
"cat %(workdir)s/log.%(task_id)s >> %(workdir)s/log",
workdir=opts.workdir,
task_id=task_id))
getstatusoutput("%s/syncLogs.py %s" % (thisPath, opts.workdir))
if not "AUTOIB SUCCESS" in log:
tagCollectorAPI.failRequest(request_id=task_id, results_url="http://cmssdt.cern.ch/SDT/tc-ib-logs/%s/log.%s.html" % (socket.gethostname(), task_id))
print log
print saveLog
sys.exit(1)
tagCollectorAPI.finishRequest(request_id=task_id, results_url="http://cmssdt.cern.ch/SDT/tc-ib-logs/%s/log.%s.html" % (socket.gethostname(), task_id))
# Here we are done processing the job. Now schedule continuations.
if not "continuations" in payload:
sys.exit(0)
continuationsSpec = payload["continuations"] or ""
continuations = [x for x in continuationsSpec.split(";")]
if len(continuations) == 0:
sys.exit(0)
if len(continuations) != 1:
# map over the params ...
self.id = int(ret['id'])
self.ref = ret['release_name']
latestIB = False
if self.ref[-2:] == '_X' : latestIB = True
if latestIB:
try:
self.ref = self.getLatestIB(self.ref)
except Exception, e:
print "ERROR when trying to find latest IB, aborting"
print str(e)
tc.commentRequest(self.id,"ERROR: Could not find latest IB for "+self.ref)
self.setStatusBuilding(self.ref)
tc.failRequest(self.id)
informRequestor([ret['email']],"Sorry, can not process your custom IB request "+str(ret['id'])+" due to missing IB "+self.ref+" for SCRAM_ARCH "+plat,plat)
return False
self.tagList = ret['tags']
self._stamp = str(self.id)
self.getUserInfo(ret)
if self.doDeBuG:
print "found request id ",self.id," for:"
print "\tIB :",self.ref
print "\tuser :"******"\tstamp:",self._stamp
print "\ttags :"
for p,t in self.tagList.items():
print "\t",p, t
"echo 'Interrupted externally' >> %(workdir)s/log.%(task_id)s",
workdir=opts.workdir,
task_id=task_id))
error, saveLog = getstatusoutput(
format(
"set -e ;\n"
"echo '#### Log file for %(task_id)s' >> %(workdir)s/log ;\n"
"cat %(workdir)s/log.%(task_id)s >> %(workdir)s/log",
workdir=opts.workdir,
task_id=task_id))
getstatusoutput("%s/syncLogs.py %s" % (thisPath, opts.workdir))
if not "AUTOIB SUCCESS" in log:
tagCollectorAPI.failRequest(
request_id=task_id,
results_url="http://cmssdt.cern.ch/SDT/tc-ib-logs/%s/log.%s.html" %
(socket.gethostname(), task_id))
print log
print saveLog
sys.exit(1)
tagCollectorAPI.finishRequest(
request_id=task_id,
results_url="http://cmssdt.cern.ch/SDT/tc-ib-logs/%s/log.%s.html" %
(socket.gethostname(), task_id))
# Here we are done processing the job. Now schedule continuations.
if not "continuations" in payload:
sys.exit(0)
continuationsSpec = payload["continuations"] or ""
continuations = [x for x in continuationsSpec.split(";")]
