def kill_processing(self, processing):
try:
if processing:
from pandatools import Client
task_id = processing.workload_id
Client.killTask(task_id)
except Exception as ex:
msg = "Failed to check the processing (%s) status: %s" % (str(
processing['processing_id']), str(ex))
raise exceptions.IDDSException(msg)
def kill_processing_force(self, processing):
try:
if processing:
from pandatools import Client
proc = processing['processing_metadata']['processing']
task_id = proc.workload_id
# task_id = processing['processing_metadata']['task_id']
Client.killTask(task_id)
# Client.finishTask(task_id, soft=True)
except Exception as ex:
msg = "Failed to check the processing (%s) status: %s" % (str(
processing['processing_id']), str(ex))
raise exceptions.IDDSException(msg)
def master_kill(self):
'''Kill jobs'''
from pandatools import Client
job = self.getJobObject()
logger.debug('Killing job %s' % job.getFQID('.'))
active_status = [
None, 'registered', 'waiting', 'defined', 'pending', 'assigning',
'ready', 'scouting', 'running', 'holding', 'merging', 'prepared',
'aborting', 'finishing'
]
#active_status = [ None, 'defined', 'unknown', 'assigned', 'waiting', 'activated', 'sent', 'starting', 'running', 'holding', 'transferring' ]
if self.id and self.status in active_status:
status, output = Client.killTask(self.id)
if status:
logger.error('Failed killing job (status = %d)', status)
return False
else:
logger.info('Killing Jedi task %s, Server returned: %s' %
(self.id, output))
else:
logger.error(
'Cannot kill Jedi job %s since it is not in active status',
self.id)
return True
def master_kill(self):
'''Kill jobs'''
from pandatools import Client
job = self.getJobObject()
logger.debug('Killing job %s' % job.getFQID('.'))
active_status = [ None, 'registered', 'waiting', 'defined', 'pending', 'assigning', 'ready', 'scouting', 'running', 'holding', 'merging', 'prepared', 'aborting', 'finishing' ]
#active_status = [ None, 'defined', 'unknown', 'assigned', 'waiting', 'activated', 'sent', 'starting', 'running', 'holding', 'transferring' ]
if self.id and self.status in active_status:
status, output = Client.killTask(self.id)
if status:
logger.error('Failed killing job (status = %d)',status)
return False
else:
logger.info('Killing Jedi task %s, Server returned: %s' %(self.id, output))
else:
logger.error('Cannot kill Jedi job %s since it is not in active status', self.id)
return True
def kill(self,JobID,useJobsetID=False):
# get logger
tmpLog = PLogger.getPandaLogger()
# check proxy
self.gridPassPhrase,self.vomsFQAN = PsubUtils.checkGridProxy(
self.gridPassPhrase,
False,
self.verbose,
useCache=True)
# force update just in case
self.status(JobID,True)
# get jobset
jobList = self.getJobIDsWithSetID(JobID)
if jobList == None:
# works only for jobsetID
if useJobsetID:
return
# works with jobID
jobList = [JobID]
else:
tmpMsg = "ID=%s is composed of JobID=" % JobID
for tmpJobID in jobList:
tmpMsg += '%s,' % tmpJobID
tmpMsg = tmpMsg[:-1]
tmpLog.info(tmpMsg)
for tmpJobID in jobList:
# get job info from local repository
job = self.getJobInfo(tmpJobID)
if job == None:
tmpLog.warning("JobID=%s not found in local repository. Synchronization may be needed" % tmpJobID)
continue
# skip frozen job
if job.dbStatus == 'frozen':
tmpLog.info('All subJobs in JobID=%s already finished/failed' % tmpJobID)
continue
if not job.isJEDI():
# get PandaID list
killJobs = job.PandaID.split(',')
# kill
tmpLog.info('Sending kill command ...')
status,output = Client.killJobs(killJobs,self.verbose)
if status != 0:
tmpLog.error(output)
tmpLog.error("Failed to kill JobID=%s" % tmpJobID)
return False
# update database
job.commandToPilot = 'tobekilled'
# update DB
try:
PdbUtils.updateJobDB(job,self.verbose)
except:
tmpLog.error("Failed to update local repository for JobID=%s" % tmpJobID)
return False
else:
# kill JEDI task
tmpLog.info('Sending killTask command ...')
status,output = Client.killTask(job.jediTaskID,self.verbose)
# communication error
if status != 0:
tmpLog.error(output)
tmpLog.error("Failed to kill JobID=%s" % tmpJobID)
return False
tmpStat,tmpDiag = output
if not tmpStat:
tmpLog.error(tmpDiag)
tmpLog.error("Failed to kill JobID=%s" % tmpJobID)
return False
tmpLog.info(tmpDiag)
# done
if job.isJEDI():
tmpLog.info('Done. TaskID=%s will be killed in 30min' % job.jediTaskID)
else:
tmpLog.info('Done. JobID=%s will be killed in 30min' % tmpJobID)
return True
task_ids = []
# task_ids = [1565, 1566, 1567, 1568, 1570, 1572, 1575, 1576, 1579, 1580, 1581, 1582, 1584, 1585, 1586, 1587, 1588, 1589, 1590, 1591, 1592, 1593, 1597, 1598, 1599, 1601, 1602, 1603, 1604, 1607, 1608, 1609, 1610, 1611, 1612, 1613, 1617]
# task_ids = [i for i in range(1091, 1104)] + [i for i in range(1274, 1392)]
# task_ids = [812]
# task_ids += [i for i in range(815, 822)]
# task_ids = [827, 830, 913, 914, 916, 917, 1030, 1031, 1033, 1034, 1036, 1048, 1090, 1392]
# task_ids += [i for i in range(833, 839)]
# task_ids += [i for i in range(1048, 1078)]
# task_ids = [i for i in range(1855, 1856)]
# task_ids = [i for i in range(1840, 1850)] + [i for i in range(1990, 2000)]
# task_ids = [2549, 2560]
# task_ids = [i for i in range(3692, 3723)]
task_ids = []
for task_id in task_ids:
print("Killing %s" % task_id)
Client.killTask(task_id)
"""
jobids = []
Client.getJobStatus(ids=jobids, verbose=False)
Client.getJediTaskDetails(taskDict,fullFlag,withTaskInfo,verbose=False)
Client.getFullJobStatus(ids, verbose=False)
"""
# getJobIDsJediTasksInTimeRange(timeRange, dn=None, minTaskID=None, verbose=False, task_type='user')
# /DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=atlpilo1/CN=614260/CN=Robot: ATLAS Pilot1
start_time = datetime.datetime.utcnow() - datetime.timedelta(hours=5)
start_time = start_time.strftime('%Y-%m-%d %H:%M:%S')
# ret = Client.getJobIDsJediTasksInTimeRange(start_time, verbose=False)
