def kill_processing(self, processing): try: if processing: from pandatools import Client task_id = processing.workload_id Client.killTask(task_id) except Exception as ex: msg = "Failed to check the processing (%s) status: %s" % (str( processing['processing_id']), str(ex)) raise exceptions.IDDSException(msg)
def kill_processing_force(self, processing): try: if processing: from pandatools import Client proc = processing['processing_metadata']['processing'] task_id = proc.workload_id # task_id = processing['processing_metadata']['task_id'] Client.killTask(task_id) # Client.finishTask(task_id, soft=True) except Exception as ex: msg = "Failed to check the processing (%s) status: %s" % (str( processing['processing_id']), str(ex)) raise exceptions.IDDSException(msg)
def master_kill(self): '''Kill jobs''' from pandatools import Client job = self.getJobObject() logger.debug('Killing job %s' % job.getFQID('.')) active_status = [ None, 'registered', 'waiting', 'defined', 'pending', 'assigning', 'ready', 'scouting', 'running', 'holding', 'merging', 'prepared', 'aborting', 'finishing' ] #active_status = [ None, 'defined', 'unknown', 'assigned', 'waiting', 'activated', 'sent', 'starting', 'running', 'holding', 'transferring' ] if self.id and self.status in active_status: status, output = Client.killTask(self.id) if status: logger.error('Failed killing job (status = %d)', status) return False else: logger.info('Killing Jedi task %s, Server returned: %s' % (self.id, output)) else: logger.error( 'Cannot kill Jedi job %s since it is not in active status', self.id) return True
def master_kill(self): '''Kill jobs''' from pandatools import Client job = self.getJobObject() logger.debug('Killing job %s' % job.getFQID('.')) active_status = [ None, 'registered', 'waiting', 'defined', 'pending', 'assigning', 'ready', 'scouting', 'running', 'holding', 'merging', 'prepared', 'aborting', 'finishing' ] #active_status = [ None, 'defined', 'unknown', 'assigned', 'waiting', 'activated', 'sent', 'starting', 'running', 'holding', 'transferring' ] if self.id and self.status in active_status: status, output = Client.killTask(self.id) if status: logger.error('Failed killing job (status = %d)',status) return False else: logger.info('Killing Jedi task %s, Server returned: %s' %(self.id, output)) else: logger.error('Cannot kill Jedi job %s since it is not in active status', self.id) return True
def kill(self,JobID,useJobsetID=False): # get logger tmpLog = PLogger.getPandaLogger() # check proxy self.gridPassPhrase,self.vomsFQAN = PsubUtils.checkGridProxy( self.gridPassPhrase, False, self.verbose, useCache=True) # force update just in case self.status(JobID,True) # get jobset jobList = self.getJobIDsWithSetID(JobID) if jobList == None: # works only for jobsetID if useJobsetID: return # works with jobID jobList = [JobID] else: tmpMsg = "ID=%s is composed of JobID=" % JobID for tmpJobID in jobList: tmpMsg += '%s,' % tmpJobID tmpMsg = tmpMsg[:-1] tmpLog.info(tmpMsg) for tmpJobID in jobList: # get job info from local repository job = self.getJobInfo(tmpJobID) if job == None: tmpLog.warning("JobID=%s not found in local repository. Synchronization may be needed" % tmpJobID) continue # skip frozen job if job.dbStatus == 'frozen': tmpLog.info('All subJobs in JobID=%s already finished/failed' % tmpJobID) continue if not job.isJEDI(): # get PandaID list killJobs = job.PandaID.split(',') # kill tmpLog.info('Sending kill command ...') status,output = Client.killJobs(killJobs,self.verbose) if status != 0: tmpLog.error(output) tmpLog.error("Failed to kill JobID=%s" % tmpJobID) return False # update database job.commandToPilot = 'tobekilled' # update DB try: PdbUtils.updateJobDB(job,self.verbose) except: tmpLog.error("Failed to update local repository for JobID=%s" % tmpJobID) return False else: # kill JEDI task tmpLog.info('Sending killTask command ...') status,output = Client.killTask(job.jediTaskID,self.verbose) # communication error if status != 0: tmpLog.error(output) tmpLog.error("Failed to kill JobID=%s" % tmpJobID) return False tmpStat,tmpDiag = output if not tmpStat: tmpLog.error(tmpDiag) tmpLog.error("Failed to kill JobID=%s" % tmpJobID) return False tmpLog.info(tmpDiag) # done if job.isJEDI(): tmpLog.info('Done. TaskID=%s will be killed in 30min' % job.jediTaskID) else: tmpLog.info('Done. JobID=%s will be killed in 30min' % tmpJobID) return True
task_ids = [] # task_ids = [1565, 1566, 1567, 1568, 1570, 1572, 1575, 1576, 1579, 1580, 1581, 1582, 1584, 1585, 1586, 1587, 1588, 1589, 1590, 1591, 1592, 1593, 1597, 1598, 1599, 1601, 1602, 1603, 1604, 1607, 1608, 1609, 1610, 1611, 1612, 1613, 1617] # task_ids = [i for i in range(1091, 1104)] + [i for i in range(1274, 1392)] # task_ids = [812] # task_ids += [i for i in range(815, 822)] # task_ids = [827, 830, 913, 914, 916, 917, 1030, 1031, 1033, 1034, 1036, 1048, 1090, 1392] # task_ids += [i for i in range(833, 839)] # task_ids += [i for i in range(1048, 1078)] # task_ids = [i for i in range(1855, 1856)] # task_ids = [i for i in range(1840, 1850)] + [i for i in range(1990, 2000)] # task_ids = [2549, 2560] # task_ids = [i for i in range(3692, 3723)] task_ids = [] for task_id in task_ids: print("Killing %s" % task_id) Client.killTask(task_id) """ jobids = [] Client.getJobStatus(ids=jobids, verbose=False) Client.getJediTaskDetails(taskDict,fullFlag,withTaskInfo,verbose=False) Client.getFullJobStatus(ids, verbose=False) """ # getJobIDsJediTasksInTimeRange(timeRange, dn=None, minTaskID=None, verbose=False, task_type='user') # /DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=atlpilo1/CN=614260/CN=Robot: ATLAS Pilot1 start_time = datetime.datetime.utcnow() - datetime.timedelta(hours=5) start_time = start_time.strftime('%Y-%m-%d %H:%M:%S') # ret = Client.getJobIDsJediTasksInTimeRange(start_time, verbose=False)