def eraseDispDatasets(ids): print "eraseDispDatasets" datasets = [] # get jobs status, jobs = Client.getJobStatus(ids) if status != 0: return # gather dispDBlcoks for job in jobs: # dispatchDS is not a DQ2 dataset in US if job.cloud == 'US': continue # erase disp datasets for production jobs only if job.prodSourceLabel != 'managed': continue for file in job.Files: if file.dispatchDBlock == 'NULL': continue if (not file.dispatchDBlock in datasets) and \ re.search('_dis\d+$',file.dispatchDBlock) != None: datasets.append(file.dispatchDBlock) # erase for dataset in datasets: print 'erase %s' % dataset status, out = ddm.DQ2.main('eraseDataset', dataset) print out
def eraseDispDatasets(ids): print "eraseDispDatasets" datasets = [] # get jobs status,jobs = Client.getJobStatus(ids) if status != 0: return # gather dispDBlcoks for job in jobs: # dispatchDS is not a DQ2 dataset in US if job.cloud == 'US': continue # erase disp datasets for production jobs only if job.prodSourceLabel != 'managed': continue for file in job.Files: if file.dispatchDBlock == 'NULL': continue if (not file.dispatchDBlock in datasets) and \ re.search('_dis\d+$',file.dispatchDBlock) != None: datasets.append(file.dispatchDBlock) # erase for dataset in datasets: print 'erase %s' % dataset status,out = ddm.DQ2.main('eraseDataset',dataset) print out
def getJobStatus(self, ids): import userinterface.Client as Client s,o = Client.getJobStatus(ids) result = {} if s != 0: _logger.error('Error response code: %s %s' %(str(s), str(o))) return result for x in o: result[x.PandaID] = x.jobStatus return result
def getJobStatus(self, ids): import userinterface.Client as Client s, o = Client.getJobStatus(ids) result = {} if s != 0: _logger.error('Error response code: %s %s' % (str(s), str(o))) return result for x in o: result[x.PandaID] = x.jobStatus return result
def getStatus(self, expectedStates): idList = [job['jobID'] for job in self.__jobList] print idList status, jobInfoList = Client.getJobStatus(idList) print jobInfoList assert status == 0, "Retrieval of job state finished with status: %s" %status for job in jobInfoList: assert job.jobStatus in expectedStates, "Recently defined job was not in states %s (PandaID: %s jobStatus: %s)" %(expectedStates, job.PandaID, job.jobStatus) return jobInfoList
def getStatus(self, expectedStates): idList = [job['jobID'] for job in self.__jobList] _logger.info("%s" % idList) status, jobInfoList = Client.getJobStatus(idList) _logger.info("%s" % jobInfoList) assert status == 0, "Retrieval of job state finished with status: %s" % status for job in jobInfoList: assert job.jobStatus in expectedStates, "Recently defined job was not in states %s (PandaID: %s jobStatus: %s)" % ( expectedStates, job.PandaID, job.jobStatus) return jobInfoList
def eraseDispDatasets(ids): datasets = [] # get jobs status,jobs = Client.getJobStatus(ids) if status != 0: return # gather dispDBlcoks for job in jobs: for file in job.Files: if not file.dispatchDBlock in datasets: datasets.append(file.dispatchDBlock) # erase for dataset in datasets: ddm.DQ2.main(['eraseDataset',datasets])
def update_status(): # Method to sync PandaDB job status and local job status # show users jobs jobs = Job.query.filter(Job.pandaid.isnot(None))\ .filter(~Job.status.in_(['finished', 'failed', 'cancelled']))\ .all() ids = [] localids = [] for job in jobs: localids.append(job.id) ids.append(job.pandaid) # get status update if len(ids) > 0: _logger.debug('getJobStatus: ' + str(ids)) s, o = Client.getJobStatus(ids) _logger.debug(o) _logger.debug(s) _logger.debug("---------------------") for job in jobs: if job.pandaid in ids: for obj in o: if obj.PandaID == job.pandaid: # Update attemptNr if changed if job.attemptnr not in [obj.attemptNr]: job.attemptnr = obj.attemptNr jobs_.save(job) # Update status if changed if job.status != obj.jobStatus: job.status = obj.jobStatus job.modification_time = datetime.utcnow() jobs_.save(job) return localids
def update_status(): # Method to sync PandaDB job status and local job status # show users jobs jobs = Job.query.filter(Job.pandaid.isnot(None))\ .filter(~Job.status.in_(['finished', 'failed', 'cancelled']))\ .all() ids = [] localids = [] for job in jobs: localids.append(job.id) ids.append(job.pandaid) # get status update if len(ids) > 0: _logger.debug('getJobStatus: ' + str(ids)) s, o = Client.getJobStatus(ids) _logger.debug(o) _logger.debug(s) _logger.debug("---------------------") for job in jobs: if job.pandaid in ids: for obj in o: if obj.PandaID == job.pandaid: # Update attemptNr if changed if job.attemptnr not in [obj.attemptNr]: job.attemptnr = obj.attemptNr jobs_.save(job) # Update status if changed if job.status != obj.jobStatus: job.status = obj.jobStatus job.modification_time = datetime.utcnow() jobs_.save(job) return localids
ids = {'pandatest.000003.dd.input._00028.junk':'6c19e1fc-ee8c-4bae-bd4c-c9e5c73aca27', 'pandatest.000003.dd.input._00033.junk':'98f79ba1-1793-4253-aac7-bdf90a51d1ee', 'pandatest.000003.dd.input._00039.junk':'33660dd5-7cef-422a-a7fc-6c24cb10deb1'} for lfn in ids.keys(): file = FileSpec() file.lfn = lfn file.GUID = ids[file.lfn] file.dataset = 'pandatest.000003.dd.input' file.type = 'input' job.addFile(file) s,o = Client.submitJobs([job]) print "---------------------" print s print o print "---------------------" s,o = Client.getJobStatus([4934, 4766, 4767, 4768, 4769]) print s if s == 0: for job in o: if job == None: continue print job.PandaID for file in job.Files: print file.lfn,file.type print "---------------------" s,o = Client.queryPandaIDs([0]) print s print o
from django.db.models import Q from prodsys.models import Task, Job print 'Getting tasks with status send and running' tasks_list = Task.objects.all().filter(Q(status='send') | Q(status='running')) print 'Got list of %s tasks' % len(tasks_list) for t in tasks_list: print 'Getting jobs with status send and running for task with status send and running' jobs_list = list( Job.objects.filter( task=t).filter(Q(status='sent') | Q(status='running')).values_list( 'panda_id', flat=True)[:50]) print 'Got list of %s jobs' % len(jobs_list) print 'Sending request to PanDA server' s, o = Client.getJobStatus(jobs_list, None) if s == 0: for x in o: print 'Getting job for PandaID=%s' % x.PandaID j_update = Job.objects.get(panda_id=x.PandaID) if j_update.status != x.jobStatus: today = datetime.datetime.today() if x.jobStatus == 'running' or x.jobStatus == 'finished' or x.jobStatus == 'failed': print 'Going to update status of job %s from %s to %s' % ( j_update.file, j_update.status, x.jobStatus) j_update.status = x.jobStatus j_update.date_updated = today if x.jobStatus == 'finished': j_update.status_merging_mdst = 'ready'
from userinterface.Client import baseURLSSL from taskbuffer.TaskBuffer import taskBuffer from brokerage.SiteMapper import SiteMapper from config import panda_config # instantiate TB taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=1) # instantiate sitemapper siteMapper = SiteMapper(taskBuffer) import httplib import commands id = sys.argv[1] s, o = Client.getJobStatus([id]) if s != 0: print "failed to get job with:%s" % s sys.exit(0) job = o[0] if job == None: print "got None" sys.exit(0) xml = """<?xml version="1.0" encoding="UTF-8" standalone="no" ?> <!-- ATLAS file meta-data catalog --> <!DOCTYPE POOLFILECATALOG SYSTEM "InMemory"> <POOLFILECATALOG>
def x_check_mdst(): logger = logging.getLogger('periodic_tasks_logger') getRotatingFileHandler(logger, 'celery.x_check_mdst.log') today = datetime.datetime.today() logger.info('Starting %s' % __name__) logger.info('Getting tasks with status send and running') tasks_list = Task.objects.all().filter( Q(status='send') | Q(status='running')) # tasks_list = Task.objects.all().filter(name='dvcs2016P07t1_mu+_part1') logger.info('Got list of %s tasks' % len(tasks_list)) for t in tasks_list: logger.info( 'Getting run numbers for task %s status merging finished and status x check no' % t.name) runs_list = Job.objects.filter(task=t).filter( status_merging_mdst='finished').filter( status_x_check='no').order_by('run_number').values_list( 'run_number', flat=True).distinct() logger.info('Got list of %s runs' % len(runs_list)) if len(runs_list) == 0: logger.info('No runs found for checking') continue rnum = 0 for run_number in runs_list: # if rnum == 1: # break logger.info('Getting all jobs of run number %s' % run_number) jobs_list_count = Job.objects.all().filter(task=t).filter( run_number=run_number).values_list( 'panda_id_merging_mdst', flat=True).distinct().count() logger.info(jobs_list_count) nEvents_run = 0 nEvents_chunks = 0 logger.info( 'Getting jobs with merging mdst status finished for task %s and run number %s' % (t.name, run_number)) merging_jobs_list = list( Job.objects.filter(task=t).filter( run_number=run_number).filter( status_merging_mdst='finished').values_list( 'panda_id_merging_mdst', flat=True).distinct()) logger.info('Got list of %s jobs' % len(merging_jobs_list)) if len(merging_jobs_list) != jobs_list_count: logger.info( 'Not all jobs of run are ready for checking, skipping') continue logger.info('Sending request to PanDA server') s, o = Client.getJobStatus(merging_jobs_list, None) if s == 0: for x in o: for j in merging_jobs_list: if j == x.PandaID: nEvents_run += x.nEvents logger.info('Got merging job with % events' % x.nEvents) logger.info( 'Going to get chunks with panda_id_merging_mdst=%s' % x.PandaID) jobs_list = list( Job.objects.filter(panda_id_merging_mdst=x. PandaID).values_list( 'panda_id', flat=True)) logger.info('Got list of %s jobs' % len(jobs_list)) logger.info('Sending request to PanDA server') s1, o1 = Client.getJobStatus(jobs_list, None) if s1 == 0: for x1 in o1: nEvents_chunks += x1.nEvents logger.info( 'Number of events in merged files of run %s and sum of events in chunks: %s - %s' % (run_number, nEvents_run, nEvents_chunks)) if nEvents_run == nEvents_chunks: logger.info( 'Number of events in merged files and run are equal') logger.info( 'Going to update x check status of jobs of run number %s to yes' % run_number) jobs_list_update = Job.objects.filter(task=t).filter( run_number=run_number).update(status_x_check='yes', date_updated=today) else: logger.info( 'Number of events in merged files and run are not equal') logger.info( 'Going to update x check status of jobs of run number %s to manual check is needed' % run_number) jobs_list_update = Job.objects.filter(task=t).filter( run_number=run_number).update( status_x_check='manual check is needed', date_updated=today) rnum += 1 logger.info('done') return True
'98f79ba1-1793-4253-aac7-bdf90a51d1ee', 'pandatest.000003.dd.input._00039.junk': '33660dd5-7cef-422a-a7fc-6c24cb10deb1' } for lfn in ids.keys(): file = FileSpec() file.lfn = lfn file.GUID = ids[file.lfn] file.dataset = 'pandatest.000003.dd.input' file.type = 'input' job.addFile(file) s, o = Client.submitJobs([job]) print "---------------------" print s print o print "---------------------" s, o = Client.getJobStatus([4934, 4766, 4767, 4768, 4769]) print s if s == 0: for job in o: if job == None: continue print job.PandaID for file in job.Files: print file.lfn, file.type print "---------------------" s, o = Client.queryPandaIDs([0]) print s print o
merge_tail = 0 print 'Getting jobs for task %s with status finished and status merging finished' % t.name merging_jobs_list = Job.objects.filter(task=t).filter( status='finished').filter(status_merging_mdst='finished').filter( status_x_check='no').order_by('panda_id_merging_mdst').values_list( 'panda_id_merging_mdst', flat=True).distinct() print 'Got list of %s jobs' % len(merging_jobs_list) if len(merging_jobs_list) == 0: print 'No merging jobs found for cross-checking' continue for j in merging_jobs_list: nEvents = 0 print 'Getting number of events of merging job %s' % j s, o = Client.getJobStatus([j], None) if s == 0: for x in o: nEvents = x.nEvents print 'nEvents = %s' % nEvents print 'Getting all jobs with panda_id_merging_mdst %s' % j jobs_list = Job.objects.all().filter(panda_id_merging_mdst=j).filter( status='finished').order_by('id') if len(jobs_list) == 0: print 'No jobs found for cross-checking' continue nEventsSum = 0 print 'Getting sum of number of events of jobs' for k in jobs_list:
from taskbuffer.TaskBuffer import taskBuffer from brokerage.SiteMapper import SiteMapper from config import panda_config # instantiate TB taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1) # instantiate sitemapper siteMapper = SiteMapper(taskBuffer) import httplib import commands id = sys.argv[1] s,o = Client.getJobStatus([id]) if s != 0: print "failed to get job with:%s" % s sys.exit(0) job = o[0] if job == None: print "got None" sys.exit(0) xml = """<?xml version="1.0" encoding="UTF-8" standalone="no" ?> <!-- ATLAS file meta-data catalog --> <!DOCTYPE POOLFILECATALOG SYSTEM "InMemory"> <POOLFILECATALOG>
#for idx,argv in enumerate(sys.argv): # if argv == '-s': # aSrvID = sys.argv[idx+1] # sys.argv = sys.argv[:idx] # break quiet = False jobs_to_query = [] for j in sys.argv[1:]: if j == "-q": quiet = True if j.isdigit(): jobs_to_query.append(j) o = Client.getJobStatus(jobs_to_query) for j in o[1]: if j is None: continue message_template = "%s: %s (%s): %s %s" % ( j.PandaID, j.jobStatus, j.jobSubStatus, j.jobParameters, j.cmtConfig) if quiet: message_template = j.jobStatus print message_template os.exit(0) message = None if j.jobStatus == 'finished': message = c(message_template).green elif j.jobStatus == 'failed':