def eraseDispDatasets(ids):
print "eraseDispDatasets"
datasets = []
# get jobs
status, jobs = Client.getJobStatus(ids)
if status != 0:
return
# gather dispDBlcoks
for job in jobs:
# dispatchDS is not a DQ2 dataset in US
if job.cloud == 'US':
continue
# erase disp datasets for production jobs only
if job.prodSourceLabel != 'managed':
continue
for file in job.Files:
if file.dispatchDBlock == 'NULL':
continue
if (not file.dispatchDBlock in datasets) and \
re.search('_dis\d+$',file.dispatchDBlock) != None:
datasets.append(file.dispatchDBlock)
# erase
for dataset in datasets:
print 'erase %s' % dataset
status, out = ddm.DQ2.main('eraseDataset', dataset)
print out
def eraseDispDatasets(ids):
print "eraseDispDatasets"
datasets = []
# get jobs
status,jobs = Client.getJobStatus(ids)
if status != 0:
return
# gather dispDBlcoks
for job in jobs:
# dispatchDS is not a DQ2 dataset in US
if job.cloud == 'US':
continue
# erase disp datasets for production jobs only
if job.prodSourceLabel != 'managed':
continue
for file in job.Files:
if file.dispatchDBlock == 'NULL':
continue
if (not file.dispatchDBlock in datasets) and \
re.search('_dis\d+$',file.dispatchDBlock) != None:
datasets.append(file.dispatchDBlock)
# erase
for dataset in datasets:
print 'erase %s' % dataset
status,out = ddm.DQ2.main('eraseDataset',dataset)
print out
def getJobStatus(self, ids):
import userinterface.Client as Client
s,o = Client.getJobStatus(ids)
result = {}
if s != 0:
_logger.error('Error response code: %s %s' %(str(s), str(o)))
return result
for x in o:
result[x.PandaID] = x.jobStatus
return result
def getJobStatus(self, ids):
import userinterface.Client as Client
s, o = Client.getJobStatus(ids)
result = {}
if s != 0:
_logger.error('Error response code: %s %s' % (str(s), str(o)))
return result
for x in o:
result[x.PandaID] = x.jobStatus
return result
def getStatus(self, expectedStates):
idList = [job['jobID'] for job in self.__jobList]
print idList
status, jobInfoList = Client.getJobStatus(idList)
print jobInfoList
assert status == 0, "Retrieval of job state finished with status: %s" %status
for job in jobInfoList:
assert job.jobStatus in expectedStates, "Recently defined job was not in states %s (PandaID: %s jobStatus: %s)" %(expectedStates, job.PandaID, job.jobStatus)
return jobInfoList
def getStatus(self, expectedStates):
idList = [job['jobID'] for job in self.__jobList]
_logger.info("%s" % idList)
status, jobInfoList = Client.getJobStatus(idList)
_logger.info("%s" % jobInfoList)
assert status == 0, "Retrieval of job state finished with status: %s" % status
for job in jobInfoList:
assert job.jobStatus in expectedStates, "Recently defined job was not in states %s (PandaID: %s jobStatus: %s)" % (
expectedStates, job.PandaID, job.jobStatus)
return jobInfoList
def eraseDispDatasets(ids):
datasets = []
# get jobs
status,jobs = Client.getJobStatus(ids)
if status != 0:
return
# gather dispDBlcoks
for job in jobs:
for file in job.Files:
if not file.dispatchDBlock in datasets:
datasets.append(file.dispatchDBlock)
# erase
for dataset in datasets:
ddm.DQ2.main(['eraseDataset',datasets])
def update_status():
# Method to sync PandaDB job status and local job status
# show users jobs
jobs = Job.query.filter(Job.pandaid.isnot(None))\
.filter(~Job.status.in_(['finished', 'failed', 'cancelled']))\
.all()
ids = []
localids = []
for job in jobs:
localids.append(job.id)
ids.append(job.pandaid)
# get status update
if len(ids) > 0:
_logger.debug('getJobStatus: ' + str(ids))
s, o = Client.getJobStatus(ids)
_logger.debug(o)
_logger.debug(s)
_logger.debug("---------------------")
for job in jobs:
if job.pandaid in ids:
for obj in o:
if obj.PandaID == job.pandaid:
# Update attemptNr if changed
if job.attemptnr not in [obj.attemptNr]:
job.attemptnr = obj.attemptNr
jobs_.save(job)
# Update status if changed
if job.status != obj.jobStatus:
job.status = obj.jobStatus
job.modification_time = datetime.utcnow()
jobs_.save(job)
return localids
def update_status():
# Method to sync PandaDB job status and local job status
# show users jobs
jobs = Job.query.filter(Job.pandaid.isnot(None))\
.filter(~Job.status.in_(['finished', 'failed', 'cancelled']))\
.all()
ids = []
localids = []
for job in jobs:
localids.append(job.id)
ids.append(job.pandaid)
# get status update
if len(ids) > 0:
_logger.debug('getJobStatus: ' + str(ids))
s, o = Client.getJobStatus(ids)
_logger.debug(o)
_logger.debug(s)
_logger.debug("---------------------")
for job in jobs:
if job.pandaid in ids:
for obj in o:
if obj.PandaID == job.pandaid:
# Update attemptNr if changed
if job.attemptnr not in [obj.attemptNr]:
job.attemptnr = obj.attemptNr
jobs_.save(job)
# Update status if changed
if job.status != obj.jobStatus:
job.status = obj.jobStatus
job.modification_time = datetime.utcnow()
jobs_.save(job)
return localids
ids = {'pandatest.000003.dd.input._00028.junk':'6c19e1fc-ee8c-4bae-bd4c-c9e5c73aca27',
'pandatest.000003.dd.input._00033.junk':'98f79ba1-1793-4253-aac7-bdf90a51d1ee',
'pandatest.000003.dd.input._00039.junk':'33660dd5-7cef-422a-a7fc-6c24cb10deb1'}
for lfn in ids.keys():
file = FileSpec()
file.lfn = lfn
file.GUID = ids[file.lfn]
file.dataset = 'pandatest.000003.dd.input'
file.type = 'input'
job.addFile(file)
s,o = Client.submitJobs([job])
print "---------------------"
print s
print o
print "---------------------"
s,o = Client.getJobStatus([4934, 4766, 4767, 4768, 4769])
print s
if s == 0:
for job in o:
if job == None:
continue
print job.PandaID
for file in job.Files:
print file.lfn,file.type
print "---------------------"
s,o = Client.queryPandaIDs([0])
print s
print o
from django.db.models import Q
from prodsys.models import Task, Job
print 'Getting tasks with status send and running'
tasks_list = Task.objects.all().filter(Q(status='send') | Q(status='running'))
print 'Got list of %s tasks' % len(tasks_list)
for t in tasks_list:
print 'Getting jobs with status send and running for task with status send and running'
jobs_list = list(
Job.objects.filter(
task=t).filter(Q(status='sent') | Q(status='running')).values_list(
'panda_id', flat=True)[:50])
print 'Got list of %s jobs' % len(jobs_list)
print 'Sending request to PanDA server'
s, o = Client.getJobStatus(jobs_list, None)
if s == 0:
for x in o:
print 'Getting job for PandaID=%s' % x.PandaID
j_update = Job.objects.get(panda_id=x.PandaID)
if j_update.status != x.jobStatus:
today = datetime.datetime.today()
if x.jobStatus == 'running' or x.jobStatus == 'finished' or x.jobStatus == 'failed':
print 'Going to update status of job %s from %s to %s' % (
j_update.file, j_update.status, x.jobStatus)
j_update.status = x.jobStatus
j_update.date_updated = today
if x.jobStatus == 'finished':
j_update.status_merging_mdst = 'ready'
from userinterface.Client import baseURLSSL
from taskbuffer.TaskBuffer import taskBuffer
from brokerage.SiteMapper import SiteMapper
from config import panda_config
# instantiate TB
taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=1)
# instantiate sitemapper
siteMapper = SiteMapper(taskBuffer)
import httplib
import commands
id = sys.argv[1]
s, o = Client.getJobStatus([id])
if s != 0:
print "failed to get job with:%s" % s
sys.exit(0)
job = o[0]
if job == None:
print "got None"
sys.exit(0)
xml = """<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
<!-- ATLAS file meta-data catalog -->
<!DOCTYPE POOLFILECATALOG SYSTEM "InMemory">
<POOLFILECATALOG>
def x_check_mdst():
logger = logging.getLogger('periodic_tasks_logger')
getRotatingFileHandler(logger, 'celery.x_check_mdst.log')
today = datetime.datetime.today()
logger.info('Starting %s' % __name__)
logger.info('Getting tasks with status send and running')
tasks_list = Task.objects.all().filter(
Q(status='send') | Q(status='running'))
# tasks_list = Task.objects.all().filter(name='dvcs2016P07t1_mu+_part1')
logger.info('Got list of %s tasks' % len(tasks_list))
for t in tasks_list:
logger.info(
'Getting run numbers for task %s status merging finished and status x check no'
% t.name)
runs_list = Job.objects.filter(task=t).filter(
status_merging_mdst='finished').filter(
status_x_check='no').order_by('run_number').values_list(
'run_number', flat=True).distinct()
logger.info('Got list of %s runs' % len(runs_list))
if len(runs_list) == 0:
logger.info('No runs found for checking')
continue
rnum = 0
for run_number in runs_list:
# if rnum == 1:
# break
logger.info('Getting all jobs of run number %s' % run_number)
jobs_list_count = Job.objects.all().filter(task=t).filter(
run_number=run_number).values_list(
'panda_id_merging_mdst', flat=True).distinct().count()
logger.info(jobs_list_count)
nEvents_run = 0
nEvents_chunks = 0
logger.info(
'Getting jobs with merging mdst status finished for task %s and run number %s'
% (t.name, run_number))
merging_jobs_list = list(
Job.objects.filter(task=t).filter(
run_number=run_number).filter(
status_merging_mdst='finished').values_list(
'panda_id_merging_mdst', flat=True).distinct())
logger.info('Got list of %s jobs' % len(merging_jobs_list))
if len(merging_jobs_list) != jobs_list_count:
logger.info(
'Not all jobs of run are ready for checking, skipping')
continue
logger.info('Sending request to PanDA server')
s, o = Client.getJobStatus(merging_jobs_list, None)
if s == 0:
for x in o:
for j in merging_jobs_list:
if j == x.PandaID:
nEvents_run += x.nEvents
logger.info('Got merging job with % events' %
x.nEvents)
logger.info(
'Going to get chunks with panda_id_merging_mdst=%s'
% x.PandaID)
jobs_list = list(
Job.objects.filter(panda_id_merging_mdst=x.
PandaID).values_list(
'panda_id', flat=True))
logger.info('Got list of %s jobs' % len(jobs_list))
logger.info('Sending request to PanDA server')
s1, o1 = Client.getJobStatus(jobs_list, None)
if s1 == 0:
for x1 in o1:
nEvents_chunks += x1.nEvents
logger.info(
'Number of events in merged files of run %s and sum of events in chunks: %s - %s'
% (run_number, nEvents_run, nEvents_chunks))
if nEvents_run == nEvents_chunks:
logger.info(
'Number of events in merged files and run are equal')
logger.info(
'Going to update x check status of jobs of run number %s to yes'
% run_number)
jobs_list_update = Job.objects.filter(task=t).filter(
run_number=run_number).update(status_x_check='yes',
date_updated=today)
else:
logger.info(
'Number of events in merged files and run are not equal')
logger.info(
'Going to update x check status of jobs of run number %s to manual check is needed'
% run_number)
jobs_list_update = Job.objects.filter(task=t).filter(
run_number=run_number).update(
status_x_check='manual check is needed',
date_updated=today)
rnum += 1
logger.info('done')
return True
'98f79ba1-1793-4253-aac7-bdf90a51d1ee',
'pandatest.000003.dd.input._00039.junk':
'33660dd5-7cef-422a-a7fc-6c24cb10deb1'
}
for lfn in ids.keys():
file = FileSpec()
file.lfn = lfn
file.GUID = ids[file.lfn]
file.dataset = 'pandatest.000003.dd.input'
file.type = 'input'
job.addFile(file)
s, o = Client.submitJobs([job])
print "---------------------"
print s
print o
print "---------------------"
s, o = Client.getJobStatus([4934, 4766, 4767, 4768, 4769])
print s
if s == 0:
for job in o:
if job == None:
continue
print job.PandaID
for file in job.Files:
print file.lfn, file.type
print "---------------------"
s, o = Client.queryPandaIDs([0])
print s
print o
merge_tail = 0
print 'Getting jobs for task %s with status finished and status merging finished' % t.name
merging_jobs_list = Job.objects.filter(task=t).filter(
status='finished').filter(status_merging_mdst='finished').filter(
status_x_check='no').order_by('panda_id_merging_mdst').values_list(
'panda_id_merging_mdst', flat=True).distinct()
print 'Got list of %s jobs' % len(merging_jobs_list)
if len(merging_jobs_list) == 0:
print 'No merging jobs found for cross-checking'
continue
for j in merging_jobs_list:
nEvents = 0
print 'Getting number of events of merging job %s' % j
s, o = Client.getJobStatus([j], None)
if s == 0:
for x in o:
nEvents = x.nEvents
print 'nEvents = %s' % nEvents
print 'Getting all jobs with panda_id_merging_mdst %s' % j
jobs_list = Job.objects.all().filter(panda_id_merging_mdst=j).filter(
status='finished').order_by('id')
if len(jobs_list) == 0:
print 'No jobs found for cross-checking'
continue
nEventsSum = 0
print 'Getting sum of number of events of jobs'
for k in jobs_list:
from taskbuffer.TaskBuffer import taskBuffer
from brokerage.SiteMapper import SiteMapper
from config import panda_config
# instantiate TB
taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1)
# instantiate sitemapper
siteMapper = SiteMapper(taskBuffer)
import httplib
import commands
id = sys.argv[1]
s,o = Client.getJobStatus([id])
if s != 0:
print "failed to get job with:%s" % s
sys.exit(0)
job = o[0]
if job == None:
print "got None"
sys.exit(0)
xml = """<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
<!-- ATLAS file meta-data catalog -->
<!DOCTYPE POOLFILECATALOG SYSTEM "InMemory">
<POOLFILECATALOG>
#for idx,argv in enumerate(sys.argv):
# if argv == '-s':
# aSrvID = sys.argv[idx+1]
# sys.argv = sys.argv[:idx]
# break
quiet = False
jobs_to_query = []
for j in sys.argv[1:]:
if j == "-q":
quiet = True
if j.isdigit():
jobs_to_query.append(j)
o = Client.getJobStatus(jobs_to_query)
for j in o[1]:
if j is None:
continue
message_template = "%s: %s (%s): %s %s" % (
j.PandaID, j.jobStatus, j.jobSubStatus, j.jobParameters, j.cmtConfig)
if quiet:
message_template = j.jobStatus
print message_template
os.exit(0)
message = None
if j.jobStatus == 'finished':
message = c(message_template).green
elif j.jobStatus == 'failed':
