def printDsLocation(ds, clean=False, anyb=False):
"""
Only printing
"""
onlycomplete = not anyb
sites = sorted(phedexClient.getBlockReplicaSites(ds, onlycomplete))
print ds
if onlycomplete:
print "block replicas (only complete):"
else:
print "All block replicas"
print ",".join(sites)
# print subscriptions only when asked for full block
if onlycomplete:
sites = sorted(phedexClient.getSubscriptionSites(ds))
print "subscriptions:"
print ",".join(sites)
# print in the clean ready-to-use format
if clean:
sites2 = []
for s in sites:
if "_MSS" in s or "_Export" in s or "_Buffer" in s:
continue
s = s.replace("_Disk", "")
sites2.append(s)
print ",".join(sites2)
# and the size
size = dbsClient.getDatasetSize(ds)
print formatSize(size)
def printDsLocation(ds, clean=False, anyb=False):
"""
Only printing
"""
onlycomplete = not anyb
sites = sorted(phedexClient.getBlockReplicaSites(ds, onlycomplete))
print ds
if onlycomplete:
print "block replicas (only complete):"
else:
print "All block replicas"
print ','.join(sites)
# print subscriptions only when asked for full block
if onlycomplete:
sites = sorted(phedexClient.getSubscriptionSites(ds))
print "subscriptions:"
print ','.join(sites)
# print in the clean ready-to-use format
if clean:
sites2 = []
for s in sites:
if '_MSS' in s or '_Export' in s or '_Buffer' in s:
continue
s = s.replace('_Disk', '')
sites2.append(s)
print ','.join(sites2)
# and the size
size = dbsClient.getDatasetSize(ds)
print formatSize(size)
def validateClosingWorkflow(url, workflow, closePercentage = 0.95, checkEqual=False,
checkDuplicates=True, checkLumiNumb=False, checkPhedex='custodial'):
"""
Validates if a workflow can be closed out, using different parameters of validation.
returns the response as a dict.
checkPhedex can be 'custodial', 'any' or False
"""
#inputDataset = reqMgrClient.getInputDataSet(url, workflow)
result = {'name':workflow.name, 'datasets': {}}
result['datasets'] = dict( (ds,{}) for ds in workflow.outputDatasets)
closeOutWorkflow = True
#check if dataset is ready
#TODO validate here if workflow is MonteCarlo from GEN with two output
for dataset in workflow.outputDatasets:
closeOutDataset = False
try:
percentage = workflow.percentageCompletion(dataset, skipInvalid=True)
except Exception as e:
print 'Error getting information from DBS', workflow, dataset
percentage = 0.0
#retrieve either custodial or all subscriptions.
try:
if checkPhedex == 'custodial':
phedexReqs = phedexClient.getCustodialSubscriptionRequestSite(dataset)
elif checkPhedex == 'any':
phedexReqs = phedexClient.getSubscriptionSites(dataset)
else:
phedexReqs = None
except Exception:
print 'Error getting phedex info,: ', dataset
phedexReqs = None
duplicate = None
correctLumis = None
transPerc = None
missingSubs = False
equalFiles = None
dbsFiles = dbs3Client.getFileCountDataset(dataset)
phdFiles = phedexClient.getFileCountDataset(url,dataset)
equalFiles = (dbsFiles == phdFiles)
#Check first percentage
if ((checkEqual and percentage == closePercentage)
or (not checkEqual and percentage >= closePercentage)
or dataset.endswith("DQMIO") ): #DQMIO are exceptions (have 0 events)
#if we need to check duplicates
if checkDuplicates:
try:
duplicate = dbs3Client.duplicateRunLumi(dataset, skipInvalid=True)
except Exception:
print "Error in checking duplicate lumis for", dataset
#if we need to check for correct lumi number
if checkLumiNumb:
correctLumis = checkCorrectLumisEventGEN(dataset)
#dataset healthy means:
# checkDuplicates -> no duplicates
# checkLumiNumb -> correct
if (not (checkDuplicates and duplicate) and
not ( checkLumiNumb and not correctLumis)):
#if phedex check not required we can closeout
if not checkPhedex:
#last check, that files are equal
closeOutDataset = equalFiles
#if phedex check is required and has it
elif checkPhedex and phedexReqs:
try:
transPerc = phedexClient.getTransferPercentage(url, dataset, phedexReqs[0])
except:
transPerc = None
#last check, that files are equal
closeOutDataset = equalFiles
else:
#TODO only missing subscription if equal # of files
missingSubs = equalFiles
#if at least one dataset is not ready wf cannot be closed out
closeOutWorkflow = closeOutWorkflow and closeOutDataset
#load results in a dict
result['datasets'][dataset]["percentage"] = percentage
result['datasets'][dataset]["duplicate"] = duplicate
result['datasets'][dataset]["phedexReqs"] = phedexReqs
result['datasets'][dataset]["closeOutDataset"] = closeOutDataset
result['datasets'][dataset]["transPerc"] = transPerc
result['datasets'][dataset]["correctLumis"] = correctLumis
result['datasets'][dataset]["missingSubs"] = missingSubs
result['datasets'][dataset]["dbsFiles"] = dbsFiles
result['datasets'][dataset]["phedexFiles"] = phdFiles
result['closeOutWorkflow'] = closeOutWorkflow
return result