if max_wf: wfs = wfs[:max_wf]
random.shuffle( wfs )
all_blocks_at_sites = defaultdict(set)
done = json.loads(open('myblock_done.json').read())
print len(wfs),"to look the output of"
for iw,wfo in enumerate(wfs):
print "%s/%s:"%(iw,len(wfs)),wfo.name
#wfi = workflowInfo(url, wfo.name)
#outs= wfi.request['OutputDatasets']
wl = getWorkLoad(url, wfo.name)
outs= wl['OutputDatasets']
for out in outs:
blocks_at_sites = getDatasetBlockAndSite(url, out, group="")
deletions = getDatasetOnGoingDeletion(url, out)
if len(deletions):
print "\t\tshould not subscribe with on-going deletions",out
continue
for site,blocks in blocks_at_sites.items():
if 'Buffer' in site or 'Export' in site or 'MSS' in site: continue
all_blocks_at_sites[site].update( blocks )
print "\t",out
print "\t\t",len(blocks_at_sites),"sites",sorted(blocks_at_sites.keys()),"with unsubscribed blocks"
if len(all_blocks_at_sites.keys())==0 and len(wfs):
## no subscription to be done at this time, let me know
sendEmail('no unsubscribed blocks','while catching up %s does not need to be there anymore'%( one_status ))
print len(all_blocks_at_sites.keys()),"sites to subscribe things at"
continue
try:
pid = filter(lambda w: w.count('-') == 2,
wqe['RequestName'].split('_'))[0]
camp = pid.split('-')[1]
except:
camp = None
if not camp: continue
#print json.dumps( wqe, indent=2)
for b in wqe['Inputs']:
if not '#' in b: continue
#b is the block
ds = b.split('#')[0]
if not ds in block_locations:
s_block_locations[ds] = getDatasetBlockAndSite(url,
ds,
complete='y')
for s in s_block_locations[ds]:
for bl in s_block_locations[ds][s]:
block_locations[ds][bl].append(s)
if not b in block_locations[ds]:
print b, "is not to be found in phedex"
continue
#block_ce = [si.SE_to_CE(s) for s in block_locations[ds][b]]
#wqe_ce = [s for s in wqe['Inputs'][b]]
## true location of the data
block_se = block_locations[ds][b]
## what the global queue thinks about the block location
wqe_se = [si.CE_to_SE(s) for s in wqe['Inputs'][b]]
## where the wf is set to be run at
not_runable_acdc.add( wf['RequestName'] )
#not_processable.add( b )
se_whitelist = sorted(set([si.CE_to_SE(s) for s in wqe['SiteWhitelist'] if s in si.sites_ready]))
missing_in_whitelist = sorted([si.SE_to_CE(s) for s in (set(acdc_location) - set(se_whitelist))])
if wqe['NoInputUpdate']==False and missing_in_whitelist: #(se_whitelist>=acdc_location):
#missing_in_whitelist = sorted([si.SE_to_CE(s) for s in (set(acdc_location) - set(se_whitelist))])
print "Should have",missing_in_whitelist,"also in the whitelist, or have xrootd enabled"
print sorted(acdc_location),"for the ACDC location"
print sorted(se_whitelist),"for the whitelist"
not_runable_acdc.add( wf['RequestName'] )
continue
#b is the block
ds = b.split('#')[0]
if not ds in block_locations:
s_block_locations[ds] = getDatasetBlockAndSite(url, ds, complete='y', vetoes=[])
for s in s_block_locations[ds]:
for bl in s_block_locations[ds][s]:
block_locations[ds][bl].append( s )
if not b in block_locations[ds]:
print b,"is not to be found in phedex, needed by",wfi.request['RequestName']
## should send a critical log
continue
#block_ce = [si.SE_to_CE(s) for s in block_locations[ds][b]]
#wqe_ce = [s for s in wqe['Inputs'][b]]
## true location of the data
block_se = block_locations[ds][b]
print len(wfs), "to look the output of"
for iw, wfn in enumerate(wfs):
if type(wfn) == dict:
wl = wfn
wfn = wl['RequestName']
else:
wl = getWorkLoad(url, wfn)
print "%s/%s:" % (iw, len(wfs)), wfn
if not wl:
continue
outs = wl['OutputDatasets']
for out in outs:
blocks_at_sites = getDatasetBlockAndSite(url, out, group="")
deletions = getDatasetOnGoingDeletion(url, out)
if len(deletions):
print "\t\tshould not subscribe with on-going deletions", out
continue
for site, blocks in blocks_at_sites.items():
if 'Buffer' in site or 'Export' in site or 'MSS' in site: continue
all_blocks_at_sites[site].update(blocks)
print "\t", out
print "\t\t", len(blocks_at_sites), "sites", sorted(
blocks_at_sites.keys()), "with unsubscribed blocks"
if len(all_blocks_at_sites.keys()) == 0 and len(wfs):
## no subscription to be done at this time, let me know
#sendEmail('no unsubscribed blocks','while catching up %s does not need to be there anymore'%( one_status ))
pass
