def batchor(url):
UC = unifiedConfiguration()
SI = global_SI()
CI = campaignInfo()
BI = batchInfo()
## get all workflows in assignment-approved with SubRequestType = relval
all_wfs = []
for user in UC.get("user_relval"):
all_wfs.extend(
getWorkflows(url,
'assignment-approved',
details=True,
user=user,
rtype='TaskChain'))
wfs = filter(
lambda r: r['SubRequestType'] == 'RelVal'
if 'SubRequestType' in r else False, all_wfs)
## need a special treatment for those
hi_wfs = filter(
lambda r: r['SubRequestType'] == 'HIRelVal'
if 'SubRequestType' in r else False, all_wfs)
by_campaign = defaultdict(set)
by_hi_campaign = defaultdict(set)
for wf in wfs:
print "Relval:", wf['RequestName'], wf['Campaign']
by_campaign[wf['Campaign']].add(wf['PrepID'])
for wf in hi_wfs:
print "HI Relval:", wf['RequestName'], wf['Campaign']
by_hi_campaign[wf['Campaign']].add(wf['PrepID'])
default_setup = {
"go": True,
"parameters": {
"SiteWhitelist": ["T1_US_FNAL"],
"MergedLFNBase": "/store/relval",
"Team": "relval",
"NonCustodialGroup": "RelVal"
},
"custodial_override": "notape",
"phedex_group": "RelVal",
"lumisize": -1,
"fractionpass": 0.0,
"maxcopies": 1
}
default_hi_setup = copy.deepcopy(default_setup)
add_on = {}
relval_routing = UC.get('relval_routing')
def pick_one_site(p):
## modify the parameters on the spot to have only one site
if "parameters" in p and "SiteWhitelist" in p["parameters"] and len(
p["parameters"]["SiteWhitelist"]) > 1:
choose_from = list(
set(p["parameters"]["SiteWhitelist"]) & set(SI.sites_ready))
picked = random.choice(choose_from)
print "picked", picked, "from", choose_from
p["parameters"]["SiteWhitelist"] = [picked]
batches = BI.all()
for campaign in by_campaign:
if campaign in batches: continue
## get a bunch of information
setup = copy.deepcopy(default_setup)
for key in relval_routing:
if key in campaign:
## augment with the routing information
augment_with = relval_routing[key]
print "Modifying the batch configuration because of keyword", key
print "with", augment_with
setup = deep_update(setup, augment_with)
pick_one_site(setup)
add_on[campaign] = setup
sendLog('batchor',
'Adding the relval campaigns %s with parameters \n%s' %
(campaign, json.dumps(setup, indent=2)),
level='critical')
BI.update(campaign, by_campaign[campaign])
for campaign in by_hi_campaign:
if campaign in batches: continue
## get a bunch of information
setup = copy.deepcopy(default_hi_setup)
possible_sites = set(["T1_DE_KIT", "T1_FR_CCIN2P3"])
hi_site = random.choice(list(possible_sites))
setup["parameters"]["SiteWhitelist"] = [hi_site]
pick_one_site(setup)
add_on[campaign] = setup
sendLog('batchor',
'Adding the HI relval campaigns %s with parameters \n%s' %
(campaign, json.dumps(setup, indent=2)),
level='critical')
BI.update(campaign, by_hi_campaign[campaign])
## only new campaigns in announcement
for new_campaign in list(
set(add_on.keys()) - set(CI.all(c_type='relval'))):
## this is new, and can be announced as such
print new_campaign, "is new stuff"
subject = "Request of RelVal samples batch %s" % new_campaign
text = """Dear all,
A new batch of relval workflows was requested.
Batch ID:
%s
Details of the workflows:
https://dmytro.web.cern.ch/dmytro/cmsprodmon/requests.php?campaign=%s
This is an automated message""" % (
new_campaign,
new_campaign,
)
print subject
print text
to = ['*****@*****.**']
sendEmail(subject, text, destination=to)
sendLog('batchor', text, level='critical')
## go through all existing campaigns and remove the ones not in use anymore ?
for old_campaign in CI.all(c_type='relval'):
all_in_batch = getWorkflowByCampaign(url, old_campaign, details=True)
if not all_in_batch: continue
is_batch_done = all(
map(
lambda s: not s in [
'completed', 'force-complete', 'running-open',
'running-closed', 'acquired', 'assigned',
'assignment-approved'
], [wf['RequestStatus'] for wf in all_in_batch]))
## check all statuses
if is_batch_done:
#print "batch",old_campaign,"can be closed or removed if necessary"
#campaigns[old_campaign]['go'] = False ## disable
CI.pop(old_campaign) ## or just drop it all together ?
BI.pop(old_campaign)
print "batch", old_campaign, " configuration was removed"
## merge all anyways
CI.update(add_on, c_type='relval')
def close(self):
if os.path.isfile('.closor_stop'):
print "The closing of workflows is shortened"
return
url = self.url
batch_go = self.batch_go
CI = self.CI
UC = self.UC
wfo = self.wfo
jump_the_line = self.jump_the_line
batch_goodness = self.batch_goodness
check_parentage_to_announce = UC.get('check_parentage_to_announce')
check_fullcopy_to_announce = UC.get('check_fullcopy_to_announce')
## what is the expected #lumis
self.wfi = workflowInfo(url, wfo.name)
wfi = self.wfi
wfo.wm_status = wfi.request['RequestStatus']
if wfi.isRelval():
has_batch_go = False
batch_name = wfi.getCampaign()
if not batch_name in batch_go:
## do the esimatation whethere this can be announced : only once per batch
in_batches = getWorkflowByCampaign(url,
batch_name,
details=True)
batch_go[batch_name] = all(
map(
lambda s: not s in [
'completed', 'running-open', 'running-closed',
'acquired', 'staged', 'staging', 'assigned',
'assignment-approved'
], [r['RequestStatus'] for r in in_batches]))
## already verified
has_batch_go = batch_go[batch_name]
if not has_batch_go:
wfi.sendLog(
'closor',
'Cannot close for now because the batch <a href=https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?campaign=%s>%s</a> is not all close'
% (batch_name, batch_name))
return
if wfi.request['RequestStatus'] in ['announced', 'normal-archived'
] and not options.force:
## manually announced ??
self.to_status = 'done'
self.to_wm_status = wfi.request['RequestStatus']
wfi.sendLog(
'closor',
'%s is announced already : %s' % (wfo.name, self.to_wm_status))
return
if jump_the_line:
wfi.sendLog('closor', 'Announcing while completing')
expected_lumis = 1
if not 'TotalInputLumis' in wfi.request:
print wfo.name, "has not been assigned yet, or the database is corrupted"
elif wfi.request['TotalInputLumis'] == 0:
print wfo.name, "is corrupted with 0 expected lumis"
else:
expected_lumis = wfi.request['TotalInputLumis']
## what are the outputs
outputs = wfi.request['OutputDatasets']
## check whether the number of lumis is as expected for each
all_OK = defaultdict(lambda: False)
stats = defaultdict(int)
#print outputs
if len(outputs):
print wfo.name, wfi.request['RequestStatus']
for out in outputs:
event_count, lumi_count = getDatasetEventsAndLumis(dataset=out)
self.outs.append(Output(datasetname=out))
odb = self.outs[-1]
odb.workflow = wfo
odb.nlumis = lumi_count
odb.nevents = event_count
odb.workfow_id = wfo.id
if odb.expectedlumis < expected_lumis:
odb.expectedlumis = expected_lumis
else:
expected_lumis = odb.expectedlumis
odb.date = time.mktime(time.gmtime())
fraction = lumi_count / float(expected_lumis) * 100.
completion_line = "%60s %d/%d = %3.2f%%" % (
out, lumi_count, expected_lumis, fraction)
wfi.sendLog('closor', "\t%s" % completion_line)
if wfi.isRelval() and fraction < batch_goodness:
self.batch_warnings[wfi.getCampaign()].add(completion_line)
if fraction < 50:
self.batch_extreme_warnings[wfi.getCampaign()].add(
completion_line)
stats[out] = lumi_count
all_OK[out] = True
## check for at least one full copy prior to moving on
#in_full = {}
for out in outputs:
all_OK[out] = True
#in_full[out] = []
#presence = getDatasetPresence( url, out )
#where = [site for site,info in presence.items() if info[0]]
#if where:
# all_OK[out] = True
# print out,"is in full at",",".join(where)
# in_full[out] = copy.deepcopy(where)
#else:
# going_to = wfi.request['NonCustodialSites']+wfi.request['CustodialSites']
# wfi.sendLog('closor',"%s is not in full anywhere. send to %s"%(out, ",".join(sorted(going_to))))
# at_destination = dict([(k,v) for (k,v) in presence.items() if k in going_to])
# else_where = dict([(k,v) for (k,v) in presence.items() if not k in going_to])
# print json.dumps( at_destination )
# print json.dumps( else_where, indent=2 )
## do the full stuck transfer study, missing files and shit !
#for there in going_to:
# late_info = findLateFiles(url, out, going_to = there )
# for l in late_info:
# l.update({"workflow":wfo.name,"dataset":out})
# self.all_late_files.extend( late_info )
# if check_fullcopy_to_announce:
## only set this false if the check is relevant
# all_OK[out] = False
## verify if we have to do harvesting
#if not options.no_harvest and not jump_the_line:
# #(OK, requests) = spawn_harvesting(url, wfi, in_full)
# sites_for_DQMHarvest = UC.get("sites_for_DQMHarvest")
# (OK, requests) = spawn_harvesting(url, wfi, sites_for_DQMHarvest)
# print "Harvesting workflow has been created and assigned to: "
# print sites_for_DQMHarvest
# all_OK.update( OK )
## only that status can let me go into announced
if all(all_OK.values()) and (
(wfi.request['RequestStatus'] in ['closed-out']) or options.force
or jump_the_line):
print wfo.name, "to be announced"
results = []
if not results:
for out in outputs:
print "dealing with", out
if out in stats and not stats[out]:
continue
_, dsn, process_string, tier = out.split('/')
if all_OK[out]:
print "setting valid"
results.append(
setDatasetStatus(out, 'VALID', withFiles=False))
if all_OK[out] and wfi.isRelval():
## make the specific relval rules and the replicas
## figure the destination(s) out
destinations = set()
if tier in UC.get("tiers_to_rucio_relval"):
wfi.sendLog(
'closor',
"Data Tier: %s is blacklisted, so skipping dataset placement for: %s"
% (tier, out))
continue
if tier != "RECO" and tier != "ALCARECO":
destinations.add('T2_CH_CERN')
if tier == "GEN-SIM":
destinations.add('T1_US_FNAL_Disk')
if tier == "GEN-SIM-DIGI-RAW":
destinations.add('T1_US_FNAL_Disk')
if tier == "GEN-SIM-RECO":
destinations.add('T1_US_FNAL_Disk')
if "RelValTTBar" in dsn and "TkAlMinBias" in process_string and tier != "ALCARECO":
destinations.add('T2_CH_CERN')
if "MinimumBias" in dsn and "SiStripCalMinBias" in process_string and tier != "ALCARECO":
destinations.add('T2_CH_CERN')
if destinations:
wfi.sendLog(
'closor', '%s to go to %s' %
(out, ', '.join(sorted(destinations))))
elif all_OK[out]:
campaign = None
try:
campaign = out.split('/')[2].split('-')[0]
except:
if 'Campaign' in wfi.request and wfi.request[
'Campaign']:
campaign = wfi.request['Campaign']
to_DDM = False
## campaign override
if campaign and campaign in CI.campaigns and 'toDDM' in CI.campaigns[
campaign] and tier in CI.campaigns[campaign][
'toDDM']:
to_DDM = True
## by typical enabling
if tier in UC.get("tiers_to_rucio_nonrelval"):
wfi.sendLog(
'closor',
"Data Tier: %s is blacklisted, so skipping dataset placement for: %s"
% (tier, out))
continue
if tier in UC.get("tiers_to_DDM"):
to_DDM = True
## check for unitarity
if not tier in UC.get("tiers_no_DDM") + UC.get(
"tiers_to_DDM"):
print "tier", tier, "neither TO or NO DDM for", out
results.append('Not recognitized tier %s' % tier)
#sendEmail("failed DDM injection","could not recognize %s for injecting in DDM"% out)
sendLog(
'closor',
"could not recognize %s for injecting in DDM" %
out,
level='critical')
continue
n_copies = 1
destinations = []
if to_DDM and campaign and campaign in CI.campaigns and 'DDMcopies' in CI.campaigns[
campaign]:
ddm_instructions = CI.campaigns[campaign][
'DDMcopies']
if type(ddm_instructions) == int:
n_copies = CI.campaigns[campaign]['DDMcopies']
elif type(ddm_instructions) == dict:
## a more fancy configuration
for ddmtier, indication in ddm_instructions.items(
):
if ddmtier == tier or ddmtier in [
'*', 'all'
]:
## this is for us
if 'N' in indication:
n_copies = indication['N']
if 'host' in indication:
destinations = indication['host']
destination_spec = ""
if destinations:
destination_spec = "--destination=" + ",".join(
destinations)
group_spec = "" ## not used yet
else:
print wfo.name, "no stats for announcing", out
results.append('No Stats')
# adding check for PrentageResolved flag from ReqMgr:
if wfi.request[
'RequestType'] == 'StepChain' and check_parentage_to_announce:
if wfi.request['ParentageResolved']:
results.append(True)
else:
wfi.sendLog(
'closor',
"Delayed announcement of %s due to unresolved Parentage dependencies"
% wfi.request['RequestName'])
results.append('No ParentageResolved')
if all(
map(lambda result: result in ['None', None, True],
results)):
if not jump_the_line:
## only announce if all previous are fine
res = reqMgrClient.announceWorkflowCascade(
url, wfo.name)
if not res in ['None', None]:
## check the status again, it might well have toggled
wl_bis = workflowInfo(url, wfo.name)
self.to_wm_status = wl_bis.request['RequestStatus']
if wl_bis.request['RequestStatus'] in [
'announced', 'normal-archived'
]:
res = None
else:
res = reqMgrClient.announceWorkflowCascade(
url, wfo.name)
results.append(res)
print results
if all(map(lambda result: result in ['None', None, True],
results)):
if jump_the_line:
if not 'announced' in wfo.status:
self.to_status = wfo.status.replace(
'announce', 'announced')
else:
self.to_status = 'done'
self.closing = True
wfi.sendLog('closor', "workflow outputs are announced")
else:
wfi.sendLog(
'closor', "Error with %s to be announced \n%s" %
(wfo.name, json.dumps(results)))
elif wfi.request['RequestStatus'] in [
'failed', 'aborted', 'aborted-archived', 'rejected',
'rejected-archived', 'aborted-completed'
]:
if wfi.isRelval():
self.to_status = 'forget'
self.to_wm_status = wfi.request['RequestStatus']
wfi.sendLog(
'closor',
"%s is %s, but will not be set in trouble to find a replacement."
% (wfo.name, self.to_wm_status))
else:
self.to_status = 'trouble'
self.to_wm_status = wfi.request['RequestStatus']
else:
print wfo.name, "not good for announcing:", wfi.request[
'RequestStatus']
wfi.sendLog('closor', "cannot be announced")
self.held.add(wfo.name)
def batchor( url ):
UC = unifiedConfiguration()
SI = global_SI()
## get all workflows in assignment-approved with SubRequestType = relval
all_wfs = []
for user in UC.get("user_relval"):
all_wfs.extend( getWorkflows(url, 'assignment-approved', details=True, user=user, rtype='TaskChain') )
wfs = filter( lambda r :r['SubRequestType'] == 'RelVal' if 'SubRequestType' in r else False, all_wfs)
## need a special treatment for those
hi_wfs = filter( lambda r :r['SubRequestType'] == 'HIRelVal' if 'SubRequestType' in r else False, all_wfs)
by_campaign = defaultdict(set)
by_hi_campaign = defaultdict(set)
for wf in wfs:
print "Relval:",wf['RequestName'], wf['Campaign']
#by_campaign[wf['Campaign']].add( wf['RequestName'] )
by_campaign[wf['Campaign']].add( wf['PrepID'] )
for wf in hi_wfs:
print "HI Relval:",wf['RequestName'], wf['Campaign']
#by_hi_campaign[wf['Campaign']].add( wf['RequestName'] )
by_hi_campaign[wf['Campaign']].add( wf['PrepID'] )
default_setup = {
"go" :True,
"parameters" : {
"SiteWhitelist": [ "T1_US_FNAL" ],
"MergedLFNBase": "/store/relval",
"Team" : "relval",
"NonCustodialGroup" : "RelVal"
},
"custodial" : "T1_US_FNAL_MSS",
"custodial_override" : ["DQMIO"],
"phedex_group" : "RelVal",
"lumisize" : -1,
"fractionpass" : 0.0,
"maxcopies" : 1
}
default_hi_setup = copy.deepcopy( default_setup )
add_on = {}
batches = json.loads( open('batches.json').read() )
relval_routing = UC.get('relval_routing')
def pick_one_site( p):
## modify the parameters on the spot to have only one site
if "parameters" in p and "SiteWhitelist" in p["parameters"] and len(p["parameters"]["SiteWhitelist"])>1:
choose_from = list(set(p["parameters"]["SiteWhitelist"]) & set(SI.sites_ready))
picked = random.choice( choose_from )
print "picked",picked,"from",choose_from
p["parameters"]["SiteWhitelist"] = [picked]
for campaign in by_campaign:
if campaign in batches: continue
## get a bunch of information
setup = copy.deepcopy( default_setup )
for key in relval_routing:
if key in campaign:
## augment with the routing information
augment_with = relval_routing[key]
print "Modifying the batch configuration because of keyword",key
print "with",augment_with
setup = deep_update( setup, augment_with )
#if 'cc7' in campaign: setup["parameters"]["SiteWhitelist"] = ["T2_US_Nebraska"]
pick_one_site( setup )
add_on[campaign] = setup
sendLog('batchor','Adding the relval campaigns %s with parameters \n%s'%( campaign, json.dumps( setup, indent=2)),level='critical')
if not campaign in batches: batches[campaign] = []
batches[campaign] = list(set(list(copy.deepcopy( by_campaign[campaign] )) + batches[campaign] ))
for campaign in by_hi_campaign:
if campaign in batches: continue
## get a bunch of information
setup = copy.deepcopy( default_hi_setup )
hi_site = random.choice(["T1_DE_KIT","T1_FR_CCIN2P3"])
setup["parameters"]["SiteWhitelist"]=[ hi_site ]
#setup["parameters"]["SiteWhitelist"]=["T1_DE_KIT","T1_FR_CCIN2P3"]
pick_one_site( setup )
add_on[campaign] = setup
sendLog('batchor','Adding the HI relval campaigns %s with parameters \n%s'%( campaign, json.dumps( setup, indent=2)),level='critical')
if not campaign in batches: batches[campaign] = []
batches[campaign] = list(set(list(copy.deepcopy( by_hi_campaign[campaign] )) + batches[campaign] ))
open('batches.json','w').write( json.dumps( batches , indent=2 ) )
## open the campaign configuration
campaigns = json.loads( open('campaigns.relval.json').read() )
## protect for overwriting ??
for new_campaign in list(set(add_on.keys())-set(campaigns.keys())):
## this is new, and can be announced as such
print new_campaign,"is new stuff"
subject = "Request of RelVal samples batch %s"% new_campaign
text="""Dear all,
A new batch of relval workflows was requested.
Batch ID:
%s
Details of the workflows:
https://dmytro.web.cern.ch/dmytro/cmsprodmon/requests.php?campaign=%s
This is an automated message"""%( new_campaign,
new_campaign,
)
print subject
print text
to = ['*****@*****.**']
sendEmail(subject, text, destination=to)
sendLog('batchor',text, level='critical')
## go through all existing campaigns and remove the ones not in use anymore ?
for old_campaign in campaigns.keys():
all_in_batch = getWorkflowByCampaign(url, old_campaign, details=True)
is_batch_done = all(map(lambda s : not s in ['completed','running-open','running-closed','acquired','assigned','assignment-approved'], [wf['RequestStatus']for wf in all_in_batch]))
## check all statuses
if is_batch_done:
#print "batch",old_campaign,"can be closed or removed if necessary"
#campaigns[old_campaign]['go'] = False ## disable
campaigns.pop( old_campaign ) ## or just drop it all together ?
print "batch",old_campaign," configuration was removed"
## merge all anyways
campaigns.update( add_on )
## write it out for posterity
open('campaigns.json.updated','w').write(json.dumps( campaigns , indent=2))
## read back
rread = json.loads(open('campaigns.json.updated').read())
os.system('mv campaigns.json.updated campaigns.relval.json')
def closor(url, specific=None, options=None):
if userLock(): return
if duplicateLock(): return
if not componentInfo().check(): return
UC = unifiedConfiguration()
CI = campaignInfo()
all_late_files = []
check_fullcopy_to_announce = UC.get('check_fullcopy_to_announce')
jump_the_line = options.announce if options else False
if jump_the_line:
print "announce option is on. Checking on things on-going ready to be announced"
wfs = session.query(Workflow).filter(
Workflow.status.contains('announce')).filter(
sqlalchemy.not_(Workflow.status.contains('announced'))).all()
else:
print "regular option. Checking on things done and to be announced"
wfs = session.query(Workflow).filter(Workflow.status == 'close').all()
wfs_n = [w.name for w in wfs]
print "unique names?"
print len(set(wfs_n)) == len(wfs_n)
held = set()
print len(wfs), "closing"
random.shuffle(wfs)
max_per_round = UC.get('max_per_round').get('closor', None)
if options.limit: max_per_round = options.limit
if max_per_round:
## order them by priority
all_closedout = sorted(getWorkflows(url, 'closed-out', details=True),
key=lambda r: r['RequestPriority'])
all_closedout = [r['RequestName'] for r in all_closedout]
def rank(wfn):
return all_closedout.index(wfn) if wfn in all_closedout else 0
wfs = sorted(wfs, key=lambda wfo: rank(wfo.name), reverse=True)
wfs = wfs[:max_per_round]
batch_go = {}
batch_warnings = defaultdict(set)
batch_goodness = UC.get("batch_goodness")
for iwfo, wfo in enumerate(wfs):
if specific and not specific in wfo.name: continue
print "Progress [%d/%d]" % (iwfo, len(wfs))
## what is the expected #lumis
wfi = workflowInfo(url, wfo.name)
wfo.wm_status = wfi.request['RequestStatus']
if wfi.isRelval():
has_batch_go = False
batch_name = wfi.getCampaign()
if not batch_name in batch_go:
## do the esimatation whethere this can be announced : only once per batch
in_batches = getWorkflowByCampaign(url,
batch_name,
details=True)
batch_go[batch_name] = all(
map(
lambda s: not s in [
'completed', 'running-open', 'running-closed',
'acquired', 'assigned', 'assignment-approved'
], [r['RequestStatus'] for r in in_batches]))
## already verified
has_batch_go = batch_go[batch_name]
if not has_batch_go:
wfi.sendLog(
'closor',
'Cannot close for now because the batch <a href=https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?campaign=%s>%s</a> is not all close'
% (batch_name, batch_name))
continue
if wfi.request['RequestStatus'] in ['announced', 'normal-archived'
] and not options.force:
## manually announced ??
wfo.status = 'done'
wfo.wm_status = wfi.request['RequestStatus']
wfi.sendLog(
'closor',
'%s is announced already : %s' % (wfo.name, wfo.wm_status))
session.commit()
if jump_the_line:
wfi.sendLog('closor', 'Announcing while completing')
expected_lumis = 1
if not 'TotalInputLumis' in wfi.request:
print wfo.name, "has not been assigned yet, or the database is corrupted"
elif wfi.request['TotalInputLumis'] == 0:
print wfo.name, "is corrupted with 0 expected lumis"
else:
expected_lumis = wfi.request['TotalInputLumis']
## what are the outputs
outputs = wfi.request['OutputDatasets']
## check whether the number of lumis is as expected for each
all_OK = defaultdict(lambda: False)
stats = defaultdict(int)
#print outputs
if len(outputs):
print wfo.name, wfi.request['RequestStatus']
for out in outputs:
event_count, lumi_count = getDatasetEventsAndLumis(dataset=out)
odb = session.query(Output).filter(
Output.datasetname == out).first()
if not odb:
print "adding an output object", out
odb = Output(datasetname=out)
odb.workflow = wfo
session.add(odb)
odb.nlumis = lumi_count
odb.nevents = event_count
odb.workfow_id = wfo.id
if odb.expectedlumis < expected_lumis:
odb.expectedlumis = expected_lumis
else:
expected_lumis = odb.expectedlumis
odb.date = time.mktime(time.gmtime())
session.commit()
fraction = lumi_count / float(expected_lumis) * 100.
completion_line = "%60s %d/%d = %3.2f%%" % (
out, lumi_count, expected_lumis, fraction)
wfi.sendLog('closor', "\t%s" % completion_line)
if wfi.isRelval() and fraction < batch_goodness:
batch_warnings[wfi.getCampaign()].add(completion_line)
stats[out] = lumi_count
all_OK[out] = True
## check for at least one full copy prior to moving on
in_full = {}
for out in outputs:
in_full[out] = []
presence = getDatasetPresence(url, out)
where = [site for site, info in presence.items() if info[0]]
if where:
all_OK[out] = True
print out, "is in full at", ",".join(where)
in_full[out] = copy.deepcopy(where)
else:
going_to = wfi.request['NonCustodialSites'] + wfi.request[
'CustodialSites']
wfi.sendLog(
'closor', "%s is not in full anywhere. send to %s" %
(out, ",".join(sorted(going_to))))
at_destination = dict([(k, v) for (k, v) in presence.items()
if k in going_to])
else_where = dict([(k, v) for (k, v) in presence.items()
if not k in going_to])
print json.dumps(at_destination)
print json.dumps(else_where, indent=2)
## do the full stuck transfer study, missing files and shit !
for there in going_to:
late_info = findLateFiles(url, out, going_to=there)
for l in late_info:
l.update({"workflow": wfo.name, "dataset": out})
all_late_files.extend(late_info)
if check_fullcopy_to_announce:
## only set this false if the check is relevant
all_OK[out] = False
## verify if we have to do harvesting
if not options.no_harvest and not jump_the_line:
(OK, requests) = spawn_harvesting(url, wfi, in_full)
all_OK.update(OK)
## only that status can let me go into announced
if all(all_OK.values()) and (
(wfi.request['RequestStatus'] in ['closed-out']) or options.force
or jump_the_line):
print wfo.name, "to be announced"
results = []
if not results:
for out in outputs:
if out in stats and not stats[out]:
continue
_, dsn, process_string, tier = out.split('/')
if all_OK[out]:
results.append(setDatasetStatus(out, 'VALID'))
if all_OK[out] and wfi.isRelval():
## make the specific relval rules and the replicas
## figure the destination(s) out
destinations = set()
if tier != "RECO" and tier != "ALCARECO":
destinations.add('T2_CH_CERN')
if tier == "GEN-SIM":
destinations.add('T1_US_FNAL_Disk')
if tier == "GEN-SIM-DIGI-RAW":
destinations.add('T1_US_FNAL_Disk')
if tier == "GEN-SIM-RECO":
destinations.add('T1_US_FNAL_Disk')
if "RelValTTBar" in dsn and "TkAlMinBias" in process_string and tier != "ALCARECO":
destinations.add('T2_CH_CERN')
if "MinimumBias" in dsn and "SiStripCalMinBias" in process_string and tier != "ALCARECO":
destinations.add('T2_CH_CERN')
if destinations:
wfi.sendLog(
'closor', '%s to go to %s' %
(out, ', '.join(sorted(destinations))))
## call to makereplicarequest under relval => done
for site in destinations:
result = makeReplicaRequest(
url,
site, [out],
'Copy for release validation consumption',
priority='normal',
approve=True,
mail=False,
group='RelVal')
try:
request_id = result['phedex'][
'request_created'][0]['id']
results.append(True)
except:
results.append('Failed relval transfer')
elif all_OK[out]:
campaign = None
try:
campaign = out.split('/')[2].split('-')[0]
except:
if 'Campaign' in wfi.request and wfi.request[
'Campaign']:
campaign = wfi.request['Campaign']
to_DDM = False
## campaign override
if campaign and campaign in CI.campaigns and 'toDDM' in CI.campaigns[
campaign] and tier in CI.campaigns[campaign][
'toDDM']:
to_DDM = True
## by typical enabling
if tier in UC.get("tiers_to_DDM"):
to_DDM = True
## check for unitarity
if not tier in UC.get("tiers_no_DDM") + UC.get(
"tiers_to_DDM"):
print "tier", tier, "neither TO or NO DDM for", out
results.append('Not recognitized tier %s' % tier)
#sendEmail("failed DDM injection","could not recognize %s for injecting in DDM"% out)
sendLog(
'closor',
"could not recognize %s for injecting in DDM" %
out,
level='critical')
continue
n_copies = 1
destinations = []
if to_DDM and campaign and campaign in CI.campaigns and 'DDMcopies' in CI.campaigns[
campaign]:
ddm_instructions = CI.campaigns[campaign][
'DDMcopies']
if type(ddm_instructions) == int:
n_copies = CI.campaigns[campaign]['DDMcopies']
elif type(ddm_instructions) == dict:
## a more fancy configuration
for ddmtier, indication in ddm_instructions.items(
):
if ddmtier == tier or ddmtier in [
'*', 'all'
]:
## this is for us
if 'N' in indication:
n_copies = indication['N']
if 'host' in indication:
destinations = indication['host']
destination_spec = ""
if destinations:
destination_spec = "--destination=" + ",".join(
destinations)
group_spec = "" ## not used yet
### should make this a campaign configuration
## inject to DDM when necessary
if to_DDM:
print "Sending", out, " to DDM"
status = pass_to_dynamo(
[out],
N=n_copies,
sites=destinations if destinations else None,
group=group_spec if group_spec else None)
results.append(status)
if status == True:
wfi.sendLog(
'closor',
'%s is send to dynamo in %s copies %s %s' %
(out, n_copies, sorted(destinations),
group_spec))
else:
sendLog('closor',
"could not add " + out +
" to dynamo pool. check closor logs.",
level='critical')
wfi.sendLog(
'closor', "could not add " + out +
" to dynamo pool. check closor logs.")
else:
print wfo.name, "no stats for announcing", out
results.append('No Stats')
if all(
map(lambda result: result in ['None', None, True],
results)):
if not jump_the_line:
## only announce if all previous are fine
res = reqMgrClient.announceWorkflowCascade(
url, wfo.name)
if not res in ['None', None]:
## check the status again, it might well have toggled
wl_bis = workflowInfo(url, wfo.name)
wfo.wm_status = wl_bis.request['RequestStatus']
session.commit()
if wl_bis.request['RequestStatus'] in [
'announced', 'normal-archived'
]:
res = None
else:
## retry ?
res = reqMgrClient.announceWorkflowCascade(
url, wfo.name)
results.append(res)
#print results
if all(map(lambda result: result in ['None', None, True],
results)):
if jump_the_line:
if not 'announced' in wfo.status:
wfo.status = wfo.status.replace(
'announce', 'announced')
else:
wfo.status = 'done'
session.commit()
wfi.sendLog('closor', "workflow outputs are announced")
else:
wfi.sendLog(
'closor', "Error with %s to be announced \n%s" %
(wfo.name, json.dumps(results)))
elif wfi.request['RequestStatus'] in [
'failed', 'aborted', 'aborted-archived', 'rejected',
'rejected-archived', 'aborted-completed'
]:
if wfi.isRelval():
wfo.status = 'forget'
wfo.wm_status = wfi.request['RequestStatus']
wfi.sendLog(
'closor',
"%s is %s, but will not be set in trouble to find a replacement."
% (wfo.name, wfo.wm_status))
else:
wfo.status = 'trouble'
wfo.wm_status = wfi.request['RequestStatus']
session.commit()
else:
print wfo.name, "not good for announcing:", wfi.request[
'RequestStatus']
wfi.sendLog('closor', "cannot be announced")
held.add(wfo.name)
days_late = 0.
retries_late = 10
really_late_files = [
info for info in all_late_files if info['retries'] >= retries_late
]
really_late_files = [
info for info in really_late_files
if info['delay'] / (60 * 60 * 24.) >= days_late
]
if really_late_files:
subject = 'These %d files are lagging for %d days and %d retries announcing dataset \n%s' % (
len(really_late_files), days_late, retries_late,
json.dumps(really_late_files, indent=2))
#sendEmail('waiting for files to announce', subject)
sendLog('closor', subject, level='warning')
sendLog('closor', subject)
print subject
open('%s/stuck_files.json' % monitor_dir,
'w').write(json.dumps(really_late_files, indent=2))
if held:
sendLog('closor',
"the workflows below are held up \n%s" %
("\n".join(sorted(held))),
level='critical')
#batches = json.loads(open('batches.json').read())
for bname, go in batch_go.items():
if go:
subject = "Release Validation Samples Batch %s" % bname
issues = ""
if batch_warnings[bname]:
issues = "The following datasets have outstanding completion (<%d%%) issues:\n\n" % batch_goodness
issues += "\n".join(sorted(batch_warnings[bname]))
issues += "\n\n"
text = """
Dear all,
a batch of release validation workflows has finished.
Batch ID:
%s
Detail of the workflows
https://dmytro.web.cern.ch/dmytro/cmsprodmon/requests.php?campaign=%s
%s
This is an automated message.
""" % (bname, bname, issues)
to = ['*****@*****.**']
sendEmail(subject, text, destination=to)
def closor(url, specific=None, options=None):
if userLock(): return
if duplicateLock(): return
if not componentInfo().check(): return
UC = unifiedConfiguration()
CI = campaignInfo()
all_late_files = []
check_fullcopy_to_announce = UC.get('check_fullcopy_to_announce')
jump_the_line = options.announce if options else False
if jump_the_line:
wfs = session.query(Workflow).filter(Workflow.status.contains('announce')).filter(sqlalchemy.not_(Workflow.status.contains('announced'))).all()
else:
wfs = session.query(Workflow).filter(Workflow.status=='close').all()
held = set()
print len(wfs),"closing"
max_per_round = UC.get('max_per_round').get('closor',None)
if options.limit: max_per_round = options.limit
random.shuffle( wfs )
if max_per_round: wfs = wfs[:max_per_round]
batch_go = {}
batch_warnings = defaultdict(set)
batch_goodness = UC.get("batch_goodness")
for wfo in wfs:
if specific and not specific in wfo.name: continue
## what is the expected #lumis
wfi = workflowInfo(url, wfo.name )
wfo.wm_status = wfi.request['RequestStatus']
if wfi.isRelval():
has_batch_go = False
batch_name = wfi.getCampaign()
if not batch_name in batch_go:
## do the esimatation whethere this can be announced : only once per batch
in_batches = getWorkflowByCampaign(url , batch_name, details=True)
batch_go[ batch_name ] = all(map(lambda s : not s in ['completed','running-open','running-closed','acquired','assigned','assignment-approved'], [r['RequestStatus'] for r in in_batches]))
## already verified
has_batch_go = batch_go[batch_name]
if not has_batch_go:
wfi.sendLog('closor', 'Cannot close for now because the batch %s is not all close'% batch_name)
continue
if wfi.request['RequestStatus'] in ['announced','normal-archived'] and not options.force:
## manually announced ??
wfo.status = 'done'
wfo.wm_status = wfi.request['RequestStatus']
wfi.sendLog('closor','%s is announced already : %s'%( wfo.name,wfo.wm_status))
session.commit()
if jump_the_line:
wfi.sendLog('closor','Announcing while completing')
expected_lumis = 1
if not 'TotalInputLumis' in wfi.request:
print wfo.name,"has not been assigned yet, or the database is corrupted"
elif wfi.request['TotalInputLumis']==0:
print wfo.name,"is corrupted with 0 expected lumis"
else:
expected_lumis = wfi.request['TotalInputLumis']
## what are the outputs
outputs = wfi.request['OutputDatasets']
## check whether the number of lumis is as expected for each
all_OK = defaultdict(lambda : False)
stats = defaultdict(int)
#print outputs
if len(outputs):
print wfo.name,wfi.request['RequestStatus']
for out in outputs:
event_count,lumi_count = getDatasetEventsAndLumis(dataset=out)
odb = session.query(Output).filter(Output.datasetname==out).first()
if not odb:
print "adding an output object",out
odb = Output( datasetname = out )
odb.workflow = wfo
session.add( odb )
odb.nlumis = lumi_count
odb.nevents = event_count
odb.workfow_id = wfo.id
if odb.expectedlumis < expected_lumis:
odb.expectedlumis = expected_lumis
else:
expected_lumis = odb.expectedlumis
odb.date = time.mktime(time.gmtime())
session.commit()
fraction = lumi_count/float(expected_lumis)*100.
completion_line = "%60s %d/%d = %3.2f%%"%(out,lumi_count,expected_lumis,fraction)
wfi.sendLog('closor',"\t%s"% completion_line)
if wfi.isRelval() and fraction < batch_goodness:
batch_warnings[ wfi.getCampaign()].add( completion_line )
stats[out] = lumi_count
all_OK[out] = True
## check for at least one full copy prior to moving on
in_full = {}
for out in outputs:
in_full[out] = []
presence = getDatasetPresence( url, out )
where = [site for site,info in presence.items() if info[0]]
if where:
all_OK[out] = True
print out,"is in full at",",".join(where)
in_full[out] = copy.deepcopy(where)
else:
going_to = wfi.request['NonCustodialSites']+wfi.request['CustodialSites']
wfi.sendLog('closor',"%s is not in full anywhere. send to %s"%(out, ",".join(sorted(going_to))))
at_destination = dict([(k,v) for (k,v) in presence.items() if k in going_to])
else_where = dict([(k,v) for (k,v) in presence.items() if not k in going_to])
print json.dumps( at_destination )
print json.dumps( else_where, indent=2 )
## do the full stuck transfer study, missing files and shit !
for there in going_to:
late_info = findLateFiles(url, out, going_to = there )
for l in late_info:
l.update({"workflow":wfo.name,"dataset":out})
all_late_files.extend( late_info )
if check_fullcopy_to_announce:
## only set this false if the check is relevant
all_OK[out] = False
## verify if we have to do harvesting
if not options.no_harvest and not jump_the_line:
(OK, requests) = spawn_harvesting(url, wfi, in_full)
all_OK.update( OK )
## only that status can let me go into announced
if all(all_OK.values()) and ((wfi.request['RequestStatus'] in ['closed-out']) or options.force or jump_the_line):
print wfo.name,"to be announced"
results=[]
if not results:
for out in outputs:
if out in stats and not stats[out]:
continue
_,dsn,process_string,tier = out.split('/')
if all_OK[out]:
results.append(setDatasetStatus(out, 'VALID'))
if all_OK[out] and wfi.isRelval():
## make the specific relval rules and the replicas
## figure the destination(s) out
destinations = set()
if tier != "RECO" and tier != "ALCARECO":
destinations.add('T2_CH_CERN')
if tier == "GEN-SIM":
destinations.add('T1_US_FNAL_Disk')
if tier == "GEN-SIM-DIGI-RAW":
destinations.add('T1_US_FNAL_Disk')
if tier == "GEN-SIM-RECO":
destinations.add('T1_US_FNAL_Disk')
if "RelValTTBar" in dsn and "TkAlMinBias" in process_string and tier != "ALCARECO":
destinations.add('T2_CH_CERN')
if "MinimumBias" in dsn and "SiStripCalMinBias" in process_string and tier != "ALCARECO":
destinations.add('T2_CH_CERN')
if destinations:
wfi.sendLog('closor', '%s to go to %s'%(out, ', '.join( sorted( destinations ))))
## call to makereplicarequest under relval => done
for site in destinations:
result = makeReplicaRequest(url, site, [out], 'Copy for release validation consumption', priority='normal', approve=True, mail=False, group='RelVal')
try:
request_id = result['phedex']['request_created'][0]['id']
results.append( True )
except:
results.append( 'Failed relval transfer' )
elif all_OK[out]:
campaign = None
try:
campaign = out.split('/')[2].split('-')[0]
except:
if 'Campaign' in wfi.request and wfi.request['Campaign']:
campaign = wfi.request['Campaign']
to_DDM = False
## campaign override
if campaign and campaign in CI.campaigns and 'toDDM' in CI.campaigns[campaign] and tier in CI.campaigns[campaign]['toDDM']:
to_DDM = True
## by typical enabling
if tier in UC.get("tiers_to_DDM"):
to_DDM = True
## check for unitarity
if not tier in UC.get("tiers_no_DDM")+UC.get("tiers_to_DDM"):
print "tier",tier,"neither TO or NO DDM for",out
results.append('Not recognitized tier %s'%tier)
#sendEmail("failed DDM injection","could not recognize %s for injecting in DDM"% out)
sendLog('closor', "could not recognize %s for injecting in DDM"% out, level='critical')
continue
n_copies = 2
destinations=[]
if to_DDM and campaign and campaign in CI.campaigns and 'DDMcopies' in CI.campaigns[campaign]:
ddm_instructions = CI.campaigns[campaign]['DDMcopies']
if type(ddm_instructions) == int:
n_copies = CI.campaigns[campaign]['DDMcopies']
elif type(ddm_instructions) == dict:
## a more fancy configuration
for ddmtier,indication in ddm_instructions.items():
if ddmtier==tier or ddmtier in ['*','all']:
## this is for us
if 'N' in indication:
n_copies = indication['N']
if 'host' in indication:
destinations = indication['host']
destination_spec = ""
if destinations:
destination_spec = "--destination="+",".join( destinations )
group_spec = "" ## not used yet
### should make this a campaign configuration
## inject to DDM when necessary
if to_DDM:
print "Sending",out," to DDM"
p = os.popen('python assignDatasetToSite.py --nCopies=%d --dataset=%s %s %s --debug 0 --exec'%(n_copies, out,destination_spec, group_spec))
ddm_text = p.read()
print ddm_text
status = p.close()
if status!=None:
print "Failed DDM, retrying to send",out,"a second time"
p = os.popen('python assignDatasetToSite.py --nCopies=%d --dataset=%s %s %s --debug 1 --exec'%(n_copies, out,destination_spec, group_spec))
ddm_text = p.read()
print ddm_text
status = p.close()
if status!=None:
#sendEmail("failed DDM injection","could not add "+out+" to DDM pool. check closor logs.")
sendLog('closor',"could not add "+out+" to DDM pool. check closor logs.", level='critical')
if options.force: status = True
results.append( status )
if status == None:
wfi.sendLog('closor',ddm_text)
wfi.sendLog('closor','%s is send to AnalysisOps DDM pool in %s copies %s'%( out, n_copies, destination_spec))
else:
print wfo.name,"no stats for announcing",out
results.append('No Stats')
if all(map(lambda result : result in ['None',None,True],results)):
if not jump_the_line:
## only announce if all previous are fine
res = reqMgrClient.announceWorkflowCascade(url, wfo.name)
if not res in ['None',None]:
## check the status again, it might well have toggled
wl_bis = workflowInfo(url, wfo.name)
wfo.wm_status = wl_bis.request['RequestStatus']
session.commit()
if wl_bis.request['RequestStatus'] in ['announced','normal-archived']:
res = None
else:
## retry ?
res = reqMgrClient.announceWorkflowCascade(url, wfo.name)
results.append( res )
#print results
if all(map(lambda result : result in ['None',None,True],results)):
if jump_the_line:
if not 'announced' in wfo.status:
wfo.status = wfo.status.replace('announce','announced')
else:
wfo.status = 'done'
session.commit()
wfi.sendLog('closor',"workflow outputs are announced")
else:
wfi.sendLog('closor',"Error with %s to be announced \n%s"%( wfo.name, json.dumps( results )))
elif wfi.request['RequestStatus'] in ['failed','aborted','aborted-archived','rejected','rejected-archived','aborted-completed']:
if wfi.isRelval():
wfo.status = 'forget'
wfo.wm_status = wfi.request['RequestStatus']
wfi.sendLog('closor',"%s is %s, but will not be set in trouble to find a replacement."%( wfo.name, wfo.wm_status))
else:
wfo.status = 'trouble'
wfo.wm_status = wfi.request['RequestStatus']
session.commit()
else:
print wfo.name,"not good for announcing:",wfi.request['RequestStatus']
wfi.sendLog('closor',"cannot be announced")
held.add( wfo.name )
days_late = 0.
retries_late = 10
really_late_files = [info for info in all_late_files if info['retries']>=retries_late]
really_late_files = [info for info in really_late_files if info['delay']/(60*60*24.)>=days_late]
if really_late_files:
subject = 'These %d files are lagging for %d days and %d retries announcing dataset \n%s'%(len(really_late_files), days_late, retries_late, json.dumps( really_late_files , indent=2) )
#sendEmail('waiting for files to announce', subject)
sendLog('closor', subject, level='warning')
sendLog('closor',subject)
print subject
open('%s/stuck_files.json'%monitor_dir,'w').write( json.dumps( really_late_files , indent=2))
if held:
sendLog('closor',"the workflows below are held up \n%s"%("\n".join( sorted(held) )), level='critical')
#batches = json.loads(open('batches.json').read())
for bname,go in batch_go.items():
if go:
subject = "Release Validation Samples Batch %s"% bname
issues=""
if batch_warnings[ bname ]:
issues="The following datasets have outstanding completion (<%d%%) issues:\n\n"% batch_goodness
issues+="\n".join( sorted( batch_warnings[ bname ] ))
issues+="\n\n"
text = """
Dear all,
a batch of release validation workflows has finished.
Batch ID:
%s
Detail of the workflows
https://dmytro.web.cern.ch/dmytro/cmsprodmon/requests.php?campaign=%s
%s
This is an automated message.
"""%( bname,
bname,
issues)
to = ['*****@*****.**']
sendEmail(subject, text, destination=to )
def close(self):
if os.path.isfile('.closor_stop'):
print "The closing of workflows is shortened"
return
url = self.url
batch_go = self.batch_go
CI = self.CI
UC = self.UC
wfo = self.wfo
jump_the_line = self.jump_the_line
batch_goodness = self.batch_goodness
check_fullcopy_to_announce = UC.get('check_fullcopy_to_announce')
## what is the expected #lumis
self.wfi = workflowInfo(url, wfo.name )
wfi = self.wfi
wfo.wm_status = wfi.request['RequestStatus']
if wfi.isRelval():
has_batch_go = False
batch_name = wfi.getCampaign()
if not batch_name in batch_go:
## do the esimatation whethere this can be announced : only once per batch
in_batches = getWorkflowByCampaign(url , batch_name, details=True)
batch_go[ batch_name ] = all(map(lambda s : not s in ['completed','running-open','running-closed','acquired','assigned','assignment-approved'], [r['RequestStatus'] for r in in_batches]))
## already verified
has_batch_go = batch_go[batch_name]
if not has_batch_go:
wfi.sendLog('closor', 'Cannot close for now because the batch <a href=https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?campaign=%s>%s</a> is not all close'%( batch_name, batch_name))
return
if wfi.request['RequestStatus'] in ['announced','normal-archived'] and not options.force:
## manually announced ??
self.to_status = 'done'
self.to_wm_status = wfi.request['RequestStatus']
wfi.sendLog('closor','%s is announced already : %s'%( wfo.name,self.to_wm_status))
return
if jump_the_line:
wfi.sendLog('closor','Announcing while completing')
expected_lumis = 1
if not 'TotalInputLumis' in wfi.request:
print wfo.name,"has not been assigned yet, or the database is corrupted"
elif wfi.request['TotalInputLumis']==0:
print wfo.name,"is corrupted with 0 expected lumis"
else:
expected_lumis = wfi.request['TotalInputLumis']
## what are the outputs
outputs = wfi.request['OutputDatasets']
## check whether the number of lumis is as expected for each
all_OK = defaultdict(lambda : False)
stats = defaultdict(int)
#print outputs
if len(outputs):
print wfo.name,wfi.request['RequestStatus']
for out in outputs:
event_count,lumi_count = getDatasetEventsAndLumis(dataset=out)
self.outs.append( Output( datasetname = out ))
odb = self.outs[-1]
odb.workflow = wfo
odb.nlumis = lumi_count
odb.nevents = event_count
odb.workfow_id = wfo.id
if odb.expectedlumis < expected_lumis:
odb.expectedlumis = expected_lumis
else:
expected_lumis = odb.expectedlumis
odb.date = time.mktime(time.gmtime())
fraction = lumi_count/float(expected_lumis)*100.
completion_line = "%60s %d/%d = %3.2f%%"%(out,lumi_count,expected_lumis,fraction)
wfi.sendLog('closor',"\t%s"% completion_line)
if wfi.isRelval() and fraction < batch_goodness:
self.batch_warnings[ wfi.getCampaign()].add( completion_line )
stats[out] = lumi_count
all_OK[out] = True
## check for at least one full copy prior to moving on
in_full = {}
for out in outputs:
in_full[out] = []
presence = getDatasetPresence( url, out )
where = [site for site,info in presence.items() if info[0]]
if where:
all_OK[out] = True
print out,"is in full at",",".join(where)
in_full[out] = copy.deepcopy(where)
else:
going_to = wfi.request['NonCustodialSites']+wfi.request['CustodialSites']
wfi.sendLog('closor',"%s is not in full anywhere. send to %s"%(out, ",".join(sorted(going_to))))
at_destination = dict([(k,v) for (k,v) in presence.items() if k in going_to])
else_where = dict([(k,v) for (k,v) in presence.items() if not k in going_to])
print json.dumps( at_destination )
print json.dumps( else_where, indent=2 )
## do the full stuck transfer study, missing files and shit !
for there in going_to:
late_info = findLateFiles(url, out, going_to = there )
for l in late_info:
l.update({"workflow":wfo.name,"dataset":out})
self.all_late_files.extend( late_info )
if check_fullcopy_to_announce:
## only set this false if the check is relevant
all_OK[out] = False
## verify if we have to do harvesting
if not options.no_harvest and not jump_the_line:
(OK, requests) = spawn_harvesting(url, wfi, in_full)
all_OK.update( OK )
## only that status can let me go into announced
if all(all_OK.values()) and ((wfi.request['RequestStatus'] in ['closed-out']) or options.force or jump_the_line):
print wfo.name,"to be announced"
results=[]
if not results:
for out in outputs:
print "dealing with",out
if out in stats and not stats[out]:
continue
_,dsn,process_string,tier = out.split('/')
if all_OK[out]:
print "setting valid"
results.append(setDatasetStatus(out, 'VALID', withFiles=False))
if all_OK[out] and wfi.isRelval():
## make the specific relval rules and the replicas
## figure the destination(s) out
destinations = set()
if tier != "RECO" and tier != "ALCARECO":
destinations.add('T2_CH_CERN')
if tier == "GEN-SIM":
destinations.add('T1_US_FNAL_Disk')
if tier == "GEN-SIM-DIGI-RAW":
destinations.add('T1_US_FNAL_Disk')
if tier == "GEN-SIM-RECO":
destinations.add('T1_US_FNAL_Disk')
if "RelValTTBar" in dsn and "TkAlMinBias" in process_string and tier != "ALCARECO":
destinations.add('T2_CH_CERN')
if "MinimumBias" in dsn and "SiStripCalMinBias" in process_string and tier != "ALCARECO":
destinations.add('T2_CH_CERN')
if destinations:
wfi.sendLog('closor', '%s to go to %s'%(out, ', '.join( sorted( destinations ))))
## call to makereplicarequest under relval => done
for site in destinations:
result = makeReplicaRequest(url, site, [out], 'Copy for release validation consumption', priority='normal', approve=True, mail=False, group='RelVal')
try:
request_id = result['phedex']['request_created'][0]['id']
results.append( True )
except:
results.append( 'Failed relval transfer' )
elif all_OK[out]:
campaign = None
try:
campaign = out.split('/')[2].split('-')[0]
except:
if 'Campaign' in wfi.request and wfi.request['Campaign']:
campaign = wfi.request['Campaign']
to_DDM = False
## campaign override
if campaign and campaign in CI.campaigns and 'toDDM' in CI.campaigns[campaign] and tier in CI.campaigns[campaign]['toDDM']:
to_DDM = True
## by typical enabling
if tier in UC.get("tiers_to_DDM"):
to_DDM = True
## check for unitarity
if not tier in UC.get("tiers_no_DDM")+UC.get("tiers_to_DDM"):
print "tier",tier,"neither TO or NO DDM for",out
results.append('Not recognitized tier %s'%tier)
#sendEmail("failed DDM injection","could not recognize %s for injecting in DDM"% out)
sendLog('closor', "could not recognize %s for injecting in DDM"% out, level='critical')
continue
n_copies = 1
destinations=[]
if to_DDM and campaign and campaign in CI.campaigns and 'DDMcopies' in CI.campaigns[campaign]:
ddm_instructions = CI.campaigns[campaign]['DDMcopies']
if type(ddm_instructions) == int:
n_copies = CI.campaigns[campaign]['DDMcopies']
elif type(ddm_instructions) == dict:
## a more fancy configuration
for ddmtier,indication in ddm_instructions.items():
if ddmtier==tier or ddmtier in ['*','all']:
## this is for us
if 'N' in indication:
n_copies = indication['N']
if 'host' in indication:
destinations = indication['host']
destination_spec = ""
if destinations:
destination_spec = "--destination="+",".join( destinations )
group_spec = "" ## not used yet
### should make this a campaign configuration
## inject to DDM when necessary
if to_DDM:
print "Sending",out," to DDM"
status = pass_to_dynamo( [out], N = n_copies, sites=destinations if destinations else None, group = group_spec if group_spec else None)
results.append( status )
if status == True:
wfi.sendLog('closor','%s is send to dynamo in %s copies %s %s'%( out, n_copies, sorted(destinations), group_spec))
else:
sendLog('closor',"could not add "+out+" to dynamo pool. check closor logs.", level='critical')
wfi.sendLog('closor',"could not add "+out+" to dynamo pool. check closor logs.")
else:
print wfo.name,"no stats for announcing",out
results.append('No Stats')
if all(map(lambda result : result in ['None',None,True],results)):
if not jump_the_line:
## only announce if all previous are fine
res = reqMgrClient.announceWorkflowCascade(url, wfo.name)
if not res in ['None',None]:
## check the status again, it might well have toggled
wl_bis = workflowInfo(url, wfo.name)
self.to_wm_status = wl_bis.request['RequestStatus']
if wl_bis.request['RequestStatus'] in ['announced','normal-archived']:
res = None
else:
res = reqMgrClient.announceWorkflowCascade(url, wfo.name)
results.append( res )
print results
if all(map(lambda result : result in ['None',None,True],results)):
if jump_the_line:
if not 'announced' in wfo.status:
self.to_status = wfo.status.replace('announce','announced')
else:
self.to_status = 'done'
self.closing = True
wfi.sendLog('closor',"workflow outputs are announced")
else:
wfi.sendLog('closor',"Error with %s to be announced \n%s"%( wfo.name, json.dumps( results )))
elif wfi.request['RequestStatus'] in ['failed','aborted','aborted-archived','rejected','rejected-archived','aborted-completed']:
if wfi.isRelval():
self.to_status = 'forget'
self.to_wm_status = wfi.request['RequestStatus']
wfi.sendLog('closor',"%s is %s, but will not be set in trouble to find a replacement."%( wfo.name, self.to_wm_status))
else:
self.to_status = 'trouble'
self.to_wm_status = wfi.request['RequestStatus']
else:
print wfo.name,"not good for announcing:",wfi.request['RequestStatus']
wfi.sendLog('closor',"cannot be announced")
self.held.add( wfo.name )