def assignor(url ,specific = None, talk=True, options=None):
if userLock(): return
if duplicateLock(): return
if not componentInfo().check(): return
UC = unifiedConfiguration()
CI = campaignInfo()
#SI = siteInfo()
SI = global_SI()
#NLI = newLockInfo()
#if not NLI.free() and not options.go: return
LI = lockInfo()
if not LI.free() and not options.go: return
n_assigned = 0
n_stalled = 0
wfos=[]
fetch_from = []
if specific or options.early:
fetch_from.extend(['considered','staging'])
if specific:
fetch_from.extend(['considered-tried'])
fetch_from.extend(['staged'])
if options.from_status:
fetch_from = options.from_status.split(',')
print "Overriding to read from",fetch_from
for status in fetch_from:
wfos.extend(session.query(Workflow).filter(Workflow.status==status).all())
## in case of partial, go for fetching a list from json ?
#if options.partial and not specific:
# pass
dataset_endpoints = json.loads(open('%s/dataset_endpoints.json'%monitor_dir).read())
aaa_mapping = json.loads(open('%s/equalizor.json'%monitor_pub_dir).read())['mapping']
all_stuck = set()
all_stuck.update( json.loads( open('%s/stuck_transfers.json'%monitor_dir).read() ))
all_stuck.update( getAllStuckDataset())
max_per_round = UC.get('max_per_round').get('assignor',None)
max_cpuh_block = UC.get('max_cpuh_block')
random.shuffle( wfos )
for wfo in wfos:
if options.limit and (n_stalled+n_assigned)>options.limit:
break
if max_per_round and (n_stalled+n_assigned)>max_per_round:
break
if specific:
if not any(map(lambda sp: sp in wfo.name,specific.split(','))): continue
#if not specific in wfo.name: continue
print "\n\n"
wfh = workflowInfo( url, wfo.name)
if options.priority and int(wfh.request['RequestPriority']) < options.priority:
continue
options_text=""
if options.early: options_text+=", early option is ON"
if options.partial:
options_text+=", partial option is ON"
options_text+=", good fraction is %.2f"%options.good_enough
wfh.sendLog('assignor',"%s to be assigned%s"%(wfo.name, options_text))
## the site whitelist takes into account siteInfo, campaignInfo, memory and cores
(lheinput,primary,parent,secondary, sites_allowed) = wfh.getSiteWhiteList()
output_tiers = list(set([o.split('/')[-1] for o in wfh.request['OutputDatasets']]))
is_stuck = (all_stuck & primary)
if is_stuck:
wfh.sendLog('assignor',"%s are stuck input"%(','.join( is_stuck)))
## check if by configuration we gave it a GO
no_go = False
if not wfh.go(log=True) and not options.go:
no_go = True
allowed_secondary = {}
assign_parameters = {}
check_secondary = False
for campaign in wfh.getCampaigns():
if campaign in CI.campaigns:
assign_parameters.update( CI.campaigns[campaign] )
if campaign in CI.campaigns and 'secondaries' in CI.campaigns[campaign]:
if CI.campaigns[campaign]['secondaries']:
allowed_secondary.update( CI.campaigns[campaign]['secondaries'] )
check_secondary = True
if campaign in CI.campaigns and 'banned_tier' in CI.campaigns[campaign]:
banned_tier = list(set(CI.campaigns[campaign]['banned_tier']) & set(output_tiers))
if banned_tier:
no_go=True
wfh.sendLog('assignor','These data tiers %s are not allowed'%(','.join( banned_tier)))
sendLog('assignor','These data tiers %s are not allowed'%(','.join( banned_tier)), level='critical')
if secondary and check_secondary:
if (set(secondary)&set(allowed_secondary.keys())!=set(secondary)):
wfh.sendLog('assignor','%s is not an allowed secondary'%(', '.join(set(secondary)-set(allowed_secondary.keys()))))
sendLog('assignor','%s is not an allowed secondary'%(', '.join(set(secondary)-set(allowed_secondary.keys()))), level='critical')
if not options.go:
no_go = True
## then get whether there is something more to be done by secondary
for sec in secondary:
if sec in allowed_secondary:# and 'parameters' in allowed_secondary[sec]:
assign_parameters.update( allowed_secondary[sec] )
if no_go:
n_stalled+=1
continue
## check on current status for by-passed assignment
if wfh.request['RequestStatus'] !='assignment-approved':
if not options.test:
wfh.sendLog('assignor',"setting %s away and skipping"%wfo.name)
## the module picking up from away will do what is necessary of it
wfo.wm_status = wfh.request['RequestStatus']
wfo.status = 'away'
session.commit()
continue
else:
print wfo.name,wfh.request['RequestStatus']
## retrieve from the schema, dbs and reqMgr what should be the next version
version=wfh.getNextVersion()
if not version:
if options and options.ProcessingVersion:
version = options.ProcessingVersion
else:
wfh.sendLog('assignor',"cannot decide on version number")
n_stalled+=1
wfo.status = 'trouble'
session.commit()
continue
original_sites_allowed = copy.deepcopy( sites_allowed )
wfh.sendLog('assignor',"Site white list %s"%sorted(sites_allowed))
override_sec_location = CI.get(wfh.request['Campaign'], 'SecondaryLocation', [])
blocks = []
if 'BlockWhitelist' in wfh.request:
blocks = wfh.request['BlockWhitelist']
if 'RunWhitelist' in wfh.request and wfh.request['RunWhitelist']:
## augment with run white list
for dataset in primary:
blocks = list(set( blocks + getDatasetBlocks( dataset, runs=wfh.request['RunWhitelist'] ) ))
wfh.sendLog('assignor',"Allowed %s"%sorted(sites_allowed))
secondary_locations=None
primary_aaa = options.primary_aaa
secondary_aaa = options.secondary_aaa
do_partial = False #options.good_enough if options.partial else 0
if 'Campaign' in wfh.request and wfh.request['Campaign'] in CI.campaigns:
assign_parameters.update( CI.campaigns[wfh.request['Campaign']] )
if 'primary_AAA' in assign_parameters:
primary_aaa = primary_aaa or assign_parameters['primary_AAA']
if 'secondary_AAA' in assign_parameters:
secondary_aaa = secondary_aaa or assign_parameters['secondary_AAA']
if 'partial_copy' in assign_parameters:
## can this only work if there is a stuck input ? maybe not
## this is a number. 0 means no
print "Could do partial disk copy assignment"
if is_stuck or options.partial:
do_partial = assign_parameters['partial_copy']
wfh.sendLog('assignor',"Overiding partial copy assignment to %.2f fraction"% do_partial)
#sendEmail('stuck input to assignment','%s is stuck for assigning %s and going fractional'%(','.join( is_stuck), wfo.name))
do_partial = options.good_enough if options.partial else do_partial
for sec in list(secondary):
if override_sec_location:
print "We don't care where the secondary is"
print "Cannot pass for now"
#sendEmail("tempting to pass sec location check","but we cannot yet IMO")
#pass
if secondary_aaa:
#just continue without checking
continue
presence = getDatasetPresence( url, sec )
print sec
print json.dumps(presence, indent=2)
one_secondary_locations = [site for (site,(there,frac)) in presence.items() if frac>98.]
#one_secondary_locations = [site for (site,(there,frac)) in presence.items() if there]
if secondary_locations==None:
secondary_locations = one_secondary_locations
else:
secondary_locations = list(set(secondary_locations) & set(one_secondary_locations))
## reduce the site white list to site with secondary only
#sites_allowed = [site for site in sites_allowed if any([osite.startswith(site) for osite in one_secondary_locations])]
sites_allowed = [site for site in sites_allowed if SI.CE_to_SE(site) in one_secondary_locations]
wfh.sendLog('assignor',"From secondary requirement, now Allowed%s"%sorted(sites_allowed))
initial_sites_allowed = copy.deepcopy( sites_allowed ) ## keep track of this, after secondary input location restriction : that's how you want to operate it
sites_all_data = copy.deepcopy( sites_allowed )
sites_with_data = copy.deepcopy( sites_allowed )
sites_with_any_data = copy.deepcopy( sites_allowed )
primary_locations = None
available_fractions = {}
set_lfn = '/store/mc' ## by default
endpoints = set()
for prim in list(primary):
if prim in dataset_endpoints:
print "endpoints from stagor",dataset_endpoints[prim]
endpoints.update( dataset_endpoints[prim] )
set_lfn = getLFNbase( prim )
presence = getDatasetPresence( url, prim , only_blocks=blocks)
if talk:
print prim
print json.dumps(presence, indent=2)
available_fractions[prim] = getDatasetBlocksFraction(url, prim, sites = [SI.CE_to_SE(site) for site in sites_allowed] , only_blocks = blocks)
#sites_all_data = [site for site in sites_with_data if any([osite.startswith(site) for osite in [psite for (psite,(there,frac)) in presence.items() if there]])]
#sites_with_data = [site for site in sites_with_data if any([osite.startswith(site) for osite in [psite for (psite,frac) in presence.items() if frac[1]>90.]])]
sites_all_data = [site for site in sites_with_data if SI.CE_to_SE(site) in [psite for (psite,(there,frac)) in presence.items() if there]]
sites_with_data = [site for site in sites_with_data if SI.CE_to_SE(site) in [psite for (psite,frac) in presence.items() if frac[1]>90.]]
sites_with_any_data = [site for site in sites_with_any_data if SI.CE_to_SE(site) in presence.keys()]
wfh.sendLog('assignor',"Holding the data but not allowed %s"%sorted(list(set([se_site for se_site in presence.keys() if not SI.SE_to_CE(se_site) in sites_allowed]))))
if primary_locations==None:
primary_locations = presence.keys()
else:
primary_locations = list(set(primary_locations) & set(presence.keys() ))
sites_with_data = list(set(sites_with_data))
sites_with_any_data = list(set(sites_with_any_data))
opportunistic_sites=[]
down_time = False
## opportunistic running where any piece of data is available
if secondary_locations or primary_locations:
## intersection of both any pieces of the primary and good IO
#opportunistic_sites = [SI.SE_to_CE(site) for site in list((set(secondary_locations) & set(primary_locations) & set(SI.sites_with_goodIO)) - set(sites_allowed))]
if secondary_locations and primary_locations:
opportunistic_sites = [SI.SE_to_CE(site) for site in list((set(secondary_locations) & set(primary_locations)) - set([SI.CE_to_SE(site) for site in sites_allowed]))]
elif primary_locations:
opportunistic_sites = [SI.SE_to_CE(site) for site in list(set(primary_locations) - set([SI.CE_to_SE(site) for site in sites_allowed]))]
else:
opportunistic_sites = []
wfh.sendLog('assignor',"We could be running in addition at %s"% sorted(opportunistic_sites))
if any([osite in SI.sites_not_ready for osite in opportunistic_sites]):
wfh.sendLog('assignor',"One of the usable site is in downtime %s"%([osite for osite in opportunistic_sites if osite in SI.sites_not_ready]))
down_time = True
## should this be send back to considered ?
## should be 2 but for the time-being let's lower it to get things going
copies_wanted,cpuh = wfh.getNCopies()
wfh.sendLog('assignor',"we need %s CPUh"%cpuh)
if cpuh>max_cpuh_block and not options.go:
#sendEmail('large workflow','that wf %s has a large number of CPUh %s, not assigning, please check the logs'%(wfo.name, cpuh))#,destination=['*****@*****.**'])
sendLog('assignor','%s requires a large numbr of CPUh %s , not assigning, please check with requester'%( wfo.name, cpuh), level='critical')
wfh.sendLog('assignor',"Requiring a large number of CPUh %s, not assigning"%cpuh)
continue
if 'Campaign' in wfh.request and wfh.request['Campaign'] in CI.campaigns and 'maxcopies' in CI.campaigns[wfh.request['Campaign']]:
copies_needed_from_campaign = CI.campaigns[wfh.request['Campaign']]['maxcopies']
copies_wanted = min(copies_needed_from_campaign, copies_wanted)
if not options.early:
less_copies_than_requested = UC.get("less_copies_than_requested")
copies_wanted = max(1,copies_wanted-less_copies_than_requested) # take one out for the efficiency
else:
## find out whether there is a site in the whitelist, that is lacking jobs and reduce to 1 copy needed to get things going
pass
wfh.sendLog('assignor',"needed availability fraction %s"% copies_wanted)
## should also check on number of sources, if large enough, we should be able to overflow most, efficiently
## default back to white list to original white list with any data
wfh.sendLog('assignor',"Allowed sites :%s"% sorted(sites_allowed))
if primary_aaa:
## remove the sites not reachable localy if not in having the data
if not sites_all_data:
wfh.sendLog('assignor',"Overiding the primary on AAA setting to Off")
primary_aaa=False
else:
aaa_grid = set(sites_all_data)
for site in list(aaa_grid):
aaa_grid.update( aaa_mapping.get(site,[]) )
sites_allowed = list(set(initial_sites_allowed) & aaa_grid)
wfh.sendLog('assignor',"Selected to read primary through xrootd %s"%sorted(sites_allowed))
if not primary_aaa:
sites_allowed = sites_with_any_data
wfh.sendLog('assignor',"Selected for any data %s"%sorted(sites_allowed))
### check on endpoints for on-going transfers
if do_partial:
if endpoints:
end_sites = [SI.SE_to_CE(s) for s in endpoints]
sites_allowed = list(set(sites_allowed + end_sites))
if down_time and not any(osite in SI.sites_not_ready for osite in end_sites):
print "Flip the status of downtime, since our destinations are good"
down_time = False
print "with added endpoints",sorted(end_sites)
else:
print "Cannot do partial assignment without knowin the endpoints"
n_stalled+=1
continue
#if not len(sites_allowed):
# if not options.early:
# wfh.sendLog('assignor',"cannot be assign with no matched sites")
# sendLog('assignor','%s has no whitelist'% wfo.name, level='critical')
# n_stalled+=1
# continue
low_pressure = SI.sites_low_pressure(0.4)
## if any of the site allowed is low pressure : reduce to 1 copy so that it gets started
allowed_and_low = sorted(set(low_pressure) & set(sites_allowed))
if allowed_and_low:
wfh.sendLog('assignor',"The workflow can run at %s under low pressure currently"%( ','.join( allowed_and_low )))
copies_wanted = max(1., copies_wanted-1.)
if available_fractions and not all([available>=copies_wanted for available in available_fractions.values()]):
not_even_once = not all([available>=1. for available in available_fractions.values()])
above_good = all([available >= do_partial for available in available_fractions.values()])
wfh.sendLog('assignor',"The input dataset is not available %s times, only %s"%( copies_wanted, available_fractions.values()))
if down_time and not options.go and not options.early:
wfo.status = 'considered'
session.commit()
wfh.sendLog('assignor',"sending back to considered because of site downtime, instead of waiting")
#sendEmail( "cannot be assigned due to downtime","%s is not sufficiently available, due to down time of a site in the whitelist. check the assignor logs. sending back to considered."% wfo.name)
sendLog('assignor','%s is not sufficiently available, due to down time of a site in the whitelist. sending back to considered.'%( wfo.name ), level='delay')
n_stalled+=1
continue
#pass
print json.dumps(available_fractions)
if (options.go and not_even_once) or not options.go:
known = []
try:
known = json.loads(open('cannot_assign.json').read())
except:
pass
if not wfo.name in known and not options.limit and not options.go and not options.early and not (do_partial and above_good):
wfh.sendLog('assignor',"cannot be assigned, %s is not sufficiently available.\n %s"%(wfo.name,json.dumps(available_fractions)))
#sendEmail( "cannot be assigned","%s is not sufficiently available.\n %s"%(wfo.name,json.dumps(available_fractions)))
known.append( wfo.name )
open('cannot_assign.json','w').write(json.dumps( known, indent=2))
if options.early:
if wfo.status == 'considered':
wfh.sendLog('assignor',"setting considered-tried")
wfo.status = 'considered-tried'
session.commit()
else:
print "tried but status is",wfo.status
if do_partial and above_good:
print "Will move on with partial locations"
else:
n_stalled+=1
continue
if not len(sites_allowed):
if not options.early:
wfh.sendLog('assignor',"cannot be assign with no matched sites")
sendLog('assignor','%s has no whitelist'% wfo.name, level='critical')
n_stalled+=1
continue
t1_only = [ce for ce in sites_allowed if ce.startswith('T1')]
if t1_only:
# try to pick from T1 only first
sites_out = [SI.pick_dSE([SI.CE_to_SE(ce) for ce in t1_only])]
else:
# then pick any otherwise
sites_out = [SI.pick_dSE([SI.CE_to_SE(ce) for ce in sites_allowed])]
wfh.sendLog('assignor',"Placing the output on %s"%sites_out)
parameters={
'SiteWhitelist' : sites_allowed,
'NonCustodialSites' : sites_out,
'AutoApproveSubscriptionSites' : list(set(sites_out)),
'AcquisitionEra' : wfh.acquisitionEra(),
'ProcessingString' : wfh.processingString(),
'MergedLFNBase' : set_lfn,
'ProcessingVersion' : version,
}
if primary_aaa:
parameters['TrustSitelists'] = True
wfh.sendLog('assignor',"Reading primary through xrootd at %s"%sorted(sites_allowed))
if secondary_aaa:
parameters['TrustPUSitelists'] = True
wfh.sendLog('assignor',"Reading secondary through xrootd at %s"%sorted(sites_allowed))
if 'parameters' in assign_parameters:
parameters.update( assign_parameters['parameters'] )
## plain assignment here
team='production'
if os.getenv('UNIFIED_TEAM'): team = os.getenv('UNIFIED_TEAM')
if options and options.team:
team = options.team
if False and 'T2_CH_CERN' in parameters['SiteWhitelist']:
## add some check on
### the amount pending to HLT
### the size of the request
### the priority of the request (maybe not if we decide to overflow during runs)
parameters['SiteWhitelist'] = ['T2_CH_CERN_HLT']
team = 'hlt'
## reduce the splitting by factor of 4, regardless of type of splitting
sendEmail("sending work to HLT","%s was assigned to HLT"%wfo.name)
##parse options entered in command line if any
if options:
for key in reqMgrClient.assignWorkflow.keys:
v=getattr(options,key)
if v!=None:
if type(v)==str and ',' in v:
parameters[key] = filter(None,v.split(','))
else:
parameters[key] = v
if lheinput:
## throttle reading LHE article
wfh.sendLog('assignor', 'Setting the number of events per job to 500k max')
parameters['EventsPerJob'] = 500000
## pick up campaign specific assignment parameters
#parameters.update( CI.parameters(wfh.request['Campaign']) )
parameters.update( assign_parameters.get('parameters',{}) )
if not options.test:
parameters['execute'] = True
split_check = wfh.checkWorkflowSplitting()
if split_check!=True:
parameters.update( split_check )
if 'NoGo' in split_check.values():
wfh.sendLog('assignor', "Failing splitting check")
sendLog('assignor','the workflow %s is failing the splitting check. Verify in the logs'% wfo.name, level='critical')
n_stalled+=1
continue
if 'EventBased' in split_check.values():
wfh.sendLog('assignor', "Falling back to event splitting.")
#sendEmail("Fallback to EventBased","the workflow %s is too heavy to be processed as it is. Fallback to EventBased splitting"%wfo.name)
sendLog('assignor','the workflow %s is too heavy to be processed as it is. Fallback to EventBased splitting ?'%wfo.name, level='critical')
## we have a problem here, that EventBased should never be used as a backup
if not options.go:
n_stalled+=1
continue
continue ## skip all together
elif 'EventsPerJob' in split_check.values():
wfh.sendLog('assignor', "Modifying the number of events per job")
#sendEmail("Modifying the job per events","the workflow %s is too heavy in number of jobs explosion"%wfo.name)
sendLog('assignor',"the workflow %s is too heavy in number of jobs explosion"%wfo.name, level='critical')
elif 'EventsPerLumi' in split_check.values():
wfh.sendLog('assignor', "Modifying the number of events per lumi to be able to process this")
# Handle run-dependent MC
pstring = wfh.processingString()
if 'PU_RD' in pstring:
numEvents = wfh.getRequestNumEvents()
eventsPerLumi = [getDatasetEventsPerLumi(prim) for prim in primary]
eventsPerLumi = sum(eventsPerLumi)/float(len(eventsPerLumi))
reqJobs = 500
if 'PU_RD2' in pstring:
reqJobs = 2000
eventsPerJob = int(numEvents/(reqJobs*1.4))
lumisPerJob = int(eventsPerJob/eventsPerLumi)
if lumisPerJob==0:
#sendEmail("issue with event splitting for run-dependent MC","%s needs to be split by event with %s per job"%(wfo.name, eventsPerJob))
sendLog('assignor', "%s needs to be split by event with %s per job"%(wfo.name, eventsPerJob), level='critical')
wfh.sendLog('assignor', "%s needs to be split by event with %s per job"%(wfo.name, eventsPerJob))
parameters['EventsPerJob'] = eventsPerJob
else:
spl = wfh.getSplittings()[0]
eventsPerJobEstimated = spl['events_per_job'] if 'events_per_job' in spl else None
eventsPerJobEstimated = spl['avg_events_per_job'] if 'avg_events_per_job' in spl else None
if eventsPerJobEstimated and eventsPerJobEstimated > eventsPerJob:
#sendEmail("setting lumi splitting for run-dependent MC","%s was assigned with %s lumis/job"%( wfo.name, lumisPerJob))
sendLog('assignor',"%s was assigned with %s lumis/job"%( wfo.name, lumisPerJob), level='critical')
wfh.sendLog('assignor',"%s was assigned with %s lumis/job"%( wfo.name, lumisPerJob))
parameters['LumisPerJob'] = lumisPerJob
else:
#sendEmail("leaving splitting untouched for PU_RD*","please check on "+wfo.name)
sendLog('assignor',"leaving splitting untouched for %s, please check on %s"%( pstring, wfo.name), level='critical')
wfh.sendLog('assignor',"leaving splitting untouched for PU_RD*, please check.")
result = reqMgrClient.assignWorkflow(url, wfo.name, team, parameters)
# set status
if not options.test:
if result:
wfo.status = 'away'
session.commit()
n_assigned+=1
wfh.sendLog('assignor',"Properly assigned\n%s"%(json.dumps( parameters, indent=2)))
try:
## refetch information and lock output
new_wfi = workflowInfo( url, wfo.name)
(_,prim,_,sec) = new_wfi.getIO()
for secure in list(prim)+list(sec)+new_wfi.request['OutputDatasets']:
## lock all outputs flat
#NLI.lock( secure )
LI.lock( secure, reason = 'assigning')
#for site in [SI.CE_to_SE(site) for site in sites_allowed]:
# for output in new_wfi.request['OutputDatasets']:
# LI.lock( output, site, 'dataset in production')
# for primary in prim:
# LI.lock( primary, site, 'dataset used in input')
# for secondary in sec:
# LI.lock( secondary, site, 'required for mixing' )
except Exception as e:
print "fail in locking output"
print str(e)
sendEmail("failed locking of output",str(e))
else:
wfh.sendLog('assignor',"Failed to assign. Please check the logs")
print "ERROR could not assign",wfo.name
else:
pass
print "Assignment summary:"
sendLog('assignor',"Assigned %d Stalled %s"%(n_assigned, n_stalled))
def assignor(url, specific=None, talk=True, options=None):
if userLock(): return
if duplicateLock(): return
if not componentInfo().check(): return
UC = unifiedConfiguration()
CI = campaignInfo()
#SI = siteInfo()
SI = global_SI()
#NLI = newLockInfo()
#if not NLI.free() and not options.go: return
LI = lockInfo()
if not LI.free() and not options.go: return
n_assigned = 0
n_stalled = 0
wfos = []
fetch_from = []
if specific or options.early:
fetch_from.extend(['considered', 'staging'])
if specific:
fetch_from.extend(['considered-tried'])
if options.early:
print "Option Early is on"
fetch_from.extend(['staged'])
if options.from_status:
fetch_from = options.from_status.split(',')
print "Overriding to read from", fetch_from
for status in fetch_from:
print "getting wf in", status
wfos.extend(
session.query(Workflow).filter(Workflow.status == status).all())
print len(wfos)
## in case of partial, go for fetching a list from json ?
#if options.partial and not specific:
# pass
dataset_endpoints = json.loads(
open('%s/dataset_endpoints.json' % monitor_dir).read())
aaa_mapping = json.loads(
open('%s/equalizor.json' % monitor_pub_dir).read())['mapping']
all_stuck = set()
all_stuck.update(
json.loads(open('%s/stuck_transfers.json' % monitor_pub_dir).read()))
all_stuck.update(getAllStuckDataset())
max_per_round = UC.get('max_per_round').get('assignor', None)
max_cpuh_block = UC.get('max_cpuh_block')
##order by priority instead of random
if options.early:
cache = sorted(getWorkflows(url, 'assignment-approved', details=True),
key=lambda r: r['RequestPriority'])
cache = [r['RequestName'] for r in cache]
def rank(wfn):
return cache.index(wfn) if wfn in cache else 0
wfos = sorted(wfos, key=lambda wfo: rank(wfo.name), reverse=True)
print "10 first", [wfo.name for wfo in wfos[:10]]
print "10 last", [wfo.name for wfo in wfos[-10:]]
else:
random.shuffle(wfos)
for wfo in wfos:
if options.limit and (n_stalled + n_assigned) > options.limit:
break
if max_per_round and (n_stalled + n_assigned) > max_per_round:
break
if specific:
if not any(map(lambda sp: sp in wfo.name, specific.split(','))):
continue
#if not specific in wfo.name: continue
print "\n\n"
wfh = workflowInfo(url, wfo.name)
if wfh.request['RequestStatus'] in [
'rejected', 'aborted', 'aborted-completed', 'aborted-archived',
'rejected-archived'
] and wfh.isRelval():
wfo.status = 'forget'
session.commit()
n_stalled += 1
continue
if options.priority and int(
wfh.request['RequestPriority']) < options.priority:
continue
options_text = ""
if options.early: options_text += ", early option is ON"
if options.partial:
options_text += ", partial option is ON"
options_text += ", good fraction is %.2f" % options.good_enough
wfh.sendLog('assignor',
"%s to be assigned%s" % (wfo.name, options_text))
## the site whitelist takes into account siteInfo, campaignInfo, memory and cores
(lheinput, primary, parent, secondary,
sites_allowed) = wfh.getSiteWhiteList()
output_tiers = list(
set([o.split('/')[-1] for o in wfh.request['OutputDatasets']]))
if not output_tiers:
n_stalled += 1
wfh.sendLog('assignor', 'There is no output at all')
sendLog('assignor',
'Workflow %s has no output at all' % (wfo.name),
level='critical')
continue
is_stuck = (all_stuck & primary)
if is_stuck:
wfh.sendLog('assignor',
"%s are stuck input" % (','.join(is_stuck)))
## check if by configuration we gave it a GO
no_go = False
if not wfh.go(log=True) and not options.go:
no_go = True
allowed_secondary = {}
assign_parameters = {}
check_secondary = (not wfh.isRelval())
for campaign in wfh.getCampaigns():
if campaign in CI.campaigns:
assign_parameters.update(CI.campaigns[campaign])
if campaign in CI.campaigns and 'secondaries' in CI.campaigns[
campaign]:
if CI.campaigns[campaign]['secondaries']:
allowed_secondary.update(
CI.campaigns[campaign]['secondaries'])
check_secondary = True
if campaign in CI.campaigns and 'banned_tier' in CI.campaigns[
campaign]:
banned_tier = list(
set(CI.campaigns[campaign]['banned_tier'])
& set(output_tiers))
if banned_tier:
no_go = True
wfh.sendLog(
'assignor', 'These data tiers %s are not allowed' %
(','.join(banned_tier)))
sendLog('assignor',
'These data tiers %s are not allowed' %
(','.join(banned_tier)),
level='critical')
if secondary and check_secondary:
if (set(secondary) & set(allowed_secondary.keys()) !=
set(secondary)):
wfh.sendLog(
'assignor', '%s is not an allowed secondary' %
(', '.join(set(secondary) -
set(allowed_secondary.keys()))))
sendLog(
'assignor',
'%s is not an allowed secondary' %
(', '.join(set(secondary) -
set(allowed_secondary.keys()))),
level='critical')
if not options.go:
no_go = True
## then get whether there is something more to be done by secondary
for sec in secondary:
if sec in allowed_secondary: # and 'parameters' in allowed_secondary[sec]:
assign_parameters.update(allowed_secondary[sec])
if no_go:
n_stalled += 1
## make a very loud noise if >100k priority stalled
continue
## check on current status for by-passed assignment
if wfh.request['RequestStatus'] != 'assignment-approved':
if not options.test:
wfh.sendLog('assignor',
"setting %s away and skipping" % wfo.name)
## the module picking up from away will do what is necessary of it
wfo.wm_status = wfh.request['RequestStatus']
wfo.status = 'away'
session.commit()
continue
else:
print wfo.name, wfh.request['RequestStatus']
## retrieve from the schema, dbs and reqMgr what should be the next version
version = wfh.getNextVersion()
if not version:
if options and options.ProcessingVersion:
version = options.ProcessingVersion
else:
wfh.sendLog('assignor', "cannot decide on version number")
n_stalled += 1
wfo.status = 'trouble'
session.commit()
continue
original_sites_allowed = copy.deepcopy(sites_allowed)
wfh.sendLog('assignor', "Site white list %s" % sorted(sites_allowed))
override_sec_location = CI.get(wfh.request['Campaign'],
'SecondaryLocation', [])
blocks = wfh.getBlockWhiteList()
rwl = wfh.getRunWhiteList()
if rwl:
## augment with run white list
for dataset in primary:
blocks = list(set(blocks +
getDatasetBlocks(dataset, runs=rwl)))
lwl = wfh.getLumiWhiteList()
if lwl:
## augment with lumi white list
for dataset in primary:
blocks = list(
set(blocks + getDatasetBlocks(dataset, lumis=lwl)))
wfh.sendLog('assignor', "Allowed %s" % sorted(sites_allowed))
secondary_locations = None
primary_aaa = options.primary_aaa
secondary_aaa = options.secondary_aaa
do_partial = False #options.good_enough if options.partial else 0
if 'Campaign' in wfh.request and wfh.request[
'Campaign'] in CI.campaigns:
assign_parameters.update(CI.campaigns[wfh.request['Campaign']])
if 'primary_AAA' in assign_parameters:
primary_aaa = primary_aaa or assign_parameters['primary_AAA']
if 'secondary_AAA' in assign_parameters:
secondary_aaa = secondary_aaa or assign_parameters['secondary_AAA']
if 'partial_copy' in assign_parameters:
## can this only work if there is a stuck input ? maybe not
## this is a number. 0 means no
print "Could do partial disk copy assignment"
if is_stuck or options.partial:
do_partial = assign_parameters['partial_copy']
wfh.sendLog(
'assignor',
"Overiding partial copy assignment to %.2f fraction" %
do_partial)
#sendEmail('stuck input to assignment','%s is stuck for assigning %s and going fractional'%(','.join( is_stuck), wfo.name))
do_partial = options.good_enough if options.partial else do_partial
for sec in list(secondary):
if override_sec_location:
print "We don't care where the secondary is"
print "Cannot pass for now"
#sendEmail("tempting to pass sec location check","but we cannot yet IMO")
#pass
presence = getDatasetPresence(url, sec)
print sec
print json.dumps(presence, indent=2)
one_secondary_locations = [
site for (site, (there, frac)) in presence.items()
if frac > 98.
]
if secondary_aaa:
if not one_secondary_locations:
sec_availability = getDatasetBlocksFraction(url, sec)
if sec_availability >= 1. and options.go:
## there is at least one copy of each block on disk. We should go ahead and let it go.
wfh.sendLog(
'assignor',
"The secondary %s is available %s times on disk, and usable"
% (sec, sec_availability))
else:
## not even a copy on disk anywhere !!!!
sites_allowed = [] ## will block the assignment
wfh.sendLog(
'assignor',
"The secondary %s is nowhere on disk" % sec)
#just continue without checking
continue
#one_secondary_locations = [site for (site,(there,frac)) in presence.items() if there]
if secondary_locations == None:
secondary_locations = one_secondary_locations
else:
secondary_locations = list(
set(secondary_locations) & set(one_secondary_locations))
## reduce the site white list to site with secondary only
#sites_allowed = [site for site in sites_allowed if any([osite.startswith(site) for osite in one_secondary_locations])]
sites_allowed = [
site for site in sites_allowed
if SI.CE_to_SE(site) in one_secondary_locations
]
wfh.sendLog(
'assignor', "From/after secondary requirement, now Allowed%s" %
sorted(sites_allowed))
initial_sites_allowed = copy.deepcopy(
sites_allowed
) ## keep track of this, after secondary input location restriction : that's how you want to operate it
sites_all_data = copy.deepcopy(sites_allowed)
sites_with_data = copy.deepcopy(sites_allowed)
sites_with_any_data = copy.deepcopy(sites_allowed)
primary_locations = None
available_fractions = {}
set_lfn = '/store/mc' ## by default
endpoints = set()
for prim in list(primary):
if prim in dataset_endpoints:
print "endpoints from stagor", dataset_endpoints[prim]
endpoints.update(dataset_endpoints[prim])
set_lfn = getLFNbase(prim)
presence = getDatasetPresence(url, prim, only_blocks=blocks)
if talk:
print prim
print json.dumps(presence, indent=2)
available_fractions[prim] = getDatasetBlocksFraction(
url,
prim,
sites=[SI.CE_to_SE(site) for site in sites_allowed],
only_blocks=blocks)
if primary_aaa:
available_fractions[prim] = getDatasetBlocksFraction(
url, prim, only_blocks=blocks)
sites_all_data = [
site for site in sites_with_data if SI.CE_to_SE(site) in [
psite for (psite, (there, frac)) in presence.items()
if there
]
]
if primary_aaa:
sites_all_data = list(
set([
SI.SE_to_CE(psite)
for (psite, (there, frac)) in presence.items() if there
]))
sites_with_data = [
site for site in sites_with_data if SI.CE_to_SE(site) in
[psite for (psite, frac) in presence.items() if frac[1] > 90.]
]
sites_with_any_data = [
site for site in sites_with_any_data
if SI.CE_to_SE(site) in presence.keys()
]
if primary_aaa:
sites_with_any_data = list(
set([SI.SE_to_CE(psite) for psite in presence.keys()]))
wfh.sendLog(
'assignor', "Holding the data but not allowed %s" % sorted(
list(
set([
se_site for se_site in presence.keys()
if not SI.SE_to_CE(se_site) in sites_allowed
]))))
if primary_locations == None:
primary_locations = presence.keys()
else:
primary_locations = list(
set(primary_locations) & set(presence.keys()))
sites_with_data = list(set(sites_with_data))
sites_with_any_data = list(set(sites_with_any_data))
opportunistic_sites = []
down_time = False
## opportunistic running where any piece of data is available
if secondary_locations or primary_locations:
## intersection of both any pieces of the primary and good IO
#opportunistic_sites = [SI.SE_to_CE(site) for site in list((set(secondary_locations) & set(primary_locations) & set(SI.sites_with_goodIO)) - set(sites_allowed))]
if secondary_locations and primary_locations:
opportunistic_sites = [
SI.SE_to_CE(site) for site in
list((set(secondary_locations) & set(primary_locations)) -
set([SI.CE_to_SE(site) for site in sites_allowed]))
]
elif primary_locations:
opportunistic_sites = [
SI.SE_to_CE(site) for site in list(
set(primary_locations) -
set([SI.CE_to_SE(site) for site in sites_allowed]))
]
else:
opportunistic_sites = []
wfh.sendLog(
'assignor', "We could be running in addition at %s" %
sorted(opportunistic_sites))
if any(
[osite in SI.sites_not_ready
for osite in opportunistic_sites]):
wfh.sendLog(
'assignor', "One of the usable site is in downtime %s" % ([
osite for osite in opportunistic_sites
if osite in SI.sites_not_ready
]))
down_time = True
## should this be send back to considered ?
## should be 2 but for the time-being let's lower it to get things going
copies_wanted, cpuh = wfh.getNCopies()
wfh.sendLog('assignor', "we need %s CPUh" % cpuh)
if cpuh > max_cpuh_block and not options.go:
#sendEmail('large workflow','that wf %s has a large number of CPUh %s, not assigning, please check the logs'%(wfo.name, cpuh))#,destination=['*****@*****.**'])
sendLog(
'assignor',
'%s requires a large numbr of CPUh %s , not assigning, please check with requester'
% (wfo.name, cpuh),
level='critical')
wfh.sendLog(
'assignor',
"Requiring a large number of CPUh %s, not assigning" % cpuh)
continue
if 'Campaign' in wfh.request and wfh.request[
'Campaign'] in CI.campaigns and 'maxcopies' in CI.campaigns[
wfh.request['Campaign']]:
copies_needed_from_campaign = CI.campaigns[
wfh.request['Campaign']]['maxcopies']
copies_wanted = min(copies_needed_from_campaign, copies_wanted)
if not options.early:
less_copies_than_requested = UC.get("less_copies_than_requested")
copies_wanted = max(
1, copies_wanted -
less_copies_than_requested) # take one out for the efficiency
else:
## find out whether there is a site in the whitelist, that is lacking jobs and reduce to 1 copy needed to get things going
pass
wfh.sendLog('assignor',
"needed availability fraction %s" % copies_wanted)
## should also check on number of sources, if large enough, we should be able to overflow most, efficiently
## default back to white list to original white list with any data
wfh.sendLog('assignor', "Allowed sites :%s" % sorted(sites_allowed))
if primary_aaa:
## remove the sites not reachable localy if not in having the data
if not sites_all_data:
wfh.sendLog('assignor',
"Overiding the primary on AAA setting to Off")
primary_aaa = False
else:
aaa_grid = set(sites_all_data)
for site in list(aaa_grid):
aaa_grid.update(aaa_mapping.get(site, []))
sites_allowed = list(set(initial_sites_allowed) & aaa_grid)
wfh.sendLog(
'assignor', "Selected to read primary through xrootd %s" %
sorted(sites_allowed))
isStoreResults = ('StoreResults' == wfh.request.setdefault(
'RequestType', None))
if isStoreResults:
if 'MergedLFNBase' in wfh.request:
set_lfn = wfh.request['MergedLFNBase']
else:
n_stalled += 1
wfh.sendLog(
'assignor',
"Cannot assign StoreResults request because MergedLFN is missing"
)
sendLog(
'assignor',
'Cannot assign StoreResults request because MergedLFN is missing',
level='critical')
continue
if not primary_aaa:
if not isStoreResults:
sites_allowed = sites_with_any_data
else:
## if we are dealing with a StoreResults request, we don't need to check dataset availability and
## should use the SiteWhiteList set in the original request
if 'SiteWhitelist' in wfh.request:
sites_allowed = wfh.request['SiteWhitelist']
else:
wfh.sendLog(
'assignor',
"Cannot assign StoreResults request because SiteWhitelist is missing"
)
sendLog(
'assignor',
'Cannot assign StoreResults request because SiteWhitelist is missing',
level='critical')
n_stalled += 1
continue
available_fractions = {}
wfh.sendLog('assignor',
"Selected for any data %s" % sorted(sites_allowed))
### check on endpoints for on-going transfers
if do_partial:
if endpoints:
end_sites = [SI.SE_to_CE(s) for s in endpoints]
sites_allowed = list(set(sites_allowed + end_sites))
if down_time and not any(osite in SI.sites_not_ready
for osite in end_sites):
print "Flip the status of downtime, since our destinations are good"
down_time = False
print "with added endpoints", sorted(end_sites)
else:
print "Cannot do partial assignment without knowin the endpoints"
n_stalled += 1
continue
#if not len(sites_allowed):
# if not options.early:
# wfh.sendLog('assignor',"cannot be assign with no matched sites")
# sendLog('assignor','%s has no whitelist'% wfo.name, level='critical')
# n_stalled+=1
# continue
low_pressure = SI.sites_low_pressure(0.4)
## if any of the site allowed is low pressure : reduce to 1 copy so that it gets started
allowed_and_low = sorted(set(low_pressure) & set(sites_allowed))
if allowed_and_low:
wfh.sendLog(
'assignor',
"The workflow can run at %s under low pressure currently" %
(','.join(allowed_and_low)))
copies_wanted = max(1., copies_wanted - 1.)
if available_fractions and not all([
available >= copies_wanted
for available in available_fractions.values()
]):
not_even_once = not all([
available >= 1. for available in available_fractions.values()
])
above_good = all([
available >= do_partial
for available in available_fractions.values()
])
wfh.sendLog(
'assignor',
"The input dataset is not available %s times, only %s" %
(copies_wanted, available_fractions.values()))
if down_time and not options.go and not options.early:
wfo.status = 'considered'
session.commit()
wfh.sendLog(
'assignor',
"sending back to considered because of site downtime, instead of waiting"
)
#sendEmail( "cannot be assigned due to downtime","%s is not sufficiently available, due to down time of a site in the whitelist. check the assignor logs. sending back to considered."% wfo.name)
sendLog(
'assignor',
'%s is not sufficiently available, due to down time of a site in the whitelist. sending back to considered.'
% (wfo.name),
level='delay')
n_stalled += 1
continue
#pass
print json.dumps(available_fractions)
if (options.go and not_even_once) or not options.go:
known = []
try:
known = json.loads(open('cannot_assign.json').read())
except:
pass
if not wfo.name in known and not options.limit and not options.go and not options.early and not (
do_partial and above_good):
wfh.sendLog(
'assignor',
"cannot be assigned, %s is not sufficiently available.\n %s"
% (wfo.name, json.dumps(available_fractions)))
#sendEmail( "cannot be assigned","%s is not sufficiently available.\n %s"%(wfo.name,json.dumps(available_fractions)))
known.append(wfo.name)
open('cannot_assign.json',
'w').write(json.dumps(known, indent=2))
if options.early:
if wfo.status == 'considered':
wfh.sendLog('assignor', "setting considered-tried")
wfo.status = 'considered-tried'
session.commit()
else:
print "tried but status is", wfo.status
if do_partial and above_good:
print "Will move on with partial locations"
else:
n_stalled += 1
continue
if not len(sites_allowed):
if not options.early:
wfh.sendLog('assignor',
"cannot be assign with no matched sites")
sendLog('assignor',
'%s has no whitelist' % wfo.name,
level='critical')
n_stalled += 1
continue
t1_only = [ce for ce in sites_allowed if ce.startswith('T1')]
if t1_only:
# try to pick from T1 only first
sites_out = [SI.pick_dSE([SI.CE_to_SE(ce) for ce in t1_only])]
else:
# then pick any otherwise
sites_out = [
SI.pick_dSE([SI.CE_to_SE(ce) for ce in sites_allowed])
]
wfh.sendLog('assignor', "Placing the output on %s" % sites_out)
parameters = {
'SiteWhitelist': sites_allowed,
'NonCustodialSites': sites_out,
'AutoApproveSubscriptionSites': list(set(sites_out)),
'AcquisitionEra': wfh.acquisitionEra(),
'ProcessingString': wfh.processingString(),
'MergedLFNBase': set_lfn,
'ProcessingVersion': version,
}
if primary_aaa:
parameters['TrustSitelists'] = True
wfh.sendLog(
'assignor',
"Reading primary through xrootd at %s" % sorted(sites_allowed))
if secondary_aaa:
parameters['TrustPUSitelists'] = True
wfh.sendLog(
'assignor', "Reading secondary through xrootd at %s" %
sorted(sites_allowed))
## plain assignment here
team = 'production'
if os.getenv('UNIFIED_TEAM'): team = os.getenv('UNIFIED_TEAM')
if options and options.team:
team = options.team
parameters['Team'] = team
if lheinput:
## throttle reading LHE article
wfh.sendLog('assignor',
'Setting the number of events per job to 500k max')
parameters['EventsPerJob'] = 500000
def pick_options(options, parameters):
##parse options entered in command line if any
if options:
for key in reqMgrClient.assignWorkflow.keys:
v = getattr(options, key)
if v != None:
if type(v) == str and ',' in v:
parameters[key] = filter(None, v.split(','))
else:
parameters[key] = v
def pick_campaign(assign_parameters, parameters):
## pick up campaign specific assignment parameters
parameters.update(assign_parameters.get('parameters', {}))
if options.force_options:
pick_campaign(assign_parameters, parameters)
pick_options(options, parameters)
else:
## campaign parameters update last
pick_options(options, parameters)
pick_campaign(assign_parameters, parameters)
if not options.test:
parameters['execute'] = True
hold_split, split_check = wfh.checkSplitting()
if hold_split and not options.go:
if split_check:
wfh.sendLog(
'assignor', 'Holding on to the change in splitting %s' %
('\n\n'.join([str(i) for i in split_check])))
else:
wfh.sendLog('assignor', 'Change of splitting is on hold')
n_stalled += 1
continue
if split_check == None or split_check == False:
n_stalled += 1
continue
elif split_check:
## operate all recommended changes
reqMgrClient.setWorkflowSplitting(url, wfo.name, split_check)
wfh.sendLog(
'assignor', 'Applying the change in splitting %s' %
('\n\n'.join([str(i) for i in split_check])))
split_check = True ## bypass completely and use the above
# Handle run-dependent MC
pstring = wfh.processingString()
if 'PU_RD' in pstring:
numEvents = wfh.getRequestNumEvents()
eventsPerLumi = [getDatasetEventsPerLumi(prim) for prim in primary]
eventsPerLumi = sum(eventsPerLumi) / float(len(eventsPerLumi))
reqJobs = 500
if 'PU_RD2' in pstring:
reqJobs = 2000
eventsPerJob = int(numEvents / (reqJobs * 1.4))
lumisPerJob = int(eventsPerJob / eventsPerLumi)
if lumisPerJob == 0:
#sendEmail("issue with event splitting for run-dependent MC","%s needs to be split by event with %s per job"%(wfo.name, eventsPerJob))
sendLog('assignor',
"%s needs to be split by event with %s per job" %
(wfo.name, eventsPerJob),
level='critical')
wfh.sendLog(
'assignor',
"%s needs to be split by event with %s per job" %
(wfo.name, eventsPerJob))
parameters['EventsPerJob'] = eventsPerJob
else:
spl = wfh.getSplittings()[0]
eventsPerJobEstimated = spl[
'events_per_job'] if 'events_per_job' in spl else None
eventsPerJobEstimated = spl[
'avg_events_per_job'] if 'avg_events_per_job' in spl else None
if eventsPerJobEstimated and eventsPerJobEstimated > eventsPerJob:
#sendEmail("setting lumi splitting for run-dependent MC","%s was assigned with %s lumis/job"%( wfo.name, lumisPerJob))
sendLog('assignor',
"%s was assigned with %s lumis/job" %
(wfo.name, lumisPerJob),
level='critical')
wfh.sendLog(
'assignor', "%s was assigned with %s lumis/job" %
(wfo.name, lumisPerJob))
parameters['LumisPerJob'] = lumisPerJob
else:
#sendEmail("leaving splitting untouched for PU_RD*","please check on "+wfo.name)
sendLog(
'assignor',
"leaving splitting untouched for %s, please check on %s"
% (pstring, wfo.name),
level='critical')
wfh.sendLog(
'assignor',
"leaving splitting untouched for PU_RD*, please check."
)
if isHEPCloudReady(url) and wfh.isGoodForNERSC():
parameters['Team'] = 'hepcloud'
parameters['SiteWhitelist'] = ['T3_US_NERSC']
if primary:
parameters['TrustSitelists'] = True
if secondary:
parameters['TrustPUSitelists'] = True
sendEmail("sending work to hepcloud",
"pleasse check on %s" % wfh.request['RequestName'],
destination=['*****@*****.**'])
## make sure to autoapprove all NonCustodialSites
parameters['AutoApproveSubscriptionSites'] = list(
set(parameters['NonCustodialSites'] +
parameters.get('AutoApproveSubscriptionSites', [])))
result = reqMgrClient.assignWorkflow(
url, wfo.name, None,
parameters) ## team is not relevant anymore here
# set status
if not options.test:
if result:
wfo.status = 'away'
session.commit()
n_assigned += 1
wfh.sendLog(
'assignor', "Properly assigned\n%s" %
(json.dumps(parameters, indent=2)))
try:
## refetch information and lock output
new_wfi = workflowInfo(url, wfo.name)
(_, prim, _, sec) = new_wfi.getIO()
for secure in list(prim) + list(
sec) + new_wfi.request['OutputDatasets']:
## lock all outputs
LI.lock(secure, reason='assigning')
except Exception as e:
print "fail in locking output"
print str(e)
sendEmail("failed locking of output", str(e))
else:
wfh.sendLog(
'assignor',
"Failed to assign %s.\n%s \n Please check the logs" %
(wfo.name, reqMgrClient.assignWorkflow.errorMessage))
sendLog('assignor',
"Failed to assign %s.\n%s \n Please check the logs" %
(wfo.name, reqMgrClient.assignWorkflow.errorMessage),
level='critical')
print "ERROR could not assign", wfo.name
else:
pass
print "Assignment summary:"
sendLog('assignor', "Assigned %d Stalled %s" % (n_assigned, n_stalled))
if n_stalled and not options.go and not options.early:
sendLog('assignor',
"%s workflows cannot be assigned. Please take a look" %
(n_stalled),
level='critical')
def stagor(url, specific=None, options=None):
if not componentInfo().check(): return
SI = siteInfo()
CI = campaignInfo()
UC = unifiedConfiguration()
TS = transferStatuses()
cached_transfer_statuses = TS.content()
transfer_statuses = {}
done_by_wf_id = {}
done_by_input = {}
completion_by_input = {}
good_enough = 100.0
lost_blocks = json.loads(
eosRead('%s/lost_blocks_datasets.json' % monitor_dir))
lost_files = json.loads(
eosRead('%s/lost_files_datasets.json' % monitor_dir))
known_lost_blocks = {}
known_lost_files = {}
for dataset in set(lost_blocks.keys() + lost_files.keys()):
b, f = findLostBlocksFiles(url, dataset)
if dataset in lost_blocks and not b:
print dataset, "has no really lost blocks"
else:
known_lost_blocks[dataset] = [i['name'] for i in b]
if dataset in lost_files and not f:
print dataset, "has no really lost files"
else:
known_lost_files[dataset] = [i['name'] for i in f]
def time_point(label="", sub_lap=False):
now = time.mktime(time.gmtime())
nows = time.asctime(time.gmtime())
print "Time check (%s) point at : %s" % (label, nows)
print "Since start: %s [s]" % (now - time_point.start)
if sub_lap:
print "Sub Lap : %s [s]" % (now - time_point.sub_lap)
time_point.sub_lap = now
else:
print "Lap : %s [s]" % (now - time_point.lap)
time_point.lap = now
time_point.sub_lap = now
time_point.sub_lap = time_point.lap = time_point.start = time.mktime(
time.gmtime())
time_point("Check cached transfer")
## collect all datasets that are needed for wf in staging, correcting the status of those that are not really in staging
wfois = []
needs = defaultdict(list)
needs_by_priority = defaultdict(list)
for wfo in session.query(Workflow).filter(
Workflow.status == 'staging').all():
wfi = workflowInfo(url, wfo.name)
if wfi.request['RequestStatus'] in [
'running-open', 'running-closed', 'completed', 'assigned',
'acquired'
]:
wfi.sendLog('stagor',
"is in status %s" % wfi.request['RequestStatus'])
wfo.status = 'away'
session.commit()
continue
if not wfi.request['RequestStatus'] in ['assignment-approved']:
## should be setting 'away' too
## that usually happens for relvals
if wfi.request['RequestStatus'] in [
'rejected', 'aborted', 'aborted-completed',
'aborted-archived', 'rejected-archived'
] and wfi.isRelval():
wfo.status = 'forget'
session.commit()
continue
else:
print wfo.name, "is", wfi.request['RequestStatus']
#sendEmail("wrong status in staging. debug","%s is in %s, should set away."%(wfo.name,wfi.request['RequestStatus']))
sendLog("stagor",
"%s is in %s, set away" %
(wfo.name, wfi.request['RequestStatus']),
level='critical')
wfo.status = 'away'
session.commit()
continue
wfois.append((wfo, wfi))
_, primaries, _, secondaries = wfi.getIO()
for dataset in list(primaries) + list(secondaries):
needs[wfo.name].append(dataset)
done_by_input[dataset] = {}
completion_by_input[dataset] = {}
needs_by_priority[wfi.request['RequestPriority']].append(dataset)
wfi.sendLog('stagor', '%s needs %s' % (wfo.name, dataset))
time_point("Check staging workflows")
open('%s/dataset_requirements.json' % monitor_dir,
'w').write(json.dumps(needs, indent=2))
for prio in needs_by_priority:
needs_by_priority[prio] = list(set(needs_by_priority[prio]))
open('%s/dataset_priorities.json' % monitor_dir,
'w').write(json.dumps(needs_by_priority, indent=2))
dataset_endpoints = defaultdict(set)
endpoint_in_downtime = defaultdict(set)
#endpoint_completed = defaultdict(set)
endpoint_incompleted = defaultdict(set)
#endpoint = defaultdict(set)
send_back_to_considered = set()
## first check if anything is inactive
all_actives = set([
transfer.phedexid for transfer in session.query(TransferImp).filter(
TransferImp.active).all()
])
for active_phedexid in all_actives:
skip = True
transfers_phedexid = session.query(TransferImp).filter(
TransferImp.phedexid == active_phedexid).all()
for imp in transfers_phedexid:
if imp.workflow.status == 'staging':
skip = False
sendLog(
'stagor', "\t%s is staging for %s" %
(imp.phedexid, imp.workflow.name))
if skip:
sendLog('stagor', "setting %s inactive" % active_phedexid)
for imp in transfers_phedexid:
imp.active = False
session.commit()
all_actives = sorted(
set([
transfer.phedexid for transfer in session.query(
TransferImp).filter(TransferImp.active).all()
]))
for phedexid in all_actives:
if specific: continue
## check on transfer completion
not_cached = False
if phedexid in cached_transfer_statuses:
### use a cache for transfer that already looked done
sendLog('stagor', "read %s from cache" % phedexid)
checks = cached_transfer_statuses[phedexid]
else:
## I actually would like to avoid that all I can
sendLog('stagor',
'Performing spurious transfer check on %s' % phedexid,
level='critical')
checks = checkTransferStatus(url, phedexid, nocollapse=True)
try:
print json.dumps(checks, indent=2)
except:
print checks
if not checks:
## this is going to bias quite heavily the rest of the code. we should abort here
#sendLog('stagor','Ending stagor because of skewed input from checkTransferStatus', level='critical')
#return False
sendLog(
'stagor',
'Stagor has got a skewed input from checkTransferStatus',
level='critical')
checks = {}
pass
else:
TS.add(phedexid, checks)
time_point("Check transfer status %s" % phedexid, sub_lap=True)
if not specific:
for dsname in checks:
if not dsname in done_by_input: done_by_input[dsname] = {}
if not dsname in completion_by_input:
completion_by_input[dsname] = {}
done_by_input[dsname][phedexid] = all(
map(lambda i: i >= good_enough, checks[dsname].values()))
completion_by_input[dsname][phedexid] = checks[dsname].values()
if checks:
sendLog(
'stagor', "Checks for %s are %s" %
(phedexid, [node.values() for node in checks.values()]))
done = all(
map(
lambda i: i >= good_enough,
list(
itertools.chain.from_iterable(
[node.values() for node in checks.values()]))))
else:
## it is empty, is that a sign that all is done and away ?
if not_cached:
print "Transfer status was not cached"
else:
print "ERROR with the scubscriptions API of ", phedexid
print "Most likely something else is overiding the transfer request. Need to work on finding the replacement automatically, if the replacement exists"
done = False
transfers_phedexid = session.query(TransferImp).filter(
TransferImp.phedexid == phedexid).all()
for imp in transfers_phedexid:
tr_wf = imp.workflow
if tr_wf: # and tr_wf.status == 'staging':
if not tr_wf.id in done_by_wf_id: done_by_wf_id[tr_wf.id] = {}
done_by_wf_id[tr_wf.id][phedexid] = done
if done:
imp.active = False
session.commit()
for ds in checks:
for s, v in checks[ds].items():
dataset_endpoints[ds].add(s)
if done:
sendLog('stagor', "%s is done" % phedexid)
TS.add(phedexid, checks)
else:
sendLog(
'stagor',
"%s is not finished %s" % (phedexid, pprint.pformat(checks)))
##pprint.pprint( checks )
## check if the destination is in down-time
for ds in checks:
sites_incomplete = [
SI.SE_to_CE(s) for s, v in checks[ds].items()
if v < good_enough
]
sites_incomplete_down = [
s for s in sites_incomplete if not s in SI.sites_ready
]
## no space means no transfer should go there : NO, it does not work in the long run
#sites_incomplete_down = [SI.SE_to_CE(s) for s,v in checks[ds].items() if (v<good_enough and (SI.disk[s]==0 or (not SI.SE_to_CE(s) in SI.sites_ready)))]
if sites_incomplete_down:
sendLog(
'stagor',
"%s are in downtime, while waiting for %s to get there"
% (",".join(sites_incomplete_down), ds))
endpoint_in_downtime[ds].update(sites_incomplete_down)
if sites_incomplete:
endpoint_incompleted[ds].update(sites_incomplete)
time_point("Check on-going transfers")
print "End points"
for k in dataset_endpoints:
dataset_endpoints[k] = list(dataset_endpoints[k])
print json.dumps(dataset_endpoints, indent=2)
print "End point in down time"
for k in endpoint_in_downtime:
endpoint_in_downtime[k] = list(endpoint_in_downtime[k])
print json.dumps(endpoint_in_downtime, indent=2)
print "End point incomplete in down time"
for k in endpoint_incompleted:
endpoint_incompleted[k] = list(endpoint_incompleted[k])
print json.dumps(endpoint_incompleted, indent=2)
#open('%s/transfer_statuses.json'%monitor_dir,'w').write( json.dumps( transfer_statuses, indent=2))
eosFile('%s/transfer_statuses.json' % monitor_dir,
'w').write(json.dumps(TS.content(), indent=2)).close()
eosFile('%s/dataset_endpoints.json' % monitor_dir,
'w').write(json.dumps(dataset_endpoints, indent=2)).close()
already_stuck = json.loads(
eosRead('%s/stuck_transfers.json' % monitor_pub_dir)).keys()
already_stuck.extend(getAllStuckDataset())
missing_in_action = defaultdict(list)
print "-" * 10, "Checking on workflows in staging", "-" * 10
#forget_about = ['/MinBias_TuneCUETP8M1_13TeV-pythia8/RunIISummer15GS-MCRUN2_71_V1-v2/GEN-SIM']
#for what in forget_about:
# if not done_by_input[what]:
# done_by_input[what] = {'fake':True}
## come back to workflows and check if they can go
available_cache = defaultdict(lambda: defaultdict(float))
presence_cache = defaultdict(dict)
time_point("Preparing for more")
for wfo, wfi in wfois:
print "#" * 30
time_point("Forward checking %s" % wfo.name, sub_lap=True)
## the site white list takes site, campaign, memory and core information
(_, primaries, _, secondaries,
sites_allowed) = wfi.getSiteWhiteList(verbose=False)
se_allowed = [SI.CE_to_SE(site) for site in sites_allowed]
se_allowed.sort()
se_allowed_key = ','.join(se_allowed)
readys = {}
for need in list(primaries) + list(secondaries):
if not need in done_by_input:
wfi.sendLog('stagor', "missing transfer report for %s" % need)
readys[need] = False
## should warn someone about this !!!
## it cannot happen, by construction
sendEmail('missing transfer report',
'%s does not have a transfer report' % (need))
continue
if not done_by_input[need] and need in list(secondaries):
wfi.sendLog(
'stagor',
"assuming it is OK for secondary %s to have no attached transfers"
% need)
readys[need] = True
done_by_input[need] = {"fake": True}
continue
if len(done_by_input[need]) and all(done_by_input[need].values()):
wfi.sendLog('stagor', "%s is ready" % need)
print json.dumps(done_by_input[need], indent=2)
readys[need] = True
else:
wfi.sendLog(
'stagor', "%s is not ready \n%s" %
(need, json.dumps(done_by_input[need], indent=2)))
readys[need] = False
if readys and all(readys.values()):
if wfo.status == 'staging':
wfi.sendLog('stagor',
"all needs are fullfilled, setting staged")
wfo.status = 'staged'
session.commit()
else:
wfi.sendLog('stagor', "all needs are fullfilled, already")
print json.dumps(readys, indent=2)
else:
wfi.sendLog('stagor', "missing requirements")
copies_needed, _ = wfi.getNCopies()
jump_ahead = False
re_transfer = False
## there is missing input let's do something more elaborated
for need in list(primaries): #+list(secondaries):
if endpoint_in_downtime[need] and endpoint_in_downtime[
need] == endpoint_incompleted[need]:
#print need,"is going to an end point in downtime"
wfi.sendLog(
'stagor',
"%s has only incomplete endpoint in downtime\n%s" %
(need, endpoint_in_downtime[need]))
re_transfer = True
if not se_allowed_key in available_cache[need]:
available_cache[need][
se_allowed_key] = getDatasetBlocksFraction(
url, need, sites=se_allowed)
if available_cache[need][se_allowed_key] >= copies_needed:
wfi.sendLog(
'stagor',
"assuming it is OK to move on like this already for %s"
% need)
jump_ahead = True
else:
wfi.sendLog(
'stagor', "Available %s times" %
available_cache[need][se_allowed_key])
missing_and_downtime = list(
set(endpoint_in_downtime[need])
& set(endpoint_incompleted[need]))
if missing_and_downtime:
wfi.sendLog(
'stagor',
"%s is incomplete at %s which is in downtime, trying to move along"
% (need, ','.join(missing_and_downtime)))
jump_ahead = True
else:
wfi.sendLog(
'stagor',
"continue waiting for transfers for optimum production performance."
)
## compute a time since staging to filter jump starting ?
# check whether the inputs is already in the stuck list ...
for need in list(primaries) + list(secondaries):
if need in already_stuck:
wfi.sendLog('stagor',
"%s is stuck, so try to jump ahead" % need)
jump_ahead = True
if jump_ahead or re_transfer:
details_text = "checking on availability for %s to jump ahead" % wfo.name
details_text += '\n%s wants %s copies' % (wfo.name,
copies_needed)
copies_needed = max(1, copies_needed - 1)
details_text += '\nlowering by one unit to %s' % copies_needed
wfi.sendLog('stagor', details_text)
all_check = True
prim_where = set()
for need in list(primaries):
if not se_allowed_key in presence_cache[need]:
presence_cache[need][
se_allowed_key] = getDatasetPresence(
url, need, within_sites=se_allowed)
presence = presence_cache[need][se_allowed_key]
prim_where.update(presence.keys())
available = available_cache[need][se_allowed_key]
this_check = (available >= copies_needed)
wfi.sendLog(
'stagor', "%s is available %s times (%s), at %s" %
(need, available, this_check, se_allowed_key))
all_check &= this_check
if not all_check: break
for need in list(secondaries):
## I do not want to check on the secon
## this below does not function because the primary could be all available, and the secondary not complete at a certain site that does not matter at that point
this_check = all(done_by_input[need].values())
wfi.sendLog(
'stagor', "%s is this much transfered %s" %
(need, json.dumps(done_by_input[need], indent=2)))
all_check &= this_check
#if not se_allowed_key in presence_cache[need]:
# presence_cache[need][se_allowed_key] = getDatasetPresence( url, need , within_sites=se_allowed)
## restrict to where the primary is
#presence = dict([(k,v) for (k,v) in presence_cache[need][se_allowed_key].items() if k in prim_where])
#this_check = all([there for (there,frac) in presence.values()])
#print need,"is present at all sites:",this_check
#all_check&= this_check
if all_check and not re_transfer:
wfi.sendLog(
'stagor',
"needs are sufficiently fullfilled, setting staged")
wfo.status = 'staged'
session.commit()
else:
print wfo.name, "has to wait a bit more"
wfi.sendLog('stagor', "needs to wait a bit more")
else:
wfi.sendLog('stagor', "not checking availability")
if re_transfer:
wfi.sendLog(
'stagor',
"Sending back to considered because of endpoint in downtime"
)
if wfo.status == 'staging':
wfo.status = 'considered'
session.commit()
send_back_to_considered.add(wfo.name)
time_point("Checked affected workflows")
if send_back_to_considered:
#sendEmail("transfer to endpoint in downtime","sending back to considered the following workflows \n%s"%('\n'.join( send_back_to_considered)))
sendLog('stagor',
"sending back to considered the following workflows \n%s" %
('\n'.join(send_back_to_considered)),
level='critical')
print "-" * 10, "Checking on non-available datasets", "-" * 10
## now check on those that are not fully available
for dsname in available_cache.keys():
## squash the se_allowed_key key
available_cache[dsname] = min(available_cache[dsname].values())
really_stuck_dataset = set()
for dsname, available in available_cache.items():
using_its = getWorkflowByInput(url, dsname)
#print using_its
using_wfos = []
for using_it in using_its:
wf = session.query(Workflow).filter(
Workflow.name == using_it).first()
if wf:
using_wfos.append(wf)
if not len(done_by_input[dsname]):
print "For dataset", dsname, "there are no transfer report. That's an issue."
for wf in using_wfos:
if wf.status == 'staging':
if UC.get("stagor_sends_back"):
print "sending", wf.name, "back to considered"
wf.status = 'considered'
session.commit()
#sendEmail( "send back to considered","%s was send back and might be trouble"% wf.name)
sendLog('stagor',
"%s was send back and might be trouble" %
wf.name,
level='critical')
else:
print "would send", wf.name, "back to considered"
#sendEmail( "subscription lagging behind","susbscriptions to get %s running are not appearing in phedex. I would have send it back to considered but that's not good."% wf.name)
sendLog(
'stagor',
"susbscriptions to get %s running are not appearing in phedex. I would have send it back to considered but that's not good."
% wf.name,
level='critical')
continue
## not compatible with checking on secondary availability
#if all([wf.status != 'staging' for wf in using_wfos]):
# ## means despite all checks that input is not needed
# continue
if available < 1.:
print "incomplete", dsname
## there is a problem in the method below that it does not account for files stuck in T1*Buffer only
lost_blocks, lost_files = findLostBlocksFiles(
url, dsname) if (not dsname.endswith('/RAW')) else ([], [])
lost_block_names = [item['name'] for item in lost_blocks]
lost_file_names = [item['name'] for item in lost_files]
if lost_blocks:
#print json.dumps( lost , indent=2 )
## estimate for how much !
fraction_loss, _, n_missing = getDatasetBlockFraction(
dsname, lost_block_names)
print "We have lost", len(
lost_block_names
), "blocks", lost_block_names, "for %f%%" % (100. *
fraction_loss)
if fraction_loss > 0.05: ## 95% completion mark
#sendEmail('we have lost too many blocks','%s is missing %d blocks, for %d events, %f %% loss'%(dsname, len(lost_block_names), n_missing, fraction_loss))
sendLog(
'stagor',
'%s is missing %d blocks, for %d events, %3.2f %% loss'
% (dsname, len(lost_block_names), n_missing,
100 * fraction_loss),
level='critical')
## the workflow should be rejected !
for wf in using_wfos:
if wf.status == 'staging':
print wf.name, "is doomed. setting to trouble"
wf.status = 'trouble'
session.commit()
sendLog(
'stagor',
'%s has too much loss on the input dataset %s. Missing %d blocks, for %d events, %3.2f %% loss'
% (wf.name, dsname, len(lost_block_names),
n_missing, 100 * fraction_loss),
level='critical')
else:
## probably enough to make a ggus and remove
if not dsname in known_lost_blocks:
#sendEmail('we have lost a few blocks', '%s is missing %d blocks, for %d events, %f %% loss\n\n%s'%(dsname, len(lost_block_names), n_missing, fraction_loss, '\n'.join( lost_block_names ) ))
sendLog(
'stagor',
'%s is missing %d blocks, for %d events, %f %% loss\n\n%s'
% (dsname, len(lost_block_names), n_missing,
fraction_loss, '\n'.join(lost_block_names)),
level='critical')
known_lost_blocks[dsname] = [
i['name'] for i in lost_blocks
]
really_stuck_dataset.add(dsname)
if lost_files:
fraction_loss, _, n_missing = getDatasetFileFraction(
dsname, lost_file_names)
print "We have lost", len(
lost_file_names
), "files", lost_file_names, "for %f%%" % fraction_loss
if fraction_loss > 0.05:
#sendEmail('we have lost too many files','%s is missing %d files, for %d events, %f %% loss'%(dsname, len(lost_file_names),n_missing, fraction_loss))
sendLog(
'stagor',
'%s is missing %d files, for %d events, %f %% loss' %
(dsname, len(lost_file_names), n_missing,
fraction_loss),
level='critical')
for wf in using_wfos:
if wf.status == 'staging':
print wf.name, "is doomed. setting to trouble"
wf.status = 'trouble'
session.commit()
else:
## probably enough to make a ggus and remove
if not dsname in known_lost_files:
#sendEmail('we have lost a few files','%s is missing %d files, for %d events, %f %% loss\n\n%s'%(dsname, len(lost_file_names),n_missing, fraction_loss, '\n'.join(lost_file_names)))
sendLog(
'stagor',
'%s is missing %d files, for %d events, %f %% loss\n\n%s'
% (dsname, len(lost_file_names), n_missing,
fraction_loss, '\n'.join(lost_file_names)),
level='critical')
known_lost_files[dsname] = [
i['name'] for i in lost_files
]
## should the status be change to held-staging and pending on a ticket
missings = [
pid for (pid, d) in done_by_input[dsname].items() if d == False
]
print "\t", done_by_input[dsname]
print "\tneeds", len(done_by_input[dsname])
print "\tgot", done_by_input[dsname].values().count(True)
print "\tmissing", missings
missing_in_action[dsname].extend(missings)
rr = eosFile('%s/lost_blocks_datasets.json' % monitor_dir, 'w')
rr.write(json.dumps(known_lost_blocks, indent=2))
rr.close()
rr = eosFile('%s/lost_files_datasets.json' % monitor_dir, 'w')
rr.write(json.dumps(known_lost_files, indent=2))
rr.close()
eosFile('%s/incomplete_transfers.json' % monitor_dir,
'w').write(json.dumps(missing_in_action, indent=2)).close()
print "Stuck transfers and datasets"
print json.dumps(missing_in_action, indent=2)
TD = transferDataset()
datasets_by_phid = defaultdict(set)
for dataset in missing_in_action:
for phid in missing_in_action[dataset]:
#print dataset,"stuck through",phid
datasets_by_phid[phid].add(dataset)
for k in datasets_by_phid:
#datasets_by_phid[k] = list(datasets_by_phid[k])
TD.add(k, list(datasets_by_phid[k]))
#eosFile('%s/datasets_by_phid.json'%base_eos_dir,'w').write( json.dumps(datasets_by_phid, indent=2 )).close()
eosFile('%s/really_stuck_dataset.json' % base_eos_dir,
'w').write(json.dumps(list(really_stuck_dataset),
indent=2)).close()
print '\n' * 2, "Datasets really stuck"
print '\n'.join(really_stuck_dataset)
#############
## not going further for what matters
#############
return
def stagor(url,specific =None, options=None):
if not componentInfo().check(): return
SI = siteInfo()
CI = campaignInfo()
UC = unifiedConfiguration()
done_by_wf_id = {}
done_by_input = {}
completion_by_input = {}
good_enough = 100.0
lost_blocks = json.loads(open('%s/lost_blocks_datasets.json'%monitor_dir).read())
lost_files = json.loads(open('%s/lost_files_datasets.json'%monitor_dir).read())
known_lost_blocks = {}
known_lost_files = {}
for dataset in set(lost_blocks.keys()+lost_files.keys()):
b,f = findLostBlocksFiles(url, dataset)
if dataset in lost_blocks and not b:
print dataset,"has no really lost blocks"
else:
known_lost_blocks[dataset] = [i['name'] for i in b]
if dataset in lost_files and not f:
print dataset,"has no really lost files"
else:
known_lost_files[dataset] = [i['name'] for i in f]
try:
cached_transfer_statuses = json.loads(open('cached_transfer_statuses.json').read())
except:
print "inexisting transfer statuses. starting fresh"
cached_transfer_statuses = {}
transfer_statuses = {}
## pop all that are now in negative values
for phedexid in cached_transfer_statuses.keys():
transfers = session.query(Transfer).filter(Transfer.phedexid==int(phedexid)).all()
if not transfers:
print phedexid,"does not look relevant to be in cache anymore. poping"
print cached_transfer_statuses.pop( phedexid )
## collect all datasets that are needed for wf in staging, correcting the status of those that are not really in staging
wfois = []
needs = defaultdict(list)
needs_by_priority = defaultdict(list)
for wfo in session.query(Workflow).filter(Workflow.status == 'staging').all():
wfi = workflowInfo(url, wfo.name)
if wfi.request['RequestStatus'] in ['running-open','running-closed','completed','assigned','acquired']:
wfi.sendLog('stagor', "is in status %s"%wfi.request['RequestStatus'])
wfi.status='away'
session.commit()
continue
if not wfi.request['RequestStatus'] in ['assignment-approved']:
## should be setting 'away' too
print wfo.name,"is",wfi.request['RequestStatus']
sendEmail("wrong status in staging. debug","%s is in %s, should set away."%(wfo.name,wfi.request['RequestStatus']))
wfois.append( (wfo,wfi) )
_,primaries,_,secondaries = wfi.getIO()
for dataset in list(primaries)+list(secondaries):
needs[wfo.name].append( dataset)
done_by_input[dataset] = {}
completion_by_input[dataset] = {}
needs_by_priority[wfi.request['RequestPriority']].append( dataset )
wfi.sendLog('stagor', '%s needs %s'%( wfo.name, dataset))
open('%s/dataset_requirements.json'%monitor_dir,'w').write( json.dumps( needs, indent=2))
for prio in needs_by_priority: needs_by_priority[prio] = list(set(needs_by_priority[prio]))
open('%s/dataset_priorities.json'%monitor_dir,'w').write( json.dumps( needs_by_priority , indent=2))
dataset_endpoints = defaultdict(set)
endpoint_in_downtime = defaultdict(set)
#endpoint_completed = defaultdict(set)
endpoint_incompleted = defaultdict(set)
#endpoint = defaultdict(set)
send_back_to_considered = set()
## phedexid are set negative when not relevant anymore
# probably there is a db schema that would allow much faster and simpler query
for transfer in session.query(Transfer).filter(Transfer.phedexid>0).all():
if specific and str(transfer.phedexid)!=str(specific): continue
skip=True
for wfid in transfer.workflows_id:
tr_wf = session.query(Workflow).get(wfid)
if tr_wf:
if tr_wf.status == 'staging':
sendLog('stagor',"\t%s is staging for %s"%(transfer.phedexid, tr_wf.name))
skip=False
if skip:
sendLog('stagor',"setting %s to negative value"%transfer.phedexid)
transfer.phedexid = -transfer.phedexid
session.commit()
continue
if transfer.phedexid<0: continue
## check the status of transfers
checks = checkTransferApproval(url, transfer.phedexid)
approved = all(checks.values())
if not approved:
sendLog('stagor', "%s is not yet approved"%transfer.phedexid)
approveSubscription(url, transfer.phedexid)
continue
## check on transfer completion
if str(transfer.phedexid) in cached_transfer_statuses:
### use a cache for transfer that already looked done
sendLog('stagor',"read %s from cache"%transfer.phedexid)
checks = cached_transfer_statuses[str(transfer.phedexid)]
else:
checks = checkTransferStatus(url, transfer.phedexid, nocollapse=True)
## just write this out
transfer_statuses[str(transfer.phedexid)] = copy.deepcopy(checks)
if not specific:
for dsname in checks:
if not dsname in done_by_input: done_by_input[dsname]={}
if not dsname in completion_by_input: completion_by_input[dsname] = {}
done_by_input[dsname][transfer.phedexid]=all(map(lambda i:i>=good_enough, checks[dsname].values()))
completion_by_input[dsname][transfer.phedexid]=checks[dsname].values()
if checks:
sendLog('stagor',"Checks for %s are %s"%( transfer.phedexid, [node.values() for node in checks.values()]))
done = all(map(lambda i:i>=good_enough,list(itertools.chain.from_iterable([node.values() for node in checks.values()]))))
else:
## it is empty, is that a sign that all is done and away ?
print "ERROR with the scubscriptions API of ",transfer.phedexid
print "Most likely something else is overiding the transfer request. Need to work on finding the replacement automatically, if the replacement exists"
done = False
## the thing above is NOT giving the right number
#done = False
for wfid in transfer.workflows_id:
tr_wf = session.query(Workflow).get(wfid)
if tr_wf:# and tr_wf.status == 'staging':
if not tr_wf.id in done_by_wf_id: done_by_wf_id[tr_wf.id]={}
done_by_wf_id[tr_wf.id][transfer.phedexid]=done
## for those that are in staging, and the destination site is in drain
#if not done and tr_wf.status == 'staging':
for ds in checks:
for s,v in checks[ds].items():
dataset_endpoints[ds].add( s )
if done:
## transfer.status = 'done'
sendLog('stagor',"%s is done"%transfer.phedexid)
cached_transfer_statuses[str(transfer.phedexid)] = copy.deepcopy(checks)
else:
sendLog('stagor',"%s is not finished %s"%(transfer.phedexid, pprint.pformat( checks )))
pprint.pprint( checks )
## check if the destination is in down-time
for ds in checks:
sites_incomplete = [SI.SE_to_CE(s) for s,v in checks[ds].items() if v<good_enough]
sites_incomplete_down = [s for s in sites_incomplete if not s in SI.sites_ready]
if sites_incomplete_down:
sendLog('stagor',"%s are in downtime, while waiting for %s to get there"%( ",".join(sites_incomplete_down), ds))
#sites_complete = [SI.SE_to_CE(s) for s,v in checks[ds].items() if v>=good_enough]
#endpoint[ds].update( sites_complete )
#endpoint[ds].update( sites_incomplete )
#endpoint_completed[ds].update( sites_complete )
endpoint_incompleted[ds].update( sites_incomplete )
endpoint_in_downtime[ds].update( sites_incomplete_down )
print "End point in down time"
for k in endpoint_in_downtime: endpoint_in_downtime[k] = list(endpoint_in_downtime[k])
for k in dataset_endpoints: dataset_endpoints[k] = list(dataset_endpoints[k])
print json.dumps( endpoint_in_downtime , indent=2)
open('cached_transfer_statuses.json','w').write( json.dumps( cached_transfer_statuses, indent=2))
open('%s/transfer_statuses.json'%monitor_dir,'w').write( json.dumps( transfer_statuses, indent=2))
open('%s/dataset_endpoints.json'%monitor_dir,'w').write( json.dumps(dataset_endpoints, indent=2))
already_stuck = json.loads( open('%s/stuck_transfers.json'%monitor_dir).read() )
already_stuck.extend( getAllStuckDataset() )
missing_in_action = defaultdict(list)
print "-"*10,"Checking on workflows in staging","-"*10
#forget_about = ['/MinBias_TuneCUETP8M1_13TeV-pythia8/RunIISummer15GS-MCRUN2_71_V1-v2/GEN-SIM']
#for what in forget_about:
# if not done_by_input[what]:
# done_by_input[what] = {'fake':True}
## come back to workflows and check if they can go
available_cache = defaultdict(lambda : defaultdict(float))
presence_cache = defaultdict(dict)
for wfo,wfi in wfois:
print "#"*30
## the site white list takes site, campaign, memory and core information
(_,primaries,_,secondaries,sites_allowed) = wfi.getSiteWhiteList(verbose=False)
se_allowed = [SI.CE_to_SE(site) for site in sites_allowed]
se_allowed.sort()
se_allowed_key = ','.join(se_allowed)
readys={}
for need in list(primaries)+list(secondaries):
if not need in done_by_input:
wfi.sendLog('stagor',"missing transfer report for %s"%need)
readys[need] = False
## should warn someone about this !!!
## it cannot happen, by construction
sendEmail('missing transfer report','%s does not have a transfer report'%(need))
continue
if not done_by_input[need] and need in list(secondaries):
wfi.sendLog('stagor',"assuming it is OK for secondary %s to have no attached transfers"% need)
readys[need] = True
done_by_input[need] = { "fake" : True }
continue
if len(done_by_input[need]) and all(done_by_input[need].values()):
wfi.sendLog('stagor',"%s is ready"%need)
print json.dumps( done_by_input[need] , indent=2)
readys[need] = True
else:
wfi.sendLog('stagor',"%s is not ready \n%s"%(need,json.dumps( done_by_input[need] , indent=2)))
readys[need] = False
if readys and all(readys.values()):
if wfo.status == 'staging':
wfi.sendLog('stagor',"all needs are fullfilled, setting staged")
wfo.status = 'staged'
session.commit()
else:
wfi.sendLog('stagor',"all needs are fullfilled, already")
print json.dumps( readys, indent=2 )
else:
wfi.sendLog('stagor',"missing requirements")
copies_needed,_ = wfi.getNCopies()
jump_ahead = False
re_transfer = False
## there is missing input let's do something more elaborated
for need in list(primaries):#+list(secondaries):
if endpoint_in_downtime[need] == endpoint_incompleted[need]:
#print need,"is going to an end point in downtime"
wfi.sendLog('stagor',"%s has only incomplete endpoint in downtime"%need)
re_transfer=True
if not se_allowed_key in available_cache[need]:
available_cache[need][se_allowed_key] = getDatasetBlocksFraction( url , need, sites=se_allowed )
if available_cache[need][se_allowed_key] >= copies_needed:
wfi.sendLog('stagor',"assuming it is OK to move on like this already for %s"%need)
jump_ahead = True
else:
wfi.sendLog('stagor',"Available %s times"% available_cache[need][se_allowed_key])
missing_and_downtime = list(set(endpoint_in_downtime[need]) & set(endpoint_incompleted[need]))
if missing_and_downtime:
wfi.sendLog('stagor',"%s is incomplete at %s which is in downtime, trying to move along"%(need, ','.join(missing_and_downtime)))
jump_ahead = True
else:
wfi.sendLog('stagor',"continue waiting for transfers for optimum production performance.")
## compute a time since staging to filter jump starting ?
# check whether the inputs is already in the stuck list ...
for need in list(primaries)+list(secondaries):
if need in already_stuck:
wfi.sendLog('stagor',"%s is stuck, so try to jump ahead"%need)
jump_ahead = True
if jump_ahead or re_transfer:
details_text = "checking on availability for %s to jump ahead"%wfo.name
details_text += '\n%s wants %s copies'%(wfo.name,copies_needed)
copies_needed = max(1,copies_needed-1)
details_text += '\nlowering by one unit to %s'%copies_needed
wfi.sendLog('stagor', details_text)
all_check = True
prim_where = set()
for need in list(primaries):
if not se_allowed_key in presence_cache[need]:
presence_cache[need][se_allowed_key] = getDatasetPresence( url, need , within_sites=se_allowed)
presence = presence_cache[need][se_allowed_key]
prim_where.update( presence.keys() )
available = available_cache[need][se_allowed_key]
this_check = (available >= copies_needed)
wfi.sendLog('stagor', "%s is available %s times (%s), at %s"%( need, available, this_check, se_allowed_key))
all_check &= this_check
if not all_check: break
for need in list(secondaries):
## I do not want to check on the secon
## this below does not function because the primary could be all available, and the secondary not complete at a certain site that does not matter at that point
this_check = all(done_by_input[need].values())
wfi.sendLog('stagor',"%s is this much transfered %s"%(need, json.dumps(done_by_input[need], indent=2)))
all_check&= this_check
#if not se_allowed_key in presence_cache[need]:
# presence_cache[need][se_allowed_key] = getDatasetPresence( url, need , within_sites=se_allowed)
## restrict to where the primary is
#presence = dict([(k,v) for (k,v) in presence_cache[need][se_allowed_key].items() if k in prim_where])
#this_check = all([there for (there,frac) in presence.values()])
#print need,"is present at all sites:",this_check
#all_check&= this_check
if all_check:
wfi.sendLog('stagor',"needs are sufficiently fullfilled, setting staged")
wfo.status = 'staged'
session.commit()
else:
print wfo.name,"has to wait a bit more"
wfi.sendLog('stagor',"needs to wait a bit more")
else:
wfi.sendLog('stagor',"not checking availability")
if re_transfer:
wfi.sendLog('stagor',"Sending back to considered because of endpoint in downtime")
if wfo.status == 'staging':
wfo.status = 'considered'
session.commit()
send_back_to_considered.add( wfo.name )
if send_back_to_considered:
#sendEmail("transfer to endpoint in downtime","sending back to considered the following workflows \n%s"%('\n'.join( send_back_to_considered)))
sendLog('stagor', "sending back to considered the following workflows \n%s"%('\n'.join( send_back_to_considered)), level='critical')
print "-"*10,"Checking on non-available datasets","-"*10
## now check on those that are not fully available
for dsname in available_cache.keys():
## squash the se_allowed_key key
available_cache[dsname] = min( available_cache[dsname].values() )
really_stuck_dataset = set()
for dsname,available in available_cache.items():
using_its = getWorkflowByInput(url, dsname)
#print using_its
using_wfos = []
for using_it in using_its:
wf = session.query(Workflow).filter(Workflow.name == using_it).first()
if wf:
using_wfos.append( wf )
if not len(done_by_input[dsname]):
print "For dataset",dsname,"there are no transfer report. That's an issue."
for wf in using_wfos:
if wf.status == 'staging':
if UC.get("stagor_sends_back"):
print "sending",wf.name,"back to considered"
wf.status = 'considered'
session.commit()
#sendEmail( "send back to considered","%s was send back and might be trouble"% wf.name)
sendLog('stagor', "%s was send back and might be trouble"% wf.name, level='critical')
else:
print "would send",wf.name,"back to considered"
#sendEmail( "subscription lagging behind","susbscriptions to get %s running are not appearing in phedex. I would have send it back to considered but that's not good."% wf.name)
sendLog('stagor', "susbscriptions to get %s running are not appearing in phedex. I would have send it back to considered but that's not good."% wf.name, level='critical')
continue
## not compatible with checking on secondary availability
#if all([wf.status != 'staging' for wf in using_wfos]):
# ## means despite all checks that input is not needed
# continue
if available < 1.:
print "incomplete",dsname
## there is a problem in the method below that it does not account for files stuck in T1*Buffer only
lost_blocks,lost_files = findLostBlocksFiles( url, dsname )
lost_block_names = [item['name'] for item in lost_blocks]
lost_file_names = [item['name'] for item in lost_files]
if lost_blocks:
#print json.dumps( lost , indent=2 )
## estimate for how much !
fraction_loss,_,n_missing = getDatasetBlockFraction(dsname, lost_block_names)
print "We have lost",len(lost_block_names),"blocks",lost_block_names,"for %f%%"%(100.*fraction_loss)
if fraction_loss > 0.05: ## 95% completion mark
#sendEmail('we have lost too many blocks','%s is missing %d blocks, for %d events, %f %% loss'%(dsname, len(lost_block_names), n_missing, fraction_loss))
sendLog('stagor', '%s is missing %d blocks, for %d events, %3.2f %% loss'%(dsname, len(lost_block_names), n_missing, 100*fraction_loss), level='critical')
## the workflow should be rejected !
for wf in using_wfos:
if wf.status == 'staging':
print wf.name,"is doomed. setting to trouble"
wf.status = 'trouble'
session.commit()
#sendEmail('doomed workflow','%s has too much loss on the input dataset %s. please check on stagor logs https://cmst2.web.cern.ch/cmst2/unified/logs/stagor/last.log'%(wf.name, dsname))
sendLog('stagor', '%s has too much loss on the input dataset %s. Missing %d blocks, for %d events, %3.2f %% loss'%(wf.name, dsname, len(lost_block_names), n_missing, 100*fraction_loss), level='critical')
else:
## probably enough to make a ggus and remove
if not dsname in known_lost_blocks:
#sendEmail('we have lost a few blocks', '%s is missing %d blocks, for %d events, %f %% loss\n\n%s'%(dsname, len(lost_block_names), n_missing, fraction_loss, '\n'.join( lost_block_names ) ))
sendLog('stagor', '%s is missing %d blocks, for %d events, %f %% loss\n\n%s'%(dsname, len(lost_block_names), n_missing, fraction_loss, '\n'.join( lost_block_names ) ), level='critical')
known_lost_blocks[dsname] = [i['name'] for i in lost_blocks]
really_stuck_dataset.add( dsname )
if lost_files:
fraction_loss,_,n_missing = getDatasetFileFraction(dsname, lost_file_names)
print "We have lost",len(lost_file_names),"files",lost_file_names,"for %f%%"%fraction_loss
if fraction_loss > 0.05:
#sendEmail('we have lost too many files','%s is missing %d files, for %d events, %f %% loss'%(dsname, len(lost_file_names),n_missing, fraction_loss))
sendLog('stagor', '%s is missing %d files, for %d events, %f %% loss'%(dsname, len(lost_file_names),n_missing, fraction_loss), level='critical')
for wf in using_wfos:
if wf.status == 'staging':
print wf.name,"is doomed. setting to trouble"
wf.status = 'trouble'
session.commit()
else:
## probably enough to make a ggus and remove
if not dsname in known_lost_files:
#sendEmail('we have lost a few files','%s is missing %d files, for %d events, %f %% loss\n\n%s'%(dsname, len(lost_file_names),n_missing, fraction_loss, '\n'.join(lost_file_names)))
sendLog('stagor', '%s is missing %d files, for %d events, %f %% loss\n\n%s'%(dsname, len(lost_file_names),n_missing, fraction_loss, '\n'.join(lost_file_names)), level='critical')
known_lost_files[dsname] = [i['name'] for i in lost_files]
## should the status be change to held-staging and pending on a ticket
missings = [pid for (pid,d) in done_by_input[dsname].items() if d==False]
print "\t",done_by_input[dsname]
print "\tneeds",len(done_by_input[dsname])
print "\tgot",done_by_input[dsname].values().count(True)
print "\tmissing",missings
missing_in_action[dsname].extend( missings )
rr= open('%s/lost_blocks_datasets.json'%monitor_dir,'w')
rr.write( json.dumps( known_lost_blocks, indent=2))
rr.close()
rr= open('%s/lost_files_datasets.json'%monitor_dir,'w')
rr.write( json.dumps( known_lost_files, indent=2))
rr.close()
open('%s/incomplete_transfers.json'%monitor_dir,'w').write( json.dumps(missing_in_action, indent=2) )
print "Stuck transfers and datasets"
print json.dumps( missing_in_action, indent=2 )
print "Going further and make a report of stuck transfers"
datasets_by_phid = defaultdict(set)
for dataset in missing_in_action:
for phid in missing_in_action[dataset]:
#print dataset,"stuck through",phid
datasets_by_phid[phid].add( dataset )
bad_destinations = defaultdict(set)
bad_sources = defaultdict(set)
report = ""
transfer_timeout = UC.get("transfer_timeout")
transfer_lowrate = UC.get("transfer_lowrate")
for phid,datasets in datasets_by_phid.items():
issues = checkTransferLag( url, phid , datasets=list(datasets) )
for dataset in issues:
for block in issues[dataset]:
for destination in issues[dataset][block]:
(block_size,destination_size,delay,rate,dones) = issues[dataset][block][destination]
## count x_Buffer and x_MSS as one source
redones=[]
for d in dones:
if d.endswith('Buffer') or d.endswith('Export'):
if d.replace('Buffer','MSS').replace('Export','MSS') in dones:
continue
else:
redones.append( d )
else:
redones.append( d )
dones = list(set( redones ))
#dones = filter(lambda s : (s.endswith('Buffer') and not s.replace('Buffer','MSS') in dones) or (not s.endswith('Buffer')) , dones)
if delay>transfer_timeout and rate<transfer_lowrate:
if len(dones)>1:
## its the destination that sucks
bad_destinations[destination].add( block )
else:
dum=[bad_sources[d].add( block ) for d in dones]
really_stuck_dataset.add( dataset )
print "add",dataset,"to really stuck"
report += "%s is not getting to %s, out of %s faster than %f [GB/s] since %f [d]\n"%(block,destination,", ".join(dones), rate, delay)
print "\n"*2
## create tickets right away ?
report+="\nbad sources "+",".join(bad_sources.keys())+"\n"
for site,blocks in bad_sources.items():
report+="\n\n%s:"%site+"\n\t".join(['']+list(blocks))
report+="\nbad destinations "+",".join(bad_destinations.keys())+"\n"
for site,blocks in bad_destinations.items():
report+="\n\n%s:"%site+"\n\t".join(['']+list(blocks))
print '\n'*2,"Datasets really stuck"
print '\n'.join( really_stuck_dataset )
print '\n'*2,"report written at https://cmst2.web.cern.ch/cmst2/unified/logs/incomplete_transfers.log"
print report
stuck_transfers = dict([(k,v) for (k,v) in missing_in_action.items() if k in really_stuck_dataset])
print '\n'*2,'Stuck dataset transfers'
print json.dumps(stuck_transfers , indent=2)
open('%s/stuck_transfers.json'%monitor_dir,'w').write( json.dumps(stuck_transfers , indent=2) )
open('%s/logs/incomplete_transfers.log'%monitor_dir,'w').write( report )
def completor(url, specific):
mlock = moduleLock(silent=True)
if mlock(): return
use_mcm = True
up = componentInfo(soft=['mcm','wtc','jira'])
if not up.check(): return
use_mcm = up.status['mcm']
if use_mcm:
mcm = McMClient(dev=False)
safe_mode = False
CI = campaignInfo()
SI = siteInfo()
UC = unifiedConfiguration()
JC = JIRAClient() if up.status.get('jira',False) else None
wfs = []
wfs.extend( session.query(Workflow).filter(Workflow.status == 'away').all() )
wfs.extend( session.query(Workflow).filter(Workflow.status.startswith('assistance')).all() )
## just take it in random order so that not always the same is seen
random.shuffle( wfs )
max_per_round = UC.get('max_per_round').get('completor',None)
if max_per_round and not specific: wfs = wfs[:max_per_round]
all_stuck = set()
## take into account what stagor was saying
for itry in range(5):
try:
all_stuck.update( json.loads( eosRead('%s/stuck_transfers.json'%monitor_pub_dir)))
break
except:
time.sleep(2)
for itry in range(5):
try:
## take into account the block that needed to be repositioned recently
all_stuck.update( [b.split('#')[0] for b in json.loads( eosRead('%s/missing_blocks.json'%monitor_dir)) ] )
break
except:
time.sleep(2)
## take into account all stuck block and dataset from transfer team
all_stuck.update( getAllStuckDataset())
good_fractions = {}
overdoing_fractions = {}
truncate_fractions = {}
timeout = {}
campaign_injection_delay = {}
for c in CI.campaigns:
if 'force-complete' in CI.campaigns[c]:
good_fractions[c] = CI.campaigns[c]['force-complete']
if 'truncate-complete' in CI.campaigns[c]:
truncate_fractions[c] = CI.campaigns[c]['truncate-complete']
if 'force-timeout' in CI.campaigns[c]:
timeout[c] = CI.campaigns[c]['force-timeout']
if 'injection-delay' in CI.campaigns[c]:
campaign_injection_delay[c] = CI.campaigns[c]['injection-delay']
if 'overdoing-complete' in CI.campaigns[c]:
overdoing_fractions[c] = CI.campaigns[c]['overdoing-complete']
long_lasting = {}
WI = wtcInfo()
overrides = WI.getForce()
if use_mcm:
## add all workflow that mcm wants to get force completed
mcm_force = mcm.get('/restapi/requests/forcecomplete')
## assuming this will be a list of actual prepids
overrides['mcm'] = mcm_force
print "can force complete on"
print json.dumps( good_fractions ,indent=2)
print "can truncate complete on"
print json.dumps( truncate_fractions ,indent=2)
print "can overide on"
print json.dumps( overrides, indent=2)
max_force = UC.get("max_force_complete")
max_priority = UC.get("max_tail_priority")
injection_delay_threshold = UC.get("injection_delay_threshold")
injection_delay_priority = UC.get("injection_delay_priority")
delay_priority_increase = UC.get("delay_priority_increase")
default_fraction_overdoing = UC.get('default_fraction_overdoing')
set_force_complete = set()
# priority and time above which to fire a JIRA
jira_priority_and_delays = { 110000 : 21,
90000 : 28,
# 80000 : 60,
#0 : 90
}
for wfo in wfs:
if specific and not specific in wfo.name: continue
print "looking at",wfo.name
## get all of the same
wfi = workflowInfo(url, wfo.name)
pids = wfi.getPrepIDs()
skip=False
campaigns = wfi.getCampaigns()
#if not any([c in good_fractions.keys() for c in campaigns]): skip=True
#if not any([c in truncate_fractions.keys() for c in campaigns]): skip=True
for user,spec in overrides.items():
if not spec: continue
spec = filter(None, spec)
if not wfi.request['RequestStatus'] in ['force-complete', 'completed']:
if any(s in wfo.name for s in spec) or (wfo.name in spec) or any(pid in spec for pid in pids) or any(s in pids for s in spec):
wfi = workflowInfo(url, wfo.name)
forceComplete(url , wfi )
skip=True
wfi.notifyRequestor("The workflow %s was force completed by request of %s"%(wfo.name,user), do_batch=False)
wfi.sendLog('completor','%s is asking for %s to be force complete'%(user,wfo.name))
break
if wfo.status.startswith('assistance'): skip = True
if skip:
continue
priority = wfi.request['RequestPriority']
if not 'Campaign' in wfi.request: continue
if not wfi.request['RequestStatus'] in ['acquired','running-open','running-closed']: continue
## until we can map the output to task ...
output_per_task = wfi.getOutputPerTask() ## can use that one, and follow mapping
good_fraction_per_out = {}
good_fraction_nodelay_per_out = {}
truncate_fraction_per_out = {}
#allowed_delay_per_out = {}
for task,outs in output_per_task.items():
task_campaign = wfi.getCampaignPerTask( task )
for out in outs:
good_fraction_per_out[out] = good_fractions.get(task_campaign,1000.)
good_fraction_nodelay_per_out[out] = overdoing_fractions.get(task_campaign,default_fraction_overdoing)
truncate_fraction_per_out[out] = truncate_fractions.get(task_campaign,1000.)
#allowed_delay_per_out[out] = timeout.get(task_campaign, 14)
#print "force at", json.dumps( good_fraction_per_out, indent=2)
#print "truncate at",json.dumps( truncate_fraction_per_out, indent=2)
now = time.mktime(time.gmtime()) / (60*60*24.)
priority_log = filter(lambda change: change['Priority'] == priority,wfi.request.get('PriorityTransition',[]))
if not priority_log:
print "\tHas no priority log"
priority_delay = 0
else:
then = max([change['UpdateTime'] for change in priority_log]) / (60.*60.*24.)
priority_delay = now - then ## in days
print "priority was set to",priority,priority_delay,"[days] ago"
running_log = filter(lambda change : change["Status"] in ["running-open","running-closed"],wfi.request['RequestTransition'])
if not running_log:
print "\tHas no running log"
delay = 0
else:
then = max([change['UpdateTime'] for change in running_log]) / (60.*60.*24.)
delay = now - then ## in days
#further check on delays
cpuh = wfi.getComputingTime(unit='d')
wfi.sendLog('completor',"Requires %.1f [CPUd], running since %.1f [day], last priority change to %d done %.1f [day] ago"%( cpuh, delay, priority, priority_delay))
if priority_delay!=0 and priority_delay < delay:
## regardless when it started running, set the delay to when priority was changed last
delay = priority_delay
## this is supposed to be the very initial request date, inherited from clones
injection_delay = None
original = wfi
if 'OriginalRequestName' in original.request:
## go up the clone chain
original = workflowInfo(url, original.request['OriginalRequestName'])
injected_log = filter(lambda change : change["Status"] in ["assignment-approved"],original.request['RequestTransition'])
if injected_log:
injected_on = injected_log[-1]['UpdateTime'] / (60.*60.*24.)
injection_delay = now - injected_on
delay_for_priority_increase = injection_delay
#delay_for_priority_increase = delay
(w,d) = divmod(delay, 7 )
print "\t"*int(w)+"Running since",delay,"[days] priority=",priority
pop_a_jira = False
ping_on_jira = 7 *(24*60*60) # 7 days
for jp,jd in jira_priority_and_delays.items():
if priority >= jp and delay >= jd: pop_a_jira = True
if pop_a_jira and JC:
j,reopened,just_created = JC.create_or_last( prepid = wfi.request['PrepID'],
priority = wfi.request['RequestPriority'],
label = 'Late',
reopen = True)
last_time = JC.last_time( j )
since_last_ping = time.mktime(time.gmtime()) - last_time
if since_last_ping > ping_on_jira or just_created:
j_comment = "Running since %.1f [days] at priority %d"%( delay, priority)
JC.comment(j.key, j_comment)
if delay_for_priority_increase!=None and delay_for_priority_increase > injection_delay_threshold and priority >= injection_delay_priority:
quantized = 5000 ## quantize priority
tail_cutting_priority = wfi.request['InitialPriority']+ int((delay_priority_increase * (delay_for_priority_increase - injection_delay_threshold) / 7) / quantized) * quantized
tail_cutting_priority += 101 ## to signal it is from this mechanism
tail_cutting_priority = min(400000, tail_cutting_priority) ## never go above 400k priority
tail_cutting_priority = max(tail_cutting_priority, priority) ## never go below the current value
if priority < tail_cutting_priority:
if max_priority:
sendLog('completor',"%s Injected since %s [days] priority=%s, increasing to %s"%(wfo.name,delay_for_priority_increase,priority, tail_cutting_priority), level='critical')
wfi.sendLog('completor','bumping priority to %d for being injected since %s'%( tail_cutting_priority, delay_for_priority_increase))
reqMgrClient.changePriorityWorkflow(url, wfo.name, tail_cutting_priority)
max_priority-=1
else:
sendLog('completor',"%s Injected since %s [days] priority=%s, would like to increase to %s"%(wfo.name,delay_for_priority_increase,priority, tail_cutting_priority), level='critical')
wfi.sendLog('completor','would like to bump priority to %d for being injected since %s'%( tail_cutting_priority, delay_for_priority_increase))
print "Could be changing the priority to higher value, but too many already were done"
_,prim,_,_ = wfi.getIO()
is_stuck = all_stuck & prim
if is_stuck: wfi.sendLog('completor','%s is stuck'%','.join(is_stuck))
monitor_delay = 7
allowed_delay = max([timeout.get(c,14) for c in campaigns])
monitor_delay = min(monitor_delay, allowed_delay)
### just skip if too early, just for the sake of not computing the completion fraction just now.
# maybe this is fast enough that we can do it for all
if delay <= monitor_delay:
print "not enough time has passed yet"
continue
long_lasting[wfo.name] = { "delay" : delay,
"injection_delay" : injection_delay }
percent_completions = wfi.getCompletionFraction(caller='completor')
if not percent_completions:
sendLog('completor','%s has no output at all'% wfo.name, level='critical')
continue
is_over_allowed_delay = (all([percent_completions[out] >= good_fraction_per_out.get(out,1000.) for out in percent_completions]) and delay >= allowed_delay)
is_over_truncation_delay = (is_stuck and (all([percent_completions[out] >= truncate_fraction_per_out.get(out,1000.) for out in percent_completions])) and delay >= allowed_delay)
is_over_completion = (all([percent_completions[out] >= good_fraction_nodelay_per_out.get(out,1000.) for out in percent_completions]))
if is_over_completion:
wfi.sendLog('completor', "all is over completed %s\n %s"%( json.dumps( good_fraction_nodelay_per_out, indent=2 ),
json.dumps( percent_completions, indent=2 )
))
elif is_over_allowed_delay:
wfi.sendLog('completor', "all is above %s \n%s"%( json.dumps(good_fraction_per_out, indent=2 ),
json.dumps( percent_completions, indent=2 )
))
elif is_over_truncation_delay:
wfi.sendLog('completor', "all is above %s truncation level, and the input is stuck\n%s"%( json.dumps(truncate_fraction_per_out, indent=2 ),
json.dumps( percent_completions, indent=2 ) ) )
else:
long_lasting[wfo.name].update({
'completion': sum(percent_completions.values()) / len(percent_completions),
'completions' : percent_completions
})
## do something about the agents this workflow is in
long_lasting[wfo.name]['agents'] = wfi.getAgents()
wfi.sendLog('completor', "%s not over bound \ncomplete at %s \n truncate at %s \nRunning %s"%(json.dumps( percent_completions, indent=2),
json.dumps(good_fraction_per_out, indent=2),
json.dumps( truncate_fraction_per_out, indent=2),
json.dumps( long_lasting[wfo.name]['agents'], indent=2) ))
continue
#for output in percent_completions:
# completions[output]['injected'] = then
ran_at = wfi.request['SiteWhitelist']
wfi.sendLog('completor',"Required %s, time spend %s"%( cpuh, delay))
##### WILL FORCE COMPLETE BELOW
# only really force complete after n days
## find ACDCs that might be running
if max_force>0:
print "going for force-complete of",wfo.name
if not safe_mode:
forceComplete(url, wfi )
set_force_complete.add( wfo.name )
wfi.sendLog('completor','going for force completing')
wfi.notifyRequestor("The workflow %s was force completed for running too long"% wfo.name)
max_force -=1
else:
sendEmail('completor', 'The workflow %s is ready for force complete, but completor is in safe mode'%wfo.name)
else:
wfi.sendLog('completor',"too many completion this round, cannot force complete")
if set_force_complete:
sendLog('completor','The followings were set force-complete \n%s'%('\n'.join(set_force_complete)))
#open('%s/completions.json'%monitor_dir,'w').write( json.dumps( completions , indent=2))
text="These have been running for long"
#open('%s/longlasting.json'%monitor_dir,'w').write( json.dumps( long_lasting, indent=2 ))
eosFile('%s/longlasting.json'%monitor_dir,'w').write( json.dumps( long_lasting, indent=2 )).close()
for wf,info in sorted(long_lasting.items(), key=lambda tp:tp[1]['delay'], reverse=True):
delay = info['delay']
text += "\n %s : %s days"% (wf, delay)
if 'completion' in info:
text += " %d%%"%( info['completion']*100 )
print text
def completor(url, specific):
mlock = moduleLock(silent=True)
if mlock(): return
use_mcm = True
up = componentInfo(soft=['mcm', 'wtc', 'jira'])
if not up.check(): return
use_mcm = up.status['mcm']
if use_mcm:
mcm = McMClient(dev=False)
safe_mode = False
CI = campaignInfo()
SI = siteInfo()
UC = unifiedConfiguration()
JC = JIRAClient() if up.status.get('jira', False) else None
wfs = []
wfs.extend(session.query(Workflow).filter(Workflow.status == 'away').all())
wfs.extend(
session.query(Workflow).filter(
Workflow.status.startswith('assistance')).all())
## just take it in random order so that not always the same is seen
random.shuffle(wfs)
max_per_round = UC.get('max_per_round').get('completor', None)
if max_per_round and not specific: wfs = wfs[:max_per_round]
all_stuck = set()
## take into account what stagor was saying
for itry in range(5):
try:
all_stuck.update(
json.loads(eosRead('%s/stuck_transfers.json' %
monitor_pub_dir)))
break
except:
time.sleep(2)
for itry in range(5):
try:
## take into account the block that needed to be repositioned recently
all_stuck.update([
b.split('#')[0] for b in json.loads(
eosRead('%s/missing_blocks.json' % monitor_dir))
])
break
except:
time.sleep(2)
## take into account all stuck block and dataset from transfer team
all_stuck.update(getAllStuckDataset())
good_fractions = {}
overdoing_fractions = {}
truncate_fractions = {}
timeout = {}
campaign_injection_delay = {}
for c in CI.campaigns:
if 'force-complete' in CI.campaigns[c]:
good_fractions[c] = CI.campaigns[c]['force-complete']
if 'truncate-complete' in CI.campaigns[c]:
truncate_fractions[c] = CI.campaigns[c]['truncate-complete']
if 'force-timeout' in CI.campaigns[c]:
timeout[c] = CI.campaigns[c]['force-timeout']
if 'injection-delay' in CI.campaigns[c]:
campaign_injection_delay[c] = CI.campaigns[c]['injection-delay']
if 'overdoing-complete' in CI.campaigns[c]:
overdoing_fractions[c] = CI.campaigns[c]['overdoing-complete']
long_lasting = {}
WI = wtcInfo()
overrides = WI.getForce()
if use_mcm:
## add all workflow that mcm wants to get force completed
mcm_force = mcm.get('/restapi/requests/forcecomplete')
## assuming this will be a list of actual prepids
overrides['mcm'] = mcm_force
print "can force complete on"
print json.dumps(good_fractions, indent=2)
print "can truncate complete on"
print json.dumps(truncate_fractions, indent=2)
print "can overide on"
print json.dumps(overrides, indent=2)
max_force = UC.get("max_force_complete")
max_priority = UC.get("max_tail_priority")
injection_delay_threshold = UC.get("injection_delay_threshold")
injection_delay_priority = UC.get("injection_delay_priority")
delay_priority_increase = UC.get("delay_priority_increase")
default_fraction_overdoing = UC.get('default_fraction_overdoing')
set_force_complete = set()
# priority and time above which to fire a JIRA
jira_priority_and_delays = {
110000: 21,
90000: 28,
# 80000 : 60,
#0 : 90
}
for wfo in wfs:
if specific and not specific in wfo.name: continue
print "looking at", wfo.name
## get all of the same
wfi = workflowInfo(url, wfo.name)
pids = wfi.getPrepIDs()
skip = False
campaigns = wfi.getCampaigns()
#if not any([c in good_fractions.keys() for c in campaigns]): skip=True
#if not any([c in truncate_fractions.keys() for c in campaigns]): skip=True
for user, spec in overrides.items():
if not spec: continue
spec = filter(None, spec)
if not wfi.request['RequestStatus'] in [
'force-complete', 'completed'
]:
if any(s in wfo.name
for s in spec) or (wfo.name in spec) or any(
pid in spec for pid in pids) or any(s in pids
for s in spec):
wfi = workflowInfo(url, wfo.name)
forceComplete(url, wfi)
skip = True
wfi.notifyRequestor(
"The workflow %s was force completed by request of %s"
% (wfo.name, user),
do_batch=False)
wfi.sendLog(
'completor',
'%s is asking for %s to be force complete' %
(user, wfo.name))
break
if wfo.status.startswith('assistance'): skip = True
if skip:
continue
priority = wfi.request['RequestPriority']
if not 'Campaign' in wfi.request: continue
if not wfi.request['RequestStatus'] in [
'acquired', 'running-open', 'running-closed'
]:
continue
## until we can map the output to task ...
output_per_task = wfi.getOutputPerTask(
) ## can use that one, and follow mapping
good_fraction_per_out = {}
good_fraction_nodelay_per_out = {}
truncate_fraction_per_out = {}
#allowed_delay_per_out = {}
for task, outs in output_per_task.items():
task_campaign = wfi.getCampaignPerTask(task)
for out in outs:
good_fraction_per_out[out] = good_fractions.get(
task_campaign, 1000.)
good_fraction_nodelay_per_out[out] = overdoing_fractions.get(
task_campaign, default_fraction_overdoing)
truncate_fraction_per_out[out] = truncate_fractions.get(
task_campaign, 1000.)
#allowed_delay_per_out[out] = timeout.get(task_campaign, 14)
#print "force at", json.dumps( good_fraction_per_out, indent=2)
#print "truncate at",json.dumps( truncate_fraction_per_out, indent=2)
now = time.mktime(time.gmtime()) / (60 * 60 * 24.)
priority_log = filter(lambda change: change['Priority'] == priority,
wfi.request.get('PriorityTransition', []))
if not priority_log:
print "\tHas no priority log"
priority_delay = 0
else:
then = max([change['UpdateTime']
for change in priority_log]) / (60. * 60. * 24.)
priority_delay = now - then ## in days
print "priority was set to", priority, priority_delay, "[days] ago"
running_log = filter(
lambda change: change["Status"
] in ["running-open", "running-closed"],
wfi.request['RequestTransition'])
if not running_log:
print "\tHas no running log"
delay = 0
else:
then = max([change['UpdateTime']
for change in running_log]) / (60. * 60. * 24.)
delay = now - then ## in days
#further check on delays
cpuh = wfi.getComputingTime(unit='d')
wfi.sendLog(
'completor',
"Requires %.1f [CPUd], running since %.1f [day], last priority change to %d done %.1f [day] ago"
% (cpuh, delay, priority, priority_delay))
if priority_delay != 0 and priority_delay < delay:
## regardless when it started running, set the delay to when priority was changed last
delay = priority_delay
## this is supposed to be the very initial request date, inherited from clones
injection_delay = None
original = wfi
if 'OriginalRequestName' in original.request:
## go up the clone chain
original = workflowInfo(url,
original.request['OriginalRequestName'])
injected_log = filter(
lambda change: change["Status"] in ["assignment-approved"],
original.request['RequestTransition'])
if injected_log:
injected_on = injected_log[-1]['UpdateTime'] / (60. * 60. * 24.)
injection_delay = now - injected_on
delay_for_priority_increase = injection_delay
#delay_for_priority_increase = delay
(w, d) = divmod(delay, 7)
print "\t" * int(
w) + "Running since", delay, "[days] priority=", priority
pop_a_jira = False
ping_on_jira = 7 * (24 * 60 * 60) # 7 days
for jp, jd in jira_priority_and_delays.items():
if priority >= jp and delay >= jd: pop_a_jira = True
if pop_a_jira and JC:
j, reopened, just_created = JC.create_or_last(
prepid=wfi.request['PrepID'],
priority=wfi.request['RequestPriority'],
label='Late',
reopen=True)
last_time = JC.last_time(j)
since_last_ping = time.mktime(time.gmtime()) - last_time
if since_last_ping > ping_on_jira or just_created:
j_comment = "Running since %.1f [days] at priority %d" % (
delay, priority)
JC.comment(j.key, j_comment)
if delay_for_priority_increase != None and delay_for_priority_increase > injection_delay_threshold and priority >= injection_delay_priority:
quantized = 5000 ## quantize priority
tail_cutting_priority = wfi.request['InitialPriority'] + int(
(delay_priority_increase *
(delay_for_priority_increase - injection_delay_threshold) / 7)
/ quantized) * quantized
tail_cutting_priority += 101 ## to signal it is from this mechanism
tail_cutting_priority = min(
400000, tail_cutting_priority) ## never go above 400k priority
tail_cutting_priority = max(
tail_cutting_priority,
priority) ## never go below the current value
if priority < tail_cutting_priority:
if max_priority:
sendLog(
'completor',
"%s Injected since %s [days] priority=%s, increasing to %s"
% (wfo.name, delay_for_priority_increase, priority,
tail_cutting_priority),
level='critical')
wfi.sendLog(
'completor',
'bumping priority to %d for being injected since %s' %
(tail_cutting_priority, delay_for_priority_increase))
reqMgrClient.changePriorityWorkflow(
url, wfo.name, tail_cutting_priority)
max_priority -= 1
else:
sendLog(
'completor',
"%s Injected since %s [days] priority=%s, would like to increase to %s"
% (wfo.name, delay_for_priority_increase, priority,
tail_cutting_priority),
level='critical')
wfi.sendLog(
'completor',
'would like to bump priority to %d for being injected since %s'
% (tail_cutting_priority, delay_for_priority_increase))
print "Could be changing the priority to higher value, but too many already were done"
_, prim, _, _ = wfi.getIO()
is_stuck = all_stuck & prim
if is_stuck:
wfi.sendLog('completor', '%s is stuck' % ','.join(is_stuck))
monitor_delay = 7
allowed_delay = max([timeout.get(c, 14) for c in campaigns])
monitor_delay = min(monitor_delay, allowed_delay)
### just skip if too early, just for the sake of not computing the completion fraction just now.
# maybe this is fast enough that we can do it for all
if delay <= monitor_delay:
print "not enough time has passed yet"
continue
long_lasting[wfo.name] = {
"delay": delay,
"injection_delay": injection_delay
}
percent_completions = wfi.getCompletionFraction(caller='completor')
if not percent_completions:
sendLog('completor',
'%s has no output at all' % wfo.name,
level='critical')
continue
is_over_allowed_delay = (all([
percent_completions[out] >= good_fraction_per_out.get(out, 1000.)
for out in percent_completions
]) and delay >= allowed_delay)
is_over_truncation_delay = (is_stuck and (all([
percent_completions[out] >= truncate_fraction_per_out.get(
out, 1000.) for out in percent_completions
])) and delay >= allowed_delay)
is_over_completion = (all([
percent_completions[out] >= good_fraction_nodelay_per_out.get(
out, 1000.) for out in percent_completions
]))
if is_over_completion:
wfi.sendLog(
'completor', "all is over completed %s\n %s" %
(json.dumps(good_fraction_nodelay_per_out, indent=2),
json.dumps(percent_completions, indent=2)))
elif is_over_allowed_delay:
wfi.sendLog(
'completor', "all is above %s \n%s" %
(json.dumps(good_fraction_per_out, indent=2),
json.dumps(percent_completions, indent=2)))
elif is_over_truncation_delay:
wfi.sendLog(
'completor',
"all is above %s truncation level, and the input is stuck\n%s"
% (json.dumps(truncate_fraction_per_out, indent=2),
json.dumps(percent_completions, indent=2)))
else:
long_lasting[wfo.name].update({
'completion':
sum(percent_completions.values()) / len(percent_completions),
'completions':
percent_completions
})
## do something about the agents this workflow is in
long_lasting[wfo.name]['agents'] = wfi.getAgents()
wfi.sendLog(
'completor',
"%s not over bound \ncomplete at %s \n truncate at %s \nRunning %s"
% (json.dumps(percent_completions, indent=2),
json.dumps(good_fraction_per_out, indent=2),
json.dumps(truncate_fraction_per_out, indent=2),
json.dumps(long_lasting[wfo.name]['agents'], indent=2)))
continue
#for output in percent_completions:
# completions[output]['injected'] = then
ran_at = wfi.request['SiteWhitelist']
wfi.sendLog('completor', "Required %s, time spend %s" % (cpuh, delay))
##### WILL FORCE COMPLETE BELOW
# only really force complete after n days
## find ACDCs that might be running
if max_force > 0:
print "going for force-complete of", wfo.name
if not safe_mode:
forceComplete(url, wfi)
set_force_complete.add(wfo.name)
wfi.sendLog('completor', 'going for force completing')
wfi.notifyRequestor(
"The workflow %s was force completed for running too long"
% wfo.name)
max_force -= 1
else:
sendEmail(
'completor',
'The workflow %s is ready for force complete, but completor is in safe mode'
% wfo.name)
else:
wfi.sendLog(
'completor',
"too many completion this round, cannot force complete")
if set_force_complete:
sendLog(
'completor', 'The followings were set force-complete \n%s' %
('\n'.join(set_force_complete)))
#open('%s/completions.json'%monitor_dir,'w').write( json.dumps( completions , indent=2))
text = "These have been running for long"
#open('%s/longlasting.json'%monitor_dir,'w').write( json.dumps( long_lasting, indent=2 ))
eosFile('%s/longlasting.json' % monitor_dir,
'w').write(json.dumps(long_lasting, indent=2)).close()
for wf, info in sorted(long_lasting.items(),
key=lambda tp: tp[1]['delay'],
reverse=True):
delay = info['delay']
text += "\n %s : %s days" % (wf, delay)
if 'completion' in info:
text += " %d%%" % (info['completion'] * 100)
print text
def completor(url, specific):
use_mcm = True
up = componentInfo(mcm=use_mcm, soft=['mcm'])
if not up.check(): return
use_mcm = up.status['mcm']
if use_mcm:
mcm = McMClient(dev=False)
CI = campaignInfo()
SI = siteInfo()
UC = unifiedConfiguration()
wfs = []
wfs.extend( session.query(Workflow).filter(Workflow.status == 'away').all() )
wfs.extend( session.query(Workflow).filter(Workflow.status.startswith('assistance')).all() )
## just take it in random order so that not always the same is seen
random.shuffle( wfs )
max_per_round = UC.get('max_per_round').get('completor',None)
if max_per_round and not specific: wfs = wfs[:max_per_round]
all_stuck = set()
## take into account what stagor was saying
all_stuck.update( json.loads( open('%s/stuck_transfers.json'%monitor_dir).read() ))
## take into account the block that needed to be repositioned recently
all_stuck.update( [b.split('#')[0] for b in json.loads( open('%s/missing_blocks.json'%monitor_dir).read()) ] )
## take into account all stuck block and dataset from transfer team
all_stuck.update( getAllStuckDataset())
## by workflow a list of fraction / timestamps
completions = json.loads( open('%s/completions.json'%monitor_dir).read())
good_fractions = {}
truncate_fractions = {}
timeout = {}
for c in CI.campaigns:
if 'force-complete' in CI.campaigns[c]:
good_fractions[c] = CI.campaigns[c]['force-complete']
if 'truncate-complete' in CI.campaigns[c]:
truncate_fractions[c] = CI.campaigns[c]['truncate-complete']
if 'force-timeout' in CI.campaigns[c]:
timeout[c] = CI.campaigns[c]['force-timeout']
long_lasting = {}
overrides = getForceCompletes()
if use_mcm:
## add all workflow that mcm wants to get force completed
mcm_force = mcm.get('/restapi/requests/forcecomplete')
## assuming this will be a list of actual prepids
overrides['mcm'] = mcm_force
print "can force complete on"
print json.dumps( good_fractions ,indent=2)
print "can truncate complete on"
print json.dumps( truncate_fractions ,indent=2)
print "can overide on"
print json.dumps( overrides, indent=2)
max_force = UC.get("max_force_complete")
#wfs_no_location_in_GQ = set()
#block_locations = defaultdict(lambda : defaultdict(list))
#wfs_no_location_in_GQ = defaultdict(list)
set_force_complete = set()
for wfo in wfs:
if specific and not specific in wfo.name: continue
print "looking at",wfo.name
## get all of the same
wfi = workflowInfo(url, wfo.name)
pids = wfi.getPrepIDs()
skip=False
if not any([c in wfo.name for c in good_fractions]) and not any([c in wfo.name for c in truncate_fractions]): skip=True
for user,spec in overrides.items():
if wfi.request['RequestStatus']!='force-complete':
if any(s in wfo.name for s in spec) or (wfo.name in spec) or any(pid in spec for pid in pids) or any(s in pids for s in spec):
wfi = workflowInfo(url, wfo.name)
forceComplete(url , wfi )
skip=True
wfi.notifyRequestor("The workflow %s was force completed by request of %s"%(wfo.name,user), do_batch=False)
wfi.sendLog('completor','%s is asking for %s to be force complete'%(user,wfo.name))
break
if wfo.status.startswith('assistance'): skip = True
if skip:
continue
priority = wfi.request['RequestPriority']
if not 'Campaign' in wfi.request: continue
if not wfi.request['RequestStatus'] in ['acquired','running-open','running-closed']: continue
if wfi.request['RequestStatus'] in ['running-open','running-closed'] and (random.random() < 0.01):
try:
parse_one(url, wfo.name )
except Exception as e:
print "failed error parsing"
print str(e)
c = wfi.request['Campaign']
#if not c in good_fractions: continue
good_fraction = good_fractions.get(c,1000.)
truncate_fraction = truncate_fractions.get(c,1000.)
print "force at",good_fraction,"truncate at",truncate_fraction
ignore_fraction = 2.
lumi_expected = None
event_expected = None
if not 'TotalInputEvents' in wfi.request:
if 'RequestNumEvents' in wfi.request:
event_expected = wfi.request['RequestNumEvents']
else:
print "truncated, cannot do anything"
continue
else:
lumi_expected = wfi.request['TotalInputLumis']
event_expected = wfi.request['TotalInputEvents']
now = time.mktime(time.gmtime()) / (60*60*24.)
running_log = filter(lambda change : change["Status"] in ["running-open","running-closed"],wfi.request['RequestTransition'])
if not running_log:
print "\tHas no running log"
# cannot figure out when the thing started running
continue
then = running_log[-1]['UpdateTime'] / (60.*60.*24.)
delay = now - then ## in days
## this is supposed to be the very initial request date, inherited from clones
try:
injected_on =time.mktime(time.strptime("-".join(map(lambda n : "%02d"%n, wfi.request['RequestDate'])), "%Y-%m-%d-%H-%M-%S")) / (60.*60.*24.)
injection_delay = now - injected_on
except:
injection_delay = None
(w,d) = divmod(delay, 7 )
print "\t"*int(w)+"Running since",delay,"[days] priority=",priority
if injection_delay!=None and injection_delay > 50 and False:
tail_cutting_priority = 200000
## bump up to 200k
print "Injected since",injection_delay,"[days] priority=",priority
if wfi.request['RequestPriority'] != tail_cutting_priority:
wfi.sendLog('completor','bumping priority to %d for being injected since'% tail_cutting_priority)
reqMgrClient.changePriorityWorkflow(url, wfo.name, tail_cutting_priority)
_,prim,_,_ = wfi.getIO()
is_stuck = all_stuck & prim
if is_stuck: wfi.sendLog('completor','%s is stuck'%','.join(is_stuck))
monitor_delay = 7
allowed_delay = 14
if c in timeout:
allowed_delay = timeout[c]
monitor_delay = min(monitor_delay, allowed_delay)
### just skip if too early
if delay <= monitor_delay: continue
long_lasting[wfo.name] = { "delay" : delay,
"injection_delay" : injection_delay }
percent_completions = {}
for output in wfi.request['OutputDatasets']:
if "/DQM" in output: continue ## that does not count
if not output in completions: completions[output] = { 'injected' : None, 'checkpoints' : [], 'workflow' : wfo.name}
## get completion fraction
event_count,lumi_count = getDatasetEventsAndLumis(dataset=output)
lumi_completion=0.
event_completion=0.
if lumi_expected:
lumi_completion = lumi_count / float( lumi_expected )
if event_expected:
event_completion = event_count / float( event_expected )
#take the less optimistic
## taking the max or not : to force-complete on over-pro
percent_completions[output] = min( lumi_completion, event_completion )
completions[output]['checkpoints'].append( (now, event_completion ) )
if all([percent_completions[out] >= good_fraction for out in percent_completions]):
wfi.sendLog('completor', "all is above %s \n%s"%( good_fraction,
json.dumps( percent_completions, indent=2 )
))
elif is_stuck and (all([percent_completions[out] >= truncate_fraction for out in percent_completions])):
wfi.sendLog('completor', "all is above %s truncation level, and the input is stuck\n%s"%( truncate_fraction,
json.dumps( percent_completions, indent=2 ) ) )
sendEmail('workflow for truncation','check on %s'%wfo.name)
##continue ## because you do not want to get this one right now
else:
long_lasting[wfo.name].update({
'completion': sum(percent_completions.values()) / len(percent_completions),
'completions' : percent_completions
})
## do something about the agents this workflow is in
long_lasting[wfo.name]['agents'] = wfi.getAgents()
wfi.sendLog('completor', "%s not over bound %.3f (complete) %.3f truncate \n%s"%(percent_completions.values(),
good_fraction, truncate_fraction,
json.dumps( long_lasting[wfo.name]['agents'], indent=2) ))
continue
if all([percent_completions[out] >= ignore_fraction for out in percent_completions]):
print "all is done, just wait a bit"
continue
for output in percent_completions:
completions[output]['injected'] = then
#further check on delays
cpuh = wfi.getComputingTime(unit='d')
ran_at = wfi.request['SiteWhitelist']
wfi.sendLog('completor',"Required %s, time spend %s"%( cpuh, delay))
##### WILL FORCE COMPLETE BELOW
# only really force complete after n days
if delay <= allowed_delay: continue
## find ACDCs that might be running
if max_force>0:
forceComplete(url, wfi )
set_force_complete.add( wfo.name )
print "going for force-complete of",wfo.name
wfi.sendLog('completor','going for force completing')
wfi.notifyRequestor("The workflow %s was force completed for running too long"% wfo.name)
max_force -=1
else:
wfi.sendLog('completor',"too many completion this round, cannot force complete")
## do it once only for testing
#break
if delay >= 40:
print wfo.name,"has been running for",dealy,"days"
## bumping up the priority?
if set_force_complete:
sendLog('completor','The followings were set force-complete \n%s'%('\n'.join(set_force_complete)))
open('%s/completions.json'%monitor_dir,'w').write( json.dumps( completions , indent=2))
text="These have been running for long"
open('%s/longlasting.json'%monitor_dir,'w').write( json.dumps( long_lasting, indent=2 ))
for wf,info in sorted(long_lasting.items(), key=lambda tp:tp[1]['delay'], reverse=True):
delay = info['delay']
text += "\n %s : %s days"% (wf, delay)
if 'completion' in info:
text += " %d%%"%( info['completion']*100 )
print text
def stagor(url,specific =None, options=None):
if not componentInfo().check(): return
SI = siteInfo()
CI = campaignInfo()
UC = unifiedConfiguration()
TS = transferStatuses()
cached_transfer_statuses = TS.content()
transfer_statuses = {}
done_by_wf_id = {}
done_by_input = {}
completion_by_input = {}
good_enough = 100.0
lost_blocks = json.loads(eosRead('%s/lost_blocks_datasets.json'%monitor_dir))
lost_files = json.loads(eosRead('%s/lost_files_datasets.json'%monitor_dir))
known_lost_blocks = {}
known_lost_files = {}
for dataset in set(lost_blocks.keys()+lost_files.keys()):
b,f = findLostBlocksFiles(url, dataset)
if dataset in lost_blocks and not b:
print dataset,"has no really lost blocks"
else:
known_lost_blocks[dataset] = [i['name'] for i in b]
if dataset in lost_files and not f:
print dataset,"has no really lost files"
else:
known_lost_files[dataset] = [i['name'] for i in f]
def time_point(label="",sub_lap=False):
now = time.mktime(time.gmtime())
nows = time.asctime(time.gmtime())
print "Time check (%s) point at : %s"%(label, nows)
print "Since start: %s [s]"% ( now - time_point.start)
if sub_lap:
print "Sub Lap : %s [s]"% ( now - time_point.sub_lap )
time_point.sub_lap = now
else:
print "Lap : %s [s]"% ( now - time_point.lap )
time_point.lap = now
time_point.sub_lap = now
time_point.sub_lap = time_point.lap = time_point.start = time.mktime(time.gmtime())
time_point("Check cached transfer")
## collect all datasets that are needed for wf in staging, correcting the status of those that are not really in staging
wfois = []
needs = defaultdict(list)
needs_by_priority = defaultdict(list)
for wfo in session.query(Workflow).filter(Workflow.status == 'staging').all():
wfi = workflowInfo(url, wfo.name)
if wfi.request['RequestStatus'] in ['running-open','running-closed','completed','assigned','acquired']:
wfi.sendLog('stagor', "is in status %s"%wfi.request['RequestStatus'])
wfo.status='away'
session.commit()
continue
if not wfi.request['RequestStatus'] in ['assignment-approved']:
## should be setting 'away' too
## that usually happens for relvals
if wfi.request['RequestStatus'] in ['rejected','aborted','aborted-completed','aborted-archived','rejected-archived'] and wfi.isRelval():
wfo.status='forget'
session.commit()
continue
else:
print wfo.name,"is",wfi.request['RequestStatus']
#sendEmail("wrong status in staging. debug","%s is in %s, should set away."%(wfo.name,wfi.request['RequestStatus']))
sendLog("stagor","%s is in %s, set away"%(wfo.name,wfi.request['RequestStatus']), level='critical')
wfo.status = 'away'
session.commit()
continue
wfois.append( (wfo,wfi) )
_,primaries,_,secondaries = wfi.getIO()
for dataset in list(primaries)+list(secondaries):
needs[wfo.name].append( dataset)
done_by_input[dataset] = {}
completion_by_input[dataset] = {}
needs_by_priority[wfi.request['RequestPriority']].append( dataset )
wfi.sendLog('stagor', '%s needs %s'%( wfo.name, dataset))
time_point("Check staging workflows")
#open('%s/dataset_requirements.json'%monitor_dir,'w').write( json.dumps( needs, indent=2))
eosFile('%s/dataset_requirements.json'%monitor_dir,'w').write( json.dumps( needs, indent=2)).close()
for prio in needs_by_priority: needs_by_priority[prio] = list(set(needs_by_priority[prio]))
#open('%s/dataset_priorities.json'%monitor_dir,'w').write( json.dumps( needs_by_priority , indent=2))
eosFile('%s/dataset_priorities.json'%monitor_dir,'w').write( json.dumps( needs_by_priority , indent=2)).close()
dataset_endpoints = defaultdict(set)
endpoint_in_downtime = defaultdict(set)
#endpoint_completed = defaultdict(set)
endpoint_incompleted = defaultdict(set)
#endpoint = defaultdict(set)
send_back_to_considered = set()
## first check if anything is inactive
all_actives = set([transfer.phedexid for transfer in session.query(TransferImp).filter(TransferImp.active).all()])
for active_phedexid in all_actives:
skip = True
transfers_phedexid = session.query(TransferImp).filter(TransferImp.phedexid == active_phedexid).all()
for imp in transfers_phedexid:
if imp.workflow.status == 'staging':
skip =False
sendLog('stagor',"\t%s is staging for %s"%(imp.phedexid, imp.workflow.name))
if skip:
sendLog('stagor',"setting %s inactive" % active_phedexid)
for imp in transfers_phedexid:
imp.active = False
session.commit()
all_actives = sorted(set([transfer.phedexid for transfer in session.query(TransferImp).filter(TransferImp.active).all()]))
for phedexid in all_actives:
if specific: continue
## check on transfer completion
not_cached = False
if phedexid in cached_transfer_statuses:
### use a cache for transfer that already looked done
sendLog('stagor',"read %s from cache"%phedexid)
checks = cached_transfer_statuses[phedexid]
else:
## I actually would like to avoid that all I can
sendLog('stagor','Performing spurious transfer check on %s'% phedexid, level='critical')
checks = checkTransferStatus(url, phedexid, nocollapse=True)
try:
print json.dumps(checks, indent=2)
except:
print checks
if not checks:
## this is going to bias quite heavily the rest of the code. we should abort here
#sendLog('stagor','Ending stagor because of skewed input from checkTransferStatus', level='critical')
#return False
sendLog('stagor','Stagor has got a skewed input from checkTransferStatus', level='critical')
checks = {}
pass
else:
TS.add( phedexid, checks)
time_point("Check transfer status %s"% phedexid, sub_lap=True)
if not specific:
for dsname in checks:
if not dsname in done_by_input: done_by_input[dsname]={}
if not dsname in completion_by_input: completion_by_input[dsname] = {}
done_by_input[dsname][phedexid]=all(map(lambda i:i>=good_enough, checks[dsname].values()))
completion_by_input[dsname][phedexid]=checks[dsname].values()
if checks:
sendLog('stagor',"Checks for %s are %s"%( phedexid, [node.values() for node in checks.values()]))
done = all(map(lambda i:i>=good_enough,list(itertools.chain.from_iterable([node.values() for node in checks.values()]))))
else:
## it is empty, is that a sign that all is done and away ?
if not_cached:
print "Transfer status was not cached"
else:
print "ERROR with the scubscriptions API of ",phedexid
print "Most likely something else is overiding the transfer request. Need to work on finding the replacement automatically, if the replacement exists"
done = False
transfers_phedexid = session.query(TransferImp).filter(TransferImp.phedexid == phedexid).all()
for imp in transfers_phedexid:
tr_wf = imp.workflow
if tr_wf:# and tr_wf.status == 'staging':
if not tr_wf.id in done_by_wf_id: done_by_wf_id[tr_wf.id]={}
done_by_wf_id[tr_wf.id][phedexid]=done
if done:
imp.active = False
session.commit()
for ds in checks:
for s,v in checks[ds].items():
dataset_endpoints[ds].add( s )
if done:
sendLog('stagor',"%s is done"%phedexid)
TS.add( phedexid, checks)
else:
sendLog('stagor',"%s is not finished %s"%(phedexid, pprint.pformat( checks )))
##pprint.pprint( checks )
## check if the destination is in down-time
for ds in checks:
sites_incomplete = [SI.SE_to_CE(s) for s,v in checks[ds].items() if v<good_enough]
sites_incomplete_down = [s for s in sites_incomplete if not s in SI.sites_ready]
## no space means no transfer should go there : NO, it does not work in the long run
#sites_incomplete_down = [SI.SE_to_CE(s) for s,v in checks[ds].items() if (v<good_enough and (SI.disk[s]==0 or (not SI.SE_to_CE(s) in SI.sites_ready)))]
if sites_incomplete_down:
sendLog('stagor',"%s are in downtime, while waiting for %s to get there"%( ",".join(sites_incomplete_down), ds))
endpoint_in_downtime[ds].update( sites_incomplete_down )
if sites_incomplete:
endpoint_incompleted[ds].update( sites_incomplete )
time_point("Check on-going transfers")
print "End points"
for k in dataset_endpoints: dataset_endpoints[k] = list(dataset_endpoints[k])
print json.dumps( dataset_endpoints , indent=2)
print "End point in down time"
for k in endpoint_in_downtime: endpoint_in_downtime[k] = list(endpoint_in_downtime[k])
print json.dumps( endpoint_in_downtime , indent=2)
print "End point incomplete in down time"
for k in endpoint_incompleted: endpoint_incompleted[k] = list(endpoint_incompleted[k])
print json.dumps( endpoint_incompleted , indent=2)
#open('%s/transfer_statuses.json'%monitor_dir,'w').write( json.dumps( transfer_statuses, indent=2))
eosFile('%s/transfer_statuses.json'%monitor_dir,'w').write( json.dumps( TS.content(), indent=2)).close()
eosFile('%s/dataset_endpoints.json'%monitor_dir,'w').write( json.dumps(dataset_endpoints, indent=2)).close()
already_stuck = json.loads( eosRead('%s/stuck_transfers.json'%monitor_pub_dir) ).keys()
already_stuck.extend( getAllStuckDataset() )
missing_in_action = defaultdict(list)
print "-"*10,"Checking on workflows in staging","-"*10
#forget_about = ['/MinBias_TuneCUETP8M1_13TeV-pythia8/RunIISummer15GS-MCRUN2_71_V1-v2/GEN-SIM']
#for what in forget_about:
# if not done_by_input[what]:
# done_by_input[what] = {'fake':True}
## come back to workflows and check if they can go
available_cache = defaultdict(lambda : defaultdict(float))
presence_cache = defaultdict(dict)
time_point("Preparing for more")
for wfo,wfi in wfois:
print "#"*30
time_point("Forward checking %s"% wfo.name,sub_lap=True)
## the site white list takes site, campaign, memory and core information
(_,primaries,_,secondaries,sites_allowed) = wfi.getSiteWhiteList(verbose=False)
se_allowed = [SI.CE_to_SE(site) for site in sites_allowed]
se_allowed.sort()
se_allowed_key = ','.join(se_allowed)
readys={}
for need in list(primaries)+list(secondaries):
if not need in done_by_input:
wfi.sendLog('stagor',"missing transfer report for %s"%need)
readys[need] = False
## should warn someone about this !!!
## it cannot happen, by construction
sendEmail('missing transfer report','%s does not have a transfer report'%(need))
continue
if not done_by_input[need] and need in list(secondaries):
wfi.sendLog('stagor',"assuming it is OK for secondary %s to have no attached transfers"% need)
readys[need] = True
done_by_input[need] = { "fake" : True }
continue
if len(done_by_input[need]) and all(done_by_input[need].values()):
wfi.sendLog('stagor',"%s is ready"%need)
print json.dumps( done_by_input[need] , indent=2)
readys[need] = True
else:
wfi.sendLog('stagor',"%s is not ready \n%s.\nWaiting for the following:"%(need,json.dumps( done_by_input[need] , indent=2)))
for request_id in done_by_input[need]:
if not done_by_input[need][request_id]:
wfi.sendLog('stagor',"https://cmsweb.cern.ch/phedex/prod/Request::View?request={}".format(request_id))
readys[need] = False
if readys and all(readys.values()):
if wfo.status == 'staging':
wfi.sendLog('stagor',"all needs are fullfilled, setting staged")
wfo.status = 'staged'
session.commit()
else:
wfi.sendLog('stagor',"all needs are fullfilled, already")
print json.dumps( readys, indent=2 )
else:
wfi.sendLog('stagor',"missing requirements")
copies_needed_from_CPUh, _ = wfi.getNCopies()
copies_needed = copies_needed_from_CPUh
if 'Campaign' in wfi.request and wfi.request['Campaign'] in CI.campaigns and 'maxcopies' in CI.campaigns[wfi.request['Campaign']]:
copies_needed_from_campaign = CI.campaigns[wfi.request['Campaign']]['maxcopies']
copies_needed = min(copies_needed_from_campaign, copies_needed_from_CPUh)
jump_ahead = False
re_transfer = False
## there is missing input let's do something more elaborated
for need in list(primaries):#+list(secondaries):
if endpoint_in_downtime[need] and endpoint_in_downtime[need] == endpoint_incompleted[need]:
#print need,"is going to an end point in downtime"
wfi.sendLog('stagor',"%s has only incomplete endpoint in downtime\n%s"%(need, endpoint_in_downtime[need] ))
re_transfer=True
if not se_allowed_key in available_cache[need]:
available_cache[need][se_allowed_key] = getDatasetBlocksFraction( url , need, sites=se_allowed )
if available_cache[need][se_allowed_key] >= copies_needed:
wfi.sendLog('stagor',"assuming it is OK to move on like this already for %s"%need)
jump_ahead = True
else:
wfi.sendLog('stagor',"Available {} times. {} needed".format(available_cache[need][se_allowed_key], copies_needed))
missing_and_downtime = list(set(endpoint_in_downtime[need]) & set(endpoint_incompleted[need]))
if missing_and_downtime:
wfi.sendLog('stagor',"%s is incomplete at %s which is in downtime, trying to move along"%(need, ','.join(missing_and_downtime)))
jump_ahead = True
else:
wfi.sendLog('stagor',"continue waiting for transfers for optimum production performance.")
## compute a time since staging to filter jump starting ?
# check whether the inputs is already in the stuck list ...
for need in list(primaries)+list(secondaries):
if need in already_stuck:
wfi.sendLog('stagor',"%s is stuck, so try to jump ahead"%need)
jump_ahead = True
if jump_ahead or re_transfer:
details_text = "checking on availability for %s to jump ahead"%wfo.name
details_text += '\n%s wants %s copies'%(wfo.name,copies_needed)
copies_needed = max(1,copies_needed-1)
details_text += '\nlowering by one unit to %s'%copies_needed
wfi.sendLog('stagor', details_text)
all_check = True
prim_where = set()
for need in list(primaries):
if not se_allowed_key in presence_cache[need]:
presence_cache[need][se_allowed_key] = getDatasetPresence( url, need , within_sites=se_allowed)
presence = presence_cache[need][se_allowed_key]
prim_where.update( presence.keys() )
available = available_cache[need][se_allowed_key]
this_check = (available >= copies_needed)
wfi.sendLog('stagor', "%s is available %s times (%s), at %s"%( need, available, this_check, se_allowed_key))
all_check &= this_check
if not all_check: break
for need in list(secondaries):
## I do not want to check on the secon
## this below does not function because the primary could be all available, and the secondary not complete at a certain site that does not matter at that point
this_check = all(done_by_input[need].values())
wfi.sendLog('stagor',"%s is this much transfered %s"%(need, json.dumps(done_by_input[need], indent=2)))
all_check&= this_check
#if not se_allowed_key in presence_cache[need]:
# presence_cache[need][se_allowed_key] = getDatasetPresence( url, need , within_sites=se_allowed)
## restrict to where the primary is
#presence = dict([(k,v) for (k,v) in presence_cache[need][se_allowed_key].items() if k in prim_where])
#this_check = all([there for (there,frac) in presence.values()])
#print need,"is present at all sites:",this_check
#all_check&= this_check
if all_check and not re_transfer:
wfi.sendLog('stagor',"needs are sufficiently fullfilled, setting staged")
wfo.status = 'staged'
session.commit()
else:
print wfo.name,"has to wait a bit more"
wfi.sendLog('stagor',"needs to wait a bit more")
else:
wfi.sendLog('stagor',"not checking availability")
if re_transfer:
wfi.sendLog('stagor',"Sending back to considered because of endpoint in downtime")
if wfo.status == 'staging':
wfo.status = 'considered'
session.commit()
send_back_to_considered.add( wfo.name )
time_point("Checked affected workflows")
if send_back_to_considered:
#sendEmail("transfer to endpoint in downtime","sending back to considered the following workflows \n%s"%('\n'.join( send_back_to_considered)))
sendLog('stagor', "sending back to considered the following workflows \n%s"%('\n'.join( send_back_to_considered)), level='critical')
print "-"*10,"Checking on non-available datasets","-"*10
## now check on those that are not fully available
for dsname in available_cache.keys():
## squash the se_allowed_key key
available_cache[dsname] = min( available_cache[dsname].values() )
really_stuck_dataset = set()
for dsname,available in available_cache.items():
using_its = getWorkflowByInput(url, dsname)
#print using_its
using_wfos = []
for using_it in using_its:
wf = session.query(Workflow).filter(Workflow.name == using_it).first()
if wf:
using_wfos.append( wf )
if not len(done_by_input[dsname]):
print "For dataset",dsname,"there are no transfer report. That's an issue."
for wf in using_wfos:
if wf.status == 'staging':
if UC.get("stagor_sends_back"):
print "sending",wf.name,"back to considered"
wf.status = 'considered'
session.commit()
#sendEmail( "send back to considered","%s was send back and might be trouble"% wf.name)
sendLog('stagor', "%s was send back and might be trouble"% wf.name, level='critical')
else:
print "would send",wf.name,"back to considered"
#sendEmail( "subscription lagging behind","susbscriptions to get %s running are not appearing in phedex. I would have send it back to considered but that's not good."% wf.name)
sendLog('stagor', "susbscriptions to get %s running are not appearing in phedex. I would have send it back to considered but that's not good."% wf.name, level='critical')
continue
## not compatible with checking on secondary availability
#if all([wf.status != 'staging' for wf in using_wfos]):
# ## means despite all checks that input is not needed
# continue
if available < 1.:
print "incomplete",dsname
## there is a problem in the method below that it does not account for files stuck in T1*Buffer only
lost_blocks,lost_files = findLostBlocksFiles( url, dsname ) if (not dsname.endswith('/RAW')) else ([],[])
lost_block_names = [item['name'] for item in lost_blocks]
lost_file_names = [item['name'] for item in lost_files]
if lost_blocks:
#print json.dumps( lost , indent=2 )
## estimate for how much !
fraction_loss,_,n_missing = getDatasetBlockFraction(dsname, lost_block_names)
print "We have lost",len(lost_block_names),"blocks",lost_block_names,"for %f%%"%(100.*fraction_loss)
if fraction_loss > 0.05: ## 95% completion mark
#sendEmail('we have lost too many blocks','%s is missing %d blocks, for %d events, %f %% loss'%(dsname, len(lost_block_names), n_missing, fraction_loss))
sendLog('stagor', '%s is missing %d blocks, for %d events, %3.2f %% loss'%(dsname, len(lost_block_names), n_missing, 100*fraction_loss), level='critical')
## the workflow should be rejected !
for wf in using_wfos:
if wf.status == 'staging':
print wf.name,"is doomed. setting to trouble"
wf.status = 'trouble'
session.commit()
sendLog('stagor', '%s has too much loss on the input dataset %s. Missing %d blocks, for %d events, %3.2f %% loss'%(wf.name, dsname, len(lost_block_names), n_missing, 100*fraction_loss), level='critical')
else:
## probably enough to make a ggus and remove
if not dsname in known_lost_blocks:
#sendEmail('we have lost a few blocks', '%s is missing %d blocks, for %d events, %f %% loss\n\n%s'%(dsname, len(lost_block_names), n_missing, fraction_loss, '\n'.join( lost_block_names ) ))
sendLog('stagor', '%s is missing %d blocks, for %d events, %f %% loss\n\n%s'%(dsname, len(lost_block_names), n_missing, fraction_loss, '\n'.join( lost_block_names ) ), level='critical')
known_lost_blocks[dsname] = [i['name'] for i in lost_blocks]
really_stuck_dataset.add( dsname )
if lost_files:
fraction_loss,_,n_missing = getDatasetFileFraction(dsname, lost_file_names)
print "We have lost",len(lost_file_names),"files",lost_file_names,"for %f%%"%fraction_loss
if fraction_loss > 0.05:
#sendEmail('we have lost too many files','%s is missing %d files, for %d events, %f %% loss'%(dsname, len(lost_file_names),n_missing, fraction_loss))
sendLog('stagor', '%s is missing %d files, for %d events, %f %% loss'%(dsname, len(lost_file_names),n_missing, fraction_loss), level='critical')
for wf in using_wfos:
if wf.status == 'staging':
print wf.name,"is doomed. setting to trouble"
wf.status = 'trouble'
session.commit()
else:
## probably enough to make a ggus and remove
if not dsname in known_lost_files:
#sendEmail('we have lost a few files','%s is missing %d files, for %d events, %f %% loss\n\n%s'%(dsname, len(lost_file_names),n_missing, fraction_loss, '\n'.join(lost_file_names)))
sendLog('stagor', '%s is missing %d files, for %d events, %f %% loss\n\n%s'%(dsname, len(lost_file_names),n_missing, fraction_loss, '\n'.join(lost_file_names)), level='critical')
known_lost_files[dsname] = [i['name'] for i in lost_files]
## should the status be change to held-staging and pending on a ticket
missings = [pid for (pid,d) in done_by_input[dsname].items() if d==False]
print "\t",done_by_input[dsname]
print "\tneeds",len(done_by_input[dsname])
print "\tgot",done_by_input[dsname].values().count(True)
print "\tmissing",missings
missing_in_action[dsname].extend( missings )
eosFile('%s/lost_blocks_datasets.json'%monitor_dir,'w').write( json.dumps( known_lost_blocks, indent=2)).close()
eosFile('%s/lost_files_datasets.json'%monitor_dir,'w').write( json.dumps( known_lost_files, indent=2)).close()
eosFile('%s/incomplete_transfers.json'%monitor_dir,'w').write( json.dumps(missing_in_action, indent=2) ).close()
print "Stuck transfers and datasets"
print json.dumps( missing_in_action, indent=2 )
TD = transferDataset()
datasets_by_phid = defaultdict(set)
for dataset in missing_in_action:
for phid in missing_in_action[dataset]:
#print dataset,"stuck through",phid
datasets_by_phid[phid].add( dataset )
for k in datasets_by_phid:
#datasets_by_phid[k] = list(datasets_by_phid[k])
TD.add( k , list(datasets_by_phid[k]))
#eosFile('%s/datasets_by_phid.json'%base_eos_dir,'w').write( json.dumps(datasets_by_phid, indent=2 )).close()
eosFile('%s/really_stuck_dataset.json'%base_eos_dir,'w').write( json.dumps(list(really_stuck_dataset), indent=2 )).close()
print '\n'*2,"Datasets really stuck"
print '\n'.join( really_stuck_dataset )
#############
## not going further for what matters
#############
return
