def collector(url, specific, options):
up = componentInfo(mcm=False, soft=['mcm'])
if not up.check(): return
SI = siteInfo()
dss = DSS()
#NL = newLockInfo()
mcm = McMClient(dev=False)
fetch_in_campaigns = ['RunIISummer15GS']
mcm_statuses=['new']#,'validation','defined','approved']
will_be_used = defaultdict(list)
secondary_used = defaultdict(list)
for campaign,status in itertools.product( fetch_in_campaigns, mcm_statuses):
queries=[]
if campaign:
print "getting for",campaign
queries.append('member_of_campaign=%s'%campaign)
if status:
print "getting for",status
queries.append('status=%s'%status)
rs = mcm.getA('requests', query='&'.join(queries))
for r in rs:
#if r['type'] != 'Prod': continue
dataset = r['input_dataset']
if dataset:
#print r['prepid'],dataset
will_be_used[dataset].append( r )
pileup = r['pileup_dataset_name']
if pileup:
secondary_used['pileup'].append( r )
all_transfers = defaultdict(list)
print len(will_be_used),"datasets that can be pre-fetched"
## for secondary we really need to have the campaign right
print len(secondary_used),"pileup will be used"
datasets = will_be_used.keys()
if options.limit:
print "Restricting to randomly picked",options.limit
random.shuffle( datasets )
datasets = datasets[:options.limit]
for dataset in datasets:
print "\tlooking at",dataset
#presence = getDatasetPresence(url, dataset)#, within_sites=['T2_CH_CERN'])
## lock all those, and pre-fecth them
#NL.lock( dataset )
## we could get the reqmgr dictionnary from McM if it was implemented and use standard workflowInfo !!!
for request in will_be_used[dataset]:
print "will be used by",request['prepid']
campaign = request['member_of_campaign']
## based on the campaign, pre-fetch a site list
sites_allowed = SI.sites_T1s + SI.sites_with_goodIO
if options.spread:
## pick up the number of copies from campaign
copies_needed = 1 ## hard coded for now
else:
copies_needed = 1 ## hard coded for now
print "Will look for",copies_needed,"of",dataset
## figure out where it is and going
destinations, all_block_names = getDatasetDestinations(url, dataset, within_sites = [SI.CE_to_SE(site) for site in sites_allowed])
print json.dumps( destinations, indent=2)
prim_location = [site for (site,info) in destinations.items() if info['completion']==100 and info['data_fraction']==1]
prim_destination = [site for site in destinations.keys() if not site in prim_location]
prim_destination = [site for site in prim_destination if not any([osite.startswith(site) for osite in SI.sites_veto_transfer])]
copies_needed = max(0,copies_needed - len(prim_location))
copies_being_made = [ sum([info['blocks'].keys().count(block) for site,info in destinations.items() if site in prim_destination]) for block in all_block_names]
prim_to_distribute = [site for site in sites_allowed if not SI.CE_to_SE(site) in prim_location]
prim_to_distribute = [site for site in prim_to_distribute if not SI.CE_to_SE(site) in prim_destination]
## take out the ones that cannot receive transfers
prim_to_distribute = [site for site in prim_to_distribute if not any([osite.startswith(site) for osite in SI.sites_veto_transfer])]
copies_needed = max(0,copies_needed - min(copies_being_made))
spreading = {}
if copies_needed:
print "needing",copies_needed
chops,sizes = getDatasetChops(dataset, chop_threshold = options.chopsize)
spreading = distributeToSites( chops, prim_to_distribute, n_copies = copies_needed, weights=SI.cpu_pledges, sizes=sizes)
else:
print "no copy needed for",dataset
for (site,items) in spreading.items():
all_transfers[site].extend( items )
print "accumulated transfers"
print json.dumps(all_transfers, indent=2)
if not options.test:
sendEmail('dataset to be fetched',
'the following datasets and location were figured from mcm up-coming requests\n%s'%( json.dumps(all_transfers, indent=2) ),
destination=['*****@*****.**'])
## now collect and make transfer request
for (site,items_to_transfer) in all_transfers.iteritems():
print "Directing at",site
items_to_transfer = list(set(items_to_transfer))
site_se = SI.CE_to_SE(site)
blocks = [it for it in items_to_transfer if '#' in it]
datasets = [it for it in items_to_transfer if not '#' in it]
print "\t",len(blocks),"blocks"
## remove blocks if full dataset is send out
blocks = [block for block in blocks if not block.split('#')[0] in datasets]
blocks_dataset = list(set([block.split('#')[0] for block in blocks]))
print "\t",len(blocks),"needed blocks for",blocks_dataset
print "\t",len(datasets),"datasets"
print "\t",datasets
items_to_transfer = blocks + datasets
total_size = 0
for dataset in datasets:
ds_size,_ = dss.get_block_size( dataset )
total_size += ds_size
for dataset in blocks_dataset:
_,bs_size = dss.get_block_size( dataset )
total_size += sum([ s for b,s in bs_size if b in blocks ])
print "For a total of",total_size,"[GB]"
if options.test:
result= {'phedex':{'request_created' : []}}
else:
##result = makeReplicaRequest(url, site_se, items_to_transfer, 'fetching pre-production', priority='normal', approve=True)
## should make sure there is something in it
pass
def transferor(url ,specific = None, talk=True, options=None):
if userLock(): return
if duplicateLock(): return
use_mcm = True
up = componentInfo(mcm=use_mcm, soft=['mcm'])
if not up.check(): return
use_mcm = up.status['mcm']
if options and options.test:
execute = False
else:
execute = True
SI = siteInfo()
CI = campaignInfo()
NLI = newLockInfo()
mcm = McMClient(dev=False)
dss = DSS()
#allowed_secondary = UC.get('')
print "counting all being handled..."
being_handled = len(session.query(Workflow).filter(Workflow.status == 'away').all())
being_handled += len(session.query(Workflow).filter(Workflow.status.startswith('stag')).all())
being_transfered = len(session.query(Workflow).filter(Workflow.status == 'staging').all())
being_handled += len(session.query(Workflow).filter(Workflow.status.startswith('assistance-')).all())
max_to_handle = options.maxworkflows
max_to_transfer = options.maxstaging
allowed_to_handle = max(0,max_to_handle - being_handled)
allowed_to_transfer = max(0,max_to_transfer - being_transfered)
wf_buffer = 5
if allowed_to_handle<=wf_buffer: ## buffer for having several wf per transfer
print "Not allowed to run more than",max_to_handle,"at a time. Currently",being_handled,"and",wf_buffer,"buffer"
else:
print being_handled,"already being handled",max_to_handle,"max allowed,",allowed_to_handle,"remaining","and",wf_buffer,"buffer"
if allowed_to_transfer <= wf_buffer:
print "Not allowed to transfer more than",max_to_transfer,"at a time. Currently",being_transfered,"and",wf_buffer,"buffer"
else:
print being_transfered,"already being transfered",max_to_transfer,"max allowed,",allowed_to_transfer,"remaining","and",wf_buffer,"buffer"
print "... done"
all_transfers=defaultdict(list)
workflow_dependencies = defaultdict(set) ## list of wf.id per input dataset
wfs_and_wfh=[]
print "getting all wf to consider ..."
cache = getWorkflows(url, 'assignment-approved', details=True)
for wfo in session.query(Workflow).filter(Workflow.status.startswith('considered')).all():
print "\t",wfo.name
if specific and not specific in wfo.name: continue
cache_r =filter(lambda d:d['RequestName']==wfo.name, cache)
if len(cache_r):
wfs_and_wfh.append( (wfo, workflowInfo( url, wfo.name, spec=False, request = cache_r[0]) ) )
else:
wfs_and_wfh.append( (wfo, workflowInfo( url, wfo.name, spec=False) ) )
print "... done"
transfers_per_sites = defaultdict(int)
input_sizes = {}
ignored_input_sizes = {}
input_cput = {}
input_st = {}
## list the size of those in transfer already
in_transfer_priority=None
min_transfer_priority=None
print "getting all wf in staging ..."
stucks = json.loads(open('%s/stuck_transfers.json'%monitor_dir).read())
for wfo in session.query(Workflow).filter(Workflow.status=='staging').all():
wfh = workflowInfo( url, wfo.name, spec=False)
#(lheinput,primary,parent,secondary) = wfh.getIO()
#sites_allowed = getSiteWhiteList( (lheinput,primary,parent,secondary) )
(lheinput,primary,parent,secondary,sites_allowed) = wfh.getSiteWhiteList()
for site in sites_allowed: ## we should get the actual transfer destination instead of the full white list
transfers_per_sites[site] += 1
#input_cput[wfo.name] = wfh.getComputingTime()
#input_st[wfo.name] = wfh.getSystemTime()
for prim in primary:
ds_s = dss.get( prim )
if prim in stucks:
sendLog('transferor', "%s appears stuck, so not counting it %s [GB]"%( prim, ds_s), wfi=wfh)
ignored_input_sizes[prim] = ds_s
else:
input_sizes[prim] = ds_s
sendLog('transferor', "%s needs %s [GB]"%( wfo.name, ds_s), wfi=wfh)
if in_transfer_priority==None:
in_transfer_priority = int(wfh.request['RequestPriority'])
else:
in_transfer_priority = max(in_transfer_priority, int(wfh.request['RequestPriority']))
if min_transfer_priority==None:
min_transfer_priority = int(wfh.request['RequestPriority'])
else:
min_transfer_priority = min(min_transfer_priority, int(wfh.request['RequestPriority']))
if min_transfer_priority==None or in_transfer_priority ==None:
print "nothing is lining up for transfer"
sendEmail("no request in staging","no request in staging")
return
pass
try:
print "Ignored input sizes"
ignored_values = list(ignored_input_sizes.items())
ignored_values.sort( key = lambda i : i[1] )
print "\n".join( map(str, ignored_values ) )
print "Considered input sizes"
considered_values = list(input_sizes.items())
considered_values.sort( key = lambda i : i[1] )
print "\n".join( map(str, considered_values) )
except Exception as e:
print "trying to print the summary of input size"
print str(e)
print "... done"
print "Max priority in transfer already",in_transfer_priority
print "Min priority in transfer already",min_transfer_priority
print "transfers per sites"
print json.dumps( transfers_per_sites, indent=2)
in_transfer_already = sum(input_sizes.values())
cput_in_transfer_already = sum(input_cput.values())
st_in_transfer_already = sum(input_st.values())
## list the size of all inputs
primary_input_per_workflow_gb = defaultdict(float)
print "getting all input sizes ..."
for (wfo,wfh) in wfs_and_wfh:
(_,primary,_,_) = wfh.getIO()
#input_cput[wfo.name] = wfh.getComputingTime()
#input_st[wfo.name] = wfh.getSystemTime()
for prim in primary:
## do not count it if it appears stalled !
prim_size = dss.get( prim )
input_sizes[prim] = prim_size
primary_input_per_workflow_gb[wfo.name] += prim_size
print "... done"
# shuffle first by name
random.shuffle( wfs_and_wfh )
# Sort smallest transfers first; allows us to transfer as many as possible workflows.
def prio_and_size( i, j):
if int(i[1].request['RequestPriority']) == int(j[1].request['RequestPriority']):
return cmp(int(primary_input_per_workflow_gb.get(j[0].name, 0)), int(primary_input_per_workflow_gb.get(i[0].name, 0)) )
else:
return cmp(int(i[1].request['RequestPriority']),int(j[1].request['RequestPriority']))
#wfs_and_wfh.sort(cmp = prio_and_size, reverse=True)
#wfs_and_wfh.sort(cmp = lambda i,j : cmp(int(primary_input_per_workflow_gb.get(i[0].name, 0)), int(primary_input_per_workflow_gb.get(j[0].name, 0)) ))
#sort by priority higher first
wfs_and_wfh.sort(cmp = lambda i,j : cmp(int(i[1].request['RequestPriority']),int(j[1].request['RequestPriority']) ), reverse=True)
cput_grand_total = sum(input_cput.values())
cput_to_transfer = cput_grand_total - cput_in_transfer_already
st_grand_total = sum(input_st.values())
st_to_transfer = st_grand_total - st_in_transfer_already
print "%15.4f [CPU h] worth already in transfer"%cput_in_transfer_already
print "%15.4f [CPU h] worth is the current requested transfer load"%cput_to_transfer
print "%15.4f [h] worth of absolute system time in transfer"%( cput_in_transfer_already / SI.availableSlots())
print "%15.4f [h] worth of absolute system time is the current requested transfer load"%( cput_to_transfer / SI.availableSlots())
print "%15.4f [h] worth of theoritical system time in transfer"%( st_in_transfer_already )
print "%15.4f [h] worth of theoritical system time is the current requested transfer load"%( st_to_transfer )
grand_total = sum(input_sizes.values())
to_transfer = grand_total - in_transfer_already
grand_transfer_limit = options.maxtransfer
#grand_transfer_limit = SI.total_disk()*0.25*1024## half of the free sapce in TB->GB
transfer_limit = grand_transfer_limit - in_transfer_already
print "%15.4f GB already being transfered"%in_transfer_already
print "%15.4f GB is the current requested transfer load"%to_transfer
print "%15.4f GB is the global transfer limit"%grand_transfer_limit
print "%15.4f GB is the available limit"%transfer_limit
max_staging_per_site = options.maxstagingpersite
# the max priority value per dataset.
max_priority = defaultdict(int)
needs_transfer=0 ## so that we can count'em
passing_along = 0
transfer_sizes={}
went_over_budget=False
destination_cache = {}
no_goes = set()
max_per_round = UC.get('max_per_round').get('transferor',None)
if max_per_round and not spec:
wfs_and_wfh = wfs_and_wfh[:max_per_round]
for (wfo,wfh) in wfs_and_wfh:
print wfo.name,"to be transfered with priority",wfh.request['RequestPriority']
if wfh.request['RequestStatus']!='assignment-approved':
if wfh.request['RequestStatus'] in ['aborted','rejected','rejected-archived','aborted-archived']:
wfo.status = 'trouble' ## so that we look or a replacement
else:
wfo.status = 'away'
wfh.sendLog('transferor', '%s in status %s, setting %s'%( wfo.name,wfh.request['RequestStatus'],wfo.status))
continue
(_,primary,_,_) = wfh.getIO()
this_load=sum([input_sizes[prim] for prim in primary])
no_budget = False
if ( this_load and (sum(transfer_sizes.values())+this_load > transfer_limit or went_over_budget ) ):
if went_over_budget:
wfh.sendLog('transferor', "Transfer has gone over bubget.")
else:
wfh.sendLog('transferor', "Transfer will go over bubget.")
wfh.sendLog('transferor', "%15.4f GB this load, %15.4f GB already this round, %15.4f GB is the available limit"%(this_load, sum(transfer_sizes.values()), transfer_limit))
#if sum(transfer_sizes.values()) > transfer_limit:
went_over_budget = True
if in_transfer_priority!=None and min_transfer_priority!=None:
if int(wfh.request['RequestPriority']) >= in_transfer_priority and min_transfer_priority!=in_transfer_priority:
wfh.sendLog('transferor',"Higher priority sample %s >= %s go-on over budget"%( wfh.request['RequestPriority'], in_transfer_priority))
else:
if not options.go:
wfh.sendLog('transferor',"%s minimum priority %s < %s : stop"%( min_transfer_priority,wfh.request['RequestPriority'],in_transfer_priority))
no_budget = True
## throtlle by campaign go
no_go = False
if not wfh.go(log=True) and not options.go:
no_go = True
no_goes.add( wfo.name )
allowed_secondary = set()
for campaign in wfh.getCampaigns():
if campaign in CI.campaigns and 'secondaries' in CI.campaigns[campaign]:
allowed_secondary.update( CI.campaigns[campaign]['secondaries'] )
if secondary:
if (secondary and allowed_secondary) and (set(secondary)&allowed_secondary!=set(secondary)):
wfh.sendLog('assignor','%s is not an allowed secondary'%(', '.join(set(secondary)-allowed_secondary)))
no_go = True
if no_go:
continue
## check if the batch is announced
def check_mcm(wfn):
announced=False
is_real=False
if not wfn.startswith('pdmvserv'):
is_real = True
try:
for b in mcm.getA('batches',query='contains=%s'% wfo.name):
is_real = True
if b['status']=='announced':
announced=True
break
except:
try:
for b in mcm.getA('batches',query='contains=%s'% wfo.name):
is_real = True
if b['status']=='announced':
announced=True
break
except:
print "could not get mcm batch announcement, assuming not real"
return announced,is_real
if not use_mcm:
announced,is_real = False,True
else:
if wfh.request['RequestType'] in ['ReReco']:
announced,is_real = True,True
else:
announced,is_real = check_mcm( wfo.name )
if not announced:
wfh.sendLog('transferor', "does not look announced.")
if not is_real:
wfh.sendLog('transferor', "does not appear to be genuine.")
## prevent any duplication. if the wf is not mentioned in any batch, regardless of status
continue
## check on a grace period
injection_time = time.mktime(time.strptime('.'.join(map(str,wfh.request['RequestDate'])),"%Y.%m.%d.%H.%M.%S")) / (60.*60.)
now = time.mktime(time.gmtime()) / (60.*60.)
if float(now - injection_time) < 4.:
if not options.go and not announced:
wfh.sendLog('transferor', "It is too soon to start transfer: %3.2fH remaining"%(now - injection_time))
continue
if passing_along >= allowed_to_handle:
#if int(wfh.request['RequestPriority']) >= in_transfer_priority and min_transfer_priority!=in_transfer_priority:
if in_transfer_priority!=None and min_transfer_priority!=None:
if int(wfh.request['RequestPriority']) >= in_transfer_priority and int(wfh.request['RequestPriority']) !=min_transfer_priority:
## higher priority, and not only this priority being transfered
wfh.sendLog('transferor',"Higher priority sample %s >= %s go-on over %s"%( wfh.request['RequestPriority'], in_transfer_priority, max_to_handle))
else:
wfh.sendLog('transferor'," Not allowed to pass more than %s at a time. Currently %s handled, and adding %s"%( max_to_handle, being_handled, passing_along))
if not options.go:
## should not allow to jump that fence
break
if this_load and needs_transfer >= allowed_to_transfer:
if in_transfer_priority!=None and min_transfer_priority!=None:
if int(wfh.request['RequestPriority']) >= in_transfer_priority and int(wfh.request['RequestPriority']) !=min_transfer_priority:
## higher priority, and not only this priority being transfered
wfh.sendLog('transferor',"Higher priority sample %s >= %s go-on over %s"%(wfh.request['RequestPriority'], in_transfer_priority,max_to_transfer))
else:
wfh.sendLog('transferor',"Not allowed to transfer more than %s at a time. Currently %s transfering, and adding %s"%( max_to_transfer, being_transfered, needs_transfer))
if not options.go:
no_budget = True
if no_budget:
continue
## the site white list considers site, campaign, memory and core information
(lheinput,primary,parent,secondary,sites_allowed) = wfh.getSiteWhiteList()
if options and options.tosites:
sites_allowed = options.tosites.split(',')
for dataset in list(primary)+list(parent)+list(secondary):
## lock everything flat
NLI.lock( dataset )
if not sites_allowed:
wfh.sendLog('transferor',"not possible site to run at")
#sendEmail("no possible sites","%s has no possible sites to run at"%( wfo.name ))
sendLog('transferor',"%s has no possible sites to run at"%( wfo.name ),level='critical')
continue
blocks = []
if 'BlockWhitelist' in wfh.request and wfh.request['BlockWhitelist']:
blocks = wfh.request['BlockWhitelist']
if 'RunWhitelist' in wfh.request and wfh.request['RunWhitelist']:
## augment with run white list
for dataset in primary:
blocks = list(set( blocks + getDatasetBlocks( dataset, runs=wfh.request['RunWhitelist'] ) ))
if 'LumiList' in wfh.request and wfh.request['LumiList']:
## augment with the lumi white list
blocks = list(set( blocks + getDatasetBlocks( dataset, lumis= wfh.request['LumiList'] ) ))
if blocks:
print "Reading",len(blocks),"in block whitelist"
can_go = True
staging=False
allowed=True
primary_destinations = set()
if primary:
copies_needed_from_CPUh,CPUh = wfh.getNCopies()
if talk:
print wfo.name,'reads',', '.join(primary),'in primary'
## chope the primary dataset
for prim in primary:
## keep track of what needs what
workflow_dependencies[prim].add( wfo.id )
max_priority[prim] = max(max_priority[prim],int(wfh.request['RequestPriority']))
wfh.sendLog('transferor',"Would make %s from cpu requirement %s"%( copies_needed_from_CPUh, CPUh))
copies_needed = copies_needed_from_CPUh
if 'Campaign' in wfh.request and wfh.request['Campaign'] in CI.campaigns and 'maxcopies' in CI.campaigns[wfh.request['Campaign']]:
copies_needed_from_campaign = CI.campaigns[wfh.request['Campaign']]['maxcopies']
copies_needed = min(copies_needed_from_campaign, copies_needed)
wfh.sendLog('transferor',"Maxed to %s by campaign configuration %s"%( copies_needed, wfh.request['Campaign']))
### new ways of making the whole thing
destinations,all_block_names = getDatasetDestinations(url, prim, within_sites = [SI.CE_to_SE(site) for site in sites_allowed], only_blocks=blocks )
print json.dumps(destinations, indent=2)
## get where the dataset is in full and completed
prim_location = [site for (site,info) in destinations.items() if info['completion']==100 and info['data_fraction']==1]
## the rest is places it is going to be
prim_destination = [site for site in destinations.keys() if not site in prim_location]
if len(prim_location) >= copies_needed:
wfh.sendLog('transferor',"The input is all fully in place at %s sites %s"%( len(prim_location), sorted(prim_location)))
continue
copies_needed = max(0,copies_needed - len(prim_location))
wfh.sendLog('transferor',"not counting existing copies ; now need %s"% copies_needed)
copies_being_made = [ sum([info['blocks'].keys().count(block) for site,info in destinations.items() if site in prim_destination]) for block in all_block_names]
latching_on_transfers = set()
[latching_on_transfers.update(info['blocks'].values()) for site,info in destinations.items() if site in prim_destination]
latching_on_transfers = list(latching_on_transfers)
#print latching_on_transfers
## figure out where all this is going to go
prim_to_distribute = [site for site in sites_allowed if not SI.CE_to_SE(site) in prim_location]
prim_to_distribute = [site for site in prim_to_distribute if not SI.CE_to_SE(site) in prim_destination]
## take out the ones that cannot receive transfers
prim_to_distribute = [site for site in prim_to_distribute if not any([osite.startswith(site) for osite in SI.sites_veto_transfer])]
wfh.sendLog('transferor',"Could be going to: %s"% sorted( prim_to_distribute))
if not prim_to_distribute or any([transfers_per_sites[site] < max_staging_per_site for site in prim_to_distribute]):
## means there is openings let me go
print "There are transfer slots available:",[(site,transfers_per_sites[site]) for site in prim_to_distribute]
#for site in sites_allowed:
# #increment accross the board, regardless of real destination: could be changed
# transfers_per_sites[site] += 1
else:
if int(wfh.request['RequestPriority']) >= in_transfer_priority and min_transfer_priority!=in_transfer_priority:
wfh.sendLog('transferor', "Higher priority sample %s >= %s go-on over transfer slots available"%(wfh.request['RequestPriority'], in_transfer_priority))
else:
wfh.sendLog('transferor',"Not allowed to transfer more than %s per site at a time. Going overboard for %s"%( max_staging_per_site, sorted([site for site in prim_to_distribute if transfers_per_sites[site]>=max_staging_per_site])))
if not options.go:
allowed = False
break
for latching in latching_on_transfers:
tfo = session.query(Transfer).filter(Transfer.phedexid == int(latching)).first()
if not tfo:
tfo = session.query(Transfer).filter(Transfer.phedexid == -int(latching)).first()
if not tfo:
tfo = Transfer( phedexid = latching)
tfo.workflows_id = []
session.add(tfo)
else:
tfo.phedexid = latching ## make it positive ever
if not wfo.id in tfo.workflows_id:
print "adding",wfo.id,"to",tfo.id,"with phedexid",latching
l = copy.deepcopy( tfo.workflows_id )
l.append( wfo.id )
tfo.workflows_id = l
if not options.test:
session.commit()
else:
session.flush() ## regardless of commit later on, we need to let the next wf feeding on this transfer to see it in query
can_go = False
transfer_sizes[prim] = input_sizes[prim]
staging = True
# reduce the number of copies required by the on-going full transfer : how do we bootstrap on waiting for them ??
#copies_needed = max(0,copies_needed - len(prim_destination))
copies_needed = max(0,copies_needed - min(copies_being_made))
wfh.sendLog('transferor', "Not counting the copies being made ; then need %s"% copies_needed)
if copies_needed == 0:
wfh.sendLog('transferor', "The output is either fully in place or getting in full somewhere with %s"% latching_on_transfers)
can_go = True
continue
elif len(prim_to_distribute)==0:
wfh.sendLog('transferor', "We are going to need extra copies, but no destinations seems available")
prim_to_distribute = [site for site in sites_allowed if not SI.CE_to_SE(site) in prim_location]
prim_to_distribute = [site for site in prim_to_distribute if not any([osite.startswith(site) for osite in SI.sites_veto_transfer])]
if len(prim_to_distribute)>0: ## maybe that a parameter we can play with to limit the
if not options or options.chop:
### hard include the tape disk andpoint ?
#tapes = [site for site in getDatasetPresence( url, prim, vetos=['T0','T2','T3','Disk']) if site.endswith('MSS')]
chops,sizes = getDatasetChops(prim, chop_threshold = options.chopsize, only_blocks=blocks)
spreading = distributeToSites( chops, prim_to_distribute, n_copies = copies_needed, weights=SI.cpu_pledges, sizes=sizes)
transfer_sizes[prim] = sum(sizes)
if not spreading:
sendLog('transferor','cannot send %s to any site, it cannot fit anywhere'% prim, level='critical')
wfh.sendLog('transferor', "cannot send to any site. %s cannot seem to fit anywhere"%(prim))
staging=False
can_go = False
else:
spreading = {}
for site in prim_to_distribute:
if blocks:
spreading[site]=blocks
else:
spreading[site]=[prim]
transfer_sizes[prim] = input_sizes[prim] ## this is approximate if blocks are specified
can_go = False
wfh.sendLog('transferor', "selected CE destinations %s"%(sorted( spreading.keys())))
for (site,items) in spreading.items():
all_transfers[site].extend( items )
transfers_per_sites[site] += 1
primary_destinations.add( site )
if not allowed:
wfh.sendLog('transferor', "Not allowed to move on with")
continue
if secondary:
override_sec_destination = []
if 'SecondaryLocation' in CI.campaigns[wfh.request['Campaign']]:
override_sec_destination = CI.campaigns[wfh.request['Campaign']]['SecondaryLocation']
print wfo.name,'reads',', '.join(secondary),'in secondary'
for sec in secondary:
workflow_dependencies[sec].add( wfo.id )
if True:
## new style, failing on minbias
if not sec in destination_cache:
## this is barbbaric, and does not show the correct picture on workflow by workflow with different whitelist
destination_cache[sec],_ = getDatasetDestinations(url, sec) ## NO SITE WHITE LIST ADDED
#destination_cache[sec],_ = getDatasetDestinations(url, sec, within_sites = [SI.CE_to_SE(site) for site in sites_allowed])
## limit to the site whitelist NOW
se_allowed = [SI.CE_to_SE(site) for site in sites_allowed]
destinations = dict([(k,v) for (k,v) in destination_cache[sec].items() if site in se_allowed])
## truncate location/destination to those making up for >90% of the dataset
bad_destinations = [destinations.pop(site) for (site,info) in destinations.items() if info['data_fraction']<0.9]
sec_location = [site for (site,info) in destinations.items() if info['completion']>=95]
sec_destination = [site for site in destinations.keys() if not site in sec_location]
else:
## old style
presence = getDatasetPresence( url, sec )
sec_location = [site for site,pres in presence.items() if pres[1]>90.] ## more than 90% of the minbias at sites
subscriptions = listSubscriptions( url ,sec )
sec_destination = [site for site in subscriptions]
sec_to_distribute = [site for site in sites_allowed if not any([osite.startswith(site) for osite in sec_location])]
sec_to_distribute = [site for site in sec_to_distribute if not any([osite.startswith(site) for osite in sec_destination])]
sec_to_distribute = [site for site in sec_to_distribute if not any([osite.startswith(site) for osite in SI.sites_veto_transfer])]
if override_sec_destination:
## intersect with where we want the PU to be
not_needed_anymore = list(set(sec_to_distribute) - set(override_sec_destination))
#sendEmail("secondary superfluous","the dataset %s could be removed from %s"%( sec, not_needed_anymore ))
sendLog('transferor', "the dataset %s could be removed from %s"%( sec, not_needed_anymore ))
sec_to_distribute = list(set(sec_to_distribute) & set(override_sec_destination))
if len( sec_to_distribute )>0:
print "secondary could go to",sorted(sec_to_distribute)
sec_size = dss.get( sec )
for site in sec_to_distribute:
site_se =SI.CE_to_SE(site)
if (SI.disk[site_se]*1024.) > sec_size:
all_transfers[site].append( sec )
can_go = False
else:
print "could not send the secondary input to",site_se,"because it is too big for the available disk",SI.disk[site_se]*1024,"GB need",sec_size
if primary_destinations and site in primary_destinations:
#sendEmail('secondary input too big','%s is too big (%s) for %s (%s)'%( sec, sec_size, site_se, SI.disk[site_se]*1024))
sendLog('transferor', '%s is too big (%s) for %s (%s)'%( sec, sec_size, site_se, SI.disk[site_se]*1024), level='critical')
else:
print "the secondary input does not have to be send to site"
## is that possible to do something more
if can_go:
## no explicit transfer required this time
if staging:
## but using existing ones
wfh.sendLog('transferor', "latches on existing transfers, and nothing else, settin staging")
wfo.status = 'staging'
needs_transfer+=1
else:
wfh.sendLog('transferor', "should just be assigned now to %s"%sorted(sites_allowed))
wfo.status = 'staged'
passing_along+=1
wfh.sendLog('transferor', "setting status to %s"%wfo.status)
session.commit()
continue
else:
## there is an explicit transfer required
if staging:
## and also using an existing one
wfh.sendLog('transferor', "latches on existing transfers")
if not options.test:
wfo.status = 'staging'
wfh.sendLog('transferor', "setting status to %s"%wfo.status)
session.commit()
wfh.sendLog('transferor',"needs a transfer")
needs_transfer+=1
passing_along+=1
if no_goes:
#sendEmail("no go for managing","No go for \n"+"\n".join( no_goes ))
sendLog('transferor', "No go for \n"+"\n".join( no_goes ), level='critical')
print "accumulated transfers"
print json.dumps(all_transfers, indent=2)
fake_id=-1
wf_id_in_prestaging=set()
for (site,items_to_transfer) in all_transfers.iteritems():
items_to_transfer = list(set(items_to_transfer))
## convert to storage element
site_se = SI.CE_to_SE(site)
## site that do not want input datasets
if site in SI.sites_veto_transfer:
print site,"does not want transfers"
continue
## throttle the transfer size to T2s ? we'd be screwed by a noPU sample properly configured.
## massage a bit the items
blocks = [it for it in items_to_transfer if '#' in it]
block_datasets = list(set([it.split('#')[0] for it in blocks]))
datasets = [it for it in items_to_transfer if not '#' in it]
details_text = "Making a replica to %s (CE) %s (SE) for"%( site, site_se)
#print "\t",len(blocks),"blocks"
## remove blocks if full dataset is send out
blocks = [block for block in blocks if not block.split('#')[0] in datasets]
#print "\t",len(blocks),"needed blocks for",list(set([block.split('#')[0] for block in blocks]))
#print "\t",len(datasets),"datasets"
#print "\t",datasets
details_text += '\n\t%d blocks'%len(blocks)
details_text += '\n\t%d needed blocks for %s'%( len(blocks), sorted(list(set([block.split('#')[0] for block in blocks]))))
details_text += '\n\t%d datasets'% len(datasets)
details_text += '\n\t%s'%sorted(datasets)
items_to_transfer = blocks + datasets
if execute:
sendLog('transferor', details_text)
else:
print "Would make a replica to",site,"(CE)",site_se,"(SE) for"
print details_text
## operate the transfer
if options and options.stop:
## ask to move-on
answer = raw_input('Continue with that ?')
if not answer.lower() in ['y','yes','go']:
continue
if execute:
priority = 'normal'
cds = [ds for ds in datasets+block_datasets if ds in max_priority]
if cds and False: ## I don't think this is working. subscription should be updated on the fly and regularly for raising the priority if needed
## decide on an overall priority : that's a bit too large though
if any([max_priority[ds]>=90000 for ds in cds]):
priority = 'high'
elif all([max_priority[ds]<80000 for ds in cds]):
priority = 'low'
result = makeReplicaRequest(url, site_se, items_to_transfer, 'prestaging', priority=priority)
else:
result= {'phedex':{'request_created' : []}}
fake_id-=1
if not result:
print "ERROR Could not make a replica request for",site,items_to_transfer,"pre-staging"
continue
for phedexid in [o['id'] for o in result['phedex']['request_created']]:
new_transfer = session.query(Transfer).filter(Transfer.phedexid == int(phedexid)).first()
if not new_transfer:
new_transfer = session.query(Transfer).filter(Transfer.phedexid == -int(phedexid)).first()
print phedexid,"transfer created"
if not new_transfer:
new_transfer = Transfer( phedexid = phedexid)
session.add( new_transfer )
else:
new_transfer.phedexid = phedexid ## make it positive again
new_transfer.workflows_id = set()
for transfering in list(set(map(lambda it : it.split('#')[0], items_to_transfer))):
new_transfer.workflows_id.update( workflow_dependencies[transfering] )
new_transfer.workflows_id = list(new_transfer.workflows_id)
wf_id_in_prestaging.update(new_transfer.workflows_id)
session.commit()
## auto approve it
if execute:
approved = approveSubscription(url, phedexid, [site_se])
for wfid in wf_id_in_prestaging:
tr_wf = session.query(Workflow).get(wfid)
if tr_wf and tr_wf.status!='staging':
if execute:
tr_wf.status = 'staging'
if talk:
print "setting",tr_wf.name,"to staging"
session.commit()
def transferor(url, specific=None, talk=True, options=None):
if userLock(): return
mlock = moduleLock()
if mlock(): return
use_mcm = True
up = componentInfo(soft=['mcm', 'wtc', 'jira'])
if not up.check(): return
use_mcm = up.status['mcm']
if options and options.test:
execute = False
else:
execute = True
SI = siteInfo()
CI = campaignInfo()
#NLI = newLockInfo()
#if not NLI.free(): return
LI = lockInfo()
if not LI.free(): return
mcm = McMClient(dev=False)
dss = DSS()
#allowed_secondary = UC.get('')
print "counting all being handled..."
being_handled = len(
session.query(Workflow).filter(Workflow.status == 'away').all())
being_handled += len(
session.query(Workflow).filter(
Workflow.status.startswith('stag')).all())
being_transfered = len(
session.query(Workflow).filter(Workflow.status == 'staging').all())
#being_handled += len(session.query(Workflow).filter(Workflow.status.startswith('assistance-')).all())
being_handled += len(
session.query(Workflow).filter(
Workflow.status.startswith('assistance-')).filter(
~Workflow.status.contains('custodial')).all())
max_to_handle = options.maxworkflows
max_to_transfer = options.maxstaging
allowed_to_handle = max(0, max_to_handle - being_handled)
allowed_to_transfer = max(0, max_to_transfer - being_transfered)
wf_buffer = 5
if allowed_to_handle <= wf_buffer: ## buffer for having several wf per transfer
print "Not allowed to run more than", max_to_handle, "at a time. Currently", being_handled, "and", wf_buffer, "buffer"
else:
print being_handled, "already being handled", max_to_handle, "max allowed,", allowed_to_handle, "remaining", "and", wf_buffer, "buffer"
if allowed_to_transfer <= wf_buffer:
print "Not allowed to transfer more than", max_to_transfer, "at a time. Currently", being_transfered, "and", wf_buffer, "buffer"
else:
print being_transfered, "already being transfered", max_to_transfer, "max allowed,", allowed_to_transfer, "remaining", "and", wf_buffer, "buffer"
print "... done"
all_transfers = defaultdict(list)
workflow_dependencies = defaultdict(
set) ## list of wf.id per input dataset
wfs_and_wfh = []
max_per_round = UC.get('max_per_round').get('transferor', None)
print "getting all wf to consider ..."
cache = getWorkflows(url, 'assignment-approved', details=True)
all_to_include = session.query(Workflow).filter(
Workflow.status.startswith('considered')).all()
if len(cache) > 2000:
max_to_include = max_per_round
random.shuffle(cache) ## randomize first by wf name
cache = sorted(cache, key=lambda r: r['RequestPriority'],
reverse=True) ## order by prio
highest = [r['RequestName'] for r in cache[:max_to_include]]
all_to_include = [wfo for wfo in all_to_include if wfo.name in highest]
print "limiting what to consider to", max_to_include, "because there is too much stuff going on. Got", len(
all_to_include)
for wfo in all_to_include:
print "\t", wfo.name
if specific and not specific in wfo.name: continue
cache_r = filter(lambda d: d['RequestName'] == wfo.name, cache)
if len(cache_r):
wfs_and_wfh.append((wfo,
workflowInfo(url,
wfo.name,
spec=False,
request=cache_r[0])))
else:
wfs_and_wfh.append((wfo, workflowInfo(url, wfo.name, spec=False)))
print "... done"
transfers_per_sites = defaultdict(int)
input_sizes = defaultdict(float)
ignored_input_sizes = defaultdict(float)
input_cput = {}
input_st = {}
## list the size of those in transfer already
in_transfer_priority = None
min_transfer_priority = None
print "getting all wf in staging ..."
#stucks = json.loads(open('%s/stuck_transfers.json'%monitor_pub_dir).read())
stucks = json.loads(eosRead('%s/stuck_transfers.json' % monitor_pub_dir))
for wfo in session.query(Workflow).filter(
Workflow.status == 'staging').all():
wfh = workflowInfo(url, wfo.name, spec=False)
#(lheinput,primary,parent,secondary) = wfh.getIO()
#sites_allowed = getSiteWhiteList( (lheinput,primary,parent,secondary) )
print wfo.name, "staging"
(lheinput, primary, parent, secondary,
sites_allowed) = wfh.getSiteWhiteList()
for site in sites_allowed: ## we should get the actual transfer destination instead of the full white list
transfers_per_sites[site] += 1
#input_cput[wfo.name] = wfh.getComputingTime()
#input_st[wfo.name] = wfh.getSystemTime()
blocks = wfh.getBlocks()
for prim in primary:
ds_s = dss.get(prim, blocks=blocks)
if prim in stucks:
wfh.sendLog(
'transferor',
"%s appears stuck, so not counting it %s [GB]" %
(prim, ds_s))
ignored_input_sizes[prim] = max(ds_s,
ignored_input_sizes[prim])
else:
input_sizes[prim] = max(ds_s, input_sizes[prim])
wfh.sendLog('transferor',
"%s needs %s [GB]" % (wfo.name, ds_s))
if in_transfer_priority == None:
in_transfer_priority = int(wfh.request['RequestPriority'])
else:
in_transfer_priority = max(in_transfer_priority,
int(wfh.request['RequestPriority']))
if min_transfer_priority == None:
min_transfer_priority = int(wfh.request['RequestPriority'])
else:
min_transfer_priority = min(min_transfer_priority,
int(wfh.request['RequestPriority']))
try:
print "Ignored input sizes"
ignored_values = list(ignored_input_sizes.items())
ignored_values.sort(key=lambda i: i[1])
print "\n".join(map(str, ignored_values))
print "Considered input sizes"
considered_values = list(input_sizes.items())
considered_values.sort(key=lambda i: i[1])
print "\n".join(map(str, considered_values))
except Exception as e:
print "trying to print the summary of input size"
print str(e)
print "... done"
print "Max priority in transfer already", in_transfer_priority
print "Min priority in transfer already", min_transfer_priority
print "transfers per sites"
print json.dumps(transfers_per_sites, indent=2)
in_transfer_already = sum(input_sizes.values())
cput_in_transfer_already = sum(input_cput.values())
st_in_transfer_already = sum(input_st.values())
## list the size of all inputs
primary_input_per_workflow_gb = defaultdict(float)
print "getting all input sizes ..."
input_blocks = {}
for (wfo, wfh) in wfs_and_wfh:
(_, primary, _, _) = wfh.getIO()
blocks = wfh.getBlocks()
input_blocks[wfo.name] = blocks
for prim in primary:
## do not count it if it appears stalled !
prim_size = dss.get(prim, blocks=blocks)
input_sizes[prim] = max(prim_size, input_sizes[prim])
primary_input_per_workflow_gb[wfo.name] += prim_size
print "... done"
# shuffle first by name
random.shuffle(wfs_and_wfh)
# Sort smallest transfers first; allows us to transfer as many as possible workflows.
def prio_and_size(i, j):
if int(i[1].request['RequestPriority']) == int(
j[1].request['RequestPriority']):
return cmp(int(primary_input_per_workflow_gb.get(j[0].name, 0)),
int(primary_input_per_workflow_gb.get(i[0].name, 0)))
else:
return cmp(int(i[1].request['RequestPriority']),
int(j[1].request['RequestPriority']))
#wfs_and_wfh.sort(cmp = prio_and_size, reverse=True)
#wfs_and_wfh.sort(cmp = lambda i,j : cmp(int(primary_input_per_workflow_gb.get(i[0].name, 0)), int(primary_input_per_workflow_gb.get(j[0].name, 0)) ))
#sort by priority higher first
wfs_and_wfh.sort(cmp=lambda i, j: cmp(int(i[1].request[
'RequestPriority']), int(j[1].request['RequestPriority'])),
reverse=True)
if min_transfer_priority == None or in_transfer_priority == None:
print "nothing is lining up for transfer"
sendLog(
"transferor",
"No request in staging, using first request to set priority limit")
if len(wfs_and_wfh):
min_transfer_priority = wfs_and_wfh[0][1].request[
'RequestPriority']
in_transfer_priority = wfs_and_wfh[0][1].request['RequestPriority']
else:
return
cput_grand_total = sum(input_cput.values())
cput_to_transfer = cput_grand_total - cput_in_transfer_already
st_grand_total = sum(input_st.values())
st_to_transfer = st_grand_total - st_in_transfer_already
print "%15.4f [CPU h] worth already in transfer" % cput_in_transfer_already
print "%15.4f [CPU h] worth is the current requested transfer load" % cput_to_transfer
print "%15.4f [h] worth of absolute system time in transfer" % (
cput_in_transfer_already / SI.availableSlots())
print "%15.4f [h] worth of absolute system time is the current requested transfer load" % (
cput_to_transfer / SI.availableSlots())
print "%15.4f [h] worth of theoritical system time in transfer" % (
st_in_transfer_already)
print "%15.4f [h] worth of theoritical system time is the current requested transfer load" % (
st_to_transfer)
grand_total = sum(input_sizes.values())
to_transfer = grand_total - in_transfer_already
grand_transfer_limit = options.maxtransfer
#grand_transfer_limit = SI.total_disk()*0.25*1024## half of the free sapce in TB->GB
transfer_limit = grand_transfer_limit - in_transfer_already
print "%15.4f GB already being transfered" % in_transfer_already
print "%15.4f GB is the current requested transfer load" % to_transfer
print "%15.4f GB is the global transfer limit" % grand_transfer_limit
print "%15.4f GB is the available limit" % transfer_limit
max_staging_per_site = options.maxstagingpersite
# the max priority value per dataset.
max_priority = defaultdict(int)
needs_transfer = 0 ## so that we can count'em
passing_along = 0
transfer_sizes = defaultdict(float)
went_over_budget = False
destination_cache = {}
no_goes = set()
if max_per_round and not spec:
wfs_and_wfh = wfs_and_wfh[:max_per_round]
for (wfo, wfh) in wfs_and_wfh:
print wfo.name, "to be transfered with priority", wfh.request[
'RequestPriority']
if wfh.request['RequestStatus'] != 'assignment-approved':
if wfh.request['RequestStatus'] in [
'aborted', 'rejected', 'rejected-archived',
'aborted-archived'
]:
if wfh.isRelval():
wfo.status = 'forget'
else:
wfo.status = 'trouble' ## so that we look or a replacement
else:
wfo.status = 'away'
wfh.sendLog(
'transferor', '%s in status %s, setting %s' %
(wfo.name, wfh.request['RequestStatus'], wfo.status))
continue
(lheinput, primary, parent, secondary,
sites_allowed) = wfh.getSiteWhiteList()
blocks = input_blocks.get(wfo.name, wfh.getBlocks())
if blocks:
print "Reading only", len(blocks), "blocks in input"
this_load = sum([dss.get(prim, blocks=blocks) for prim in primary])
no_budget = False
if (this_load
and (sum(transfer_sizes.values()) + this_load > transfer_limit
or went_over_budget)):
if went_over_budget:
wfh.sendLog('transferor', "Transfer has gone over bubget.")
else:
wfh.sendLog('transferor', "Transfer will go over bubget.")
wfh.sendLog(
'transferor',
"%15.4f GB this load, %15.4f GB already this round, %15.4f GB is the available limit"
% (this_load, sum(transfer_sizes.values()), transfer_limit))
#if sum(transfer_sizes.values()) > transfer_limit:
went_over_budget = True
if in_transfer_priority != None and min_transfer_priority != None:
if int(
wfh.request['RequestPriority']
) >= in_transfer_priority and min_transfer_priority != in_transfer_priority:
wfh.sendLog(
'transferor',
"Higher priority sample %s >= %s go-on over budget" %
(wfh.request['RequestPriority'], in_transfer_priority))
else:
if not options.go:
wfh.sendLog(
'transferor',
"%s minimum priority %s < %s : stop" %
(min_transfer_priority,
wfh.request['RequestPriority'],
in_transfer_priority))
no_budget = True
## throtlle by campaign go
no_go = False
if not wfh.go(log=True) and not options.go:
no_go = True
no_goes.add(wfo.name)
allowed_secondary = {}
overide_parameters = {}
check_secondary = (not wfh.isRelval())
output_tiers = list(
set([o.split('/')[-1] for o in wfh.request['OutputDatasets']]))
for campaign in wfh.getCampaigns():
if campaign in CI.campaigns:
overide_parameters.update(CI.campaigns[campaign])
if campaign in CI.campaigns and 'secondaries' in CI.campaigns[
campaign]:
if CI.campaigns[campaign]['secondaries']:
allowed_secondary.update(
CI.campaigns[campaign]['secondaries'])
check_secondary = True
if campaign in CI.campaigns and 'banned_tier' in CI.campaigns[
campaign]:
banned_tier = list(
set(CI.campaigns[campaign]['banned_tier'])
& set(output_tiers))
if banned_tier:
no_go = True
wfh.sendLog(
'transferor', 'These data tiers %s are not allowed' %
(','.join(banned_tier)))
sendLog('transferor',
'These data tiers %s are not allowed in %s' %
(','.join(banned_tier), wfo.name),
level='critical')
if secondary and check_secondary:
if (set(secondary) & set(allowed_secondary.keys()) !=
set(secondary)):
msg = '%s is not an allowed secondary' % (
', '.join(set(secondary) - set(allowed_secondary.keys())))
wfh.sendLog('transferor', msg)
critical_msg = msg + '\nWorkflow URL: https://dmytro.web.cern.ch/dmytro/cmsprodmon/workflows.php?prep_id=task_{}'.format(
wfh.getPrepIDs()[0])
sendLog('transferor', critical_msg, level='critical')
if not options.go:
no_go = True
for sec in secondary:
if sec in allowed_secondary:
overide_parameters.update(allowed_secondary[sec])
if 'SiteWhitelist' in overide_parameters:
sites_allowed = list(
set(sites_allowed) & set(overide_parameters['SiteWhitelist']))
wfh.sendLog(
'transferor',
'Intersecting with the overriding whitelist parameters, allowed sites become {}'
.format(sites_allowed))
if no_go:
continue
if passing_along >= allowed_to_handle:
#if int(wfh.request['RequestPriority']) >= in_transfer_priority and min_transfer_priority!=in_transfer_priority:
if in_transfer_priority != None and min_transfer_priority != None:
if int(wfh.request['RequestPriority']
) >= in_transfer_priority and int(
wfh.request['RequestPriority']
) != min_transfer_priority:
## higher priority, and not only this priority being transfered
wfh.sendLog(
'transferor',
"Higher priority sample %s >= %s go-on over %s" %
(wfh.request['RequestPriority'], in_transfer_priority,
max_to_handle))
else:
wfh.sendLog(
'transferor',
" Not allowed to pass more than %s at a time. Currently %s handled, and adding %s"
% (max_to_handle, being_handled, passing_along))
if not options.go:
## should not allow to jump that fence
break
if this_load and needs_transfer >= allowed_to_transfer:
if in_transfer_priority != None and min_transfer_priority != None:
if int(wfh.request['RequestPriority']
) >= in_transfer_priority and int(
wfh.request['RequestPriority']
) != min_transfer_priority:
## higher priority, and not only this priority being transfered
wfh.sendLog(
'transferor',
"Higher priority sample %s >= %s go-on over %s" %
(wfh.request['RequestPriority'], in_transfer_priority,
max_to_transfer))
else:
wfh.sendLog(
'transferor',
"Not allowed to transfer more than %s at a time. Currently %s transfering, and adding %s"
% (max_to_transfer, being_transfered, needs_transfer))
if not options.go:
no_budget = True
if no_budget:
continue
# break ## try this for a while to make things faster
## the site white list considers site, campaign, memory and core information
if options and options.tosites:
sites_allowed = options.tosites.split(',')
for dataset in list(primary) + list(parent) + list(secondary):
LI.lock(dataset, reason='staging')
if not sites_allowed:
wfh.sendLog('transferor', "not possible site to run at")
#sendEmail("no possible sites","%s has no possible sites to run at"%( wfo.name ))
sendLog('transferor',
"%s has no possible sites to run at" % (wfo.name),
level='critical')
continue
can_go = True
staging = False
allowed = True
primary_destinations = set()
if primary:
copies_needed_from_CPUh, CPUh = wfh.getNCopies()
if talk:
print wfo.name, 'reads', ', '.join(primary), 'in primary'
## chope the primary dataset
for prim in primary:
## keep track of what needs what
workflow_dependencies[prim].add(wfo.id)
max_priority[prim] = max(max_priority[prim],
int(wfh.request['RequestPriority']))
wfh.sendLog(
'transferor', "Would make %s from cpu requirement %s" %
(copies_needed_from_CPUh, CPUh))
copies_needed = copies_needed_from_CPUh
if 'Campaign' in wfh.request and wfh.request[
'Campaign'] in CI.campaigns and 'maxcopies' in CI.campaigns[
wfh.request['Campaign']]:
copies_needed_from_campaign = CI.campaigns[
wfh.request['Campaign']]['maxcopies']
copies_needed = min(copies_needed_from_campaign,
copies_needed)
wfh.sendLog(
'transferor',
"Maxed to %s by campaign configuration %s" %
(copies_needed, wfh.request['Campaign']))
if blocks:
print "limiting to blocks", "\n".join(sorted(blocks))
### new ways of making the whole thing
destinations, all_block_names = getDatasetDestinations(
url,
prim,
within_sites=[SI.CE_to_SE(site) for site in sites_allowed],
only_blocks=blocks)
print json.dumps(destinations, indent=2)
## get where the dataset is in full and completed
prim_location = [
site for (site, info) in destinations.items()
if info['completion'] == 100 and info['data_fraction'] == 1
]
## the rest is places it is going to be
#prim_destination = [site for site in destinations.keys() if not site in prim_location]
prim_destination = [
site for (site, info) in destinations.items()
if info['data_fraction'] == 1 and info['completion'] != 100
]
## veto the site with no current disk space, for things that are not relval
prim_destination = [
site for site in prim_destination
if (SI.disk[site] or wfh.isRelval())
]
if len(prim_location) >= copies_needed:
wfh.sendLog(
'transferor',
"The input is all fully in place at %s sites %s" %
(len(prim_location), sorted(prim_location)))
continue
copies_needed = max(0, copies_needed - len(prim_location))
wfh.sendLog(
'transferor',
"Counting existing copies ; now need %s" % copies_needed)
copies_being_made = [
sum([
info['blocks'].keys().count(block)
for site, info in destinations.items()
if site in prim_destination
]) for block in all_block_names
]
latching_on_transfers = set()
[
latching_on_transfers.update(info['blocks'].values())
for site, info in destinations.items()
if site in prim_destination
]
latching_on_transfers = list(latching_on_transfers)
#print latching_on_transfers
## figure out where all this is going to go
prim_to_distribute = [
site for site in sites_allowed
if not SI.CE_to_SE(site) in prim_location
]
prim_to_distribute = [
site for site in prim_to_distribute
if not SI.CE_to_SE(site) in prim_destination
]
## take out the ones that cannot receive transfers
potential_destinations = len(prim_to_distribute)
#prim_to_distribute = [site for site in prim_to_distribute if not SI.CE_to_SE(site) in SI.sites_veto_transfer]
prim_to_distribute = [
site for site in prim_to_distribute
if (SI.disk[SI.CE_to_SE(site)] or wfh.isRelval())
]
## do we want to restrict transfers if the amount of site in vetoe are too large ?
wfh.sendLog(
'transferor',
"Could be going to: %s" % sorted(prim_to_distribute))
if not prim_to_distribute or any([
transfers_per_sites[site] < max_staging_per_site
for site in prim_to_distribute
]):
## means there is openings let me go
print "There are transfer slots available:", [
(site, transfers_per_sites[site])
for site in prim_to_distribute
]
else:
if int(
wfh.request['RequestPriority']
) >= in_transfer_priority and min_transfer_priority != in_transfer_priority:
wfh.sendLog(
'transferor',
"Higher priority sample %s >= %s go-on over transfer slots available"
% (wfh.request['RequestPriority'],
in_transfer_priority))
else:
wfh.sendLog(
'transferor',
"Not allowed to transfer more than %s per site at a time. Going overboard for %s"
% (max_staging_per_site,
sorted([
site for site in prim_to_distribute
if transfers_per_sites[site] >=
max_staging_per_site
])))
if not options.go:
allowed = False
break
for latching in latching_on_transfers:
existings = session.query(TransferImp).filter(
TransferImp.phedexid == int(latching)).filter(
TransferImp.workflow_id == wfo.id).all()
if not existings:
tri = TransferImp(phedexid=int(latching), workflow=wfo)
print "adding", wfo.id, "with phedexid", latching
session.add(tri)
else:
for existing in existings:
existing.active = True
session.flush()
can_go = False
transfer_sizes[prim] = max(this_load, transfer_sizes[prim])
staging = True
# reduce the number of copies required by the on-going full transfer : how do we bootstrap on waiting for them ??
#copies_needed = max(0,copies_needed - len(prim_destination))
copies_needed = max(0, copies_needed - min(copies_being_made))
wfh.sendLog(
'transferor',
"Counting the copies being made ; then need %s" %
copies_needed)
if copies_needed == 0:
wfh.sendLog(
'transferor',
"The input is either fully in place or getting in full somewhere with %s"
% latching_on_transfers)
can_go = True
continue
elif len(prim_to_distribute) == 0:
wfh.sendLog(
'transferor',
"We are going to need extra copies of %s, but no destinations seems available"
% (prim))
sendLog(
'transferor',
"We are going to need extra copies of %s, but no destinations seems available"
% (prim),
level='critical')
print json.dumps(prim_to_distribute, indent=2)
print json.dumps(prim_location, indent=2)
print json.dumps(prim_destination, indent=2)
prim_to_distribute = [
site for site in sites_allowed
if not SI.CE_to_SE(site) in prim_location
]
#prim_to_distribute = [site for site in prim_to_distribute if not SI.CE_to_SE(site) in SI.sites_veto_transfer ]
prim_to_distribute = [
site for site in prim_to_distribute
if (SI.disk[SI.CE_to_SE(site)] or wfh.isRelval())
]
print "changed to"
print json.dumps(prim_to_distribute, indent=2)
if len(
prim_to_distribute
) > 0: ## maybe that a parameter we can play with to limit the
if not options or options.chop:
### hard include the tape disk andpoint ?
#tapes = [site for site in getDatasetPresence( url, prim, vetos=['T0','T2','T3','Disk']) if site.endswith('MSS')]
chops, sizes = getDatasetChops(
prim,
chop_threshold=options.chopsize,
only_blocks=blocks)
spreading = distributeToSites(chops,
prim_to_distribute,
n_copies=copies_needed,
weights=SI.cpu_pledges,
sizes=sizes)
## prune the blocks/destination that are already in the making, so that subscription don't overlap
for site in spreading:
for block in list(spreading[site]):
if site in destinations and block in destinations[
site]['blocks'].keys():
## prune it
spreading[site].remove(block)
transfer_sizes[prim] = sum(sizes)
if not spreading:
sendLog(
'transferor',
'cannot send %s to any site, it cannot fit anywhere'
% prim,
level='critical')
wfh.sendLog(
'transferor',
"cannot send to any site. %s cannot seem to fit anywhere"
% (prim))
staging = False
can_go = False
else:
spreading = {}
for site in prim_to_distribute:
if blocks:
spreading[site] = blocks
else:
spreading[site] = [prim]
transfer_sizes[prim] = max(this_load,
transfer_sizes[prim])
can_go = False
wfh.sendLog(
'transferor', "selected CE destinations %s" %
(sorted(spreading.keys())))
for (site, items) in spreading.items():
all_transfers[site].extend(items)
transfers_per_sites[site] += 1
primary_destinations.add(site)
else:
can_go = False
allowed = False
if not allowed:
wfh.sendLog('transferor', "Not allowed to move on with")
continue
if secondary:
override_sec_destination = []
if 'SecondaryLocation' in CI.campaigns[wfh.request['Campaign']]:
override_sec_destination = CI.campaigns[
wfh.request['Campaign']]['SecondaryLocation']
if 'SecondaryLocation' in overide_parameters:
override_sec_destination = overide_parameters[
'SecondaryLocation']
print wfo.name, 'reads', ', '.join(secondary), 'in secondary'
for sec in secondary:
workflow_dependencies[sec].add(wfo.id)
if True:
## new style, failing on minbias
if not sec in destination_cache:
## this is barbbaric, and does not show the correct picture on workflow by workflow with different whitelist
destination_cache[sec], _ = getDatasetDestinations(
url, sec) ## NO SITE WHITE LIST ADDED
#destination_cache[sec],_ = getDatasetDestinations(url, sec, within_sites = [SI.CE_to_SE(site) for site in sites_allowed])
## limit to the site whitelist NOW
se_allowed = set(
[SI.CE_to_SE(site) for site in sites_allowed])
destinations = dict([
(k, v) for (k, v) in destination_cache[sec].items()
if k in se_allowed
])
## truncate location/destination to those making up for >90% of the dataset
bad_destinations = [
destinations.pop(site)
for (site, info) in destinations.items()
if info['data_fraction'] < 0.9
]
print sec, json.dumps(destinations, indent=2)
sec_location = [
site for (site, info) in destinations.items()
if info['completion'] >= 95
]
sec_destination = [
site for site in destinations.keys()
if not site in sec_location
] ## this is in SE
else:
## old style
presence = getDatasetPresence(url, sec)
sec_location = [
site for site, pres in presence.items()
if pres[1] > 90.
] ## more than 90% of the minbias at sites
subscriptions = listSubscriptions(url, sec)
sec_destination = [site for site in subscriptions]
## how to make unified understand that it has to wait for the secondary if the sec_destination and
#sec_to_distribute = [site for site in sites_allowed if not any([osite.startswith(site) for osite in sec_location])]
sec_to_distribute = [
site for site in sites_allowed
if not SI.CE_to_SE(site) in sec_location
]
#sec_to_distribute = [site for site in sec_to_distribute if not any([osite.startswith(site) for osite in sec_destination])]
sec_to_distribute = [
site for site in sec_to_distribute
if not SI.CE_to_SE(site) in sec_destination
]
presitespace_sec_to_distribute = copy.deepcopy(
sec_to_distribute)
#sec_to_distribute = [site for site in sec_to_distribute if not any([osite.startswith(site) for osite in SI.sites_veto_transfer])]
#sec_to_distribute = [site for site in sec_to_distribute if not SI.CE_to_SE(site) in SI.sites_veto_transfer]
sec_to_distribute = [
site for site in sec_to_distribute
if (SI.disk[SI.CE_to_SE(site)] or wfh.isRelval())
]
## at this point you have a problem
if len(sec_to_distribute) == 0 and len(
presitespace_sec_to_distribute):
sendLog(
'transferor',
'%s is getting no possible destinations because of lack of space. To be decided what to do in general'
% (sec),
level='critical')
if override_sec_destination:
## intersect with where we want the PU to be
not_needed_anymore = list(
set(sec_to_distribute) - set(override_sec_destination))
#sendEmail("secondary superfluous","the dataset %s could be removed from %s"%( sec, not_needed_anymore ))
sendLog(
'transferor',
"the dataset %s could be removed from %s" %
(sec, not_needed_anymore))
sec_to_distribute = list(
set(sec_to_distribute) & set(override_sec_destination))
if len(sec_to_distribute) > 0:
print "secondary could go to", sorted(sec_to_distribute)
sec_size = dss.get(sec)
for site in sec_to_distribute:
site_se = SI.CE_to_SE(site)
if (SI.disk[site_se] *
1024.) > sec_size or wfh.isRelval():
wfh.sendLog('transferor',
'Sending %s to %s' % (sec, site))
all_transfers[site].append(sec)
can_go = False
else:
print "could not send the secondary input to", site_se, "because it is too big for the available disk", SI.disk[
site_se] * 1024, "GB need", sec_size
if primary_destinations and site in primary_destinations:
#sendEmail('secondary input too big','%s is too big (%s) for %s (%s)'%( sec, sec_size, site_se, SI.disk[site_se]*1024))
sendLog(
'transferor',
'%s is too big (%s) for %s (%s). %s will not be able to run there.'
% (sec, sec_size, site_se,
SI.disk[site_se] * 1024, wfo.name),
level='critical')
wfh.sendLog(
'transferor',
'%s is too big (%s) for %s (%s). will not be able to run there.'
% (sec, sec_size, site_se,
SI.disk[site_se] * 1024))
else:
## this is bas overall
print "the secondary input does not have to be send to site"
## is that possible to do something more
if can_go:
## no explicit transfer required this time
if staging:
## but using existing ones
wfh.sendLog(
'transferor',
"latches on existing transfers, and nothing else, settin staging"
)
wfo.status = 'staging'
needs_transfer += 1
else:
wfh.sendLog(
'transferor', "should just be assigned now to %s" %
sorted(sites_allowed))
wfo.status = 'staged'
passing_along += 1
wfh.sendLog('transferor',
"setting %s status to %s" % (wfo.name, wfo.status))
#session.commit()
continue
else:
## there is an explicit transfer required
if staging:
## and also using an existing one
wfh.sendLog('transferor', "latches on existing transfers")
if not options.test:
wfo.status = 'staging'
wfh.sendLog(
'transferor',
"setting %s status to %s" % (wfo.name, wfo.status))
#session.commit()
wfh.sendLog('transferor', "needs a transfer")
needs_transfer += 1
passing_along += 1
if no_goes:
#sendEmail("no go for managing","No go for \n"+"\n".join( no_goes ))
sendLog('transferor',
"No go for \n" + "\n".join(sorted(no_goes)),
level='critical')
print "accumulated transfers"
print json.dumps(all_transfers, indent=2)
fake_id = -1
wf_id_in_prestaging = set()
for (site, items_to_transfer) in all_transfers.iteritems():
items_to_transfer = list(set(items_to_transfer))
## convert to storage element
site_se = SI.CE_to_SE(site)
## site that do not want input datasets
#if site in SI.sites_veto_transfer:
# print site,"does not want transfers"
# continue
## throttle the transfer size to T2s ? we'd be screwed by a noPU sample properly configured.
## massage a bit the items
blocks = [it for it in items_to_transfer if '#' in it]
block_datasets = list(set([it.split('#')[0] for it in blocks]))
datasets = [it for it in items_to_transfer if not '#' in it]
details_text = "Making a replica to %s (CE) %s (SE) for" % (site,
site_se)
#print "\t",len(blocks),"blocks"
## remove blocks if full dataset is send out
blocks = [
block for block in blocks if not block.split('#')[0] in datasets
]
#print "\t",len(blocks),"needed blocks for",list(set([block.split('#')[0] for block in blocks]))
#print "\t",len(datasets),"datasets"
#print "\t",datasets
details_text += '\n\t%d blocks' % len(blocks)
details_text += '\n\t%d needed blocks for %s' % (
len(blocks),
sorted(list(set([block.split('#')[0] for block in blocks]))))
details_text += '\n\t%d datasets' % len(datasets)
details_text += '\n\t%s' % sorted(datasets)
items_to_transfer = blocks + datasets
if execute:
sendLog('transferor', details_text)
else:
print "Would make a replica to", site, "(CE)", site_se, "(SE) for"
print details_text
## operate the transfer
if options and options.stop:
## ask to move-on
answer = raw_input('Continue with that ?')
if not answer.lower() in ['y', 'yes', 'go']:
continue
transfered_items = defaultdict(set)
if execute:
priority = 'normal'
cds = [
ds for ds in set(datasets + block_datasets)
if ds in max_priority
]
## bucketize the transfers by priority of workflows
prioritized_items = defaultdict(set)
for item in items_to_transfer:
d = item.split('#')[0]
p = max_priority.get(d, 80000)
q = 'normal'
if p > 100000:
q = 'reserved'
elif p < 70000:
q = 'low'
prioritized_items[q].add(item)
for priority, items in prioritized_items.items():
result = makeReplicaRequest(url,
site_se,
list(items),
'prestaging',
priority=priority,
approve=True)
if result:
these_transfers = [
o['id'] for o in result['phedex']['request_created']
]
#phedexids.extend( these_transfers )
for ph in these_transfers:
transfered_items[ph].update(items)
else:
sendLog(
'transferor',
'Could not make a replica request for items %s to site %s'
% (items, site_se),
level='critical')
#result = makeReplicaRequest(url, site_se, items_to_transfer, 'prestaging', priority=priority, approve=True)
#phedexids = [o['id'] for o in result['phedex']['request_created']]:
#else:
# #result= {'phedex':{'request_created' : []}}
# phedexids = []
# fake_id-=1
if not transfered_items:
sendLog(
'transferor',
'Could not make a replica request for items %s to site %s' %
(items_to_transfer, site),
level='critical')
continue
for phedexid, items in transfered_items.items():
print phedexid, "transfer created"
for transfering in list(
set(map(lambda it: it.split('#')[0], items))):
for wfid in workflow_dependencies[transfering]:
new_transfer = session.query(TransferImp).filter(
TransferImp.phedexid == int(phedexid)).filter(
TransferImp.workflow_id == wfid).first()
if not new_transfer:
new_transfer = TransferImp(
phedexid=phedexid,
workflow=session.query(Workflow).get(wfid))
session.add(new_transfer)
else:
new_transfer.active = True
wf_id_in_prestaging.add(wfid)
#session.commit()
for wfid in wf_id_in_prestaging:
tr_wf = session.query(Workflow).get(wfid)
if tr_wf and tr_wf.status != 'staging':
if execute:
tr_wf.status = 'staging'
if talk:
print "setting", tr_wf.name, "to staging"
#session.commit()
## one big session commit at the end that everything went fine
session.commit()
def transferor(url, specific=None, talk=True, options=None):
if userLock(): return
if duplicateLock(): return
use_mcm = True
up = componentInfo(mcm=use_mcm, soft=['mcm'])
if not up.check(): return
use_mcm = up.status['mcm']
if options and options.test:
execute = False
else:
execute = True
SI = siteInfo()
CI = campaignInfo()
NLI = newLockInfo()
mcm = McMClient(dev=False)
dss = DSS()
#allowed_secondary = UC.get('')
print "counting all being handled..."
being_handled = len(
session.query(Workflow).filter(Workflow.status == 'away').all())
being_handled += len(
session.query(Workflow).filter(
Workflow.status.startswith('stag')).all())
being_transfered = len(
session.query(Workflow).filter(Workflow.status == 'staging').all())
being_handled += len(
session.query(Workflow).filter(
Workflow.status.startswith('assistance-')).all())
max_to_handle = options.maxworkflows
max_to_transfer = options.maxstaging
allowed_to_handle = max(0, max_to_handle - being_handled)
allowed_to_transfer = max(0, max_to_transfer - being_transfered)
wf_buffer = 5
if allowed_to_handle <= wf_buffer: ## buffer for having several wf per transfer
print "Not allowed to run more than", max_to_handle, "at a time. Currently", being_handled, "and", wf_buffer, "buffer"
else:
print being_handled, "already being handled", max_to_handle, "max allowed,", allowed_to_handle, "remaining", "and", wf_buffer, "buffer"
if allowed_to_transfer <= wf_buffer:
print "Not allowed to transfer more than", max_to_transfer, "at a time. Currently", being_transfered, "and", wf_buffer, "buffer"
else:
print being_transfered, "already being transfered", max_to_transfer, "max allowed,", allowed_to_transfer, "remaining", "and", wf_buffer, "buffer"
print "... done"
all_transfers = defaultdict(list)
workflow_dependencies = defaultdict(
set) ## list of wf.id per input dataset
wfs_and_wfh = []
print "getting all wf to consider ..."
cache = getWorkflows(url, 'assignment-approved', details=True)
for wfo in session.query(Workflow).filter(
Workflow.status.startswith('considered')).all():
print "\t", wfo.name
if specific and not specific in wfo.name: continue
cache_r = filter(lambda d: d['RequestName'] == wfo.name, cache)
if len(cache_r):
wfs_and_wfh.append((wfo,
workflowInfo(url,
wfo.name,
spec=False,
request=cache_r[0])))
else:
wfs_and_wfh.append((wfo, workflowInfo(url, wfo.name, spec=False)))
print "... done"
transfers_per_sites = defaultdict(int)
input_sizes = {}
ignored_input_sizes = {}
input_cput = {}
input_st = {}
## list the size of those in transfer already
in_transfer_priority = None
min_transfer_priority = None
print "getting all wf in staging ..."
stucks = json.loads(open('%s/stuck_transfers.json' % monitor_dir).read())
for wfo in session.query(Workflow).filter(
Workflow.status == 'staging').all():
wfh = workflowInfo(url, wfo.name, spec=False)
#(lheinput,primary,parent,secondary) = wfh.getIO()
#sites_allowed = getSiteWhiteList( (lheinput,primary,parent,secondary) )
(lheinput, primary, parent, secondary,
sites_allowed) = wfh.getSiteWhiteList()
for site in sites_allowed: ## we should get the actual transfer destination instead of the full white list
transfers_per_sites[site] += 1
#input_cput[wfo.name] = wfh.getComputingTime()
#input_st[wfo.name] = wfh.getSystemTime()
for prim in primary:
ds_s = dss.get(prim)
if prim in stucks:
sendLog('transferor',
"%s appears stuck, so not counting it %s [GB]" %
(prim, ds_s),
wfi=wfh)
ignored_input_sizes[prim] = ds_s
else:
input_sizes[prim] = ds_s
sendLog('transferor',
"%s needs %s [GB]" % (wfo.name, ds_s),
wfi=wfh)
if in_transfer_priority == None:
in_transfer_priority = int(wfh.request['RequestPriority'])
else:
in_transfer_priority = max(in_transfer_priority,
int(wfh.request['RequestPriority']))
if min_transfer_priority == None:
min_transfer_priority = int(wfh.request['RequestPriority'])
else:
min_transfer_priority = min(min_transfer_priority,
int(wfh.request['RequestPriority']))
if min_transfer_priority == None or in_transfer_priority == None:
print "nothing is lining up for transfer"
sendEmail("no request in staging", "no request in staging")
return
pass
try:
print "Ignored input sizes"
ignored_values = list(ignored_input_sizes.items())
ignored_values.sort(key=lambda i: i[1])
print "\n".join(map(str, ignored_values))
print "Considered input sizes"
considered_values = list(input_sizes.items())
considered_values.sort(key=lambda i: i[1])
print "\n".join(map(str, considered_values))
except Exception as e:
print "trying to print the summary of input size"
print str(e)
print "... done"
print "Max priority in transfer already", in_transfer_priority
print "Min priority in transfer already", min_transfer_priority
print "transfers per sites"
print json.dumps(transfers_per_sites, indent=2)
in_transfer_already = sum(input_sizes.values())
cput_in_transfer_already = sum(input_cput.values())
st_in_transfer_already = sum(input_st.values())
## list the size of all inputs
primary_input_per_workflow_gb = defaultdict(float)
print "getting all input sizes ..."
for (wfo, wfh) in wfs_and_wfh:
(_, primary, _, _) = wfh.getIO()
#input_cput[wfo.name] = wfh.getComputingTime()
#input_st[wfo.name] = wfh.getSystemTime()
for prim in primary:
## do not count it if it appears stalled !
prim_size = dss.get(prim)
input_sizes[prim] = prim_size
primary_input_per_workflow_gb[wfo.name] += prim_size
print "... done"
# shuffle first by name
random.shuffle(wfs_and_wfh)
# Sort smallest transfers first; allows us to transfer as many as possible workflows.
def prio_and_size(i, j):
if int(i[1].request['RequestPriority']) == int(
j[1].request['RequestPriority']):
return cmp(int(primary_input_per_workflow_gb.get(j[0].name, 0)),
int(primary_input_per_workflow_gb.get(i[0].name, 0)))
else:
return cmp(int(i[1].request['RequestPriority']),
int(j[1].request['RequestPriority']))
#wfs_and_wfh.sort(cmp = prio_and_size, reverse=True)
#wfs_and_wfh.sort(cmp = lambda i,j : cmp(int(primary_input_per_workflow_gb.get(i[0].name, 0)), int(primary_input_per_workflow_gb.get(j[0].name, 0)) ))
#sort by priority higher first
wfs_and_wfh.sort(cmp=lambda i, j: cmp(int(i[1].request[
'RequestPriority']), int(j[1].request['RequestPriority'])),
reverse=True)
cput_grand_total = sum(input_cput.values())
cput_to_transfer = cput_grand_total - cput_in_transfer_already
st_grand_total = sum(input_st.values())
st_to_transfer = st_grand_total - st_in_transfer_already
print "%15.4f [CPU h] worth already in transfer" % cput_in_transfer_already
print "%15.4f [CPU h] worth is the current requested transfer load" % cput_to_transfer
print "%15.4f [h] worth of absolute system time in transfer" % (
cput_in_transfer_already / SI.availableSlots())
print "%15.4f [h] worth of absolute system time is the current requested transfer load" % (
cput_to_transfer / SI.availableSlots())
print "%15.4f [h] worth of theoritical system time in transfer" % (
st_in_transfer_already)
print "%15.4f [h] worth of theoritical system time is the current requested transfer load" % (
st_to_transfer)
grand_total = sum(input_sizes.values())
to_transfer = grand_total - in_transfer_already
grand_transfer_limit = options.maxtransfer
#grand_transfer_limit = SI.total_disk()*0.25*1024## half of the free sapce in TB->GB
transfer_limit = grand_transfer_limit - in_transfer_already
print "%15.4f GB already being transfered" % in_transfer_already
print "%15.4f GB is the current requested transfer load" % to_transfer
print "%15.4f GB is the global transfer limit" % grand_transfer_limit
print "%15.4f GB is the available limit" % transfer_limit
max_staging_per_site = options.maxstagingpersite
# the max priority value per dataset.
max_priority = defaultdict(int)
needs_transfer = 0 ## so that we can count'em
passing_along = 0
transfer_sizes = {}
went_over_budget = False
destination_cache = {}
no_goes = set()
max_per_round = UC.get('max_per_round').get('transferor', None)
if max_per_round and not spec:
wfs_and_wfh = wfs_and_wfh[:max_per_round]
for (wfo, wfh) in wfs_and_wfh:
print wfo.name, "to be transfered with priority", wfh.request[
'RequestPriority']
if wfh.request['RequestStatus'] != 'assignment-approved':
if wfh.request['RequestStatus'] in [
'aborted', 'rejected', 'rejected-archived',
'aborted-archived'
]:
wfo.status = 'trouble' ## so that we look or a replacement
else:
wfo.status = 'away'
wfh.sendLog(
'transferor', '%s in status %s, setting %s' %
(wfo.name, wfh.request['RequestStatus'], wfo.status))
continue
(_, primary, _, _) = wfh.getIO()
this_load = sum([input_sizes[prim] for prim in primary])
no_budget = False
if (this_load
and (sum(transfer_sizes.values()) + this_load > transfer_limit
or went_over_budget)):
if went_over_budget:
wfh.sendLog('transferor', "Transfer has gone over bubget.")
else:
wfh.sendLog('transferor', "Transfer will go over bubget.")
wfh.sendLog(
'transferor',
"%15.4f GB this load, %15.4f GB already this round, %15.4f GB is the available limit"
% (this_load, sum(transfer_sizes.values()), transfer_limit))
#if sum(transfer_sizes.values()) > transfer_limit:
went_over_budget = True
if in_transfer_priority != None and min_transfer_priority != None:
if int(
wfh.request['RequestPriority']
) >= in_transfer_priority and min_transfer_priority != in_transfer_priority:
wfh.sendLog(
'transferor',
"Higher priority sample %s >= %s go-on over budget" %
(wfh.request['RequestPriority'], in_transfer_priority))
else:
if not options.go:
wfh.sendLog(
'transferor',
"%s minimum priority %s < %s : stop" %
(min_transfer_priority,
wfh.request['RequestPriority'],
in_transfer_priority))
no_budget = True
## throtlle by campaign go
no_go = False
if not wfh.go(log=True) and not options.go:
no_go = True
no_goes.add(wfo.name)
allowed_secondary = set()
for campaign in wfh.getCampaigns():
if campaign in CI.campaigns and 'secondaries' in CI.campaigns[
campaign]:
allowed_secondary.update(CI.campaigns[campaign]['secondaries'])
if secondary:
if (secondary and allowed_secondary) and (
set(secondary) & allowed_secondary != set(secondary)):
wfh.sendLog(
'assignor', '%s is not an allowed secondary' %
(', '.join(set(secondary) - allowed_secondary)))
no_go = True
if no_go:
continue
## check if the batch is announced
def check_mcm(wfn):
announced = False
is_real = False
if not wfn.startswith('pdmvserv'):
is_real = True
try:
for b in mcm.getA('batches', query='contains=%s' % wfo.name):
is_real = True
if b['status'] == 'announced':
announced = True
break
except:
try:
for b in mcm.getA('batches',
query='contains=%s' % wfo.name):
is_real = True
if b['status'] == 'announced':
announced = True
break
except:
print "could not get mcm batch announcement, assuming not real"
return announced, is_real
if not use_mcm:
announced, is_real = False, True
else:
if wfh.request['RequestType'] in ['ReReco']:
announced, is_real = True, True
else:
announced, is_real = check_mcm(wfo.name)
if not announced:
wfh.sendLog('transferor', "does not look announced.")
if not is_real:
wfh.sendLog('transferor', "does not appear to be genuine.")
## prevent any duplication. if the wf is not mentioned in any batch, regardless of status
continue
## check on a grace period
injection_time = time.mktime(
time.strptime('.'.join(map(str, wfh.request['RequestDate'])),
"%Y.%m.%d.%H.%M.%S")) / (60. * 60.)
now = time.mktime(time.gmtime()) / (60. * 60.)
if float(now - injection_time) < 4.:
if not options.go and not announced:
wfh.sendLog(
'transferor',
"It is too soon to start transfer: %3.2fH remaining" %
(now - injection_time))
continue
if passing_along >= allowed_to_handle:
#if int(wfh.request['RequestPriority']) >= in_transfer_priority and min_transfer_priority!=in_transfer_priority:
if in_transfer_priority != None and min_transfer_priority != None:
if int(wfh.request['RequestPriority']
) >= in_transfer_priority and int(
wfh.request['RequestPriority']
) != min_transfer_priority:
## higher priority, and not only this priority being transfered
wfh.sendLog(
'transferor',
"Higher priority sample %s >= %s go-on over %s" %
(wfh.request['RequestPriority'], in_transfer_priority,
max_to_handle))
else:
wfh.sendLog(
'transferor',
" Not allowed to pass more than %s at a time. Currently %s handled, and adding %s"
% (max_to_handle, being_handled, passing_along))
if not options.go:
## should not allow to jump that fence
break
if this_load and needs_transfer >= allowed_to_transfer:
if in_transfer_priority != None and min_transfer_priority != None:
if int(wfh.request['RequestPriority']
) >= in_transfer_priority and int(
wfh.request['RequestPriority']
) != min_transfer_priority:
## higher priority, and not only this priority being transfered
wfh.sendLog(
'transferor',
"Higher priority sample %s >= %s go-on over %s" %
(wfh.request['RequestPriority'], in_transfer_priority,
max_to_transfer))
else:
wfh.sendLog(
'transferor',
"Not allowed to transfer more than %s at a time. Currently %s transfering, and adding %s"
% (max_to_transfer, being_transfered, needs_transfer))
if not options.go:
no_budget = True
if no_budget:
continue
## the site white list considers site, campaign, memory and core information
(lheinput, primary, parent, secondary,
sites_allowed) = wfh.getSiteWhiteList()
if options and options.tosites:
sites_allowed = options.tosites.split(',')
for dataset in list(primary) + list(parent) + list(secondary):
## lock everything flat
NLI.lock(dataset)
if not sites_allowed:
wfh.sendLog('transferor', "not possible site to run at")
#sendEmail("no possible sites","%s has no possible sites to run at"%( wfo.name ))
sendLog('transferor',
"%s has no possible sites to run at" % (wfo.name),
level='critical')
continue
blocks = []
if 'BlockWhitelist' in wfh.request and wfh.request['BlockWhitelist']:
blocks = wfh.request['BlockWhitelist']
if 'RunWhitelist' in wfh.request and wfh.request['RunWhitelist']:
## augment with run white list
for dataset in primary:
blocks = list(
set(blocks + getDatasetBlocks(
dataset, runs=wfh.request['RunWhitelist'])))
if 'LumiList' in wfh.request and wfh.request['LumiList']:
## augment with the lumi white list
blocks = list(
set(blocks +
getDatasetBlocks(dataset, lumis=wfh.request['LumiList'])))
if blocks:
print "Reading", len(blocks), "in block whitelist"
can_go = True
staging = False
allowed = True
primary_destinations = set()
if primary:
copies_needed_from_CPUh, CPUh = wfh.getNCopies()
if talk:
print wfo.name, 'reads', ', '.join(primary), 'in primary'
## chope the primary dataset
for prim in primary:
## keep track of what needs what
workflow_dependencies[prim].add(wfo.id)
max_priority[prim] = max(max_priority[prim],
int(wfh.request['RequestPriority']))
wfh.sendLog(
'transferor', "Would make %s from cpu requirement %s" %
(copies_needed_from_CPUh, CPUh))
copies_needed = copies_needed_from_CPUh
if 'Campaign' in wfh.request and wfh.request[
'Campaign'] in CI.campaigns and 'maxcopies' in CI.campaigns[
wfh.request['Campaign']]:
copies_needed_from_campaign = CI.campaigns[
wfh.request['Campaign']]['maxcopies']
copies_needed = min(copies_needed_from_campaign,
copies_needed)
wfh.sendLog(
'transferor',
"Maxed to %s by campaign configuration %s" %
(copies_needed, wfh.request['Campaign']))
### new ways of making the whole thing
destinations, all_block_names = getDatasetDestinations(
url,
prim,
within_sites=[SI.CE_to_SE(site) for site in sites_allowed],
only_blocks=blocks)
print json.dumps(destinations, indent=2)
## get where the dataset is in full and completed
prim_location = [
site for (site, info) in destinations.items()
if info['completion'] == 100 and info['data_fraction'] == 1
]
## the rest is places it is going to be
prim_destination = [
site for site in destinations.keys()
if not site in prim_location
]
if len(prim_location) >= copies_needed:
wfh.sendLog(
'transferor',
"The input is all fully in place at %s sites %s" %
(len(prim_location), sorted(prim_location)))
continue
copies_needed = max(0, copies_needed - len(prim_location))
wfh.sendLog(
'transferor',
"not counting existing copies ; now need %s" %
copies_needed)
copies_being_made = [
sum([
info['blocks'].keys().count(block)
for site, info in destinations.items()
if site in prim_destination
]) for block in all_block_names
]
latching_on_transfers = set()
[
latching_on_transfers.update(info['blocks'].values())
for site, info in destinations.items()
if site in prim_destination
]
latching_on_transfers = list(latching_on_transfers)
#print latching_on_transfers
## figure out where all this is going to go
prim_to_distribute = [
site for site in sites_allowed
if not SI.CE_to_SE(site) in prim_location
]
prim_to_distribute = [
site for site in prim_to_distribute
if not SI.CE_to_SE(site) in prim_destination
]
## take out the ones that cannot receive transfers
prim_to_distribute = [
site for site in prim_to_distribute if not any([
osite.startswith(site)
for osite in SI.sites_veto_transfer
])
]
wfh.sendLog(
'transferor',
"Could be going to: %s" % sorted(prim_to_distribute))
if not prim_to_distribute or any([
transfers_per_sites[site] < max_staging_per_site
for site in prim_to_distribute
]):
## means there is openings let me go
print "There are transfer slots available:", [
(site, transfers_per_sites[site])
for site in prim_to_distribute
]
#for site in sites_allowed:
# #increment accross the board, regardless of real destination: could be changed
# transfers_per_sites[site] += 1
else:
if int(
wfh.request['RequestPriority']
) >= in_transfer_priority and min_transfer_priority != in_transfer_priority:
wfh.sendLog(
'transferor',
"Higher priority sample %s >= %s go-on over transfer slots available"
% (wfh.request['RequestPriority'],
in_transfer_priority))
else:
wfh.sendLog(
'transferor',
"Not allowed to transfer more than %s per site at a time. Going overboard for %s"
% (max_staging_per_site,
sorted([
site for site in prim_to_distribute
if transfers_per_sites[site] >=
max_staging_per_site
])))
if not options.go:
allowed = False
break
for latching in latching_on_transfers:
tfo = session.query(Transfer).filter(
Transfer.phedexid == int(latching)).first()
if not tfo:
tfo = session.query(Transfer).filter(
Transfer.phedexid == -int(latching)).first()
if not tfo:
tfo = Transfer(phedexid=latching)
tfo.workflows_id = []
session.add(tfo)
else:
tfo.phedexid = latching ## make it positive ever
if not wfo.id in tfo.workflows_id:
print "adding", wfo.id, "to", tfo.id, "with phedexid", latching
l = copy.deepcopy(tfo.workflows_id)
l.append(wfo.id)
tfo.workflows_id = l
if not options.test:
session.commit()
else:
session.flush(
) ## regardless of commit later on, we need to let the next wf feeding on this transfer to see it in query
can_go = False
transfer_sizes[prim] = input_sizes[prim]
staging = True
# reduce the number of copies required by the on-going full transfer : how do we bootstrap on waiting for them ??
#copies_needed = max(0,copies_needed - len(prim_destination))
copies_needed = max(0, copies_needed - min(copies_being_made))
wfh.sendLog(
'transferor',
"Not counting the copies being made ; then need %s" %
copies_needed)
if copies_needed == 0:
wfh.sendLog(
'transferor',
"The output is either fully in place or getting in full somewhere with %s"
% latching_on_transfers)
can_go = True
continue
elif len(prim_to_distribute) == 0:
wfh.sendLog(
'transferor',
"We are going to need extra copies, but no destinations seems available"
)
prim_to_distribute = [
site for site in sites_allowed
if not SI.CE_to_SE(site) in prim_location
]
prim_to_distribute = [
site for site in prim_to_distribute if not any([
osite.startswith(site)
for osite in SI.sites_veto_transfer
])
]
if len(
prim_to_distribute
) > 0: ## maybe that a parameter we can play with to limit the
if not options or options.chop:
### hard include the tape disk andpoint ?
#tapes = [site for site in getDatasetPresence( url, prim, vetos=['T0','T2','T3','Disk']) if site.endswith('MSS')]
chops, sizes = getDatasetChops(
prim,
chop_threshold=options.chopsize,
only_blocks=blocks)
spreading = distributeToSites(chops,
prim_to_distribute,
n_copies=copies_needed,
weights=SI.cpu_pledges,
sizes=sizes)
transfer_sizes[prim] = sum(sizes)
if not spreading:
sendLog(
'transferor',
'cannot send %s to any site, it cannot fit anywhere'
% prim,
level='critical')
wfh.sendLog(
'transferor',
"cannot send to any site. %s cannot seem to fit anywhere"
% (prim))
staging = False
can_go = False
else:
spreading = {}
for site in prim_to_distribute:
if blocks:
spreading[site] = blocks
else:
spreading[site] = [prim]
transfer_sizes[prim] = input_sizes[
prim] ## this is approximate if blocks are specified
can_go = False
wfh.sendLog(
'transferor', "selected CE destinations %s" %
(sorted(spreading.keys())))
for (site, items) in spreading.items():
all_transfers[site].extend(items)
transfers_per_sites[site] += 1
primary_destinations.add(site)
if not allowed:
wfh.sendLog('transferor', "Not allowed to move on with")
continue
if secondary:
override_sec_destination = []
if 'SecondaryLocation' in CI.campaigns[wfh.request['Campaign']]:
override_sec_destination = CI.campaigns[
wfh.request['Campaign']]['SecondaryLocation']
print wfo.name, 'reads', ', '.join(secondary), 'in secondary'
for sec in secondary:
workflow_dependencies[sec].add(wfo.id)
if True:
## new style, failing on minbias
if not sec in destination_cache:
## this is barbbaric, and does not show the correct picture on workflow by workflow with different whitelist
destination_cache[sec], _ = getDatasetDestinations(
url, sec) ## NO SITE WHITE LIST ADDED
#destination_cache[sec],_ = getDatasetDestinations(url, sec, within_sites = [SI.CE_to_SE(site) for site in sites_allowed])
## limit to the site whitelist NOW
se_allowed = [SI.CE_to_SE(site) for site in sites_allowed]
destinations = dict([
(k, v) for (k, v) in destination_cache[sec].items()
if site in se_allowed
])
## truncate location/destination to those making up for >90% of the dataset
bad_destinations = [
destinations.pop(site)
for (site, info) in destinations.items()
if info['data_fraction'] < 0.9
]
sec_location = [
site for (site, info) in destinations.items()
if info['completion'] >= 95
]
sec_destination = [
site for site in destinations.keys()
if not site in sec_location
]
else:
## old style
presence = getDatasetPresence(url, sec)
sec_location = [
site for site, pres in presence.items()
if pres[1] > 90.
] ## more than 90% of the minbias at sites
subscriptions = listSubscriptions(url, sec)
sec_destination = [site for site in subscriptions]
sec_to_distribute = [
site for site in sites_allowed if
not any([osite.startswith(site) for osite in sec_location])
]
sec_to_distribute = [
site for site in sec_to_distribute if not any(
[osite.startswith(site) for osite in sec_destination])
]
sec_to_distribute = [
site for site in sec_to_distribute if not any([
osite.startswith(site)
for osite in SI.sites_veto_transfer
])
]
if override_sec_destination:
## intersect with where we want the PU to be
not_needed_anymore = list(
set(sec_to_distribute) - set(override_sec_destination))
#sendEmail("secondary superfluous","the dataset %s could be removed from %s"%( sec, not_needed_anymore ))
sendLog(
'transferor',
"the dataset %s could be removed from %s" %
(sec, not_needed_anymore))
sec_to_distribute = list(
set(sec_to_distribute) & set(override_sec_destination))
if len(sec_to_distribute) > 0:
print "secondary could go to", sorted(sec_to_distribute)
sec_size = dss.get(sec)
for site in sec_to_distribute:
site_se = SI.CE_to_SE(site)
if (SI.disk[site_se] * 1024.) > sec_size:
all_transfers[site].append(sec)
can_go = False
else:
print "could not send the secondary input to", site_se, "because it is too big for the available disk", SI.disk[
site_se] * 1024, "GB need", sec_size
if primary_destinations and site in primary_destinations:
#sendEmail('secondary input too big','%s is too big (%s) for %s (%s)'%( sec, sec_size, site_se, SI.disk[site_se]*1024))
sendLog('transferor',
'%s is too big (%s) for %s (%s)' %
(sec, sec_size, site_se,
SI.disk[site_se] * 1024),
level='critical')
else:
print "the secondary input does not have to be send to site"
## is that possible to do something more
if can_go:
## no explicit transfer required this time
if staging:
## but using existing ones
wfh.sendLog(
'transferor',
"latches on existing transfers, and nothing else, settin staging"
)
wfo.status = 'staging'
needs_transfer += 1
else:
wfh.sendLog(
'transferor', "should just be assigned now to %s" %
sorted(sites_allowed))
wfo.status = 'staged'
passing_along += 1
wfh.sendLog('transferor', "setting status to %s" % wfo.status)
session.commit()
continue
else:
## there is an explicit transfer required
if staging:
## and also using an existing one
wfh.sendLog('transferor', "latches on existing transfers")
if not options.test:
wfo.status = 'staging'
wfh.sendLog('transferor',
"setting status to %s" % wfo.status)
session.commit()
wfh.sendLog('transferor', "needs a transfer")
needs_transfer += 1
passing_along += 1
if no_goes:
#sendEmail("no go for managing","No go for \n"+"\n".join( no_goes ))
sendLog('transferor',
"No go for \n" + "\n".join(no_goes),
level='critical')
print "accumulated transfers"
print json.dumps(all_transfers, indent=2)
fake_id = -1
wf_id_in_prestaging = set()
for (site, items_to_transfer) in all_transfers.iteritems():
items_to_transfer = list(set(items_to_transfer))
## convert to storage element
site_se = SI.CE_to_SE(site)
## site that do not want input datasets
if site in SI.sites_veto_transfer:
print site, "does not want transfers"
continue
## throttle the transfer size to T2s ? we'd be screwed by a noPU sample properly configured.
## massage a bit the items
blocks = [it for it in items_to_transfer if '#' in it]
block_datasets = list(set([it.split('#')[0] for it in blocks]))
datasets = [it for it in items_to_transfer if not '#' in it]
details_text = "Making a replica to %s (CE) %s (SE) for" % (site,
site_se)
#print "\t",len(blocks),"blocks"
## remove blocks if full dataset is send out
blocks = [
block for block in blocks if not block.split('#')[0] in datasets
]
#print "\t",len(blocks),"needed blocks for",list(set([block.split('#')[0] for block in blocks]))
#print "\t",len(datasets),"datasets"
#print "\t",datasets
details_text += '\n\t%d blocks' % len(blocks)
details_text += '\n\t%d needed blocks for %s' % (
len(blocks),
sorted(list(set([block.split('#')[0] for block in blocks]))))
details_text += '\n\t%d datasets' % len(datasets)
details_text += '\n\t%s' % sorted(datasets)
items_to_transfer = blocks + datasets
if execute:
sendLog('transferor', details_text)
else:
print "Would make a replica to", site, "(CE)", site_se, "(SE) for"
print details_text
## operate the transfer
if options and options.stop:
## ask to move-on
answer = raw_input('Continue with that ?')
if not answer.lower() in ['y', 'yes', 'go']:
continue
if execute:
priority = 'normal'
cds = [
ds for ds in datasets + block_datasets if ds in max_priority
]
if cds and False: ## I don't think this is working. subscription should be updated on the fly and regularly for raising the priority if needed
## decide on an overall priority : that's a bit too large though
if any([max_priority[ds] >= 90000 for ds in cds]):
priority = 'high'
elif all([max_priority[ds] < 80000 for ds in cds]):
priority = 'low'
result = makeReplicaRequest(url,
site_se,
items_to_transfer,
'prestaging',
priority=priority)
else:
result = {'phedex': {'request_created': []}}
fake_id -= 1
if not result:
print "ERROR Could not make a replica request for", site, items_to_transfer, "pre-staging"
continue
for phedexid in [o['id'] for o in result['phedex']['request_created']]:
new_transfer = session.query(Transfer).filter(
Transfer.phedexid == int(phedexid)).first()
if not new_transfer:
new_transfer = session.query(Transfer).filter(
Transfer.phedexid == -int(phedexid)).first()
print phedexid, "transfer created"
if not new_transfer:
new_transfer = Transfer(phedexid=phedexid)
session.add(new_transfer)
else:
new_transfer.phedexid = phedexid ## make it positive again
new_transfer.workflows_id = set()
for transfering in list(
set(map(lambda it: it.split('#')[0], items_to_transfer))):
new_transfer.workflows_id.update(
workflow_dependencies[transfering])
new_transfer.workflows_id = list(new_transfer.workflows_id)
wf_id_in_prestaging.update(new_transfer.workflows_id)
session.commit()
## auto approve it
if execute:
approved = approveSubscription(url, phedexid, [site_se])
for wfid in wf_id_in_prestaging:
tr_wf = session.query(Workflow).get(wfid)
if tr_wf and tr_wf.status != 'staging':
if execute:
tr_wf.status = 'staging'
if talk:
print "setting", tr_wf.name, "to staging"
session.commit()
def transferor(url ,specific = None, talk=True, options=None):
if userLock('transferor'): return
if options and options.test:
execute = False
else:
execute = True
SI = siteInfo()
CI = campaignInfo()
mcm = McMClient(dev=False)
dss = DSS()
print "counting all being handled..."
being_handled = len(session.query(Workflow).filter(Workflow.status == 'away').all())
being_handled += len(session.query(Workflow).filter(Workflow.status.startswith('stag')).all())
being_handled += len(session.query(Workflow).filter(Workflow.status.startswith('assistance')).all())
max_to_handle = options.maxworkflows
allowed_to_handle = max(0,max_to_handle - being_handled)
wf_buffer = 5
if allowed_to_handle<=wf_buffer: ## buffer for having several wf per transfer
print "Not allowed to run more than",max_to_handle,"at a time. Currently",being_handled,"and",wf_buffer,"buffer"
else:
print being_handled,"already being handled",max_to_handle,"max allowed,",allowed_to_handle,"remaining","and",wf_buffer,"buffer"
print "... done"
all_transfers=defaultdict(list)
workflow_dependencies = defaultdict(set) ## list of wf.id per input dataset
wfs_and_wfh=[]
print "getting all wf to consider ..."
cache = getWorkflows(url, 'assignment-approved', details=True)
for wfo in session.query(Workflow).filter(Workflow.status=='considered').all():
if specific and not specific in wfo.name: continue
cache_r =filter(lambda d:d['RequestName']==wfo.name, cache)
if len(cache_r):
wfs_and_wfh.append( (wfo, workflowInfo( url, wfo.name, spec=False, request = cache_r[0]) ) )
else:
wfs_and_wfh.append( (wfo, workflowInfo( url, wfo.name, spec=False) ) )
print "... done"
input_sizes = {}
## list the size of those in transfer already
in_transfer_priority=0
min_transfer_priority=100000000
print "getting all wf in staging ..."
for wfo in session.query(Workflow).filter(Workflow.status=='staging').all():
wfh = workflowInfo( url, wfo.name, spec=False)
(_,primary,_,_) = wfh.getIO()
for prim in primary:
input_sizes[prim] = dss.get( prim )
in_transfer_priority = max(in_transfer_priority, int(wfh.request['RequestPriority']))
min_transfer_priority = min(min_transfer_priority, int(wfh.request['RequestPriority']))
print "... done"
print "Max priority in transfer already",in_transfer_priority
print "Min priority in transfer already",min_transfer_priority
in_transfer_already = sum(input_sizes.values())
#sort by priority higher first
wfs_and_wfh.sort(cmp = lambda i,j : cmp(int(i[1].request['RequestPriority']),int(j[1].request['RequestPriority']) ), reverse=True)
## list the size of all inputs
print "getting all input sizes ..."
for (wfo,wfh) in wfs_and_wfh:
(_,primary,_,_) = wfh.getIO()
for prim in primary:
input_sizes[prim] = dss.get( prim )
print "... done"
grand_total = sum(input_sizes.values())
to_transfer = grand_total - in_transfer_already
grand_transfer_limit = options.maxtransfer
transfer_limit = grand_transfer_limit - in_transfer_already
print "%15.4f GB already being transfered"%in_transfer_already
print "%15.4f GB is the current requested transfer load"%to_transfer
print "%15.4f GB is the global transfer limit"%grand_transfer_limit
print "%15.4f GB is the available limit"%transfer_limit
# the max priority value per dataset.
max_priority = defaultdict(int)
needs_transfer=0 ## so that we can count'em
passing_along = 0
transfer_sizes={}
went_over_budget=False
for (wfo,wfh) in wfs_and_wfh:
print wfh.request['RequestPriority']
print wfo.name,"to be transfered"
#wfh = workflowInfo( url, wfo.name)
(_,primary,_,_) = wfh.getIO()
this_load=sum([input_sizes[prim] for prim in primary])
if ( this_load and (sum(transfer_sizes.values())+this_load > transfer_limit or went_over_budget ) ):
if went_over_budget:
print "Transfer has gone over bubget."
else:
print "Transfer will go over bubget."
print "%15.4f GB this load"%this_load
print "%15.4f GB already this round"%sum(transfer_sizes.values())
print "%15.4f GB is the available limit"%transfer_limit
went_over_budget=True
if int(wfh.request['RequestPriority']) >= in_transfer_priority and min_transfer_priority!=in_transfer_priority:
print "Higher priority sample",wfh.request['RequestPriority'],">=",in_transfer_priority,"go-on over budget"
else:
if not options.go:
print min_transfer_priority,"minimum priority",wfh.request['RequestPriority'],"<",in_transfer_priority,"stop"
continue
## throtlle by campaign go
if not CI.go( wfh.request['Campaign'] ):
print "No go for",wfh.request['Campaign']
if not options.go: continue
## check if the batch is announced
announced=False
is_real=False
for b in mcm.getA('batches',query='contains=%s'% wfo.name):
is_real = True
if b['status']=='announced':
announced=True
break
if not announced:
print wfo.name,"does not look announced."# skipping?, rejecting?, reporting?"
if not is_real:
print wfo.name,"does not appear to be genuine."
## prevent any duplication. if the wf is not mentioned in any batch, regardless of status
continue
## check on a grace period
injection_time = time.mktime(time.strptime('.'.join(map(str,wfh.request['RequestDate'])),"%Y.%m.%d.%H.%M.%S")) / (60.*60.)
now = time.mktime(time.gmtime()) / (60.*60.)
if float(now - injection_time) < 4.:
if not options.go and not announced:
print "It is too soon to start transfer: %3.2fH remaining"%(now - injection_time)
continue
passing_along += 1
if passing_along >= allowed_to_handle:
if int(wfh.request['RequestPriority']) >= in_transfer_priority and min_transfer_priority!=in_transfer_priority:
print "Higher priority sample",wfh.request['RequestPriority'],">=",in_transfer_priority,"go-on over",max_to_handle
else:
print "Not allowed to pass more than",max_to_handle,"at a time. Currently",being_handled,"handled, and adding",passing_along
break
(lheinput,primary,parent,secondary) = wfh.getIO()
if options and options.tosites:
sites_allowed = options.tosites.split(',')
else:
sites_allowed = getSiteWhiteList( (lheinput,primary,parent,secondary) )
if 'SiteWhitelist' in CI.parameters(wfh.request['Campaign']):
sites_allowed = CI.parameters(wfh.request['Campaign'])['SiteWhitelist']
blocks = []
if 'BlockWhitelist' in wfh.request and wfh.request['BlockWhitelist']:
blocks = wfh.request['BlockWhitelist']
can_go = True
staging=False
if primary:
if talk:
print wfo.name,'reads',', '.join(primary),'in primary'
## chope the primary dataset
for prim in primary:
max_priority[prim] = max(max_priority[prim],int(wfh.request['RequestPriority']))
sites_really_allowed = [site for site in sites_allowed if not any([osite.startswith(site) for osite in SI.sites_veto_transfer])]
print "Sites allowed minus the vetoed transfer"
print sites_really_allowed
copies_needed = int(0.35*len(sites_really_allowed))+1 ## should just go for a fixed number based if the white list grows that big
print "Would make",copies_needed,"copies"
if options.maxcopy>0:
copies_needed = min(options.maxcopy,copies_needed)
## remove the sites that do not want transfers
print "need",copies_needed
workflow_dependencies[prim].add( wfo.id )
presence = getDatasetPresence( url, prim )
prim_location = [site for site,pres in presence.items() if pres[0]==True]
if len(prim_location) >= copies_needed:
print "The output is all fully in place at",len(prim_location),"sites"
continue
# reduce the number of copies required by existing full copies
copies_needed = max(0,copies_needed - len(prim_location))
print "now need",copies_needed
subscriptions = listSubscriptions( url , prim )
prim_destination = list(set([site for (site,(tid,decision)) in subscriptions.items() if decision and not any([site.endswith(veto) for veto in ['MSS','Export','Buffer']])]))
## need to reject from that list the ones with a full copy already: i.e the transfer corresponds to the copy in place
prim_destination = [site for site in prim_destination if not site in prim_location]
## add transfer dependencies
latching_on_transfers = list(set([ tid for (site,(tid,decision)) in subscriptions.items() if decision and site in prim_destination and not any([site.endswith(veto) for veto in ['MSS','Export','Buffer']])]))
print latching_on_transfers
for latching in latching_on_transfers:
tfo = session.query(Transfer).filter(Transfer.phedexid == latching).first()
if not tfo:
tfo = Transfer( phedexid = latching)
tfo.workflows_id = []
session.add(tfo)
if not wfo.id in tfo.workflows_id:
print "adding",wfo.id,"to",tfo.id,"with phedexid",latching
l = copy.deepcopy( tfo.workflows_id )
l.append( wfo.id )
tfo.workflows_id = l
if not options.test:
session.commit()
else:
session.flush() ## regardless of commit later on, we need to let the next wf feeding on this transfer to see it in query
can_go = False
transfer_sizes[prim] = input_sizes[prim]
staging = True
# reduce the number of copies required by the on-going full transfer : how do we bootstrap on waiting for them ??
copies_needed = max(0,copies_needed - len(prim_destination))
print "then need",copies_needed
if copies_needed == 0:
print "The output is either fully in place or getting in full somewhere with",latching_on_transfers
can_go = True
continue
prim_to_distribute = [site for site in sites_allowed if not any([osite.startswith(site) for osite in prim_location])]
prim_to_distribute = [site for site in prim_to_distribute if not any([osite.startswith(site) for osite in prim_destination])]
## take out the ones that cannot receive transfers
prim_to_distribute = [site for site in prim_to_distribute if not any([osite.startswith(site) for osite in SI.sites_veto_transfer])]
if len(prim_to_distribute)>0: ## maybe that a parameter we can play with to limit the
if not options or options.chop:
spreading = distributeToSites( getDatasetChops(prim), prim_to_distribute, n_copies = copies_needed, weights=SI.cpu_pledges)
else:
spreading = {}
for site in prim_to_distribute: spreading[site]=[prim]
can_go = False
transfer_sizes[prim] = input_sizes[prim]
for (site,items) in spreading.items():
all_transfers[site].extend( items )
if secondary:
if talk:
print wfo.name,'reads',', '.join(secondary),'in secondary'
for sec in secondary:
workflow_dependencies[sec].add( wfo.id )
presence = getDatasetPresence( url, sec )
sec_location = [site for site,pres in presence.items() if pres[1]>90.] ## more than 90% of the minbias at sites
subscriptions = listSubscriptions( url ,sec )
sec_destination = [site for site in subscriptions]
sec_to_distribute = [site for site in sites_allowed if not any([osite.startswith(site) for osite in sec_location])]
sec_to_distribute = [site for site in sec_to_distribute if not any([osite.startswith(site) for osite in sec_destination])]
sec_to_distribute = [site for site in sec_to_distribute if not any([osite.startswith(site) for osite in SI.sites_veto_transfer])]
if len( sec_to_distribute )>0:
for site in sec_to_distribute:
all_transfers[site].append( sec )
can_go = False
## is that possible to do something more
if can_go:
## no explicit transfer required this time
if staging:
## but using existing ones
print wfo.name,"latches on existing transfers, and nothing else"
wfo.status = 'staging'
else:
print wfo.name,"should just be assigned NOW to",sites_allowed
wfo.status = 'staged'
print "setting status to",wfo.status
session.commit()
continue
else:
## there is an explicit transfer required
if staging:
## and also using an existing one
print wfo.name,"latches on existing transfers"
if not options.test:
wfo.status = 'staging'
print "setting status to",wfo.status
session.commit()
print wfo.name,"needs a transfer"
needs_transfer+=1
#print json.dumps(all_transfers)
fake_id=-1
wf_id_in_prestaging=set()
for (site,items_to_transfer) in all_transfers.iteritems():
items_to_transfer = list(set(items_to_transfer))
## convert to storage element
site_se = SI.CE_to_SE(site)
## site that do not want input datasets
if site in SI.sites_veto_transfer:
print site,"does not want transfers"
continue
## throttle the transfer size to T2s ? we'd be screwed by a noPU sample properly configured.
## massage a bit the items
blocks = [it for it in items_to_transfer if '#' in it]
datasets = [it for it in items_to_transfer if not '#' in it]
if execute:
print "Making a replica to",site,"(CE)",site_se,"(SE) for"
else:
print "Would make a replica to",site,"(CE)",site_se,"(SE) for"
print "\t",len(blocks),"blocks"
## remove blocks if full dataset is send out
blocks = [block for block in blocks if not block.split('#')[0] in datasets]
print "\t",len(blocks),"needed blocks for",list(set([block.split('#')[0] for block in blocks]))
print "\t",len(datasets),"datasets"
print "\t",datasets
items_to_transfer = blocks + datasets
## operate the transfer
if options and options.stop:
## ask to move-on
answer = raw_input('Continue with that ?')
if not answer.lower() in ['y','yes','go']:
continue
if execute:
result = makeReplicaRequest(url, site_se, items_to_transfer, 'prestaging', priority='normal')
## make use of max_priority dataset:priority to set the subscriptions priority
"""
## does not function
once = True
for item in items_to_transfer:
bds = item.split('#')[0]
if max_priority[bds] >= 90000:
if once:
w=10
print "waiting",w,"s before raising priority"
time.sleep(w)
once=False
## raise it to high priority
print item,"subscription priority raised to high at",site_se
#print "This does not work yet properly it seems"
print updateSubscription(url, site_se, item, priority='high')
"""
else:
#result= {'phedex':{'request_created' : [{'id' : fake_id}]}}
result= {'phedex':{'request_created' : []}}
fake_id-=1
if not result:
print "ERROR Could not make a replica request for",site,items_to_transfer,"pre-staging"
continue
for phedexid in [o['id'] for o in result['phedex']['request_created']]:
new_transfer = session.query(Transfer).filter(Transfer.phedexid == phedexid).first()
print phedexid,"transfer created"
if not new_transfer:
new_transfer = Transfer( phedexid = phedexid)
session.add( new_transfer )
new_transfer.workflows_id = set()
for transfering in list(set(map(lambda it : it.split('#')[0], items_to_transfer))):
new_transfer.workflows_id.update( workflow_dependencies[transfering] )
new_transfer.workflows_id = list(new_transfer.workflows_id)
wf_id_in_prestaging.update(new_transfer.workflows_id)
session.commit()
## auto approve it
if execute:
approved = approveSubscription(url, phedexid, [site_se])
for wfid in wf_id_in_prestaging:
tr_wf = session.query(Workflow).get(wfid)
if tr_wf and tr_wf.status!='staging':
if execute:
tr_wf.status = 'staging'
if talk:
print "setting",tr_wf.name,"to staging"
session.commit()
print "will make",copies_needed,"of all"
read_file = 'legacy-reco-raw_prio.txt'
for l in open(read_file).read().split('\n'):
if not l: continue
if l.startswith('#'): continue
all_ds = filter(lambda w : w.count('/')==3 and w.startswith('/'), l.split())
dss.update( all_ds )
print sorted(dss)
for ds in dss:
availability = getDatasetBlocksFraction(url, ds )
if availability>=1: continue
blocks= []
chops,sizes = getDatasetChops(ds, chop_threshold = 4000, only_blocks=blocks)
spreading = distributeToSites( chops, prim_to_distribute, n_copies = copies_needed, weights=SI.cpu_pledges, sizes=sizes)
print json.dumps(spreading, indent=2)
for site,items in spreading.items():
full_spread[site].update( items )
to_lock.add( ds )
addHocLocks = set(json.loads( open('addhoc_lock.json').read()))
addHocLocks.update( to_lock )
if act:
open('addhoc_lock.json','w').write( json.dumps( sorted(addHocLocks), indent=2 ) )
for l in to_lock:
if act:
LI.lock(l, 'add HOC')
def collector(url, specific, options):
up = componentInfo(mcm=False, soft=['mcm'])
if not up.check(): return
SI = siteInfo()
dss = DSS()
#NL = newLockInfo()
mcm = McMClient(dev=False)
fetch_in_campaigns = ['RunIISummer15GS']
mcm_statuses = ['new'] #,'validation','defined','approved']
will_be_used = defaultdict(list)
secondary_used = defaultdict(list)
for campaign, status in itertools.product(fetch_in_campaigns,
mcm_statuses):
queries = []
if campaign:
print "getting for", campaign
queries.append('member_of_campaign=%s' % campaign)
if status:
print "getting for", status
queries.append('status=%s' % status)
rs = mcm.getA('requests', query='&'.join(queries))
for r in rs:
#if r['type'] != 'Prod': continue
dataset = r['input_dataset']
if dataset:
#print r['prepid'],dataset
will_be_used[dataset].append(r)
pileup = r['pileup_dataset_name']
if pileup:
secondary_used['pileup'].append(r)
all_transfers = defaultdict(list)
print len(will_be_used), "datasets that can be pre-fetched"
## for secondary we really need to have the campaign right
print len(secondary_used), "pileup will be used"
datasets = will_be_used.keys()
if options.limit:
print "Restricting to randomly picked", options.limit
random.shuffle(datasets)
datasets = datasets[:options.limit]
for dataset in datasets:
print "\tlooking at", dataset
#presence = getDatasetPresence(url, dataset)#, within_sites=['T2_CH_CERN'])
## lock all those, and pre-fecth them
#NL.lock( dataset )
## we could get the reqmgr dictionnary from McM if it was implemented and use standard workflowInfo !!!
for request in will_be_used[dataset]:
print "will be used by", request['prepid']
campaign = request['member_of_campaign']
## based on the campaign, pre-fetch a site list
sites_allowed = SI.sites_T1s + SI.sites_with_goodIO
if options.spread:
## pick up the number of copies from campaign
copies_needed = 1 ## hard coded for now
else:
copies_needed = 1 ## hard coded for now
print "Will look for", copies_needed, "of", dataset
## figure out where it is and going
destinations, all_block_names = getDatasetDestinations(
url,
dataset,
within_sites=[SI.CE_to_SE(site) for site in sites_allowed])
print json.dumps(destinations, indent=2)
prim_location = [
site for (site, info) in destinations.items()
if info['completion'] == 100 and info['data_fraction'] == 1
]
prim_destination = [
site for site in destinations.keys()
if not site in prim_location
]
prim_destination = [
site for site in prim_destination if not any([
osite.startswith(site) for osite in SI.sites_veto_transfer
])
]
copies_needed = max(0, copies_needed - len(prim_location))
copies_being_made = [
sum([
info['blocks'].keys().count(block)
for site, info in destinations.items()
if site in prim_destination
]) for block in all_block_names
]
prim_to_distribute = [
site for site in sites_allowed
if not SI.CE_to_SE(site) in prim_location
]
prim_to_distribute = [
site for site in prim_to_distribute
if not SI.CE_to_SE(site) in prim_destination
]
## take out the ones that cannot receive transfers
prim_to_distribute = [
site for site in prim_to_distribute if not any([
osite.startswith(site) for osite in SI.sites_veto_transfer
])
]
copies_needed = max(0, copies_needed - min(copies_being_made))
spreading = {}
if copies_needed:
print "needing", copies_needed
chops, sizes = getDatasetChops(dataset,
chop_threshold=options.chopsize)
spreading = distributeToSites(chops,
prim_to_distribute,
n_copies=copies_needed,
weights=SI.cpu_pledges,
sizes=sizes)
else:
print "no copy needed for", dataset
for (site, items) in spreading.items():
all_transfers[site].extend(items)
print "accumulated transfers"
print json.dumps(all_transfers, indent=2)
if not options.test:
sendEmail(
'dataset to be fetched',
'the following datasets and location were figured from mcm up-coming requests\n%s'
% (json.dumps(all_transfers, indent=2)),
destination=['*****@*****.**'])
## now collect and make transfer request
for (site, items_to_transfer) in all_transfers.iteritems():
print "Directing at", site
items_to_transfer = list(set(items_to_transfer))
site_se = SI.CE_to_SE(site)
blocks = [it for it in items_to_transfer if '#' in it]
datasets = [it for it in items_to_transfer if not '#' in it]
print "\t", len(blocks), "blocks"
## remove blocks if full dataset is send out
blocks = [
block for block in blocks if not block.split('#')[0] in datasets
]
blocks_dataset = list(set([block.split('#')[0] for block in blocks]))
print "\t", len(blocks), "needed blocks for", blocks_dataset
print "\t", len(datasets), "datasets"
print "\t", datasets
items_to_transfer = blocks + datasets
total_size = 0
for dataset in datasets:
ds_size, _ = dss.get_block_size(dataset)
total_size += ds_size
for dataset in blocks_dataset:
_, bs_size = dss.get_block_size(dataset)
total_size += sum([s for b, s in bs_size if b in blocks])
print "For a total of", total_size, "[GB]"
if options.test:
result = {'phedex': {'request_created': []}}
else:
##result = makeReplicaRequest(url, site_se, items_to_transfer, 'fetching pre-production', priority='normal', approve=True)
## should make sure there is something in it
pass
def transferor(url ,specific = None, talk=True, options=None):
if options and options.test:
execute = False
else:
execute = True
SI = siteInfo()
CI = campaignInfo()
all_transfers=defaultdict(list)
workflow_dependencies = defaultdict(set) ## list of wf.id per input dataset
data_to_wf = {}
for wfo in session.query(Workflow).filter(Workflow.status=='considered').all():
if specific and not specific in wfo.name: continue
print wfo.name,"to be transfered"
wfh = workflowInfo( url, wfo.name)
#injection_time = time.mktime(time.strptime('.'.join(map(str,wfh.request['RequestDate'])),"%Y.%m.%d.%H.%M.%S")) / (60.*60.)
#now = time.mktime(time.gmtime()) / (60.*60.)
#if float(now - injection_time) < 4.:
# print "It is too soon to transfer", now, injection_time
# continue
(lheinput,primary,parent,secondary) = wfh.getIO()
if options and options.tosites:
sites_allowed = options.tosites.split(',')
else:
sites_allowed = getSiteWhiteList( (lheinput,primary,parent,secondary) )
if 'SiteWhitelist' in CI.parameters(wfh.request['Campaign']):
sites_allowed = CI.parameters(wfh.request['Campaign'])['SiteWhitelist']
blocks = []
if 'BlockWhitelist' in wfh.request and wfh.request['BlockWhitelist']:
blocks = wfh.request['BlockWhitelist']
can_go = True
if primary:
if talk:
print wfo.name,'reads',', '.join(primary),'in primary'
## chope the primary dataset
for prim in primary:
workflow_dependencies[prim].add( wfo.id )
presence = getDatasetPresence( url, prim )
prim_location = [site for site,pres in presence.items() if pres[0]==True]
subscriptions = listSubscriptions( url , prim )
prim_destination = [site for site in subscriptions]
prim_to_distribute = [site for site in sites_allowed if not any([osite.startswith(site) for osite in prim_location])]
prim_to_distribute = [site for site in prim_to_distribute if not any([osite.startswith(site) for osite in prim_destination])]
if len(prim_to_distribute)>0: ## maybe that a parameter we can play with to limit the
if not options or options.chop:
spreading = distributeToSites( [[prim]]+getDatasetChops(prim), prim_to_distribute, n_copies = 3, weights=SI.cpu_pledges)
else:
spreading = {}
for site in prim_to_distribute: spreading[site]=[prim]
can_go = False
for (site,items) in spreading.items():
all_transfers[site].extend( items )
if secondary:
if talk:
print wfo.name,'reads',', '.join(secondary),'in secondary'
for sec in secondary:
workflow_dependencies[sec].add( wfo.id )
presence = getDatasetPresence( url, sec )
sec_location = [site for site,pres in presence.items() if pres[1]>90.] ## more than 90% of the minbias at sites
subscriptions = listSubscriptions( url ,sec )
sec_destination = [site for site in subscriptions]
sec_to_distribute = [site for site in sites_allowed if not any([osite.startswith(site) for osite in sec_location])]
sec_to_distribute = [site for site in sec_to_distribute if not any([osite.startswith(site) for osite in sec_destination])]
if len( sec_to_distribute )>0:
for site in sec_to_distribute:
all_transfers[site].append( sec )
can_go = False
## is that possible to do something more
if can_go:
print wfo.name,"should just be assigned NOW to",sites_allowed
wfo.status = 'staged'
session.commit()
continue
else:
print wfo.name,"needs a transfer"
#print json.dumps(all_transfers)
fake_id=-1
wf_id_in_prestaging=set()
for (site,items_to_transfer) in all_transfers.iteritems():
items_to_transfer = list(set(items_to_transfer))
## convert to storage element
site_se = SI.CE_to_SE(site)
## massage a bit the items
blocks = [it for it in items_to_transfer if '#' in it]
datasets = [it for it in items_to_transfer if not '#' in it]
print "Making a replica to",site,"(CE)",site_se,"(SE) for"
print "\t",len(blocks),"blocks"
## remove blocks if full dataset is send out
blocks = [block for block in blocks if not block.split('#')[0] in datasets]
print "\t",len(blocks),"needed blocks"
print "\t",len(datasets),"datasets"
print "\t",datasets
items_to_transfer = blocks + datasets
## operate the transfer
if options and options.stop:
## ask to move-on
answer = raw_input('Continue with that ?')
if not answer.lower() in ['y','yes','go']:
continue
if execute:
result = makeReplicaRequest(url, site_se, items_to_transfer, 'prestaging')
else:
#result= {'phedex':{'request_created' : [{'id' : fake_id}]}}
result= {'phedex':{'request_created' : []}}
fake_id-=1
if not result:
print "ERROR Could not make a replica request for",site,items_to_transfer,"pre-staging"
continue
for phedexid in [o['id'] for o in result['phedex']['request_created']]:
new_transfer = session.query(Transfer).filter(Transfer.phedexid == phedexid).first()
if not new_transfer:
new_transfer = Transfer( phedexid = phedexid)
session.add( new_transfer )
new_transfer.workflows_id = set()
for transfering in list(set(map(lambda it : it.split('#')[0], items_to_transfer))):
new_transfer.workflows_id.update( workflow_dependencies[transfering] )
new_transfer.workflows_id = list(new_transfer.workflows_id)
wf_id_in_prestaging.update(new_transfer.workflows_id)
session.commit()
## auto approve it
if execute:
approved = approveSubscription(url, phedexid, [site_se])
for wfid in wf_id_in_prestaging:
tr_wf = session.query(Workflow).get(wfid)
if tr_wf and tr_wf.status!='staging':
if execute:
tr_wf.status = 'staging'
if talk:
print "setting",tr_wf.name,"to staging"
session.commit()
def transferor(url ,specific = None, talk=True, options=None):
if userLock(): return
if duplicateLock(): return
use_mcm = True
up = componentInfo(mcm=use_mcm, soft=['mcm'])
if not up.check(): return
use_mcm = up.status['mcm']
if options and options.test:
execute = False
else:
execute = True
SI = siteInfo()
CI = campaignInfo()
LI = lockInfo()
NLI = newLockInfo()
mcm = McMClient(dev=False)
dss = DSS()
print "counting all being handled..."
being_handled = len(session.query(Workflow).filter(Workflow.status == 'away').all())
being_handled += len(session.query(Workflow).filter(Workflow.status.startswith('stag')).all())
being_transfered = len(session.query(Workflow).filter(Workflow.status == 'staging').all())
being_handled += len(session.query(Workflow).filter(Workflow.status.startswith('assistance-')).all())
max_to_handle = options.maxworkflows
max_to_transfer = options.maxstaging
allowed_to_handle = max(0,max_to_handle - being_handled)
allowed_to_transfer = max(0,max_to_transfer - being_transfered)
wf_buffer = 5
if allowed_to_handle<=wf_buffer: ## buffer for having several wf per transfer
print "Not allowed to run more than",max_to_handle,"at a time. Currently",being_handled,"and",wf_buffer,"buffer"
else:
print being_handled,"already being handled",max_to_handle,"max allowed,",allowed_to_handle,"remaining","and",wf_buffer,"buffer"
if allowed_to_transfer <= wf_buffer:
print "Not allowed to transfer more than",max_to_transfer,"at a time. Currently",being_transfered,"and",wf_buffer,"buffer"
else:
print being_transfered,"already being transfered",max_to_transfer,"max allowed,",allowed_to_transfer,"remaining","and",wf_buffer,"buffer"
print "... done"
all_transfers=defaultdict(list)
needing_locks=defaultdict(list)
workflow_dependencies = defaultdict(set) ## list of wf.id per input dataset
wfs_and_wfh=[]
print "getting all wf to consider ..."
cache = getWorkflows(url, 'assignment-approved', details=True)
for wfo in session.query(Workflow).filter(Workflow.status=='considered').all():
print "\t",wfo.name
if specific and not specific in wfo.name: continue
cache_r =filter(lambda d:d['RequestName']==wfo.name, cache)
if len(cache_r):
wfs_and_wfh.append( (wfo, workflowInfo( url, wfo.name, spec=False, request = cache_r[0]) ) )
else:
wfs_and_wfh.append( (wfo, workflowInfo( url, wfo.name, spec=False) ) )
print "... done"
transfers_per_sites = defaultdict(int)
input_sizes = {}
input_cput = {}
input_st = {}
## list the size of those in transfer already
in_transfer_priority=0
min_transfer_priority=100000000
print "getting all wf in staging ..."
for wfo in session.query(Workflow).filter(Workflow.status=='staging').all():
wfh = workflowInfo( url, wfo.name, spec=False)
(lheinput,primary,parent,secondary) = wfh.getIO()
sites_allowed = getSiteWhiteList( (lheinput,primary,parent,secondary) )
for site in sites_allowed: ## we should get the actual transfer destination instead of the full white list
transfers_per_sites[site] += 1
#input_cput[wfo.name] = wfh.getComputingTime()
#input_st[wfo.name] = wfh.getSystemTime()
for prim in primary:
input_sizes[prim] = dss.get( prim )
print "\t",wfo.name,"needs",input_sizes[prim],"GB"
in_transfer_priority = max(in_transfer_priority, int(wfh.request['RequestPriority']))
min_transfer_priority = min(min_transfer_priority, int(wfh.request['RequestPriority']))
print "... done"
print "Max priority in transfer already",in_transfer_priority
print "Min priority in transfer already",min_transfer_priority
print "transfers per sites"
print json.dumps( transfers_per_sites, indent=2)
in_transfer_already = sum(input_sizes.values())
cput_in_transfer_already = sum(input_cput.values())
st_in_transfer_already = sum(input_st.values())
# shuffle first by name
random.shuffle( wfs_and_wfh )
#sort by priority higher first
wfs_and_wfh.sort(cmp = lambda i,j : cmp(int(i[1].request['RequestPriority']),int(j[1].request['RequestPriority']) ), reverse=True)
## list the size of all inputs
print "getting all input sizes ..."
for (wfo,wfh) in wfs_and_wfh:
(_,primary,_,_) = wfh.getIO()
#input_cput[wfo.name] = wfh.getComputingTime()
#input_st[wfo.name] = wfh.getSystemTime()
for prim in primary:
input_sizes[prim] = dss.get( prim )
print "... done"
cput_grand_total = sum(input_cput.values())
cput_to_transfer = cput_grand_total - cput_in_transfer_already
st_grand_total = sum(input_st.values())
st_to_transfer = st_grand_total - st_in_transfer_already
print "%15.4f [CPU h] worth already in transfer"%cput_in_transfer_already
print "%15.4f [CPU h] worth is the current requested transfer load"%cput_to_transfer
print "%15.4f [h] worth of absolute system time in transfer"%( cput_in_transfer_already / SI.availableSlots())
print "%15.4f [h] worth of absolute system time is the current requested transfer load"%( cput_to_transfer / SI.availableSlots())
print "%15.4f [h] worth of theoritical system time in transfer"%( st_in_transfer_already )
print "%15.4f [h] worth of theoritical system time is the current requested transfer load"%( st_to_transfer )
grand_total = sum(input_sizes.values())
to_transfer = grand_total - in_transfer_already
grand_transfer_limit = options.maxtransfer
transfer_limit = grand_transfer_limit - in_transfer_already
print "%15.4f GB already being transfered"%in_transfer_already
print "%15.4f GB is the current requested transfer load"%to_transfer
print "%15.4f GB is the global transfer limit"%grand_transfer_limit
print "%15.4f GB is the available limit"%transfer_limit
max_staging_per_site = options.maxstagingpersite
# the max priority value per dataset.
max_priority = defaultdict(int)
needs_transfer=0 ## so that we can count'em
passing_along = 0
transfer_sizes={}
went_over_budget=False
destination_cache = {}
for (wfo,wfh) in wfs_and_wfh:
print wfh.request['RequestPriority']
print wfo.name,"to be transfered"
#wfh = workflowInfo( url, wfo.name)
(_,primary,_,_) = wfh.getIO()
this_load=sum([input_sizes[prim] for prim in primary])
if ( this_load and (sum(transfer_sizes.values())+this_load > transfer_limit or went_over_budget ) ):
if went_over_budget:
print "Transfer has gone over bubget."
else:
print "Transfer will go over bubget."
print "%15.4f GB this load"%this_load
print "%15.4f GB already this round"%sum(transfer_sizes.values())
print "%15.4f GB is the available limit"%transfer_limit
went_over_budget=True
if int(wfh.request['RequestPriority']) >= in_transfer_priority and min_transfer_priority!=in_transfer_priority:
print "Higher priority sample",wfh.request['RequestPriority'],">=",in_transfer_priority,"go-on over budget"
else:
if not options.go:
print min_transfer_priority,"minimum priority",wfh.request['RequestPriority'],"<",in_transfer_priority,"stop"
continue
## throtlle by campaign go
if not CI.go( wfh.request['Campaign'] ):
print "No go for",wfh.request['Campaign']
if not options.go:
sendEmail("no go for managing","No go for "+wfh.request['Campaign'])
continue
## check if the batch is announced
def check_mcm(wfn):
announced=False
is_real=False
if not wfn.startswith('pdmvserv'):
is_real = True
try:
for b in mcm.getA('batches',query='contains=%s'% wfo.name):
is_real = True
if b['status']=='announced':
announced=True
break
except:
try:
for b in mcm.getA('batches',query='contains=%s'% wfo.name):
is_real = True
if b['status']=='announced':
announced=True
break
except:
print "could not get mcm batch announcement, assuming not real"
return announced,is_real
if not use_mcm:
announced,is_real = False,True
else:
announced,is_real = check_mcm( wfo.name )
if not announced:
print wfo.name,"does not look announced."# skipping?, rejecting?, reporting?"
if not is_real:
print wfo.name,"does not appear to be genuine."
## prevent any duplication. if the wf is not mentioned in any batch, regardless of status
continue
## check on a grace period
injection_time = time.mktime(time.strptime('.'.join(map(str,wfh.request['RequestDate'])),"%Y.%m.%d.%H.%M.%S")) / (60.*60.)
now = time.mktime(time.gmtime()) / (60.*60.)
if float(now - injection_time) < 4.:
if not options.go and not announced:
print "It is too soon to start transfer: %3.2fH remaining"%(now - injection_time)
continue
if passing_along >= allowed_to_handle:
#if int(wfh.request['RequestPriority']) >= in_transfer_priority and min_transfer_priority!=in_transfer_priority:
if int(wfh.request['RequestPriority']) >= in_transfer_priority and int(wfh.request['RequestPriority']) !=min_transfer_priority:
## higher priority, and not only this priority being transfered
print "Higher priority sample",wfh.request['RequestPriority'],">=",in_transfer_priority,"go-on over",max_to_handle
else:
print "Not allowed to pass more than",max_to_handle,"at a time. Currently",being_handled,"handled, and adding",passing_along
if not options.go: break
if this_load and needs_transfer >= allowed_to_transfer:
#if int(wfh.request['RequestPriority']) >= in_transfer_priority and min_transfer_priority!=in_transfer_priority:
if int(wfh.request['RequestPriority']) >= in_transfer_priority and int(wfh.request['RequestPriority']) !=min_transfer_priority:
## higher priority, and not only this priority being transfered
print "Higher priority sample",wfh.request['RequestPriority'],">=",in_transfer_priority,"go-on over",max_to_transfer
else:
print "Not allowed to transfer more than",max_to_transfer,"at a time. Currently",being_transfered,"transfering, and adding",needs_transfer
if not options.go: continue
(lheinput,primary,parent,secondary) = wfh.getIO()
for dataset in list(primary)+list(parent)+list(secondary):
## lock everything flat
NLI.lock( dataset )
if options and options.tosites:
sites_allowed = options.tosites.split(',')
else:
sites_allowed = getSiteWhiteList( (lheinput,primary,parent,secondary) )
if 'SiteWhitelist' in CI.parameters(wfh.request['Campaign']):
sites_allowed = CI.parameters(wfh.request['Campaign'])['SiteWhitelist']
if 'SiteBlacklist' in CI.parameters(wfh.request['Campaign']):
sites_allowed = list(set(sites_allowed) - set(CI.parameters(wfh.request['Campaign'])['SiteBlacklist']))
## reduce right away to sites in case of memory limitation
memory_allowed = SI.sitesByMemory( wfh.request['Memory'] )
if memory_allowed!=None:
print "sites allowing", wfh.request['Memory'],"are",memory_allowed
sites_allowed = list(set(sites_allowed) & set(memory_allowed))
if not sites_allowed:
print wfo.name,"has no possible sites to run at"
print "available for",wfh.request['Memory'],"are",memory_allowed
sendEmail("no possible sites","%s has no possible sites to run at"%( wfo.name ))
continue
blocks = []
if 'BlockWhitelist' in wfh.request and wfh.request['BlockWhitelist']:
blocks = wfh.request['BlockWhitelist']
if 'RunWhitelist' in wfh.request and wfh.request['RunWhitelist']:
## should make the block selection here
pass
if 'LumiList' in wfh.request and wfh.request['LumiList']:
## same, we could be doing the white list here too
pass
if blocks:
print "Reading",len(blocks),"in whitelist"
can_go = True
staging=False
allowed=True
if primary:
copies_needed_from_CPUh,CPUh = wfh.getNCopies()
if talk:
print wfo.name,'reads',', '.join(primary),'in primary'
## chope the primary dataset
for prim in primary:
max_priority[prim] = max(max_priority[prim],int(wfh.request['RequestPriority']))
sites_allowed = [site for site in sites_allowed if not any([osite.startswith(site) for osite in SI.sites_veto_transfer])]
print "Sites allowed minus the vetoed transfer"
print sorted(sites_allowed)
copies_needed_from_site = int(0.35*len(sites_allowed))+1 ## should just go for a fixed number based if the white list grows that big
print "Would make",copies_needed_from_site,"copies from site white list"
copies_needed = copies_needed_from_site
print "Would make",copies_needed_from_CPUh,"from cpu requirement",CPUh
copies_needed = copies_needed_from_CPUh
if options.maxcopy>0:
## stop maxing things out ??
#copies_needed = min(options.maxcopy,copies_needed)
#print "Maxed to",copies_needed
if copies_needed_from_CPUh > options.maxcopy:
sendEmail('An example of more than three copies','for %s it could have been beneficial to make %s copies'%( wfo.name, copies_needed_from_CPUh))
if 'Campaign' in wfh.request and wfh.request['Campaign'] in CI.campaigns and 'maxcopies' in CI.campaigns[wfh.request['Campaign']]:
copies_needed_from_campaign = CI.campaigns[wfh.request['Campaign']]['maxcopies']
copies_needed = min(copies_needed_from_campaign,copies_needed_from_site)
print "Maxed to",copies_needed,"by campaign configuration",wfh.request['Campaign']
## remove the sites that do not want transfers
workflow_dependencies[prim].add( wfo.id )
#####################################
###### JR 3/8/15 #### deprecating this
"""
presence = getDatasetPresence( url, prim , within_sites = [SI.CE_to_SE(site) for site in sites_allowed])
prim_location = [site for site,pres in presence.items() if pres[0]==True]
prim_parts = [site for site,pres in presence.items() if pres[0]==False]
if len(prim_location) >= copies_needed:
print "The output is all fully in place at",len(prim_location),"sites"
continue
# reduce the number of copies required by existing full copies
copies_needed = max(0,copies_needed - len(prim_location))
print "now need",copies_needed
subscriptions = listSubscriptions( url , prim , sites_allowed )
prim_destination = list(set([site for (site,(tid,decision)) in subscriptions.items() if decision and not any([site.endswith(veto) for veto in ['MSS','Export','Buffer']])]))
## remove the subscription where the dataset is in parts at
#prim_destination = list(set([site for (site,(tid,decision)) in subscriptions.items() if decision and not any([site.endswith(veto) for veto in ['MSS','Export','Buffer']]) and not site in prim_parts]))
## need to reject from that list the ones with a full copy already: i.e the transfer corresponds to the copy in place
prim_destination = [site for site in prim_destination if not site in prim_location]
## add transfer dependencies
latching_on_transfers = list(set([ tid for (site,(tid,decision)) in subscriptions.items() if decision and site in prim_destination and not any([site.endswith(veto) for veto in ['MSS','Export','Buffer']])]))
print latching_on_transfers
"""
###### JR 3/8/15 #### deprecating this
#####################################
### new ways of making the whole thing
destinations,all_block_names = getDatasetDestinations(url, prim, within_sites = [SI.CE_to_SE(site) for site in sites_allowed], only_blocks=blocks )
#destinations,all_block_names = getDatasetDestinations(url, prim, within_sites = [SI.CE_to_SE(site) for site in sites_allowed], only_blocks=blocks, group='DataOps')
#anaops_destinations,anaops_all_block_names = getDatasetDestinations(url, prim, within_sites = [SI.CE_to_SE(site) for site in sites_allowed], only_blocks=blocks, group='AnalysisOps' )
print json.dumps(destinations, indent=2)
## get where the dataset is in full and completed
prim_location = [site for (site,info) in destinations.items() if info['completion']==100 and info['data_fraction']==1]
## the rest is places it is going to be
prim_destination = [site for site in destinations.keys() if not site in prim_location]
## need to take out the transfer veto
prim_destination = [site for site in prim_destination if not any([osite.startswith(site) for osite in SI.sites_veto_transfer])]
for dsite in prim_destination:
needing_locks[dsite].append( prim )
if len(prim_location) >= copies_needed:
print "The output is all fully in place at",len(prim_location),"sites",prim_location
continue
copies_needed = max(0,copies_needed - len(prim_location))
print "now need",copies_needed
copies_being_made = [ sum([info['blocks'].keys().count(block) for site,info in destinations.items() if site in prim_destination]) for block in all_block_names]
latching_on_transfers = set()
[latching_on_transfers.update(info['blocks'].values()) for site,info in destinations.items() if site in prim_destination]
latching_on_transfers = list(latching_on_transfers)
#print latching_on_transfers
## figure out where all this is going to go
prim_to_distribute = [site for site in sites_allowed if not SI.CE_to_SE(site) in prim_location]
prim_to_distribute = [site for site in prim_to_distribute if not SI.CE_to_SE(site) in prim_destination]
## take out the ones that cannot receive transfers
prim_to_distribute = [site for site in prim_to_distribute if not any([osite.startswith(site) for osite in SI.sites_veto_transfer])]
if any([transfers_per_sites[site] < max_staging_per_site for site in prim_to_distribute]):
## means there is openings let me go
print "There are transfer slots available:",[(site,transfers_per_sites[site]) for site in prim_to_distribute]
for site in sites_allowed:
#increment accross the board, regardless of real destination: could be changed
transfers_per_sites[site] += 1
else:
if int(wfh.request['RequestPriority']) >= in_transfer_priority and min_transfer_priority!=in_transfer_priority:
print "Higher priority sample",wfh.request['RequestPriority'],">=",in_transfer_priority,"go-on over transfer slots available"
else:
print "Not allowed to transfer more than",max_staging_per_site," per site at a time. Going overboard for",[site for site in prim_to_distribute if transfers_per_sites[site]>=max_staging_per_site]
if not options.go:
allowed = False
break
for latching in latching_on_transfers:
tfo = session.query(Transfer).filter(Transfer.phedexid == latching).first()
if not tfo:
tfo = Transfer( phedexid = latching)
tfo.workflows_id = []
session.add(tfo)
if not wfo.id in tfo.workflows_id:
print "adding",wfo.id,"to",tfo.id,"with phedexid",latching
l = copy.deepcopy( tfo.workflows_id )
l.append( wfo.id )
tfo.workflows_id = l
if not options.test:
session.commit()
else:
session.flush() ## regardless of commit later on, we need to let the next wf feeding on this transfer to see it in query
can_go = False
transfer_sizes[prim] = input_sizes[prim]
staging = True
# reduce the number of copies required by the on-going full transfer : how do we bootstrap on waiting for them ??
#copies_needed = max(0,copies_needed - len(prim_destination))
copies_needed = max(0,copies_needed - min(copies_being_made))
print "then need",copies_needed
if copies_needed == 0:
print "The output is either fully in place or getting in full somewhere with",latching_on_transfers
can_go = True
continue
if len(prim_to_distribute)>0: ## maybe that a parameter we can play with to limit the
if not options or options.chop:
chops,sizes = getDatasetChops(prim, chop_threshold = options.chopsize, only_blocks=blocks)
spreading = distributeToSites( chops, prim_to_distribute, n_copies = copies_needed, weights=SI.cpu_pledges, sizes=sizes)
transfer_sizes[prim] = sum(sizes)
else:
spreading = {}
for site in prim_to_distribute:
if blocks:
spreading[site]=blocks
else:
spreading[site]=[prim]
transfer_sizes[prim] = input_sizes[prim] ## this is approximate if blocks are specified
can_go = False
print "selected CE destinations",spreading.keys()
for (site,items) in spreading.items():
all_transfers[site].extend( items )
if not allowed:
print "Not allowed to move on with",wfo.name
continue
if secondary:
if talk:
print wfo.name,'reads',', '.join(secondary),'in secondary'
for sec in secondary:
workflow_dependencies[sec].add( wfo.id )
if False:
## new style, failing on minbias
if not sec in destination_cache:
## this is barbbaric, and does not show the correct picture on workflow by workflow with different whitelist
destination_cache[sec],_ = getDatasetDestinations(url, sec, within_sites = [SI.CE_to_SE(site) for site in sites_allowed])
destinations = destination_cache[sec]
## truncate location/destination to those making up for >90% of the dataset
bad_destinations = [destinations.pop(site) for (site,info) in destinations.items() if info['data_fraction']<0.9]
sec_location = [site for (site,info) in destinations.items() if info['completion']>=95]
sec_destination = [site for site in destinations.keys() if not site in sec_location]
else:
## old style
presence = getDatasetPresence( url, sec )
sec_location = [site for site,pres in presence.items() if pres[1]>90.] ## more than 90% of the minbias at sites
subscriptions = listSubscriptions( url ,sec )
sec_destination = [site for site in subscriptions]
for site in sec_location:
needing_locks[site].append( sec )
for site in sec_destination:
needing_locks[site].append( sec )
sec_to_distribute = [site for site in sites_allowed if not any([osite.startswith(site) for osite in sec_location])]
sec_to_distribute = [site for site in sec_to_distribute if not any([osite.startswith(site) for osite in sec_destination])]
sec_to_distribute = [site for site in sec_to_distribute if not any([osite.startswith(site) for osite in SI.sites_veto_transfer])]
if len( sec_to_distribute )>0:
sec_size = dss.get( sec )
for site in sec_to_distribute:
site_se =SI.CE_to_SE(site)
if (SI.disk[site_se]*1024.) > sec_size:
all_transfers[site].append( sec )
can_go = False
else:
print "could not send the secondary input to",site_se,"because it is too big for the available disk",SI.disk[site_se]*1024,"GB need",sec_size
#sendEmail('secondary input too big','%s is too big (%s) for %s (%s)'%( sec, sec_size, site_se, SI.disk[site_se]*1024))
## is that possible to do something more
if can_go:
## no explicit transfer required this time
if staging:
## but using existing ones
print wfo.name,"latches on existing transfers, and nothing else"
wfo.status = 'staging'
needs_transfer+=1
else:
print wfo.name,"should just be assigned NOW to",sites_allowed
wfo.status = 'staged'
passing_along+=1
print "setting status to",wfo.status
session.commit()
continue
else:
## there is an explicit transfer required
if staging:
## and also using an existing one
print wfo.name,"latches on existing transfers"
if not options.test:
wfo.status = 'staging'
print "setting status to",wfo.status
session.commit()
print wfo.name,"needs a transfer"
needs_transfer+=1
passing_along+=1
print "accumulated locks of dataset in place"
print json.dumps(needing_locks, indent=2)
for site,items in needing_locks.items():
for item in items:
LI.lock( item, SI.CE_to_SE(site), 'usable input')
print "accumulated transfers"
print json.dumps(all_transfers, indent=2)
fake_id=-1
wf_id_in_prestaging=set()
for (site,items_to_transfer) in all_transfers.iteritems():
items_to_transfer = list(set(items_to_transfer))
## convert to storage element
site_se = SI.CE_to_SE(site)
## site that do not want input datasets
if site in SI.sites_veto_transfer:
print site,"does not want transfers"
continue
## throttle the transfer size to T2s ? we'd be screwed by a noPU sample properly configured.
## massage a bit the items
blocks = [it for it in items_to_transfer if '#' in it]
datasets = [it for it in items_to_transfer if not '#' in it]
if execute:
print "Making a replica to",site,"(CE)",site_se,"(SE) for"
else:
print "Would make a replica to",site,"(CE)",site_se,"(SE) for"
print "\t",len(blocks),"blocks"
## remove blocks if full dataset is send out
blocks = [block for block in blocks if not block.split('#')[0] in datasets]
print "\t",len(blocks),"needed blocks for",list(set([block.split('#')[0] for block in blocks]))
print "\t",len(datasets),"datasets"
print "\t",datasets
items_to_transfer = blocks + datasets
## operate the transfer
if options and options.stop:
## ask to move-on
answer = raw_input('Continue with that ?')
if not answer.lower() in ['y','yes','go']:
continue
if execute:
result = makeReplicaRequest(url, site_se, items_to_transfer, 'prestaging', priority='normal')
## make use of max_priority dataset:priority to set the subscriptions priority
"""
## does not function
once = True
for item in items_to_transfer:
bds = item.split('#')[0]
if max_priority[bds] >= 90000:
if once:
w=10
print "waiting",w,"s before raising priority"
time.sleep(w)
once=False
## raise it to high priority
print item,"subscription priority raised to high at",site_se
#print "This does not work yet properly it seems"
print updateSubscription(url, site_se, item, priority='high')
"""
#for item in list(set([it.split('#')[0] for it in items_to_transfer])):
for item in items_to_transfer:
LI.lock( item, site_se, 'pre-staging')
else:
#result= {'phedex':{'request_created' : [{'id' : fake_id}]}}
result= {'phedex':{'request_created' : []}}
fake_id-=1
if not result:
print "ERROR Could not make a replica request for",site,items_to_transfer,"pre-staging"
continue
for phedexid in [o['id'] for o in result['phedex']['request_created']]:
new_transfer = session.query(Transfer).filter(Transfer.phedexid == phedexid).first()
print phedexid,"transfer created"
if not new_transfer:
new_transfer = Transfer( phedexid = phedexid)
session.add( new_transfer )
new_transfer.workflows_id = set()
for transfering in list(set(map(lambda it : it.split('#')[0], items_to_transfer))):
new_transfer.workflows_id.update( workflow_dependencies[transfering] )
new_transfer.workflows_id = list(new_transfer.workflows_id)
wf_id_in_prestaging.update(new_transfer.workflows_id)
session.commit()
## auto approve it
if execute:
approved = approveSubscription(url, phedexid, [site_se])
for wfid in wf_id_in_prestaging:
tr_wf = session.query(Workflow).get(wfid)
if tr_wf and tr_wf.status!='staging':
if execute:
tr_wf.status = 'staging'
if talk:
print "setting",tr_wf.name,"to staging"
session.commit()
print "will make",copies_needed,"of all"
read_file = 'legacy-reco-raw_prio.txt'
for l in open(read_file).read().split('\n'):
if not l: continue
if l.startswith('#'): continue
all_ds = filter(lambda w : w.count('/')==3 and w.startswith('/'), l.split())
dss.update( all_ds )
print sorted(dss)
for ds in dss:
availability = getDatasetBlocksFraction(url, ds )
if availability>=1: continue
blocks= []
chops,sizes = getDatasetChops(ds, chop_threshold = 4000, only_blocks=blocks)
spreading = distributeToSites( chops, prim_to_distribute, n_copies = copies_needed, weights=SI.cpu_pledges, sizes=sizes)
print json.dumps(spreading, indent=2)
for site,items in spreading.items():
full_spread[site].update( items )
to_lock.add( ds )
addHocLocks = set(json.loads( open('addhoc_lock.json').read()))
addHocLocks.update( to_lock )
if act:
open('addhoc_lock.json','w').write( json.dumps( sorted(addHocLocks), indent=2 ) )
for l in to_lock:
if act:
LI.lock(l, 'add HOC')
def transferor(url, specific=None, talk=True, options=None):
if userLock('transferor'): return
if options and options.test:
execute = False
else:
execute = True
SI = siteInfo()
CI = campaignInfo()
mcm = McMClient(dev=False)
dss = DSS()
print "counting all being handled..."
being_handled = len(
session.query(Workflow).filter(Workflow.status == 'away').all())
being_handled += len(
session.query(Workflow).filter(
Workflow.status.startswith('stag')).all())
being_handled += len(
session.query(Workflow).filter(
Workflow.status.startswith('assistance')).all())
max_to_handle = options.maxworkflows
allowed_to_handle = max(0, max_to_handle - being_handled)
wf_buffer = 5
if allowed_to_handle <= wf_buffer: ## buffer for having several wf per transfer
print "Not allowed to run more than", max_to_handle, "at a time. Currently", being_handled, "and", wf_buffer, "buffer"
else:
print being_handled, "already being handled", max_to_handle, "max allowed,", allowed_to_handle, "remaining", "and", wf_buffer, "buffer"
print "... done"
all_transfers = defaultdict(list)
workflow_dependencies = defaultdict(
set) ## list of wf.id per input dataset
wfs_and_wfh = []
print "getting all wf to consider ..."
cache = getWorkflows(url, 'assignment-approved', details=True)
for wfo in session.query(Workflow).filter(
Workflow.status == 'considered').all():
if specific and not specific in wfo.name: continue
cache_r = filter(lambda d: d['RequestName'] == wfo.name, cache)
if len(cache_r):
wfs_and_wfh.append((wfo,
workflowInfo(url,
wfo.name,
spec=False,
request=cache_r[0])))
else:
wfs_and_wfh.append((wfo, workflowInfo(url, wfo.name, spec=False)))
print "... done"
input_sizes = {}
## list the size of those in transfer already
in_transfer_priority = 0
min_transfer_priority = 100000000
print "getting all wf in staging ..."
for wfo in session.query(Workflow).filter(
Workflow.status == 'staging').all():
wfh = workflowInfo(url, wfo.name, spec=False)
(_, primary, _, _) = wfh.getIO()
for prim in primary:
input_sizes[prim] = dss.get(prim)
in_transfer_priority = max(in_transfer_priority,
int(wfh.request['RequestPriority']))
min_transfer_priority = min(min_transfer_priority,
int(wfh.request['RequestPriority']))
print "... done"
print "Max priority in transfer already", in_transfer_priority
print "Min priority in transfer already", min_transfer_priority
in_transfer_already = sum(input_sizes.values())
#sort by priority higher first
wfs_and_wfh.sort(cmp=lambda i, j: cmp(int(i[1].request[
'RequestPriority']), int(j[1].request['RequestPriority'])),
reverse=True)
## list the size of all inputs
print "getting all input sizes ..."
for (wfo, wfh) in wfs_and_wfh:
(_, primary, _, _) = wfh.getIO()
for prim in primary:
input_sizes[prim] = dss.get(prim)
print "... done"
grand_total = sum(input_sizes.values())
to_transfer = grand_total - in_transfer_already
grand_transfer_limit = options.maxtransfer
transfer_limit = grand_transfer_limit - in_transfer_already
print "%15.4f GB already being transfered" % in_transfer_already
print "%15.4f GB is the current requested transfer load" % to_transfer
print "%15.4f GB is the global transfer limit" % grand_transfer_limit
print "%15.4f GB is the available limit" % transfer_limit
# the max priority value per dataset.
max_priority = defaultdict(int)
needs_transfer = 0 ## so that we can count'em
passing_along = 0
transfer_sizes = {}
went_over_budget = False
for (wfo, wfh) in wfs_and_wfh:
print wfh.request['RequestPriority']
print wfo.name, "to be transfered"
#wfh = workflowInfo( url, wfo.name)
(_, primary, _, _) = wfh.getIO()
this_load = sum([input_sizes[prim] for prim in primary])
if (this_load
and (sum(transfer_sizes.values()) + this_load > transfer_limit
or went_over_budget)):
if went_over_budget:
print "Transfer has gone over bubget."
else:
print "Transfer will go over bubget."
print "%15.4f GB this load" % this_load
print "%15.4f GB already this round" % sum(transfer_sizes.values())
print "%15.4f GB is the available limit" % transfer_limit
went_over_budget = True
if int(
wfh.request['RequestPriority']
) >= in_transfer_priority and min_transfer_priority != in_transfer_priority:
print "Higher priority sample", wfh.request[
'RequestPriority'], ">=", in_transfer_priority, "go-on over budget"
else:
if not options.go:
print min_transfer_priority, "minimum priority", wfh.request[
'RequestPriority'], "<", in_transfer_priority, "stop"
continue
## throtlle by campaign go
if not CI.go(wfh.request['Campaign']):
print "No go for", wfh.request['Campaign']
if not options.go: continue
## check if the batch is announced
announced = False
is_real = False
for b in mcm.getA('batches', query='contains=%s' % wfo.name):
is_real = True
if b['status'] == 'announced':
announced = True
break
if not announced:
print wfo.name, "does not look announced." # skipping?, rejecting?, reporting?"
if not is_real:
print wfo.name, "does not appear to be genuine."
## prevent any duplication. if the wf is not mentioned in any batch, regardless of status
continue
## check on a grace period
injection_time = time.mktime(
time.strptime('.'.join(map(str, wfh.request['RequestDate'])),
"%Y.%m.%d.%H.%M.%S")) / (60. * 60.)
now = time.mktime(time.gmtime()) / (60. * 60.)
if float(now - injection_time) < 4.:
if not options.go and not announced:
print "It is too soon to start transfer: %3.2fH remaining" % (
now - injection_time)
continue
passing_along += 1
if passing_along >= allowed_to_handle:
if int(
wfh.request['RequestPriority']
) >= in_transfer_priority and min_transfer_priority != in_transfer_priority:
print "Higher priority sample", wfh.request[
'RequestPriority'], ">=", in_transfer_priority, "go-on over", max_to_handle
else:
print "Not allowed to pass more than", max_to_handle, "at a time. Currently", being_handled, "handled, and adding", passing_along
break
(lheinput, primary, parent, secondary) = wfh.getIO()
if options and options.tosites:
sites_allowed = options.tosites.split(',')
else:
sites_allowed = getSiteWhiteList(
(lheinput, primary, parent, secondary))
if 'SiteWhitelist' in CI.parameters(wfh.request['Campaign']):
sites_allowed = CI.parameters(
wfh.request['Campaign'])['SiteWhitelist']
blocks = []
if 'BlockWhitelist' in wfh.request and wfh.request['BlockWhitelist']:
blocks = wfh.request['BlockWhitelist']
can_go = True
staging = False
if primary:
if talk:
print wfo.name, 'reads', ', '.join(primary), 'in primary'
## chope the primary dataset
for prim in primary:
max_priority[prim] = max(max_priority[prim],
int(wfh.request['RequestPriority']))
sites_really_allowed = [
site for site in sites_allowed if not any([
osite.startswith(site)
for osite in SI.sites_veto_transfer
])
]
print "Sites allowed minus the vetoed transfer"
print sites_really_allowed
copies_needed = int(
0.35 * len(sites_really_allowed)
) + 1 ## should just go for a fixed number based if the white list grows that big
print "Would make", copies_needed, "copies"
if options.maxcopy > 0:
copies_needed = min(options.maxcopy, copies_needed)
## remove the sites that do not want transfers
print "need", copies_needed
workflow_dependencies[prim].add(wfo.id)
presence = getDatasetPresence(url, prim)
prim_location = [
site for site, pres in presence.items() if pres[0] == True
]
if len(prim_location) >= copies_needed:
print "The output is all fully in place at", len(
prim_location), "sites"
continue
# reduce the number of copies required by existing full copies
copies_needed = max(0, copies_needed - len(prim_location))
print "now need", copies_needed
subscriptions = listSubscriptions(url, prim)
prim_destination = list(
set([
site
for (site, (tid, decision)) in subscriptions.items()
if decision and not any([
site.endswith(veto)
for veto in ['MSS', 'Export', 'Buffer']
])
]))
## need to reject from that list the ones with a full copy already: i.e the transfer corresponds to the copy in place
prim_destination = [
site for site in prim_destination
if not site in prim_location
]
## add transfer dependencies
latching_on_transfers = list(
set([
tid
for (site, (tid, decision)) in subscriptions.items()
if decision and site in prim_destination and not any([
site.endswith(veto)
for veto in ['MSS', 'Export', 'Buffer']
])
]))
print latching_on_transfers
for latching in latching_on_transfers:
tfo = session.query(Transfer).filter(
Transfer.phedexid == latching).first()
if not tfo:
tfo = Transfer(phedexid=latching)
tfo.workflows_id = []
session.add(tfo)
if not wfo.id in tfo.workflows_id:
print "adding", wfo.id, "to", tfo.id, "with phedexid", latching
l = copy.deepcopy(tfo.workflows_id)
l.append(wfo.id)
tfo.workflows_id = l
if not options.test:
session.commit()
else:
session.flush(
) ## regardless of commit later on, we need to let the next wf feeding on this transfer to see it in query
can_go = False
transfer_sizes[prim] = input_sizes[prim]
staging = True
# reduce the number of copies required by the on-going full transfer : how do we bootstrap on waiting for them ??
copies_needed = max(0, copies_needed - len(prim_destination))
print "then need", copies_needed
if copies_needed == 0:
print "The output is either fully in place or getting in full somewhere with", latching_on_transfers
can_go = True
continue
prim_to_distribute = [
site for site in sites_allowed if not any(
[osite.startswith(site) for osite in prim_location])
]
prim_to_distribute = [
site for site in prim_to_distribute if not any(
[osite.startswith(site) for osite in prim_destination])
]
## take out the ones that cannot receive transfers
prim_to_distribute = [
site for site in prim_to_distribute if not any([
osite.startswith(site)
for osite in SI.sites_veto_transfer
])
]
if len(
prim_to_distribute
) > 0: ## maybe that a parameter we can play with to limit the
if not options or options.chop:
spreading = distributeToSites(getDatasetChops(prim),
prim_to_distribute,
n_copies=copies_needed,
weights=SI.cpu_pledges)
else:
spreading = {}
for site in prim_to_distribute:
spreading[site] = [prim]
can_go = False
transfer_sizes[prim] = input_sizes[prim]
for (site, items) in spreading.items():
all_transfers[site].extend(items)
if secondary:
if talk:
print wfo.name, 'reads', ', '.join(secondary), 'in secondary'
for sec in secondary:
workflow_dependencies[sec].add(wfo.id)
presence = getDatasetPresence(url, sec)
sec_location = [
site for site, pres in presence.items() if pres[1] > 90.
] ## more than 90% of the minbias at sites
subscriptions = listSubscriptions(url, sec)
sec_destination = [site for site in subscriptions]
sec_to_distribute = [
site for site in sites_allowed if
not any([osite.startswith(site) for osite in sec_location])
]
sec_to_distribute = [
site for site in sec_to_distribute if not any(
[osite.startswith(site) for osite in sec_destination])
]
sec_to_distribute = [
site for site in sec_to_distribute if not any([
osite.startswith(site)
for osite in SI.sites_veto_transfer
])
]
if len(sec_to_distribute) > 0:
for site in sec_to_distribute:
all_transfers[site].append(sec)
can_go = False
## is that possible to do something more
if can_go:
## no explicit transfer required this time
if staging:
## but using existing ones
print wfo.name, "latches on existing transfers, and nothing else"
wfo.status = 'staging'
else:
print wfo.name, "should just be assigned NOW to", sites_allowed
wfo.status = 'staged'
print "setting status to", wfo.status
session.commit()
continue
else:
## there is an explicit transfer required
if staging:
## and also using an existing one
print wfo.name, "latches on existing transfers"
if not options.test:
wfo.status = 'staging'
print "setting status to", wfo.status
session.commit()
print wfo.name, "needs a transfer"
needs_transfer += 1
#print json.dumps(all_transfers)
fake_id = -1
wf_id_in_prestaging = set()
for (site, items_to_transfer) in all_transfers.iteritems():
items_to_transfer = list(set(items_to_transfer))
## convert to storage element
site_se = SI.CE_to_SE(site)
## site that do not want input datasets
if site in SI.sites_veto_transfer:
print site, "does not want transfers"
continue
## throttle the transfer size to T2s ? we'd be screwed by a noPU sample properly configured.
## massage a bit the items
blocks = [it for it in items_to_transfer if '#' in it]
datasets = [it for it in items_to_transfer if not '#' in it]
if execute:
print "Making a replica to", site, "(CE)", site_se, "(SE) for"
else:
print "Would make a replica to", site, "(CE)", site_se, "(SE) for"
print "\t", len(blocks), "blocks"
## remove blocks if full dataset is send out
blocks = [
block for block in blocks if not block.split('#')[0] in datasets
]
print "\t", len(blocks), "needed blocks for", list(
set([block.split('#')[0] for block in blocks]))
print "\t", len(datasets), "datasets"
print "\t", datasets
items_to_transfer = blocks + datasets
## operate the transfer
if options and options.stop:
## ask to move-on
answer = raw_input('Continue with that ?')
if not answer.lower() in ['y', 'yes', 'go']:
continue
if execute:
result = makeReplicaRequest(url,
site_se,
items_to_transfer,
'prestaging',
priority='normal')
## make use of max_priority dataset:priority to set the subscriptions priority
"""
## does not function
once = True
for item in items_to_transfer:
bds = item.split('#')[0]
if max_priority[bds] >= 90000:
if once:
w=10
print "waiting",w,"s before raising priority"
time.sleep(w)
once=False
## raise it to high priority
print item,"subscription priority raised to high at",site_se
#print "This does not work yet properly it seems"
print updateSubscription(url, site_se, item, priority='high')
"""
else:
#result= {'phedex':{'request_created' : [{'id' : fake_id}]}}
result = {'phedex': {'request_created': []}}
fake_id -= 1
if not result:
print "ERROR Could not make a replica request for", site, items_to_transfer, "pre-staging"
continue
for phedexid in [o['id'] for o in result['phedex']['request_created']]:
new_transfer = session.query(Transfer).filter(
Transfer.phedexid == phedexid).first()
print phedexid, "transfer created"
if not new_transfer:
new_transfer = Transfer(phedexid=phedexid)
session.add(new_transfer)
new_transfer.workflows_id = set()
for transfering in list(
set(map(lambda it: it.split('#')[0], items_to_transfer))):
new_transfer.workflows_id.update(
workflow_dependencies[transfering])
new_transfer.workflows_id = list(new_transfer.workflows_id)
wf_id_in_prestaging.update(new_transfer.workflows_id)
session.commit()
## auto approve it
if execute:
approved = approveSubscription(url, phedexid, [site_se])
for wfid in wf_id_in_prestaging:
tr_wf = session.query(Workflow).get(wfid)
if tr_wf and tr_wf.status != 'staging':
if execute:
tr_wf.status = 'staging'
if talk:
print "setting", tr_wf.name, "to staging"
session.commit()