def spawn_harvesting(url, wfi, in_full): SI = global_SI() all_OK = {} requests = [] outputs = wfi.request['OutputDatasets'] if ('EnableHarvesting' in wfi.request and not wfi.request['EnableHarvesting']) and ( 'DQMConfigCacheID' in wfi.request and wfi.request['DQMConfigCacheID']): if not 'MergedLFNBase' in wfi.request: print "f****d up" sendEmail('screwed up wl cache', '%s wl cache is bad' % (wfi.request['RequestName'])) all_OK['fake'] = False return all_OK, requests wfi = workflowInfo(url, wfi.request['RequestName']) dqms = [out for out in outputs if '/DQM' in out] if not all([in_full[dqm_input] for dqm_input in dqms]): wfi.sendLog( 'closor', "will not be able to assign the harvesting: holding up") for dqm_input in dqms: all_OK[dqm_input] = False ## raise the subscription to high priority sites = set(wfi.request['NonCustodialSites']) for site in sites: res = updateSubscription(url, site, dqm_input, priority='reserved') print "increased priority", res ## make a subscription somewhere else in the T1s at random each time, hoping it will get there eventually #pick = SI.CE_to_SE(random.choice(SI.sites_T1s)) #print "replicating",dqm_input,"to",pick #res = makeReplicaRequest(url, pick, [dqm_input], "Replicating DQM for harvesting", priority='normal', approve=True, mail=False) return all_OK, requests for dqm_input in dqms: ## handle it properly harvesting_schema = { 'Requestor': os.getenv('USER'), 'RequestType': 'DQMHarvest', 'Group': 'DATAOPS' } copy_over = [ 'AcquisitionEra', 'ProcessingString', 'DQMUploadUrl', 'CMSSWVersion', 'CouchDBName', 'CouchWorkloadDBName', 'ConfigCacheUrl', 'DbsUrl', 'inputMode', 'DQMConfigCacheID', 'OpenRunningTimeout', 'ScramArch', 'CMSSWVersion', 'Campaign', 'Memory', #dummy 'SizePerEvent', #dummy 'GlobalTag', #dummy ] for item in copy_over: if item in wfi.request: harvesting_schema[item] = copy.deepcopy(wfi.request[item]) else: print item, "is not in initial schema" harvesting_schema['InputDataset'] = dqm_input harvesting_schema['TimePerEvent'] = 1 harvesting_schema['PrepID'] = 'Harvest-' + wfi.request['PrepID'] if len(wfi.request['RequestString']) > 60: wfi.request['RequestString'] = wfi.request[ 'RequestString'][:60] print "truncating request string", wfi.request['RequestString'] harvesting_schema[ 'RequestString'] = 'HARVEST-' + wfi.request['RequestString'] harvesting_schema['DQMHarvestUnit'] = 'byRun' harvesting_schema['RequestPriority'] = min( wfi.request['RequestPriority'] * 10, 999999) harvest_request = reqMgrClient.submitWorkflow( url, harvesting_schema) if not harvest_request: print "Error in making harvesting for", wfi.request[ 'RequestName'] print "schema" print json.dumps(harvesting_schema, indent=2) harvest_request = reqMgrClient.submitWorkflow( url, harvesting_schema) if not harvest_request: print "Error twice in harvesting for", wfi.request[ 'RequestName'] print "schema" print json.dumps(harvesting_schema, indent=2) if harvest_request: requests.append(harvest_request) ## should we protect for setting approved ? no, it's notified below, assignment will fail, likely data = reqMgrClient.setWorkflowApproved(url, harvest_request) print "created", harvest_request, "for harvesting of", dqm_input wfi.sendLog( 'closor', "created %s for harvesting of %s" % (harvest_request, dqm_input)) ## assign it directly team = wfi.request['Team'] parameters = { 'SiteWhitelist': [ SI.SE_to_CE(se) for se in wfi.request['NonCustodialSites'] ], 'AcquisitionEra': wfi.acquisitionEra(), 'ProcessingString': wfi.processingString(), 'MergedLFNBase': wfi.request['MergedLFNBase'], 'ProcessingVersion': wfi.request['ProcessingVersion'], 'execute': True } if in_full[dqm_input]: print "using full copy at", in_full[dqm_input] parameters['SiteWhitelist'] = [ SI.SE_to_CE(se) for se in in_full[dqm_input] ] else: print "cannot do anything if not having a full copy somewhere" all_OK[dqm_input] = False continue result = reqMgrClient.assignWorkflow(url, harvest_request, team, parameters) if not result: #sendEmail('harvesting request created','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch']) wfi.sendLog( 'closor', '%s was created at announcement of %s in %s, failed to assign' % (harvest_request, dqm_input, wfi.request['RequestName'])) sendLog( 'closor', '%s was created at announcement of %s in %s, failed to assign' % (harvest_request, dqm_input, wfi.request['RequestName']), level='critical') else: #sendEmail('harvesting request assigned','%s was created at announcement of %s in %s, and assigned'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch']) wfi.sendLog( 'closor', '%s was created at announcement of %s in %s, and assigned' % (harvest_request, dqm_input, wfi.request['RequestName'])) else: #print "could not make the harvesting for",wfo.name,"not announcing" wfi.sendLog('closor', "could not make the harvesting request") sendLog('closor', "could not make the harvesting request for %s" % wfi.request['RequestName'], level='critical') all_OK[dqm_input] = False return (all_OK, requests)
def spawn_harvesting(url, wfi , in_full): #SI = siteInfo() SI = global_SI() all_OK = {} requests = [] outputs = wfi.request['OutputDatasets'] if ('EnableHarvesting' in wfi.request and wfi.request['EnableHarvesting']) or ('DQMConfigCacheID' in wfi.request and wfi.request['DQMConfigCacheID']): if not 'MergedLFNBase' in wfi.request: print "f****d up" sendEmail('screwed up wl cache','%s wl cache is bad'%(wfi.request['RequestName'])) all_OK['fake'] = False return all_OK,requests wfi = workflowInfo(url, wfi.request['RequestName']) dqms = [out for out in outputs if '/DQM' in out] if not all([in_full[dqm_input] for dqm_input in dqms]): wfi.sendLog('closor',"will not be able to assign the harvesting: holding up") for dqm_input in dqms: all_OK[dqm_input] = False ## raise the subscription to high priority sites = set(wfi.request['NonCustodialSites']) for site in sites: res = updateSubscription(url, site, dqm_input, priority='high') print "increased priority",res return all_OK,requests for dqm_input in dqms: ## handle it properly harvesting_schema = { 'Requestor': os.getenv('USER'), 'RequestType' : 'DQMHarvest', 'Group' : 'DATAOPS' } copy_over = [ 'AcquisitionEra', 'ProcessingString', 'DQMUploadUrl', 'CMSSWVersion', 'CouchDBName', 'CouchWorkloadDBName', 'CouchURL', 'DbsUrl', 'inputMode', 'DQMConfigCacheID', 'OpenRunningTimeout', 'ScramArch', 'CMSSWVersion', 'Campaign', 'Memory', #dummy 'SizePerEvent', #dummy 'GlobalTag', #dummy ] for item in copy_over: if item in wfi.request: harvesting_schema[item] = copy.deepcopy(wfi.request[item]) else: print item,"is not in initial schema" harvesting_schema['InputDataset'] = dqm_input harvesting_schema['TimePerEvent'] = 1 harvesting_schema['PrepID'] = 'Harvest-'+wfi.request['PrepID'] if len(wfi.request['RequestString'])>60: wfi.request['RequestString']= wfi.request['RequestString'][:60] print "truncating request string",wfi.request['RequestString'] harvesting_schema['RequestString'] = 'HARVEST-'+wfi.request['RequestString'] harvesting_schema['DQMHarvestUnit'] = 'byRun' harvesting_schema['ConfigCacheUrl'] = harvesting_schema['CouchURL'] ## uhm, how stupid is that ? harvesting_schema['RequestPriority'] = wfi.request['RequestPriority']*10 harvest_request = reqMgrClient.submitWorkflow(url, harvesting_schema) if not harvest_request: print "Error in making harvesting for",wfi.request['RequestName'] print "schema" print json.dumps( harvesting_schema, indent = 2) harvest_request = reqMgrClient.submitWorkflow(url, harvesting_schema) if not harvest_request: print "Error twice in harvesting for",wfi.request['RequestName'] print "schema" print json.dumps( harvesting_schema, indent = 2) if harvest_request: requests.append( harvest_request ) ## should we protect for setting approved ? no, it's notified below, assignment will fail, likely data = reqMgrClient.setWorkflowApproved(url, harvest_request) print "created",harvest_request,"for harvesting of",dqm_input wfi.sendLog('closor',"created %s for harvesting of %s"%( harvest_request, dqm_input)) ## assign it directly team = wfi.request['Teams'][0] parameters={ 'SiteWhitelist' : [SI.SE_to_CE(se) for se in wfi.request['NonCustodialSites']], 'AcquisitionEra' : wfi.acquisitionEra(), 'ProcessingString' : wfi.processingString(), 'MergedLFNBase' : wfi.request['MergedLFNBase'], 'ProcessingVersion' : wfi.request['ProcessingVersion'], 'execute' : True } if in_full[dqm_input]: print "using full copy at",in_full[dqm_input] parameters['SiteWhitelist'] = [SI.SE_to_CE(se) for se in in_full[dqm_input]] else: print "cannot do anything if not having a full copy somewhere" all_OK[dqm_input]=False continue result = reqMgrClient.assignWorkflow(url, harvest_request, team, parameters) if not result: #sendEmail('harvesting request created','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch']) wfi.sendLog('closor','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName'])) sendLog('closor','%s was created at announcement of %s in %s, failed to assign'%(harvest_request, dqm_input, wfi.request['RequestName']), level='critical') else: #sendEmail('harvesting request assigned','%s was created at announcement of %s in %s, and assigned'%(harvest_request, dqm_input, wfi.request['RequestName']), destination=[wfi.request['Requestor']+'@cern.ch']) wfi.sendLog('closor','%s was created at announcement of %s in %s, and assigned'%(harvest_request, dqm_input, wfi.request['RequestName'])) else: #print "could not make the harvesting for",wfo.name,"not announcing" wfi.sendLog('closor',"could not make the harvesting request") sendLog('closor',"could not make the harvesting request for %s"% wfi.request['RequestName'], level='critical') all_OK[dqm_input]=False return (all_OK, requests)