def submit(self):
try:
from modules.wma import makeRequest,approveRequest
from wmcontrol import random_sleep
print '\n\tFound wmcontrol\n'
except:
print '\n\tUnable to find wmcontrol modules. Please include it in your python path\n'
if not self.testMode:
print '\n\t QUIT\n'
sys.exit(-17)
import pprint
for (n,d) in self.chainDicts.items():
if self.testMode:
print "Only viewing request",n
print pprint.pprint(d)
else:
#submit to wmagent each dict
print "For eyes before submitting",n
print pprint.pprint(d)
print "Submitting",n,"..........."
workFlow=makeRequest(self.wmagent,d,encodeDict=True)
approveRequest(self.wmagent,workFlow)
print "...........",n,"submitted"
random_sleep()
def submit(self):
try:
from modules.wma import makeRequest, approveRequest
from wmcontrol import random_sleep
print '\n\tFound wmcontrol\n'
except:
print '\n\tUnable to find wmcontrol modules. Please include it in your python path\n'
if not self.testMode:
print '\n\t QUIT\n'
sys.exit(-17)
import pprint
for (n, d) in self.chainDicts.items():
if self.testMode:
print "Only viewing request", n
print pprint.pprint(d)
else:
#submit to wmagent each dict
print "For eyes before submitting", n
print pprint.pprint(d)
print "Submitting", n, "..........."
workFlow = makeRequest(self.wmagent, d, encodeDict=True)
approveRequest(self.wmagent, workFlow)
print "...........", n, "submitted"
random_sleep()
def approveRequest(options):
if options.workflows == '':
print 'No workflows found'
sys.exit(-1)
workflows = set(options.workflows.split(','))
print 'Approving requests: %s' % workflows
if options.wmtest:
wma.testbed(options.wmtesturl)
for workflow in workflows:
tries = 1
while tries < 3:
try:
if(wma.getWorkflowStatus(wma.WMAGENT_URL, workflow) == 'new'):
wma.approveRequest(wma.WMAGENT_URL, workflow)
break
except Exception, e:
time.sleep(1)
print 'Something went wrong: %s Try number: %s' % (str(e), tries)
tries += 1
def loop_and_submit(cfg):
'''
Loop on all the sections of the configparser, build and submit the request.
This is the orchestra director function.
'''
pp = pprint.PrettyPrinter(indent=4)
for section in cfg.configparser.sections():
# Warning muted
#print '\n---> Processing request "%s"' %section
# build the dictionary for the request
params,service_params = build_params_dict(section,cfg)
dataset_runs_dict = get_dataset_runs_dict (section,cfg)
if not dataset_runs_dict:
sys.stderr.write("[wmcontrol exception] No dataset_runs_dict provided")
sys.exit(-1)
# Submit request!
for dataset in sorted(dataset_runs_dict.keys()):
params['InputDataset']=dataset
runs=[]
new_blocks=[]
for item in dataset_runs_dict[dataset]:
if isinstance(item,str) and '#' in item:
if item.startswith('#'):
new_blocks.append(dataset+item)
else:
new_blocks.append(item)
else:
runs.append(item)
params['RunWhitelist']=runs
if params.has_key("BlockWhitelist"):
if params['BlockWhitelist']==[]:
params['BlockWhitelist']=new_blocks
if params['BlockWhitelist']!=[] and new_blocks!=[]:
print "WARNING: a different set of blocks was made available in the input dataset and in the blocks option."
print "Keeping the blocks option (%s) instead of (%s)" % (str(sorted(new_blocks)), str(sorted(params['BlockWhitelist'])))
params['BlockWhitelist']=new_blocks
params['RequestString']= make_request_string(params,service_params,section)
if service_params['request_type'] in ['MonteCarlo','LHEStepZero']:
params.pop('InputDataset')
params.pop('RunWhitelist')
elif service_params['request_type'] == 'TaskChain':
if params['InputDataset']:
params['Task1']['InputDataset'] = params['InputDataset']
if params['RunWhitelist']:
params['Task1']['RunWhitelist'] = params['RunWhitelist']
if 'RunWhitelist' in params: #if params has key we remove it
params.pop('RunWhitelist') #because it was set as Task parameter
if 'InputDataset' in params:
params.pop('InputDataset')
elif ('RequestNumEvents' in params and
('RunWhitelist' not in params or params['RunWhitelist']==[])):
if service_params['request_type'] in ['ReDigi','ReReco']:
events = float(params['RequestNumEvents'])
elif service_params['request_type'] in ['MonteCarloFromGEN']:
events = float(params['RequestNumEvents'] / float(params['FilterEfficiency']))
if events:
if test_mode:
t = time.time()
espl = helper.SubsetByLumi(params['InputDataset'],
float(service_params['margin']))
split, details = espl.run(int(events),
service_params['brute_force'],
service_params['force_lumis'])
# https://github.com/dmwm/DBS/issues/280
# https://github.com/dmwm/DBS/issues/428
if split == 'blocks':
params['BlockWhitelist'] = details
elif split == 'lumis':
params['LumiList'] = details
params['SplittingAlgo'] = 'LumiBased'
# params.pop('RunWhitelist')
if test_mode:
print "Finished in", int((time.time()-t)*1000), "\bms"
# just print the parameters of the request you would have injected
if test_mode:
pp.pprint(params)
else: # do it for real!
try:
workflow = wma.makeRequest(wma.WMAGENT_URL,params,encodeDict=(service_params['request_type']=='TaskChain'))
except:
random_sleep()
#just try a second time
workflow = wma.makeRequest(wma.WMAGENT_URL,params,encodeDict=(service_params['request_type']=='TaskChain'))
try:
wma.approveRequest(wma.WMAGENT_URL,workflow)
except:
random_sleep()
#just try a second time
wma.approveRequest(wma.WMAGENT_URL,workflow)
random_sleep()
