'dbsurl':dbs_url_g,

'keep_step1':False,

'keep_step2':False,

'priority':181983,

'request_type':'ReReco',

'scramarch':'slc5_amd64_gcc462',

'includeparents': 'False'

ACTIONS = optparse.Option.ACTIONS + ("extend",)

STORE_ACTIONS = optparse.Option.STORE_ACTIONS + ("extend",)

TYPED_ACTIONS = optparse.Option.TYPED_ACTIONS + ("extend",)

ALWAYS_TYPED_ACTIONS = optparse.Option.ALWAYS_TYPED_ACTIONS + ("extend",)

def take_action(self, action, dest, opt, value, values, parser):

if action == "extend":

lvalue = value.split(",")

values.ensure_value(dest, []).extend(lvalue)

else:

'''

A class that offers a common interface to get parameters out of a

optionParser (command line) or a ConfigParser (ini cfg).

The key is to build a ConfigParser object with a single section out

of the option parser.

'''

default_section= '__OptionParser__'

def __init__ (self, parser):

# assume you have a .conf input file, first...

# see : https://docs.python.org/2/library/configparser.html

self.configparser=ConfigParser.SafeConfigParser()

try:

options,args = parser.parse_args()

except SystemExit:

print "Error in parsing options"

sys.stderr.write("[wmcontrol exception] Error in parsing options")

sys.exit(-1)

global test_mode

test_mode = test_mode or options.test

if options.wmtest:

print "Setting to injection in cmswebtest : ", options.wmtesturl

wma.testbed(options.wmtesturl)

if options.req_file != '' and options.req_file !=None:

cfg_filename=options.req_file

print "We have a configfile: %s." %cfg_filename

self.configparser.read(cfg_filename)

else: # ... otherwise, we have to convert the command line option parser to a .conf, and populate self.configparser

print "We have a commandline."

self.__fill_configparser(options)

def __fill_configparser(self,options):

'''

Convert the option parser (from command line) into a configparser (as if it was .conf file).

'''

# loop on all option parser parameters and fille the cp

self.configparser.add_section(self.__class__.default_section)

for param,param_value in options.__dict__.items():

if param_value == None:

param_value = "__NOT-DEFINED__"

#print "Setting params in cfg: %s with default %s" %(param,param_value)

### HOLLY SHIT THE SYSTEMATIC RECASTING TO STR

self.configparser.set(self.__class__.default_section, param, str(param_value))

#with open('example.cfg', 'wb') as configfile:

#self.ConfigParser.write(configfile)

def get_param(self,name,default=None,section=default_section,verbose=False):

'''

I am astonished that such function does not exist in Python!!

'''

ret_val=None

if verbose:

print "I am looking for section %s and option %s, the default is #%s#" %(name, section, default)

if self.configparser.has_section(section):

if self.configparser.has_option(section,name):

#print "Getting %s %s" %(section,name)

ret_val = self.configparser.get(section,name)

# We had a cfg file and the default was not given

if ret_val == "__NOT-DEFINED__":

if verbose:

print "I was reading parameter %s and I put the default %s" %(name, default)

ret_val = default

# we have both: read and return!

else:

pass

else:

# We don't have the option, try to return the default

if default!=None:

# Case 1, we have the default

ret_val = default

else:

# Case 2, we do not have the default, exception

raise Exception ("Parameter %s cannot be found in section %s and no default is given." %(name,section))

else:

# No section found, just rais e an exception

raise Exception ("No section %s found in configuration." %section)

if verbose:

print "I am returning the value #%s#" %(ret_val)

'''

Get the runs from the DBS via the DBS interface

'''

print "Looking for runs in DBS for %s" %dset_name

minrun=int(minrun)

maxrun=int(maxrun)

# check if cmssw is set up for the dbs command

if not os.environ.has_key("CMSSW_BASE"):

raise Exception("No CMSSW environment set. You need it to query dbs.")

return_data = json.loads(wma.generic_get(wma.WMAGENT_URL, wma.DBS3_URL+"runs?dataset=%s" %(dset_name)))

run_list = return_data[0]["run_num"]

if minrun >= 0:

run_list = filter(lambda n: n >= minrun,run_list)

if maxrun >= 0:

run_list = filter(lambda n: n <= maxrun,run_list)

# not clear if this custodial method is actually used anywhere; otherwise clean it up ?

if test_mode:

return "custodialSite1"

allSites = 1

# datasetpath = None

# lfn = None

# allSites = 0

# try:

# opts, args = getopt.getopt(sys.argv[1:], "", ["lfn=","dataset=","allSites"])

# except getopt.GetoptError:

# print 'Please specify dataset with --dataset'

# print 'Specify --allSites to show all site location, otherwise show only T1 sites'

# sys.exit(2)

# check command line parameter

# for opt, arg in opts :

# if opt == "--dataset" :

# datasetpath = arg

# if opt == "--lfn" :

# lfn = arg

# if opt == "--allSites" :

# allSites = 1

# if datasetpath == None and lfn == None:

# print 'Please specify dataset with --dataset'

# sys.exit(2)

custodial = {}

non_custodial = {}

# if lfn == None :

url=wma.PHEDEX_ADDR %datasetpath

try:

result = json.load(urllib.urlopen(url))

except:

print 'Problems with url',url

try:

for block in result['phedex']['block']:

name = block['name']

for replica in block['replica']:

files = replica['files']

site = str(replica['node'])

is_custodial = replica['custodial']

if allSites == 1 or ( site[0:2] == 'T1' or site[0:2] == 'T0') :

if is_custodial == 'y' :

if site not in custodial.keys() :

custodial[site] = int(files)

else :

custodial[site] += int(files)

else :

if site not in non_custodial.keys() :

non_custodial[site] = int(files)

else :

non_custodial[site] += int(files)

except:

print 'Problems with dataset:',datasetpath

custodial_sites = custodial.keys()

non_custodial_sites = non_custodial.keys()

custodial_sites.sort()

non_custodial_sites.sort()

if len(custodial_sites) == 0 :

custodial['NONE'] = 0

if len(non_custodial_sites) == 0 :

non_custodial['NONE'] = 0

if len(custodial_sites) == 1 and custodial_sites[0].count('T0') > 0 :

custodial['NONE'] = 0

if len(non_custodial_sites) == 1 and non_custodial_sites[0].count('T0') > 0 :

non_custodial['NONE'] = 0

sites = ''

custsites = ''

for site in non_custodial_sites :

if site not in custodial_sites :

sites = sites + site + '(' + str(non_custodial[site]) + '),'

if sites[-1:] == ',' :

sites = sites[:-1]

for custsite in custodial_sites :

# custsites = custsites + custsite + '(' + str(custodial[custsite]) + '),'

custsites = custsites + custsite

if custsites[-1:] == ',' :

custsites = custsites[:-1]

# if lfn == None :

print 'dataset:',datasetpath,'custodial:',custsites,'non-custodial:',sites

# else :

# print 'lfn:',lfn,'custodial:',custsites,'non-custodial:',sites

"""

Sleep for a random time in order not to choke the server submitting too many

requests in a small time interval

"""

rnd=abs(random.gauss(0,sigma))

sleep_time=min_sleep+rnd

#print "Sleeping %s seconds" %sleep_time

"""

Create a nickname out of a dataset name. Some heuristic tricks are used to

make the nicks shorter.

"""

#print dataset

nick=dataset

if "/" in dataset:

c1,c2,c3=dataset[1:].split('/')

nick="%s_%s_%s" %(c1,c2,c3)

nick=nick.replace("Electron","El")

nick=nick.replace("Single","Sing")

nick=nick.replace("Double","Dbl")

'''

If present, eval it, if not just use the plain dataset name input and [] !

'''

dataset_runs_dict={}

run_list = []

try:

dataset_runs_dict = eval(cfg.get_param('dset_run_dict','',section))

for key in dataset_runs_dict.keys():

if isinstance(dataset_runs_dict[key], str):

if os.path.exists(dataset_runs_dict[key]):

try:

json_file = open(dataset_runs_dict[key])

json_info = json.load(json_file)

json_file.close()

except ValueError:

print "Error in JSON file: ", dataset_runs_dict[key], " Exiting..."

print traceback.format_exc()

#sys.exit()

return False

for run_number in json_info:

run_list.append(int(run_number))

run_list.sort()

dataset_runs_dict[key] = run_list

run_list = []

else:

print "JSON file doesn't exists. ", os.path.join(os.getcwd(), dataset_runs_dict[key]), " Exiting..."

#sys.exit()

return False

except:

dataset_runs_dict[cfg.get_param('input_name','',section)]=[]

request_type = service_params['request_type']

identifier=''

dataset=params['InputDataset']

# Make a string a la prep if needed: #old version

# if request == Configuration.default_section:

# # Request ID string

# joinString = ""

# if custodial(params['InputDataset']):

# joinString = "_v"

# identifier = "%s_%s%s%s" %(params['PrepID'],

# custodial(dataset),

# joinString,

# service_params['version'])

#

# elif service_params['req_name'] != '' :

# identifier=service_params['req_name']

if request == Configuration.default_section:

# Request ID string

if 'PrepID' in params:

pid=params['PrepID']

else:

pid=service_params['pid']

joinString = ""

if request_type in ['MonteCarloFromGEN', 'MonteCarlo', 'LHEStepZero', 'TaskChain']:

joinString = "_v"

identifier = "%s_%s%s%s_%s" %(pid,

service_params['batch'],

joinString,

service_params['version'],

service_params['process_string'])

else:

#if custodial(params['InputDataset']):

# custname = custodial(dataset)

#else:

# custname = "No_custT1"

joinString = "_v"

identifier = "%s_%s%s%s_%s" %(pid,

service_params['batch'],

joinString,

service_params['version'],

service_params['process_string'])

elif service_params['req_name'] != '':

identifier=service_params['req_name']

else:

dset_nick = get_dset_nick(dataset)

cmssw_version=params["CMSSWVersion"]

cmssw_version=cmssw_version.replace("CMSSW","")

cmssw_version=cmssw_version.replace("_","")

cmssw_version=cmssw_version.replace("patch","p")

identifier = "%s_%s_%s" %(service_params["section"],cmssw_version,dset_nick)

if len(identifier)>20:

identifier="%s_%s"%(service_params["section"],cmssw_version)

'''

Loop on all the sections of the configparser, build and submit the request.

This is the orchestra director function.

'''

pp = pprint.PrettyPrinter(indent=4)

for section in cfg.configparser.sections():

# Warning muted

#print '\n---> Processing request "%s"' %section

# build the dictionary for the request

params,service_params = build_params_dict(section,cfg)

dataset_runs_dict = get_dataset_runs_dict (section,cfg)

if not dataset_runs_dict:

sys.stderr.write("[wmcontrol exception] No dataset_runs_dict provided")

sys.exit(-1)

# Submit request!

for dataset in sorted(dataset_runs_dict.keys()):

params['InputDataset']=dataset

runs=[]

new_blocks=[]

for item in dataset_runs_dict[dataset]:

if isinstance(item,str) and '#' in item:

if item.startswith('#'):

new_blocks.append(dataset+item)

else:

new_blocks.append(item)

else:

runs.append(item)

params['RunWhitelist']=runs

if params.has_key("BlockWhitelist"):

if params['BlockWhitelist']==[]:

params['BlockWhitelist']=new_blocks

if params['BlockWhitelist']!=[] and new_blocks!=[]:

print "WARNING: a different set of blocks was made available in the input dataset and in the blocks option."

print "Keeping the blocks option (%s) instead of (%s)" % (str(sorted(new_blocks)), str(sorted(params['BlockWhitelist'])))

params['BlockWhitelist']=new_blocks

params['RequestString']= make_request_string(params,service_params,section)

if service_params['request_type'] in ['MonteCarlo','LHEStepZero']:

params.pop('InputDataset')

params.pop('RunWhitelist')

elif service_params['request_type'] == 'TaskChain':

if params['InputDataset']:

params['Task1']['InputDataset'] = params['InputDataset']

if params['RunWhitelist']:

params['Task1']['RunWhitelist'] = params['RunWhitelist']

if 'RunWhitelist' in params: #if params has key we remove it

params.pop('RunWhitelist') #because it was set as Task parameter

if 'InputDataset' in params:

params.pop('InputDataset')

elif ('RequestNumEvents' in params and

('RunWhitelist' not in params or params['RunWhitelist']==[])):

if service_params['request_type'] in ['ReDigi','ReReco']:

events = float(params['RequestNumEvents'])

elif service_params['request_type'] in ['MonteCarloFromGEN']:

events = float(params['RequestNumEvents'] / float(params['FilterEfficiency']))

if events:

if test_mode:

t = time.time()

espl = helper.SubsetByLumi(params['InputDataset'],

float(service_params['margin']))

split, details = espl.run(int(events),

service_params['brute_force'],

service_params['force_lumis'])

# https://github.com/dmwm/DBS/issues/280

# https://github.com/dmwm/DBS/issues/428

if split == 'blocks':

params['BlockWhitelist'] = details

elif split == 'lumis':

params['LumiList'] = details

params['SplittingAlgo'] = 'LumiBased'

# params.pop('RunWhitelist')

if test_mode:

print "Finished in", int((time.time()-t)*1000), "\bms"

# just print the parameters of the request you would have injected

if test_mode:

pp.pprint(params)

else: # do it for real!

try:

workflow = wma.makeRequest(wma.WMAGENT_URL,params,encodeDict=(service_params['request_type']=='TaskChain'))

except:

random_sleep()

#just try a second time

workflow = wma.makeRequest(wma.WMAGENT_URL,params,encodeDict=(service_params['request_type']=='TaskChain'))

try:

wma.approveRequest(wma.WMAGENT_URL,workflow)

except:

random_sleep()

#just try a second time

wma.approveRequest(wma.WMAGENT_URL,workflow)

'''

opens the file if there and reads its parameters

Skip lines beginning with #

returns a dictionary with the cfgnames as keys and the docids as values

Example config:

config_0_1_cfg.py 8216d7bbd56664bc2a5853fb4f0aa36e

config_0_2_cfg.py 8216d7bb57e664bc2a5853fb4a0aeb28

config_0_3_cfg.py 8216d7bbd5bc64bc2a5853fb4f0aed61

'''

if filename =='':

return {}

print "Building a cfg-docID dictionary.."

cfg_db_file=None

try:

cfg_db_file=open(filename,'r')

except:

raise Exception ('Problems in opening %s: does it exist? is it corrupted? do you have the right permissions?' %filename)

# at this point we have the file, let's interpret it!

cfg_docid_dict={}

for line in cfg_db_file:

# remove a trailing comment if there and skip the lines

line = line[:line.find('#')]

line = line.strip()

if line == '' or line[0]=='#':

continue # this is a comment line

# small sanity check, one could add regexp

if line.count(' ') > 1:

raise Exception('Could not interpret line %s. Too many spaces.' %line)

# split the line

cfg_name,docid = line.split(' ')

if test_mode:

print "Name, DocID in file %s: %s %s"%(filename, cfg_name,docid)

cfg_docid_dict[cfg_name] = docid

cfg_db_file.close()

# try to use the environment variables if nothing provided

user_env_name ='WMCONTROL_USER'

group_env_name='WMCONTROL_GROUP'

user_default =''

group_default=''

if os.environ.has_key(user_env_name):

user_default=os.environ[user_env_name]

if os.environ.has_key(group_env_name):

group_default=os.environ[group_env_name]

#print os.environ.has_key(user_env_name)

#print "*", user_default, "*", group_default

user = cfg.get_param('user',user_default,section)

group = cfg.get_param('group',group_default,section)

#print "*", user, "*", group

global couch_pass

'''

Build the parameters dictionary for the request. Assumes the presence of an input .conf file or commandline.

For the moment the defaults of the parameters are stored here.

Put a dictionary on top?

'''

#wm testing

wmtest = cfg.get_param('wmtest', False, section)

url_dict = cfg.get_param('url_dict',"",section)

# fetch some important parameters

#this trick is to make the uniformation smoother and be able to read old cfgfiles

doc_id = step1_docID = ''

doc_id = step1_docID = cfg.get_param('docID','',section)

dummy = cfg.get_param('step1_docID','',section)

if dummy!='':

doc_id = step1_docID = dummy

step2_docid = cfg.get_param('step2_docID','',section)

#print step2_docid

step3_docid = cfg.get_param('step3_docID','',section)

#print step3_docid

# elaborate the file containing the name docid pairs

cfg_db_file = cfg.get_param('cfg_db_file','',section)

#print cfg_db_file

cfg_docid_dict = make_cfg_docid_dict(cfg_db_file)

release = cfg.get_param('release','',section)

globaltag = cfg.get_param('globaltag','',section)

pileup_dataset = cfg.get_param('pu_dataset','',section)

primary_dataset = cfg.get_param('primary_dataset','',section)

filter_eff = cfg.get_param('filter_eff','',section)

if not filter_eff:

filter_eff = 1.0

number_events = int(cfg.get_param('number_events',0,section))

#number_events = cfg.get_param('number_events',0,section)

version = cfg.get_param('version','',section)

##new values for renewed Request Agent

time_event = float(cfg.get_param('time_event',20,section))

size_memory = int(float(cfg.get_param('size_memory',2300,section)))

size_event = int(float(cfg.get_param('size_event',2000,section)))

if size_event <0:

size_event = 2000

# parameters with fallback

scramarch = cfg.get_param('scramarch',default_parameters['scramarch'],section)

#group = cfg.get_param('group',default_parameters['group'],section)

#requestor = cfg.get_param('requestor',default_parameters['requestor'],section)

identifier = cfg.get_param('identifier','',section)

dbsurl = cfg.get_param('dbsurl',default_parameters['dbsurl'],section)

includeparents=cfg.get_param('includeparents',default_parameters['includeparents'],section)

req_name=cfg.get_param('req_name','',section)

process_string = cfg.get_param('process_string','',section)

processing_string = cfg.get_param('processing_string','',section)

batch = cfg.get_param('batch','',section)

open_running_timeout = int(float(cfg.get_param('open_running_timeout','43200',section))) # 12h is legacy

# for the user and group

user,group = get_user_group(cfg,section)

# for the skims

skim_cfg = cfg.get_param('skim_cfg','',section)

skim_docid = cfg.get_param('skim_docID','',section)

skim_name = cfg.get_param('skim_name','',section)

skim_input = cfg.get_param('skim_input','RECOoutput',section)

if not skim_docid and skim_cfg:

if cfg_docid_dict.has_key(skim_cfg):

skim_docid=cfg_docid_dict[skim_cfg]

else:

skim_docid=wma.upload_to_couch(skim_cfg, section, user, group,test_mode)

# priority

priority = cfg.get_param('priority',default_parameters['priority'],section)

#blocks

blocks = cfg.get_param('blocks', [], section)

# Now the service ones

# Service

step1_cfg = cfg_path = ''

step1_cfg = cfg_path = cfg.get_param('cfg_path','',section)

dummy = cfg.get_param('step1_cfg','',section)

if dummy != '':

step1_cfg = cfg_path = dummy

harvest_cfg = cfg.get_param('harvest_cfg','',section)

harvest_docID = cfg.get_param('harvest_docID','',section)

step1_output = cfg.get_param('step1_output','',section)

keep_step1 = cfg.get_param('keep_step1',False,section)

step2_cfg = cfg.get_param('step2_cfg','',section)

step2_docID = cfg.get_param('step2_docID','',section)

step2_output = cfg.get_param('step2_output','',section)

keep_step2 = cfg.get_param('keep_step2',False,section)

step3_cfg = cfg.get_param('step3_cfg','',section)

step3_docID = cfg.get_param('step3_docID','',section)

step3_output = cfg.get_param('step3_output','',section)

transient_output = cfg.get_param('transient_output',[],section)

request_type = cfg.get_param('request_type',default_parameters['request_type'],section)

request_id = cfg.get_param('request_id','',section)

events_per_job = cfg.get_param('events_per_job','',section)

events_per_lumi = int(float(cfg.get_param('events_per_lumi',100,section))) # 100 is legacy

force_lumis = cfg.get_param('force_lumis', False, section)

brute_force = cfg.get_param('brute_force', False, section)

margin = cfg.get_param('margin', 0.05, section)

# Upload to couch if needed or check in the cfg dict if there

docIDs=[step1_docID,step2_docID,step3_docID]

cfgs=[step1_cfg,step2_cfg,step3_cfg]

for step in xrange(3):

step_cfg_name= cfgs[step]

step_docid = docIDs[step]

#print step_cfg_name, step_docid

if step_cfg_name!='' and step_docid=='' :

#print step_cfg_name, step_docid

# try to see if it is in the cfg name dict

if cfg_docid_dict.has_key(step_cfg_name):

print "Using the one in the cfg-docid dictionary."

docIDs[step] = cfg_docid_dict[step_cfg_name]

else:

print "No DocId found for section %s. Uploading the cfg to the couch." %section

docIDs[step]= wma.upload_to_couch(step_cfg_name, section, user, group,test_mode)

step1_docID,step2_docID,step3_docID=docIDs

if harvest_docID=='' and harvest_cfg!='':

harvest_docID= wma.upload_to_couch(harvest_cfg , section, user, group,test_mode)

# check if the request is valid

if step1_docID=='' and url_dict=="":

print "Invalid request, no docID configuration specified."

sys.stderr.write("[wmcontrol exception] Invalid request, no docID configuration specified.")

sys.exit(-1)

# Extract Campaign from PREP-ID if necessary

campaign = cfg.get_param('campaign','',section)

if campaign =="" and request_id=="":

print "Campaign and request-id are not set. Provide at least the Campaign."

elif campaign =="" and request_id!="":

campaign = re.match(".*-(.*)-.*",request_id).group(1)

elif campaign !="" and request_id!="":

print "Campaign and request-id are set. Using %s as campaign." %campaign

time_per_campaign=wma.time_per_events(campaign)

if time_per_campaign:

time_event=time_per_campaign

service_params={"section": section,

"version": version,

"request_type": request_type,

"step1_cfg": step1_cfg,

"step1_output": step1_output,

"keep_step1":keep_step1,

#

"step2_cfg": step2_cfg,

"step2_output": step2_output,

"keep_step2":keep_step2,

#

"step3_cfg": step3_cfg,

"step3_output": step3_output,

#

'cfg_docid_dict' : cfg_docid_dict,

'req_name': req_name,

"batch": batch,

"process_string": process_string,

'force_lumis': force_lumis,

'brute_force': brute_force,

'margin': margin

}

# According to the rerquest type, cook a request!

params={"CMSSWVersion": release,

"ScramArch": scramarch,

"RequestPriority": priority,

"RunWhitelist": ['Will Be replaced'],

"InputDataset": 'Will Be replaced',

"RunBlacklist": [],

"BlockWhitelist": blocks,

"BlockBlacklist": [],

"DbsUrl": dbsurl,

"RequestType": request_type,

"GlobalTag": globaltag,

"inputMode": "couchDB",

"RequestString": "Will Be dynamically created",

"Group": group,

"Requestor": user,

"Campaign": campaign,

"Memory": size_memory,

"SizePerEvent": size_event,

"TimePerEvent": time_event,

"OpenRunningTimeout" : open_running_timeout,

#"ConfigCacheUrl": wma.COUCH_DB_ADDRESS,

#"EnableHarvesting" : False

"ProcessingString": processing_string

}

if url_dict != "":

#print "This is the url",url_dict,"to get the dict from"

params = json.loads(os.popen('curl -s --insecure %s'%(url_dict)).read())

#print params

service_params["request_type"] = params["RequestType"]

service_params["version"] = params["ProcessingVersion"]

service_params["process_string"] = "T"

service_params["pid"] = params["RequestString"]

params["DbsUrl"] = "https://"+wma.WMAGENT_URL+wma.DBS3_URL

params["CouchURL"] = wma.COUCH_DB_ADDRESS

params["ConfigCacheURL"] = wma.COUCH_DB_ADDRESS

elif request_type == "ReReco":

if number_events:

if blocks:

## cannot perform automatic block selection

print "\n\n\n WARNING number_events is not functionnal because you specified blocks in input\n\n\n"

else:

print "\n\n\n WARNING automated block selection performed \n\n\n"

params.update({"RequestNumEvents" : number_events})

params.update({"ConfigCacheID": step1_docID,

"Scenario": "pp",

"IncludeParents" : includeparents,

"TransientOutputModules":transient_output})

if skim_docid != '':

print "This is a skim"

params.update({"SkimName1" : skim_name,

"SkimInput1" : skim_input,

"Skim1ConfigCacheID" : skim_docid,

"nskims" : 1})

elif request_type == 'MonteCarlo':

params.update({"RequestString": identifier,

"FirstEvent": 1,

"FirstLumi": 1,

"TimePerEvent": time_event,

"FilterEfficiency": filter_eff,

"LheInputFiles" : cfg.get_param('lhe_input',False,section),

"RequestNumEvents": number_events,

"ConfigCacheID": step1_docID,

"PrimaryDataset": primary_dataset,

"PrepID": request_id,

}

)

events_per_lumi = int(float( events_per_lumi ) / float(filter_eff))

params.update({

"EventsPerLumi" : events_per_lumi,

})

if wmtest:

params.pop("EventsPerLumi")

if params["LheInputFiles"]=='True' or params["LheInputFiles"]==True:

#max out to 500K for "lhe step zero"

print "Setting events per job here !!!!",type(params["LheInputFiles"]),params["LheInputFiles"]

events_per_job=500000

if wmtest:

events_per_job=15000

if events_per_job and int(events_per_job):

params.update({

"EventsPerJob" : int(events_per_job)

})

params.pop('BlockBlacklist')

params.pop('BlockWhitelist')

params.pop('InputDataset')

params.pop('RunBlacklist')

elif request_type == 'MonteCarloFromGEN':

params.update({"TimePerEvent": time_event,

"FilterEfficiency": filter_eff,

"ConfigCacheID": step1_docID,

"PrepID": request_id,

"TotalTime": 28800 })

if primary_dataset:

params.update({"PrimaryDataset": primary_dataset})

if int(number_events):

params.update({"RequestNumEvents": number_events})

elif request_type == 'LHEStepZero':

params.update({"RequestString": identifier,

"TimePerEvent": time_event,

"FirstEvent": 1,

"FirstLumi": 1,

"LheInputFiles" : cfg.get_param('lhe_input',False,section),

"Memory": 2300,

"SizePerEvent": size_event,

"ConfigCacheID": step1_docID,

"RequestNumEvents": number_events,

"PrimaryDataset": primary_dataset,

"PrepID": request_id,

"TotalTime": 28800 ,

"EventsPerLumi":300,

"ProdJobSplitAlgo" : "EventBased",

"ProdJobSplitArgs" : {"events_per_job": int(events_per_job),"events_per_lumi": int(events_per_lumi)}

})

params.pop('BlockBlacklist')

params.pop('BlockWhitelist')

params.pop('InputDataset')

params.pop('RunBlacklist')

params.pop('RunWhitelist')

elif request_type == 'ReDigi':

if number_events:

if blocks:

## cannot perform automatic block selection

print "\n\n\n WARNING number_events is not functionnal because you specified blocks in input\n\n\n"

else:

print "\n\n\n WARNING automated block selection performed \n\n\n"

params.update({"RequestNumEvents" : number_events})

params.update({"RequestString": identifier,

"StepOneConfigCacheID": step1_docID,

"KeepStepOneOutput": keep_step1,

"StepOneOutputModuleName": step1_output,

#"DataPileup": "",

"MCPileup": pileup_dataset,

#"Scenario": "pp",

"PrepID": request_id})

if primary_dataset:

params.update({"PrimaryDataset": primary_dataset})

if step2_cfg != '' or step2_docID !='':

params.update({"StepTwoConfigCacheID": step2_docID,

"KeepStepTwoOutput": keep_step2,

"StepTwoOutputModuleName": step2_output})

if step3_cfg !='' or step3_docID!='':

params['StepThreeConfigCacheID'] = step3_docID

else:

if not keep_step2:

print 'Request not keeping its step 2 output'

raise Exception("The request has a second step, no third step and not keeping it's second output")

else:

if not keep_step1:

print 'Request not keeping anything'

raise Exception('The request has one step and not keeping anything')

elif request_type == 'TaskChain':

params.pop('RunBlacklist')

params.pop('BlockWhitelist')

params.pop('BlockBlacklist')

task1_dict={'SplittingAlgorithm': 'LumiBased',

'SplittingArguments': {'lumis_per_job': 8},

'TaskName':'Task1'

}

task1_dict['GlobalTag'] = cfg.get_param('step1_globaltag',globaltag,section)

task1_dict['ConfigCacheID'] = step1_docID

task1_dict['KeepOutput'] = keep_step1

params['Task1']=task1_dict

params['TaskChain']=1

if step2_cfg or step2_docID:

task2_dict={'SplittingAlgorithm': 'LumiBased',

'SplittingArguments': {'lumis_per_job': 8},

'TaskName':'Task2'

}

task2_dict['GlobalTag'] = cfg.get_param('step2_globaltag',globaltag,section)

task2_dict['ConfigCacheID'] = step2_docID

task2_dict['InputFromOutputModule'] = step2_output

task2_dict['InputTask'] = cfg.get_param('step2_input','Task1',section)

#task2_dict['KeepOutput'] = keep_step2

params['Task2']=task2_dict

params['TaskChain']=2

if step3_cfg or step3_docID:

task3_dict={'SplittingAlgorithm': 'LumiBased',

'SplittingArguments': {'lumis_per_job': 8},

'TaskName':'Task3'

}

task3_dict['GlobalTag'] = cfg.get_param('step3_globaltag',globaltag,section)

task3_dict['ConfigCacheID'] = step3_docID

task3_dict['InputFromOutputModule'] = step3_output

task3_dict['InputTask'] = cfg.get_param('step3_input','Task2',section)

#task3_dict['KeepOutput'] = keep_step3

params['Task3']=task3_dict

params['TaskChain']=3

#from pprint import pformat

#print "\n current dictionnary \n",pformat(params),'\n\n'

###raise Exception('Unknown request type, aborting')

else:

print "Request type chose: "+str(request_type)

raise Exception('Unknown request type, aborting')

if harvest_docID:

##setup automatic harvesting

params.update({"EnableDQMHarvest" : 1,

"DQMUploadUrl" : "https://cmsweb.cern.ch/dqm/offline",

"DQMConfigCacheID" : harvest_docID})

## pop any empty parameters

for (param,value) in params.items():

if value in ["",[]]:

params.pop(param)

'''

No defaults given here,but set them in build_params_dict .

'''