def SplitByFilesAndEvents(listoffiles, evtsperjob):
""" Group the input files
"""
mylist = []
total_evts = 0
for files in listoffiles:
myfdict = {}
info = getNumberOfevents([files])
if not "nbevts" in info:
return S_ERROR("The file %s does not have attached number of events, cannot split" % files)
myfdict['file'] = files
myfdict['nbevts'] = info['nbevts']
mylist.append(myfdict)
total_evts += info['nbevts']
#nb_jobs = total_evts/evtsperjob
joblist = []
jdict = {}
startfromevt = 0
cur_events = 0
for event in range(total_evts):
cur_events +=1
if not len(mylist):
break
item = mylist[0]
nb_evts_in_file = item['nbevts']
if cur_events == evtsperjob :
jdict['startFrom'] = startfromevt
if not 'files' in jdict:
jdict['files'] = []
jdict['files'].append(item['file'])
startfromevt = int(cur_events)
cur_events = 0
mylist.remove(item)
joblist.append(jdict)
continue
if cur_events > nb_evts_in_file:
if not 'files' in jdict:
jdict['files'] = []
jdict['files'].append(item['file'])
mylist.remove(item)
continue
#joblist.append(jdict)
#print "final jdict", jdict
return S_OK(joblist)
def resolveInputVariables(self):
""" Common utility for all sub classes, resolve the workflow parameters
for the current step. Module parameters are resolved directly.
"""
if self.workflow_commons.has_key("SystemConfig"):
self.systemConfig = self.workflow_commons["SystemConfig"]
if self.workflow_commons.has_key("IgnoreAppError"):
self.ignoreapperrors = self.workflow_commons["IgnoreAppError"]
if self.step_commons.has_key("applicationName"):
self.applicationName = self.step_commons["applicationName"]
if self.step_commons.has_key("applicationVersion"):
self.applicationVersion = self.step_commons["applicationVersion"]
if self.step_commons.has_key("applicationLog"):
self.applicationLog = self.step_commons["applicationLog"]
if self.step_commons.has_key("SteeringFile"):
self.SteeringFile = self.step_commons["SteeringFile"]
if self.workflow_commons.has_key("JobType"):
self.jobType = self.workflow_commons["JobType"]
if self.workflow_commons.has_key("Energy"):
self.energy = self.workflow_commons["Energy"]
if self.workflow_commons.has_key("NbOfEvts"):
if self.workflow_commons["NbOfEvts"] > 0:
self.NumberOfEvents = self.workflow_commons["NbOfEvts"]
if self.step_commons.has_key("InputFile"):
### This must stay, otherwise, linking between steps is impossible: OutputFile is a string
inputf = self.step_commons["InputFile"]
if not type(inputf) == types.ListType:
if len(inputf):
inputf = inputf.split(";")
else:
inputf = []
self.InputFile = inputf
if self.step_commons.has_key("ForgetInput"):
self.ignoremissingInput = self.step_commons["ForgetInput"]
if self.workflow_commons.has_key("InputData"):
inputdata = self.workflow_commons["InputData"]
if not type(inputdata) == types.ListType:
if len(inputdata):
self.InputData = inputdata.split(";")
if self.workflow_commons.has_key("ParametricInputData"):
paramdata = self.workflow_commons["ParametricInputData"]
if not type(paramdata) == types.ListType:
if len(paramdata):
self.InputData = paramdata.split(";")
if not self.OutputFile:
if self.step_commons.has_key("OutputFile"):
self.OutputFile = self.step_commons["OutputFile"]
# Next is also a module parameter, should be already set
if self.step_commons.has_key("debug"):
self.debug = self.step_commons["debug"]
if "ILDConfigPackage" in self.workflow_commons:
config_dir = self.workflow_commons["ILDConfigPackage"]
# try to copy everything from there to here:
res = getSoftwareFolder(config_dir)
if not res["OK"]:
self.log.error("Cannot find %s" % config_dir, res["Message"])
return S_ERROR("Failed to locate %s as config dir" % config_dir)
path = res["Value"]
list_f = os.listdir(path)
for f in list_f:
try:
if os.path.isdir(os.path.join(path, f)):
shutil.copytree(os.path.join(path, f), "./" + f)
else:
shutil.copy2(os.path.join(path, f), "./" + f)
except Exception as e:
self.log.error("Could not copy %s here because %s!" % (f, str(e)))
if self.InputData:
res = getNumberOfevents(self.InputData)
self.inputdataMeta.update(res["AdditionalMeta"])
if res["nbevts"]:
if self.NumberOfEvents > res["nbevts"] or self.NumberOfEvents == 0:
self.NumberOfEvents = res["nbevts"]
res = self.applicationSpecificInputs()
if not res["OK"]:
return res
return S_OK("Parameters resolved")