def defGridJob(jobName, jobGroup, inputFile):
'''Defines Grid job'''
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
job = UserJob()
job.setName(jobName)
job.setJobGroup(jobGroup)
job.setCPUTime(86400)
job.dontPromptMe()
job.setBannedSites([
'LCG.IN2P3-CC.fr', 'OSG.UConn.us', 'LCG.Cracow.pl', 'OSG.MIT.us',
'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk'
])
job.setInputData(inputFile)
job.setInputSandbox([
'LFN:/ilc/user/l/lstroem/topasymmetry/lib_ilcsoft_2017-05-30_gcc62.tar.gz'
]) #only need LFN: for InputSandbox
job.setOutputSandbox(["*.out", "*.log"])
outputFile = jobName + ".slcio"
rootFile = jobName + ".root"
#job.setOutputData([outputFile, rootFile], jobGroup, "CERN-DST-EOS") #keep some files on the GRID for further processing
job.setOutputData([rootFile], jobGroup, "CERN-DST-EOS")
return job, outputFile, rootFile
def subDDSim():
# Decide parameters for a job
outputSE = "KEK-SRM"
outputSE = "KEK-DISK"
isLocal = _clip.isLocal
nbevts = 10 if _clip.numberOfEvents == 0 else _clip.numberOfEvents
outputFile = "ddsim_example.slcio" if _clip.outputFile == "" else _clip.outputFile
outputDir = _clip.outputDir
inputFile = _clip.inputFile
if inputFile == "":
gLogger.error("Input file for ddsim does not given.")
exit(-1)
# Create DIRAC objects for job submission
dIlc = DiracILC()
job = UserJob()
job.setJobGroup("myddsimjob")
job.setName("myddsim")
job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml'])
job.setILDConfig("v02-00-02")
# job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"])
# job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination
job.setBannedSites(["LCG.UKI-SOUTHGRID-RALPP.uk"
]) # a list of sites not to submit job
# job.setCPUTime( cputime_limit_in_seconds_by_dirac_units )
ddsim = DDSim()
ddsim.setVersion("ILCSoft-02-00-02_gcc49")
ddsim.setDetectorModel("ILD_l5_v02")
ddsim.setInputFile(inputFile)
ddsim.setNumberOfEvents(nbevts)
extraCLIArguments = " --steeringFile ddsim_steer.py "
extraCLIArguments += " --outputFile %s " % outputFile
extraCLIArguments += " --vertexSigma 0.0 0.0 0.1968 0.0 --vertexOffset 0.0 0.0 0.0 0.0 "
ddsim.setExtraCLIArguments(extraCLIArguments)
# ddsim.setRandomSeed(1234565)
# ddsim.setStartFrom(20) # Number of events to skip before starting ddsim
job.append(ddsim)
if outputDir != "":
job.setOutputData([outputFile],
OutputPath=outputDir,
OutputSE=outputSE)
if isLocal:
job.submit(dIlc, mode="local")
else:
job.submit(dIlc)
def create_job(inputData, saveName, outputDir, dontPromptMe):
slcioFile = saveName + '.slcio'
rootFile = saveName + '.root'
if check_file_existence(outputDir, slcioFile, dontPromptMe):
remove_file(outputDir, slcioFile, dontPromptMe)
if check_file_existence(outputDir, rootFile, dontPromptMe):
remove_file(outputDir, rootFile, dontPromptMe)
dIlc = DiracILC()
job = UserJob()
job.setOutputSandbox(['*.out', '*.log', '*.sh', '*.py', '*.xml'])
if SAVE_SLCIO:
job.setOutputData([slcioFile, rootFile],
OutputPath=outputDir,
OutputSE=STORAGE_SE)
else:
job.setOutputData(rootFile, OutputPath=outputDir, OutputSE=STORAGE_SE)
job.setJobGroup('myMarlinRun1')
job.setName('MyMarlinJob1')
# job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk'])
job.setInputSandbox(['LFN:/ilc/user/a/amaier/mylibs.tar.gz'])
marl = Marlin()
marl.setVersion('ILCSoft-2017-11-15_gcc62')
marl.setInputFile(inputData)
job.setInputData(list(map(lambda x: x.lstrip('LFN:'), inputData)))
# marl.setInputFile(['LFN:/ilc/prod/clic/1.4tev/qq_ln/ILD/DST/00003249/010/qq_ln_dst_3249_10000.slcio'])
marl.setSteeringFile('marlin/FullChain.xml')
# marl.setSteeringFile('marlin/FullChainNewDetModel.xml')
marl.setOutputFile(slcioFile)
gearFile = '/afs/cern.ch/user/a/amaier/projects/CLIC_analysis/grid/marlin/clic_ild_cdr.gear'
if not os.path.isfile(gearFile):
print('Error: gear file', gearFile,
'does not exist! Abort submission.')
return
marl.setGearFile(gearFile)
marl.setExtraCLIArguments(
"--MyNtupleMaker.OutputFileName={rootOutfile}".format(
rootOutfile=rootFile))
# marl.setNumberOfEvents(1000)
job.append(marl)
if dontPromptMe:
job.dontPromptMe()
job.submit(dIlc)
return False
def getJob(jobid, jobpara):
iser=jobid+1
outdst = "toto-ovl-%5.5i.dst.slcio"%iser
outrec = "toto-ovl-%5.5i.rec.slcio"%iser
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinOverlayParametric%i"%iser)
j.setInputSandbox(jobpara["setting_file"])
## Define the overlay
ov = OverlayInput()
ov.setMachine("ilc_dbd")
ov.setEnergy(energy)
ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"]))
ov.setBXOverlay(int(jobpara["BXOverlay"]))
ov.setGGToHadInt(float(jobpara["GGToHadInt500"]))
ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
ov.setDetectorModel("ILD_o1_v05")
res = j.append(ov)
if not res['OK']:
print res['Message']
exit(1)
## Define Marlin job
ma = Marlin()
ma.setDebug()
ma.setVersion("ILCSoft-01-17-09")
ma.setSteeringFile("marlin_ovl_stdreco.xml")
ma.setGearFile("GearOutput.xml")
# ma.setInputFile(simfile)
ma.setInputFile(simlists[jobid])
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit(1)
j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
j.setCPUTime(10000)
j.dontPromptMe()
return j
def getJob(self):
""" Define a generic job, it should be always the same
"""
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
myjob = UserJob()
myjob.setName("Testing")
myjob.setJobGroup("Tests")
myjob.setCPUTime(30000)
myjob.dontPromptMe()
myjob.setLogLevel("VERBOSE")
myjob.setPlatform("x86_64-slc5-gcc43-opt")
myjob.setOutputSandbox(["*.log","*.xml", "*.sh"])
myjob._addParameter( myjob.workflow, 'TestFailover', 'String', True, 'Test failoverRequest')
myjob._addParameter( myjob.workflow, 'Platform', 'JDL', "x86_64-slc5-gcc43-opt", 'OS Platform')
if self.ildConfig:
myjob.setILDConfig(self.ildConfig)
return myjob
def subWhizard2():
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin, Whizard2
# Decide parameters for a job
outputSE = "KEK-SRM"
isLocal = _clip.isLocal
nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents
outputFile = "E500.P2f_bB.GWhizard2.I100000.e0.p0.n001.slcio" if _clip.outputFile == "" else _clip.outputFile
outputDir = _clip.outputDir
# Create DIRAC objects for job submission
dIlc = DiracILC()
job = UserJob()
job.setInputSandbox(["pythia6-parameters.sin", "P2f_qqbar.sin"])
job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml'])
job.setOutputData([outputFile])
job.setJobGroup("mywhiz2")
job.setName("mywhizard2")
whiz = Whizard2()
whiz.setVersion("2.7.0")
whiz.setNumberOfEvents(nbevts)
whiz.setEvtType("P2f_bB")
whiz.setProcessVariables("P2f_bB")
# whiz.setRandomSeed(15)
whiz.setSinFile("P2f_qqbar.sin")
whiz.setOutputFile(outputFile)
job.append(whiz)
if outputDir != "":
print " outputDir = " + outputDir
print " outputSE = " + outputSE
job.setOutputData([outputFile],
OutputPath=outputDir,
OutputSE=outputSE)
if isLocal:
job.submit(dIlc, mode="local")
else:
job.submit(dIlc)
def subDDSim(clip1):
# Decide parameters for a job
outputSE = "KEK-SRM"
isLocal = clip1.isLocal
nbevts = 0 if clip1.numberOfEvents == 0 else clip1.numberOfEvents
#print('inside subddsim(): nbevts ', nbevts)
outputFile = "" if clip1.outputFile == "" else clip1.outputFile
#print('inside subddsim outfile ', outputFile)
outputDir = clip1.outputDir
#print('inside subddsim outdir ', outputDir)
inputFile = clip1.inputFile
#print('inside subddsim inputFile ', inputFile)
if inputFile == "":
gLogger.error("Input file for ddsim is not given.")
exit(-1)
# Create DIRAC objects for job submission
dIlc = DiracILC()
job = UserJob()
job.setJobGroup("myddsimjob")
job.setName("myddsim")
job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml'])
job.setILDConfig("v02-00-02")
# job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"])
# job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination
job.setBannedSites(["LCG.UKI-SOUTHGRID-RALPP.uk"
]) # a list of sites not to submit job
# job.setCPUTime( cputime_limit_in_seconds_by_dirac_units )
ddsim = DDSim()
ddsim.setVersion("ILCSoft-02-00-02_gcc49")
ddsim.setDetectorModel("ILD_l5_v05")
ddsim.setInputFile(inputFile)
ddsim.setNumberOfEvents(nbevts)
extraCLIArguments = " --steeringFile ddsim_steer_July26.py"
extraCLIArguments += " --outputFile %s " % outputFile
extraCLIArguments += " --vertexSigma 0.0 0.0 0.1968 0.0 --vertexOffset 0.0 0.0 0.0 0.0 "
ddsim.setExtraCLIArguments(extraCLIArguments)
return ddsim
def getJob(self):
""" Define a generic job, it should be always the same
"""
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
myjob = UserJob()
myjob.setName("Testing")
myjob.setJobGroup("Tests")
myjob.setCPUTime(30000)
myjob.dontPromptMe()
myjob.setLogLevel("VERBOSE")
myjob.setPlatform("x86_64-slc5-gcc43-opt")
myjob.setOutputSandbox(["*.log", "*.xml", "*.sh"])
myjob._addParameter(myjob.workflow, 'TestFailover', 'String', True,
'Test failoverRequest')
myjob._addParameter(myjob.workflow, 'Platform', 'JDL',
"x86_64-slc5-gcc43-opt", 'OS Platform')
if self.ildConfig:
myjob.setILDConfig(self.ildConfig)
return myjob
#####################################################################
# #####################################################################
#prepare file for running marlin
with open(templateSteeringMarlin) as f:
open(nameSteeringMarlin,"w").write(f.read().replace(templateOutRoot,rootFile))
# #####################################################################
#####################################################################
#job definition
job = UserJob() #use UserJob unless recommended differently
job.setName(nameJob)
job.setJobGroup(nameJobGroup)
job.setCPUTime(86400)
job.setBannedSites(['LCG.UKI-LT2-IC-HEP.uk','LCG.KEK.jp','LCG.IN2P3-CC.fr','LCG.Tau.il','Weizmann.il','LCG.Weizmann.il','OSG.MIT.us','OSG.FNAL_FERMIGRID.us','OSG.GridUNESP_CENTRAL.br','OSG.SPRACE.br'])
job.setInputSandbox([nameSteeringMarlin,'LFN:/ilc/user/o/oviazlo/FCCee_o5/ilcsoft_2017-06-21/lcgeo_28_06_2017_v3.tgz',detectorModel,'LFN:/ilc/user/o/oviazlo/PandoraSettings.tar.gz','LFN:/ilc/user/o/oviazlo/FCCee_o5/marlin_lib_simHits_v3.tgz'])
job.setOutputSandbox(["*.log"]) #files that should be brought back when retrieving the job outputs
job.setOutputData([rootFile],nameDir,"CERN-DST-EOS")
#####################################################################
#####################################################################
#ddsim
ddsim = DDSim()
ddsim.setVersion(ddsimVersion)
def all_jobs(name):
d = DiracILC(True, "repo.rep")
################################################
j = UserJob()
j.setJobGroup("PM1")
j.setName("Exec1")
banned_sites = [
"OSG.BNL.us", "LCG.UKI-NORTHGRID-LIV-HEP.uk", "OSG.UCSDT2.us",
"LCG.SCOTGRIDDURHAM.uk", "LCG.NIKHEF.nl", "LCG.UKI-SOUTHGRID-RALPP.uk",
"LCG.GRIF.fr", "LCG.Manchester.uk", "LCG.UKI-LT2-IC-HEP.uk",
"LCG.Weizmann.il"
]
j.setBannedSites(banned_sites)
caindir = name
#print('Cain directory is ',caindir)
indata = [
'LFN:/ilc/user/a/amustahid/cain.exe',
str(caindir), 'LFN:/ilc/user/a/amustahid/runcain.sh',
'LFN:/ilc/user/a/amustahid/convert_pairs_lcio.py',
'LFN:/ilc/user/a/amustahid/pyLCIO.tar.gz',
'/home/belle2/mustahid/useful/my.sh', './splitInput.py',
'./subddsim.py', './ddsim_steer_July26.py', './ILD_l5_v05.xml',
'./my2.sh', './dbd_500GeV.nung_1.xml',
'LFN:/ilc/user/a/amustahid/myProcessors.tar.gz', './create_dir.py',
'./conf.py', './util.py', './testcain.sh', './beam_250.i'
]
j.setInputSandbox(indata)
################################################
#app = GenericApplication()
#app.setScript("create_dir.py")
#app.setInputFile("testcain.sh")
#logf = 'create_dir.log'
#app.setLogFile(logf)
#app.setDebug(debug=True)
#create_dirname = 'create_dir'
#app.setName(create_dirname)
#res=j.append(app)
#if not res['OK']:
# print res['Message']
# exit(1)
################################################
appre = GenericApplication()
name = name.split('/')
#print(name)
cain_name = name[-1]
subdir = name[-2]
dirname = name[-3]
#print('Cain file name ', cain_name)
appre.setScript("LFN:/ilc/user/a/amustahid/runcain.sh")
#appre.setScript("testcain.sh")
ifile = cain_name.split('.')
ifile = ifile[0] + '.' + ifile[1] + '.' + ifile[2]
#print('ifile ',ifile)
appre.setArguments(ifile)
#direc = 'LFN:/ilc/user/a/amustahid/'
#appre.setInputFile(ifile+".i")
#appre.setArguments("This is input arguments")
logf = ifile + '_' + subdir + '.log'
appre.setLogFile(logf)
appre.setDebug(debug=True)
name = 'CAIN'
appre.setName(name)
res = j.append(appre)
if not res['OK']:
print res['Message']
exit(1)
################################################
################################################
#appost = GenericApplication()
#appost.setScript("myanal.sh")
#appost.setArguments("This is my analysis step")
#res=j.append(appost)
#if not res['OK']:
# print res['Message']
# exit(1)
ap = GenericApplication()
ap.setScript('my.sh')
logf = 'my.log'
ap.setLogFile(logf)
ap.setDebug(debug=True)
name = 'my'
ap.setName(name)
res = j.append(ap)
if not res['OK']:
print res['Message']
exit(1)
outfile = 'incoherent_pair.dat'
appre.setOutputFile(outfile)
################################################
direc = 'incoherent_pair'
inputFile = direc + '/' + 'inco_pair_split.slcio'
# global variables to hold command line parameters
# ######################################
base = '.'
#outdir=base+'/'+dirname+'/slcio_test_2ndrun'
outdir = base + '/' + dirname + '/Run_7'
#print('outdir'+' '+str(outdir))
geant_name = ifile
outputFile = geant_name + '_' + subdir + '.slcio'
#_clip = _Params(False,1,inputFile,outputFile,outdir)
nbevents = 100
clip = _Params(nbevents, inputFile, outputFile, outdir)
ddsim = subDDSim(clip)
################################################
res = j.append(ddsim)
if not res['OK']:
print res['Message']
exit(1)
j.setOutputData(outputFile, outdir, "KEK-SRM")
j.setOutputSandbox(["*.log", "*.dat", "*.slcio"])
j.dontPromptMe()
res = j.submit(d)
#res = j.submit(d, mode='local')
if res['OK']:
print str(res["Value"])
#print "Dirac job, "+str(res["Value"])+", was submitted."
else:
print "Failed to submit Dirac job. return message was as follows."
pprint.pprint(res)
class Submit():
#*****************************************#
# Function name : __init__ #
# role : create a DIRAC job, an instance #
# of DiracILC, and filter the output #
#*****************************************#
def __init__(self):
self.dIlc = DiracILC(False)
self.job = UserJob()
self.job.setJobGroup("FCC")
self.job.setName("FCC APP")
self.job.setOutputSandbox(["*.log", '*.root'])
#self.job.setDestination('LCG.DESY-HH.de')
#EOS public location
self.EOS_MGM_URL = 'root://eospublic.cern.ch'
#EOS environment
self.setEOS = 'export EOS_MGM_URL=' + self.EOS_MGM_URL
self.myclient = client.FileSystem(self.EOS_MGM_URL + ':1094')
#sandbox
self.InputSandbox = []
self.folders_to_upload = []
self.filtered_extensions = []
self.excludes_or_includes = []
self.temp_cwd = os.path.join(os.getcwd(), 'fcc_temp_dirac')
#********************************************#
# Function name : read_from_file #
# input : file_name #
# role : read a file and return its content #
#********************************************#
def read_from_file(self, file_name):
try:
with open(file_name, 'r') as f:
content = f.read()
return content
except:
return False
#*********************************************************************************#
# Function name : create_temp_tree_from_files #
# role : given a relative tree path of file to the local FCCSW folder of the user #
# it looks for this file, recreates the relative tree in a temporary folder #
# containing only this file and not all files of the source folder #
# Finally, the folder will be added to the DIRAC sandbox #
#*********************************************************************************#
def create_temp_tree_from_files(self, files, fccsw_path):
if files:
for file in files:
tree = os.path.dirname(file)
tree_full_path = os.path.join(self.temp_cwd, tree)
if not os.path.exists(tree_full_path):
os.makedirs(tree_full_path)
root_folder = tree.split(os.path.sep)[0]
root_folder_full_path = os.path.join(self.temp_cwd,
root_folder)
if root_folder_full_path not in self.folders_to_upload:
self.folders_to_upload += [root_folder_full_path]
source = os.path.join(fccsw_path, file)
destination = os.path.join(self.temp_cwd, file)
if not os.path.exists(source):
print '\nThe file : ' + source + ' does not exist\n'
quit()
else:
copyfile(source, destination)
#***************************************************#
# Function name : compress #
# role : it compress all folders, the job #
# will need once in the grid #
#***************************************************#
def compress(self, temp_folder, actual_folder, tar_extension,
filtered_extension, exclude_or_include):
#print filtered_extension,exclude_or_include
exclude_func = None
if False is not filtered_extension:
if exclude_or_include:
exclude_func = lambda filename: filename.find(
filtered_extension) >= 0
else:
exclude_func = lambda filename: os.path.isfile(
filename) and filename.find(filtered_extension) < 0
tar = tarfile.open(temp_folder + tar_extension, "w:gz")
for name in os.listdir(actual_folder):
#print name
renamed = os.path.join(actual_folder, name)
#print renamed
#print os.path.basename(renamed)
tar.add(renamed,
arcname=os.path.basename(renamed),
exclude=exclude_func)
tar.close()
#*************************************************#
# Function name : find_eos_file #
# input : file_name #
# role : check if file exists on EOS #
# before sending the job to DIRAC #
#*************************************************#
def find_eos_file(self, file_name):
#then the file is in EOS
eos_file_full_path = self.EOS_MGM_URL + '/' + file_name
with client.File() as eosFile:
file_status = eosFile.open(eos_file_full_path, OpenFlags.UPDATE)
#problem with file created directly on eos
#no problem with uploded files with xrdcp cmd
#print eos_file_full_path
#print file_status
status = self.XRootDStatus2Dictionnary(file_status)
if 'False' == status[' ok'] or False == status:
return file_name, False
else:
return eos_file_full_path, True
#*************************************************#
# Function name : find_eos_folder #
# input : folder_name #
# role : check if folder exists on eos #
# before sending the job to the worker #
#*************************************************#
def find_eos_folder(self, folder_name):
#then the file is in eos
eos_folder_full_path = self.EOS_MGM_URL + '/' + folder_name
status, listing = self.myclient.dirlist(folder_name, DirListFlags.STAT)
if None == listing:
return folder_name, False
else:
return eos_folder_full_path, True
#*************************************************#
# Function name : find_path #
# input : file_name #
# role : check if file/folder exists on afs #
# before checking on eos #
#*************************************************#
def find_path(self, path, file_or_dir='file'):
#we suppose that the user enter absolute or relative afs path
#or only absolute eos path
if not path.startswith('/eos/'):
#afs path are absolute or relative
#because software are stored in this filesystem
#and users generally submit their job from lxplus
#absolute path provided
#os.isabs not cross platform
#i used startwith...
if path.startswith('/afs/') and os.path.exists(path):
return os.path.abspath(path), True
#if relative (does not start with /afs/)
#add cwd
elif os.path.exists(os.path.abspath(path)):
return os.path.abspath(path), True
#maybe local user machine, print upload error message
else:
return path, False
#absolute path
elif path.startswith('/eos/'):
#print "the file is in eos"
#eos path are absolute
file_path, is_file_exist = self.find_eos_file(path)
folder_path, is_folder_exist = self.find_eos_folder(path)
if is_file_exist:
return file_path, is_file_exist
elif is_folder_exist:
return folder_path, is_folder_exist
else:
return path, False
else: #other file system
return path, False
#********************************************************#
# Function name : upload_sandbox_with_application_files #
# role : upload all extra folders or files #
# specified by the user for an application #
# For now, we do not check if file exists on cvmfs #
#********************************************************#
def upload_sandbox_with_application_files(self, paths):
upload_path_message = " does not exist\nPlease ensure that your path exist in an accessible file system (EOS or AFS)\n"
for path in paths:
if not path.startswith('/cvmfs/'):
path, is_exist = self.find_path(path)
if False == is_exist:
message = "\nThe path '" + path + "'" + upload_path_message
else:
if path.startswith('/afs/'):
print '\nWARNING : STORING FILES ON AFS IS DEPRECATED\n'
print '\nYou plan to upload :' + path + ' which is stored on AFS\n'
#files are directly added to the sandbox
if os.path.isfile(path):
self.InputSandbox += [path]
else: #folders are compressed before
self.folders_to_upload += [path]
#********************************************************#
# Function name : upload_sandbox_with_fccsw_files #
# role : upload files called in FCCSW configuration file #
# and needed folders relative to FCCSW #
#********************************************************#
def upload_sandbox_with_fccsw_files(self, fccsw_path, fcc_conf_file):
InstallArea_folder = os.path.join(fccsw_path, 'InstallArea')
Detector_folder = os.path.join(fccsw_path, 'Detector')
fccsw_folders = [InstallArea_folder, Detector_folder]
#explanation
#InstallArea_folder : dbg files are excluded
#Detector_folder : only xml files are included
self.filtered_extensions += ['.dbg', '.xml']
self.excludes_or_includes += [True, False]
content = self.read_from_file(fcc_conf_file)
if False is content:
print "\nError in reading configuration file :\n" + fcc_conf_file
quit()
#xml_files = re.findall(r'file:(.*.xml)',content)
txt_files = re.findall(r'="(.*.txt)', content)
cmd_files = re.findall(r'filename="(.*.cmd)', content)
#print txt_files
#print cmd_files
folders = self.create_temp_tree_from_files(txt_files, fccsw_path)
if None != folders:
fccsw_folders += folders
folders = self.create_temp_tree_from_files(cmd_files, fccsw_path)
if None != folders:
fccsw_folders += folders
self.folders_to_upload = fccsw_folders + self.folders_to_upload
#********************************************************#
# Function name : update_sandbox #
# role : check the files and folders required by the job #
# and add them to the sandbox #
# Indeed, it calls upload_sandbox_with_* functions #
#********************************************************#
def update_sandbox(self, fccsw_path, paths, fcc_conf_file):
#first, it creates a specific local working directory for the sandbox
if not os.path.exists(self.temp_cwd):
os.makedirs(self.temp_cwd)
#update sandbox with application files
if '' != paths:
self.upload_sandbox_with_application_files(paths)
#update sandbox with FCCSW application files
if '' != fccsw_path:
self.upload_sandbox_with_fccsw_files(fccsw_path, fcc_conf_file)
#********************************************************#
# Function name : compress_sandbox_subfolders #
# role : compress all local folders required by the job #
#********************************************************#
def compress_sandbox_subfolders(self):
compressed_folders = []
for idx, actual_folder in enumerate(self.folders_to_upload):
tar_extension = '.tgz'
if idx < len(self.filtered_extensions):
filtered_extension = self.filtered_extensions[idx]
exclude_or_include = self.excludes_or_includes[idx]
else:
filtered_extension = False
exclude_or_include = False
temp_folder = os.path.join(self.temp_cwd,
os.path.basename(actual_folder))
#DIRAC already compressed the sandbox before submitting the job
self.compress(temp_folder, actual_folder, tar_extension,
filtered_extension, exclude_or_include)
compressed_folders += [temp_folder + tar_extension]
self.InputSandbox += compressed_folders
#**************************************************#
# Function name : submit #
# role : create DIRAC generic application for each #
# FCC application and submit the DIRAC job #
# containing these generic DIRAC applications #
#**************************************************#
def submit(self, applications, script_to_source):
fcc_execution_module = os.path.join(os.getcwd(),
"fcc_execution_module.py")
fcc_installation_module = os.path.join(os.getcwd(),
'fcc_installation_module.py')
fcc_environment = script_to_source
#Initialization of the sandbox
paths = [fcc_execution_module, fcc_environment]
self.upload_sandbox_with_application_files(paths)
for application in applications:
#application specification
job_specification = application.job_specification
#application specification details
fcc_executable = job_specification['fcc_executable']
fccsw_path = job_specification['fccsw_path']
paths = job_specification['paths']
fcc_conf_file = job_specification['fcc_conf_file']
fcc_input_files = '' if '' == job_specification[
'fcc_input_files'] else ' --ifiles ' + ' '.join(
job_specification['fcc_input_files'])
fcc_output_file = '' if '' == job_specification[
'fcc_output_file'] else ' --ofile ' + job_specification[
'fcc_output_file']
fccsw = '' if '' == job_specification[
'fccsw_path'] else ' --fccsw ' + job_specification['fccsw_path']
number_of_events = '' if '' == job_specification[
'number_of_events'] else ' -N ' + job_specification[
'number_of_events']
#**************************************************MINIMUM REQUIREMENTS CHECKING*******************************************************#
if '' == fcc_executable or '' == fcc_conf_file or '' == fcc_environment:
print "\nError in parsing applications :\n" + fcc_executable
print "\nYou have to provide at least an executable, a configuration file and a script to source for each application\n"
quit()
#**************************************************MINIMUM REQUIREMENTS CHECKING*******************************************************#
#we update the sandbox according to the specification of the application
self.update_sandbox(fccsw_path, paths, fcc_conf_file)
#we instanciate a generic DIRAC application
generic_dirac_application = GenericApplication()
#we set the the installation module as the main script
generic_dirac_application.setScript(fcc_installation_module)
#we set the arguments of the script
arguments = '--source ' + fcc_environment + ' --exec ' + fcc_executable + ' --conf ' + fcc_conf_file + fcc_input_files + fcc_output_file + fccsw + number_of_events
generic_dirac_application.setExtraCLIArguments(arguments)
#we add the generic DIRAC application to the DIRAC job
try:
res = self.job.append(generic_dirac_application)
if not res['OK']:
print res['Message']
quit()
except:
print "\nPlease, configure your proxy before submitting a job from DIRAC"
print "If you do not set up a proxy, refer to the manual or maybe you have to refresh it"
print "by typing :"
print "dirac-proxy-init\n"
quit()
#before submitting the DIRAC job, we compress all folders of the sandbox
self.compress_sandbox_subfolders()
print '\n**********************************HERE THE CONTENT OF YOUR SANDBOX*********************************\n'
print self.InputSandbox
print '\n**********************************HERE THE CONTENT OF YOUR SANDBOX*********************************\n'
#we set the sandbox
self.job.setInputSandbox(self.InputSandbox)
#-------------------------------now we submit the job containing all applications-------------------------------------------#
res = self.job.submit(self.dIlc)
if not res['OK']:
print 'Please check your application requirements'
print res['Message']
quit()
else:
print "The Job you submited has the following ID : " + str(
res['JobID'])
genfile="/gpfs/home/ilc/miyamoto/ILDProd/examples/ddsim-v01-17-10/E0500-TDR_ws.Pea_lvv.Gwhizard-1.95.eL.pB.I37494.01_000.stdhep"
else:
genfile="LFN:/ilc/prod/ilc/ild/test/temp1/gensplit/500-TDR_ws/3f/run002/E0500-TDR_ws.Pea_lvv.Gwhizard-1.95.eL.pB.I37494.01_000.stdhep"
now = datetime.now()
simfile="ddsim-data-%s.slcio" % now.strftime("%Y%m%d-%H%M%S")
steeringfile="/cvmfs/ilc.desy.de/sw/ILDConfig/v01-17-10-p01/StandardConfig/lcgeo_current/ddsim_steer.py"
d= DiracILC(True,"repo.rep")
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("DDSim-example") #%i)
# j.setInputSandbox(setting_file)
ddsim = DDSim()
ddsim.setVersion("ILCSoft-01-17-10")
ddsim.setDetectorModel("ILD_o1_v05")
ddsim.setInputFile(genfile)
ddsim.setRandomSeed(12345)
# ddsim.setStartFrom(1)
ddsim.setNumberOfEvents(5) # Number of events should not exceed number of events in file.
# Otherwise, G4exception is thrown
# ddsim.setDebug()
ddsim.setSteeringFile(steeringfile)
ddsim.setOutputFile(simfile)
res = j.append(ddsim)
ov.setDetectorType("ILD")
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(1)
res = j.append(ov)
if not res["OK"]:
print res["Message"]
dexit(1)
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma.setGearFile("clic_ild_cdr.gear")
ma.getInputFromApp(mo)
ma.setDebug(True)
res = j.append(ma)
if not res["OK"]:
print res["Message"]
dexit(1)
# print appplication's attributes.
ma.listAttributes()
j.setName("test")
j.setOutputSandbox("*.log")
res = dirac.checkparams(j)
if not res["OK"]:
print res["Message"]
dexit(1)
#jobParams = [('slicTest8_mu+_theta90.mac','diracTest8_mu+_theta90.slcio',50),('slicTest7_mu+_theta_5-175.mac','diracTest_mu+_theta_5-175.slcio',50),('slicTest3_e+.mac','diracTest3_e+.slcio',10),('slicTest2_pi+.mac','diractTest2_pi+.slcio',10)]
#jobParams = [('slicTest10_mu+_100gev_theta70_testNewGeom.mac','diracTest10_mu+_100gev_theta70_testNewGeom.slcio',10),('slicTest10_mu+_100gev_theta90_testNewGeom.mac','diracTest10_mu+_100gev_theta90_testNewGeom.slcio',10)]
jobParams = [
('/users/detector/ssetru/SiDSim/detectors/detector_vtx_matbudghalf_nonsensitivelayer/slicmacros/slicTest8_mu+_100gev_theta60.mac',
'diracTest_100gev_theta60_vtx_matbudghalf_nonsensitivelayer.slcio', 100),
('/users/detector/ssetru/SiDSim/detectors/detector_vtx_matbudghalf_nonsensitivelayer/slicmacros/slicTest8_mu+_10gev_theta60.mac',
'diracTest_10gev_theta60_vtx_matbudghalf_nonsensitivelayer.slcio', 100)
]
#slicMacros = ['slicTest8_mu+_theta90.mac','slicTest7_mu+_theta_5-175.mac','slicTest3_e+.mac','slicTest2_pi+.mac']
#fileOutputs = ['diracTest2Loop1.slcio','diracTest2Loop2.slcio','diracTest2Loop3.slcio','diractTest2Loop4.slcio']
#slicNumEvents = [100,100,10,10]
for macro, output, nEvts in jobParams:
job = UserJob()
job.setName("ssetru_dirac_test1")
job.setJobGroup("tests")
job.setCPUTime(86400)
#below ten mb, specified local path
#larger input files are put into a grid storage unit, specified with grid path
#job.setInputSandbox(["newDetector.zip"])
job.setInputSandbox(["alias.properties"])
#'/afs/cern.ch/user/s/ssetru/www/newDetector.zip'
#job.setInputSandbox.append('/afs/cern.ch/user/s/ssetru/www/newDetector.zip')
#has log files, also may want to specify *.xml, generally short term data
job.setOutputSandbox(["*.log", "*.mac", "*.xml"])
#stored forever, in grid storage until you delete, path specified goes after your user directory in dirac
job.setOutputData(output, "test_vtx_matbudghalf_nonsensitivelayer",
"CERN-SRM")
#for index in xrange(0,len(slicNumEvents)):
basedir="/cvmfs/ilc.desy.de/sw/ILDConfig/v01-16-p05_500/StandardConfig/current/"
pandoraLikelihoodData=basedir+"PandoraLikelihoodData9EBin.xml"
bg_aver=basedir+"bg_aver.sv01-14-01-p00.mILD_o1_v05.E500-TDR_ws.PBeamstr-pairs.I230000.root"
simfile="LFN:/ilc/prod/ilc/mc-dbd/ild/sim/500-TDR_ws/6f_ttbar/ILD_o1_v05/v01-14-01-p00/sv01-14-01-p00.mILD_o1_v05.E500-TDR_ws.I37623.P6f_bbcyyc.eR.pL-00001.slcio"
gearfile=basedir+"GearOutput.xml"
outdst = "toto.dst.slcio" #% i
outrec = "toto.rec.slcio" #% i
d= DiracILC(True,"repo.rep")
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinExample")#%i)
ma = Marlin()
ma.setDebug()
# ma.setLogLevel("verbose")
# ma.setILDConfig("v01-16-p05_500")
ma.setVersion("v01-16-02")
ma.setSteeringFile("marlin_stdreco.xml")
ma.setGearFile(gearfile)
ma.setInputFile([simfile, pandoraLikelihoodData, bg_aver])
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
ma.setDetectorModel(detectorModel)
ma.setSteeringFile("MarlinStdReco.xml")
ma.setExtraCLIArguments(" --constant.DetectorModel=%s "%(detectorModel) )
ma.setLogFile("marlin.log")
#ma.getInputFromApp(sim)
ma.setInputFile([lcoutputSIM])
ma.setEnergy(energy)
ma.setNumberOfEvents(evtsPerRun)
ma.setOutputDstFile(lcoutputDST)
ma.setOutputRecFile(lcoutputREC)
RECoutput.append(lcoutputDST)
RECoutput.append(lcoutputREC)
job = UserJob()
job.setName(jobname)
job.setJobGroup(jobGrName)
job.setILDConfig(ILDConfigVer)
job.setCPUTime(86400)
job.setInputData([lcinputREC])
job.setInputSandbox(["runRecoSplit_all_Tmp.py"])
job.setOutputSandbox(["*.log","*.sh","MarlinStdRecoParsed.xml","marlin*.xml","*.py "])
#job.setOutputSandbox(["*.log","*.sh","MarlinStdRecoParsed.xml","marlin*.xml","*.py ","*.root"])
#job.setDestinationCE('lyogrid07.in2p3.fr')
job.dontPromptMe()
job.setBannedSites(['LCG.QMUL.uk'])
#job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.DESYZN.de','LCG.DESY-HH.de','LCG.KEK.jp','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us'])
# run Malrin reco jobs
mares = job.append(ma)
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
#### Software Versions ####
softVersions = ["v3r0p3", "HEAD", "ILC_DBD",
"0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# Check the --runs and --split arguments to make sure they are compatible, if not exit...
if not check_events_arguments(args.events, args.split):
sys.exit(1)
# Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
lfn_check, lfn = check_input_LFN(args.stdhepInput)
if not lfn_check:
sys.exit(1)
# Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
dirac = DiracILC(True,
setup_repository_name(args.stdhepInput, args.detector))
# Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
# into the input sandbox
inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(path.basename(args.stdhepInput))
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent / nOutputEvents
print "Job ", fileNumber
outputFiles = setup_output_dict(args.stdhepInput, args.detector,
fileNumber, args.outputPath,
softVersions)
slicOutput = outputFiles['slicOutput']
prePandoraOutput = outputFiles['prePandoraOutput']
pandoraOutput = outputFiles['pandoraOutput']
vertexingOutput = outputFiles['vertexingOutput']
lcsimRecOutput = outputFiles['lcsimRecOutput']
lcsimDstOutput = outputFiles['lcsimDstOutput']
flavortagOutput = outputFiles['flavortagOutput']
diracOutput = outputFiles['diracOutput']
################## SLIC ##################################################
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(slicOutput)
slic.setDetectorModel(args.detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (digitization and tracking) #####################
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile(
'steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml'
) # Another version is included in /steeringFiles
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy(
'steeringFiles/sidloi3_trackingStrategies_default.xml')
# lcsim.setAliasProperties('alias.properties')
lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
lcsim.setOutputFile(prePandoraOutput)
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(args.detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(pandoraOutput)
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/sidloi3.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(vertexingOutput)
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
# lcsimDst.setAliasProperties('alias.properties')
lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
lcsimDst.setOutputRecFile(lcsimRecOutput)
lcsimDst.setOutputDstFile(lcsimDstOutput)
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if args.flavortag:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/sidloi3.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(flavortagOutput)
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites([
'LCG.IN2P3-CC.fr',
'LCG.RAL-LCG2.uk',
'LCG.DESY-HH.de',
'LCG.DESYZN.de',
'LCG.KEK.jp',
'OSG.PNNL.us',
])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
if args.flavortag:
job.setOutputData(flavortagOutput, diracOutput, args.SE)
else:
job.setOutputData(lcsimDstOutput, diracOutput, args.SE)
job.setOutputSandbox(outputSandbox)
job.setInputData(lfn)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0
def subOverlay():
# Decide parameters for a job
outputSE = "KEK-SRM"
isLocal = _clip.isLocal
nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents
nbevts = 0 # To analize all input events
outputFilePrefix = "overlay_example" if _clip.outputFilePrefix == "" else _clip.outputFilePrefix
outputDir = _clip.outputDir
inputFile = _clip.inputFile
if inputFile == "":
gLogger.error("Input file for ddsim does not given.")
exit(-1)
recfile = outputFilePrefix + ".rec.slcio"
dstfile = outputFilePrefix + ".dst.slcio"
detector_model = "ILD_l5_o1_v02"
key = detector_model.split('_')
sim_detectorModel = "_".join([key[0], key[1], key[3]])
# Create DIRAC objects for job submission
dIlc = DiracILC()
job = UserJob()
job.setJobGroup("myoverlayjob")
job.setName("myoverlay")
job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml'])
job.setILDConfig("v02-00-02")
# job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"])
# job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination
# job.setBannedSites([]) # a list of sites not to submit job
# job.setCPUTime( cputime_limit_in_seconds_by_dirac_units )
# Create Overlay application
ovldata = [{
"ProcessorName": "BgOverlayWW",
"evttype": "aa_lowpt_WW",
"ProdID": 10237,
"expBG": 0.211,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayWB",
"evttype": "aa_lowpt_WB",
"ProdID": 10241,
"expBG": 0.24605,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayBW",
"evttype": "aa_lowpt_BW",
"ProdID": 10239,
"expBG": 0.243873,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayBB",
"evttype": "aa_lowpt_BB",
"ProdID": 10235,
"expBG": 0.35063,
"subdir": "000"
}, {
"ProcessorName": "PairBgOverlay",
"evttype": "seeablepairs",
"ProdID": 10233,
"expBG": 1.0,
"subdir": "100"
}]
BXOverlay = 1
NbSigEvtsPerJob = 100
numberOfSignalEvents = NbSigEvtsPerJob
basebkgpath = "/ilc/prod/ilc/mc-opt-3/ild/sim/500-TDR_ws"
energy = "500"
for ovl in ovldata:
print "### OverlayInput ... " + ovl["ProcessorName"]
ovlapp = OverlayInput()
ovlpath = "%s/%s/%s/v02-00-01/%8.8d/%s" % \
( basebkgpath, ovl["evttype"], sim_detectorModel, ovl["ProdID"] , ovl["subdir"] )
print " OverlayPath ... " + ovlpath
ovlapp.setMachine("ilc_dbd")
# ovlapp.setEnergy(energy)
# ovlapp.setDetectorModel(sim_detectorModel)
ovlapp.setProcessorName(ovl["ProcessorName"])
ovlapp.setBkgEvtType(ovl["evttype"])
ovlapp.setPathToFiles(ovlpath)
ovlapp.setGGToHadInt(ovl["expBG"])
ovlapp.setBXOverlay(BXOverlay)
ovlapp.setNbSigEvtsPerJob(NbSigEvtsPerJob)
ovlapp.setNumberOfSignalEventsPerJob(numberOfSignalEvents)
res = job.append(ovlapp)
if not res['OK']:
print res['Message']
exit(1)
# Create Marlin application
marlin = Marlin()
marlin.setVersion("ILCSoft-02-00-02_gcc49")
marlin.setDetectorModel(detector_model)
marlin.setSteeringFile("MarlinStdReco.xml")
marlin.setInputFile(inputFile)
marlin.setNumberOfEvents(nbevts)
marlin.setOutputDstFile(dstfile)
marlin.setOutputRecFile(recfile)
extraCLIArguments = " --constant.DetectorModel=%s " % detector_model
extraCLIArguments += " --constant.RunOverlay=true --constant.CMSEnergy=%s " % str(
energy)
extraCLIArguments += " --global.Verbosity=MESSAGE "
marlin.setExtraCLIArguments(extraCLIArguments)
job.append(marlin)
if outputDir != "":
job.setOutputData([dstfile, recfile],
OutputPath=outputDir,
OutputSE=outputSE)
if isLocal:
job.submit(dIlc, mode="local")
else:
job.submit(dIlc)
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
#### Software Versions ####
softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# Check the --runs and --split arguments to make sure they are compatible, if not exit...
if not check_events_arguments(args.events, args.split):
sys.exit(1)
# Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
lfn_check, lfn = check_input_LFN(args.stdhepInput)
if not lfn_check:
sys.exit(1)
# Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
dirac = DiracILC(True, setup_repository_name(args.stdhepInput, args.detector))
# Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
# into the input sandbox
inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(path.basename(args.stdhepInput))
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent/nOutputEvents
print "Job ", fileNumber
outputFiles = setup_output_dict(args.stdhepInput, args.detector, fileNumber, args.outputPath, softVersions)
slicOutput=outputFiles['slicOutput']
prePandoraOutput=outputFiles['prePandoraOutput']
pandoraOutput=outputFiles['pandoraOutput']
vertexingOutput=outputFiles['vertexingOutput']
lcsimRecOutput=outputFiles['lcsimRecOutput']
lcsimDstOutput=outputFiles['lcsimDstOutput']
flavortagOutput=outputFiles['flavortagOutput']
diracOutput=outputFiles['diracOutput']
################## SLIC ##################################################
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(slicOutput)
slic.setDetectorModel(args.detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (digitization and tracking) #####################
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile('steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml') # Another version is included in /steeringFiles
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
# lcsim.setAliasProperties('alias.properties')
lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
lcsim.setOutputFile(prePandoraOutput)
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(args.detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(pandoraOutput)
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/sidloi3.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(vertexingOutput)
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
# lcsimDst.setAliasProperties('alias.properties')
lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
lcsimDst.setOutputRecFile(lcsimRecOutput)
lcsimDst.setOutputDstFile(lcsimDstOutput)
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if args.flavortag:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/sidloi3.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(flavortagOutput)
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
'OSG.PNNL.us',])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
if args.flavortag:
job.setOutputData(flavortagOutput, diracOutput, args.SE)
else:
job.setOutputData(lcsimDstOutput, diracOutput, args.SE)
job.setOutputSandbox(outputSandbox)
job.setInputData(lfn)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0;
ma.setOutputDstFile("mydst_no_ov_%s.slcio"%i)
res = j.append(ma)
if not res['OK']:
print res['Message']
break
ov = OverlayInput()
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(n_evts_per_job)
ov.setDetectorType("ILD")
res = j.append(ov)
if not res['OK']:
print res['Message']
break
ma2 = Marlin()
ma2.setVersion("v0111Prod")
ma2.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma2.getInputFromApp(mo)
ma2.setOutputDstFile("mydst_ov_%s.slcio"%i)
res = j.append(ma2)
if not res['OK']:
print res['Message']
break
j.setOutputSandbox(["mydst_no_ov_%s.slcio"%i,"mydst_ov_%s.slcio"%i,"*.log"])
j.setName("SingleElectron_%s"%i)
j.setJobGroup("singleElectrons")
j.submit(d)
dexit(0)
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
d= DiracILC(True,"repo.rep")
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("example")#%i)
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.setGearFile("clic_ild_cdr.gear")
ma.setInputFile("LFN:/ilc/prod/clic/3tev/gghad/ILD/SIM/00000187/000/gghad_sim_187_97.slcio")
outdst = "toto.dst.slcio" #% i
outrec = "toto.rec.slcio" #% i
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit(1)
j.setOutputData([outdst,outrec],"some/path","KEK-SRM")
j.setOutputSandbox("*.log")
j.dontPromptMe()
if ov:
steeringf = "clic_ild_cdr_steering_overlay.xml"
res = j.append(ovi)
if not res['OK']:
print(res['Message'])
continue
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setGearFile("clic_ild_cdr.gear")
ma.setSteeringFile(steeringf)
ma.setInputFile("LFN:"+lfn)
ma.setNbEvts(10)
ma.setEnergy(500.)
ma.setOutputRecFile("myrec_overlay_%s.slcio"%ov)
ma.setOutputDstFile("mydst_overlay_%s.slcio"%ov)
res =j.append(ma)
if not res['OK']:
print(res['Message'])
exit()
j.setCPUTime(86400)
j.setOutputData("myrec_overlay_%s.slcio"%ov,"some/path")
j.setName("SomeName")
j.setJobGroup("SomeGroup")
res = d.checkparams(j)
if not res['OK']:
print(res['Message'])
exit()
j.submit(d)
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
print args.chain[0]
# softVersions = ["v3r0p3", "3.0-SNAPSHOT", "ILC_DBD", "0116"]
softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILCSoft-01-17-08", "0116"]
check_events_arguments(args.events, args.split)
detector = args.detector
alias_properties(detector)
outputPath, outputBase, repoName = input_output(args.Input, detector, args.chain, args.digiSteering)
inputSandbox, outputSandbox = setup_sandboxes(args.macFile)
dirac = DiracILC(True, repoName)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(outputBase)
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent/nOutputEvents
print "Job ---> ", fileNumber
################## SLIC ##################################################
if 1 in args.chain:
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'))
slic.setDetectorModel(detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Overlay ###############################################
if 2 in args.chain:
'''
#Add the gghad background overlay.
gghad = OverlayInput()
#gghad.setProdID(1767)
gghad.setEnergy(500.0)
gghad.setBXOverlay('args.bunches')
gghad.setGGToHadInt( 4.1 )
gghad.setNbSigEvtsPerJob(nOutputEvents)
gghad.setMachine('ilc_dbd')
gghad.setDetectorModel('sidloi3')
gghad.setBkgEvtType('aa_lowpt')
result = job.append( gghad )
if not result['OK']:
print result['Message']
sys.exit(2)
#Add the pair background overlay.
pairs = OverlayInput()
pairs.setProdID(2)
pairs.setEnergy(500.0)
pairs.setBXOverlay('args.bunches')
pairs.setGGToHadInt(1.)
pairs.setNbSigEvtsPerJob(nOutputEvents)
pairs.setMachine('ilc_dbd')
pairs.setDetectorModel('sidloi3')
pairs.setBkgEvtType('eepairs')
result = job.append( pairs )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
gghad = OverlayInput()
gghad.setPathToFiles('/ilc/user/j/jstrube/gghadron_lowpt/sidloi3/')
gghad.setBXOverlay(int(args.bunches))
gghad.setGGToHadInt( 4.1 )
gghad.setNbSigEvtsPerJob(nOutputEvents)
gghad.setBkgEvtType('aa_lowpt')
result = job.append( gghad )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
pairs = OverlayInput()
pairs.setPathToFiles('/ilc/user/j/jstrube/GuineaPig/sidloi3/')
pairs.setBXOverlay(int(args.bunches))
pairs.setGGToHadInt(1.)
pairs.setBkgEvtType('eepairs')
pairs.setNbSigEvtsPerJob(nOutputEvents)
result = job.append( pairs )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
################## lcsim (digitization and tracking) #####################
if 3 in args.chain:
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile(args.digiSteering) # Another version is included in /steeringFiles
if 1 in args.chain:
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
lcsim.setAliasProperties('steeringFiles/alias.properties')
lcsim.setDetectorModel(detector+".zip")
#lcsim.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'))
lcsim.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio')) #NEED TO CHANGE!!!
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
if 4 in args.chain:
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'))
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
if 5 in args.chain:
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/' + detector + '.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_vertexing.slcio'))
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
lcsimDst.setAliasProperties('steeringFiles/alias.properties')
lcsimDst.setDetectorModel(detector+".zip")
lcsimDst.setOutputRecFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_Rec.slcio'))
lcsimDst.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'))
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if 6 in args.chain:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/' + detector + '.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(outputBase.replace('.slcio', '_' + '_flavortag.slcio'))
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
'OSG.PNNL.us','OSG.UConn.us','OSG.GridUNESP_CENTRAL.br','LCG.SCOTGRIDDURHAM.uk',
'LCG.TECHNIONself.il','LCG.UKI-SOUTHGRID-RALPP.uk','OSG.FNAL_FERMIGRID.us','LCG.UKI-LT2-IC-HEP.uk'])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
outputLevel = max(args.chain)
if outputLevel == 1:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 3:
#job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'), outputPath, 'CERN-SRM')
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 4:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 5:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 6:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_flavortag.slcio'), outputPath, 'CERN-SRM')
job.setOutputSandbox(outputSandbox)
job.setInputData(args.Input)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0;
job = UserJob()
job.setDestination('LCG.CERN.ch')
job.setInputSandbox([
"LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/PandoraLikelihoodData9EBin_CLIC_ILD.xml",
"LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/PandoraSettingsFast.xml",
"LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/MarlinRecoRootFiles/steering.xml",
"LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/clic_ild_cdr.gear",
"LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/MarlinRecoRootFiles/lib.tar.gz"
])
job.setInputData(
"LFN:/ilc/user/k/kacarevic/hgamgam/Marlin/newPandora/aa/aa_%d.slcio" %
i)
job.setOutputSandbox(
["*.log", "*.sh", "*.py", "*.out", "*.xml", "*.steer "])
job.setJobGroup("myRoot")
job.setName("root_aa_%d" % i)
marl = Marlin()
marl.setVersion('ILCSoft-2016-09-27_gcc48')
marl.setInputFile([
"LFN:/ilc/user/k/kacarevic/hgamgam/Marlin/newPandora/aa/aa_%d.slcio" %
i
])
# marl.setNumberOfEvents(10)
marl.setSteeringFile("steering.xml")
marl.setGearFile("clic_ild_cdr.gear")
marl.setExtraCLIArguments("--MyProcessor.RootFilename=%s" % lcoutput)
res = job.append(marl)
if not res['OK']:
print res['Message']
quit() #do something, like quit
job.setOutputData(lcoutput, "hgamgam/Marlin/rootFiles/aa/workingVersion",
def execute(self):
""" First we update the site list and banned site
"""
res = getProxyInfo(False, False)
if not res['OK']:
self.log.error("submitTasks: Failed to determine credentials for submission", res['Message'])
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
self.log.info("submitTasks: Jobs will be submitted with the credentials %s:%s" % (owner, ownerGroup))
sites = self.diracadmin.getSiteMask()['Value']
for site in sites:
res = self.ppc.changeSiteStatus( {'SiteName' : site, 'Status' : 'OK'} )
if not res['OK']:
self.log.error('Cannot add or update site %s' % site)
banned_sites = self.diracadmin.getBannedSites()['Value']
for banned_site in banned_sites:
self.ppc.changeSiteStatus( {'SiteName' : banned_site, 'Status' : 'Banned'} )
if not res['OK']:
self.log.error('Cannot mark as banned site %s' % banned_site)
##Then we need to get new installation tasks
res = self.ppc.getInstallSoftwareTask()
if not res['OK']:
self.log.error('Failed to obtain task')
task_dict = res['Value']
for softdict in task_dict.values():
self.log.info('Will install %s %s at %s' % (softdict['AppName'], softdict['AppVersion'], softdict['Sites']))
for site in softdict['Sites']:
j = UserJob()
j.setPlatform(softdict['Platform'])
j.dontPromptMe()
j.setDestination(site)
j.setJobGroup("Installation")
j.setName('install_%s' % site)
j._addSoftware(softdict['AppName'], softdict['AppVersion'])
#Add the application here somehow.
res = j.append(SoftwareInstall())
if not res['OK']:
self.log.error(res['Message'])
continue
res = j.submit(self.dirac)
#res = self.dirac.submit(j)
if not res['OK']:
self.log.error('Could not create the job')
continue
jobdict = {}
jobdict['AppName'] = softdict['AppName']
jobdict['AppVersion'] = softdict['AppVersion']
jobdict['Platform'] = softdict['Platform']
jobdict['JobID'] = res['Value']
jobdict['Status'] = 'Waiting'
jobdict['Site'] = site
res = self.ppc.addOrUpdateJob(jobdict)
if not res['OK']:
self.log.error('Could not add job %s: %s' % (jobdict['JobID'], res['Message']))
##Monitor jobs
jobs = {}
res = self.ppc.getJobs()
if not res['OK']:
self.log.error('Could not retrieve jobs')
else:
jobs = res['Value']
for job in jobs:
res = self.dirac.status(job['JobID'])
if res['OK']:
jobstatuses = res['Value']
job['Status'] = jobstatuses['JobID']['Status']
res = self.ppc.addOrUpdateJob(job)
if not res['OK']:
self.log.error("Failed to updated job %s: %s" % (job['JobID'], res['Message']))
else:
self.log.error("Failed to update job %s status" % job['JobID'])
return S_OK()
ov.setNbSigEvtsPerJob(1)
ov.setBkgEvtType("gghad")
res = j.append(ov)
if not res['OK']:
print res['Message']
dexit(1)
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma.setGearFile("clic_ild_cdr.gear")
ma.getInputFromApp(mo)
ma.setDebug(True)
res = j.append(ma)
if not res['OK']:
print res['Message']
dexit(1)
#print appplication's attributes.
ma.listAttributes()
j.setName("test")
j.setOutputSandbox("*.log")
res = dirac.checkparams(j)
if not res['OK']:
print res['Message']
dexit(1)
n_events_per_job=100
energy=500
GGToHadInt500=1.7
BXOverlay=1
outdst = "toto-ovl.dst.slcio" #% i
outrec = "toto-ovl.rec.slcio" #% i
d= DiracILC(True,"repo.rep")
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinOverlayExample")#%i)
j.setInputSandbox([setting_file])
## Define the overlay
ov = OverlayInput()
ov.setMachine("ilc_dbd")
ov.setEnergy(energy)
ov.setNumberOfSignalEventsPerJob(n_events_per_job)
ov.setBXOverlay(BXOverlay)
ov.setGGToHadInt(GGToHadInt500)
ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
ov.setDetectorModel("ILD_o1_v05")
res = j.append(ov)
if not res['OK']:
print res['Message']
break
ov = OverlayInput()
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(n_evts_per_job)
ov.setBkgEvtType("gghad")
ov.setDetectorModel("CLIC_ILD_CDR")
res = j.append(ov)
if not res['OK']:
print res['Message']
break
ma2 = Marlin()
ma2.setVersion("v0111Prod")
ma2.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma2.getInputFromApp(mo)
ma2.setOutputDstFile("mydst_ov_%s.slcio" % i)
res = j.append(ma2)
if not res['OK']:
print res['Message']
break
j.setOutputSandbox(
["mydst_no_ov_%s.slcio" % i,
"mydst_ov_%s.slcio" % i, "*.log"])
j.setName("SingleElectron_%s" % i)
j.setJobGroup("singleElectrons")
j.submit(d)
dexit(0)
def getJob(dirac, jobid, jobpara):
iser=jobid+100
outdir = "/ilc/user/a/amiyamoto/myprod2/test/"
outdst = "toto-ovl-%5.5i.dst.slcio"%iser
outrec = "toto-ovl-%5.5i.rec.slcio"%iser
dstlfn = outdir+"dst/"+outdst
reclfn = outdir+"rec/"+outrec
outsrm = "CERN-SRM"
###In case one wants a loop: comment the folowing.
#for i in range(2):
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("MarlinOverlayParametric%i"%iser)
j.setInputSandbox(jobpara["setting_file"])
## Define the overlay
ov = OverlayInput()
ov.setMachine("ilc_dbd")
ov.setEnergy(energy)
ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"]))
ov.setBXOverlay(int(jobpara["BXOverlay"]))
ov.setGGToHadInt(float(jobpara["GGToHadInt500"]))
ov.setBkgEvtType("aa_lowpt")
# ov.setBackgroundType("aa_lowpt")
ov.setDetectorModel("ILD_o1_v05")
res = j.append(ov)
if not res['OK']:
print res['Message']
exit(1)
## Define Marlin job
ma = Marlin()
ma.setDebug()
ma.setVersion("ILCSoft-01-17-09")
ma.setSteeringFile("marlin_stdreco.xml")
ma.setGearFile("GearOutput.xml")
# ma.setInputFile(simfile)
ma.setInputFile(simlists[jobid])
ma.setOutputDstFile(outdst)
ma.setOutputRecFile(outrec)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit(1)
# Upload files to different directories
upload_script="upload%i.sh"%iser
upload = GenericApplication()
# Create a script to upload files.
shfile = open(upload_script,"w")
shfile.write("#!/bin/bash\n")
shfile.write("/bin/ls -l \n")
shfile.write("dirac-dms-add-file -ddd "+dstlfn+" "+outdst+" "+outsrm+" \n")
shfile.write("dirac-dms-add-file -ddd "+reclfn+" "+outrec+" "+outsrm+" \n")
shfile.close()
os.chmod(upload_script,0755)
upload.setScript(upload_script)
res = j.append(upload)
if not res['OK'] :
print res['Message']
exit(1)
# j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM")
j.setInputSandbox([ setting_file, upload_script ] )
j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"])
j.setCPUTime(10000)
j.dontPromptMe()
res = j.submit(dirac)
if not res["OK"] :
print "Failed submit job, jobid=%s" %jobid
print res
os.remove(upload_script)
return j
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import GenericApplication
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
datadir = "/ilc/prod/ilc/mc-opt-3/ild/dst-merged/500-TDR_ws/higgs_ffh/ILD_l5_o1_v02/v02-00-01/"
filepref = "rv02-00-01.sv02-00-01.mILD_l5_o1_v02.E500-TDR_ws.I106523.Pnnh.eL.pR.n001.d_dstm_10763_"
indata = [datadir + filepref + "0.slcio", datadir + filepref + "1.slcio"]
d = DiracILC(True, "repo.rep")
################################################
j = UserJob()
j.setJobGroup("Tutorial")
j.setName("GenericExec")
j.setInputSandbox(["mypre.sh", "myanal.sh"])
j.setInputData(indata)
j._setSoftwareTags(["marlin.ILCSoft-02-00-02_gcc49"])
# j._setSoftwareTags(["lcio.ILCSoft-02-00-01_gcc49"])
j.setILDConfig("v02-00-02")
################################################
appre = GenericApplication()
appre.setScript("mypre.sh")
appre.setArguments("This is input arguments")
res = j.append(appre)
if not res['OK']:
print res['Message']
exit(1)
if ov:
steeringf = "clic_ild_cdr_steering_overlay.xml"
res = j.append(ovi)
if not res['OK']:
print res['Message']
continue
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setGearFile("clic_ild_cdr.gear")
ma.setSteeringFile(steeringf)
ma.setInputFile("LFN:" + lfn)
ma.setNbEvts(10)
ma.setEnergy(500.)
ma.setOutputRecFile("myrec_overlay_%s.slcio" % ov)
ma.setOutputDstFile("mydst_overlay_%s.slcio" % ov)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit()
j.setCPUTime(86400)
j.setOutputData("myrec_overlay_%s.slcio" % ov, "some/path")
j.setName("SomeName")
j.setJobGroup("SomeGroup")
res = d.checkparams(j)
if not res['OK']:
print res['Message']
exit()
j.submit(d)
def main(argv):
# Input arguments
ildconfig_version = "$ILDCONFIGVER"
ilcsoft_version = "$ILCSOFTVER"
evts_per_run = $EVTSPERRUN
detector_model = "$DETECTOR"
sim_input = "$SIMINPUT"
process_name = "$PROCESS"
index = $IND
sim_input = diracpath_from_pnfspath( sim_input )
sim_detector_model = detector_model_wo_option( detector_model )
job_group = ilcsoft_version + "_" + ildconfig_version + "_" + process_name + "_" + detector_model
dirac = DiracILC(True,job_group+".rep")
# outputs to be saved onto grid SE
RECoutput = []
# DDSim
evtStart = (index-1)*evts_per_run
evtEnd = index*evts_per_run - 1
RandSeed = random.randrange(11623, 99999)
lcinputSIM = "LFN:" + sim_input
lcoutputSIM = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_SIM.slcio"%(str(evtStart),(str)(evtEnd))
sim = DDSim()
sim.setVersion(ilcsoft_version)
sim.setDetectorModel(sim_detector_model)
sim.setInputFile(lcinputSIM)
sim.setSteeringFile("ddsim_steer.py")
sim.setNumberOfEvents(evts_per_run)
sim.setRandomSeed(RandSeed)
sim.setEnergy(1000)
sim.setStartFrom(evtStart)
sim.setOutputFile(lcoutputSIM)
# Marlin
lcoutputDST = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_DST.slcio"%(str(evtStart),(str)(evtEnd))
ma = Marlin()
ma.setVersion(ilcsoft_version)
ma.setDetectorModel(detector_model)
ma.setSteeringFile("MarlinStdReco.xml")
ma.setExtraCLIArguments( "--constant.lcgeo_DIR=$lcgeo_DIR --constant.DetectorModel={} --global.MaxRecordNumber=0".format(detector_model) )
ma.setLogFile("marlin.log")
ma.getInputFromApp(sim)
ma.setEnergy(1000)
ma.setOutputDstFile(lcoutputDST)
RECoutput.append(lcoutputDST)
# ILCDirac user job
job = UserJob()
job.setName("user_sim_reco")
job.setJobGroup(job_group)
job.setILDConfig(ildconfig_version)
job.setCPUTime(86400)
tmp_file_name = process_name + "_sim_reco_job_tmp.py"
job.setInputSandbox([tmp_file_name])
job.setOutputSandbox(["*.log","MarlinStdRecoParsed.xml","marlin*.xml","*.py "])
job.dontPromptMe()
job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us'])
# run simulation job
simres = job.append(sim)
if not simres['OK']:
print simres['Not ok appending ddsim to job']
quit()
# run Malrin reco jobs
mares = job.append(ma)
if not mares['OK']:
print mares['Not ok appending Marlin to job']
quit()
job.setOutputData(RECoutput,"ILDPerformance/WWZZSeparation/{}_ILDConfig_{}_{}".format(ilcsoft_version,ildconfig_version,detector_model),"DESY-SRM")
print RECoutput
submit_output = job.submit(dirac)
print submit_output
