def subDDSim(): # Decide parameters for a job outputSE = "KEK-SRM" outputSE = "KEK-DISK" isLocal = _clip.isLocal nbevts = 10 if _clip.numberOfEvents == 0 else _clip.numberOfEvents outputFile = "ddsim_example.slcio" if _clip.outputFile == "" else _clip.outputFile outputDir = _clip.outputDir inputFile = _clip.inputFile if inputFile == "": gLogger.error("Input file for ddsim does not given.") exit(-1) # Create DIRAC objects for job submission dIlc = DiracILC() job = UserJob() job.setJobGroup("myddsimjob") job.setName("myddsim") job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml']) job.setILDConfig("v02-00-02") # job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"]) # job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination job.setBannedSites(["LCG.UKI-SOUTHGRID-RALPP.uk" ]) # a list of sites not to submit job # job.setCPUTime( cputime_limit_in_seconds_by_dirac_units ) ddsim = DDSim() ddsim.setVersion("ILCSoft-02-00-02_gcc49") ddsim.setDetectorModel("ILD_l5_v02") ddsim.setInputFile(inputFile) ddsim.setNumberOfEvents(nbevts) extraCLIArguments = " --steeringFile ddsim_steer.py " extraCLIArguments += " --outputFile %s " % outputFile extraCLIArguments += " --vertexSigma 0.0 0.0 0.1968 0.0 --vertexOffset 0.0 0.0 0.0 0.0 " ddsim.setExtraCLIArguments(extraCLIArguments) # ddsim.setRandomSeed(1234565) # ddsim.setStartFrom(20) # Number of events to skip before starting ddsim job.append(ddsim) if outputDir != "": job.setOutputData([outputFile], OutputPath=outputDir, OutputSE=outputSE) if isLocal: job.submit(dIlc, mode="local") else: job.submit(dIlc)
def create_job(inputData, saveName, outputDir, dontPromptMe): slcioFile = saveName + '.slcio' rootFile = saveName + '.root' if check_file_existence(outputDir, slcioFile, dontPromptMe): remove_file(outputDir, slcioFile, dontPromptMe) if check_file_existence(outputDir, rootFile, dontPromptMe): remove_file(outputDir, rootFile, dontPromptMe) dIlc = DiracILC() job = UserJob() job.setOutputSandbox(['*.out', '*.log', '*.sh', '*.py', '*.xml']) if SAVE_SLCIO: job.setOutputData([slcioFile, rootFile], OutputPath=outputDir, OutputSE=STORAGE_SE) else: job.setOutputData(rootFile, OutputPath=outputDir, OutputSE=STORAGE_SE) job.setJobGroup('myMarlinRun1') job.setName('MyMarlinJob1') # job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk']) job.setInputSandbox(['LFN:/ilc/user/a/amaier/mylibs.tar.gz']) marl = Marlin() marl.setVersion('ILCSoft-2017-11-15_gcc62') marl.setInputFile(inputData) job.setInputData(list(map(lambda x: x.lstrip('LFN:'), inputData))) # marl.setInputFile(['LFN:/ilc/prod/clic/1.4tev/qq_ln/ILD/DST/00003249/010/qq_ln_dst_3249_10000.slcio']) marl.setSteeringFile('marlin/FullChain.xml') # marl.setSteeringFile('marlin/FullChainNewDetModel.xml') marl.setOutputFile(slcioFile) gearFile = '/afs/cern.ch/user/a/amaier/projects/CLIC_analysis/grid/marlin/clic_ild_cdr.gear' if not os.path.isfile(gearFile): print('Error: gear file', gearFile, 'does not exist! Abort submission.') return marl.setGearFile(gearFile) marl.setExtraCLIArguments( "--MyNtupleMaker.OutputFileName={rootOutfile}".format( rootOutfile=rootFile)) # marl.setNumberOfEvents(1000) job.append(marl) if dontPromptMe: job.dontPromptMe() job.submit(dIlc) return False
def getJob(jobid, jobpara): iser=jobid+1 outdst = "toto-ovl-%5.5i.dst.slcio"%iser outrec = "toto-ovl-%5.5i.rec.slcio"%iser ###In case one wants a loop: comment the folowing. #for i in range(2): j = UserJob() j.setJobGroup("Tutorial") j.setName("MarlinOverlayParametric%i"%iser) j.setInputSandbox(jobpara["setting_file"]) ## Define the overlay ov = OverlayInput() ov.setMachine("ilc_dbd") ov.setEnergy(energy) ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"])) ov.setBXOverlay(int(jobpara["BXOverlay"])) ov.setGGToHadInt(float(jobpara["GGToHadInt500"])) ov.setBkgEvtType("aa_lowpt") # ov.setBackgroundType("aa_lowpt") ov.setDetectorModel("ILD_o1_v05") res = j.append(ov) if not res['OK']: print res['Message'] exit(1) ## Define Marlin job ma = Marlin() ma.setDebug() ma.setVersion("ILCSoft-01-17-09") ma.setSteeringFile("marlin_ovl_stdreco.xml") ma.setGearFile("GearOutput.xml") # ma.setInputFile(simfile) ma.setInputFile(simlists[jobid]) ma.setOutputDstFile(outdst) ma.setOutputRecFile(outrec) res = j.append(ma) if not res['OK']: print res['Message'] exit(1) j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM") j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"]) j.setCPUTime(10000) j.dontPromptMe() return j
def subWhizard2(): from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin, Whizard2 # Decide parameters for a job outputSE = "KEK-SRM" isLocal = _clip.isLocal nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents outputFile = "E500.P2f_bB.GWhizard2.I100000.e0.p0.n001.slcio" if _clip.outputFile == "" else _clip.outputFile outputDir = _clip.outputDir # Create DIRAC objects for job submission dIlc = DiracILC() job = UserJob() job.setInputSandbox(["pythia6-parameters.sin", "P2f_qqbar.sin"]) job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml']) job.setOutputData([outputFile]) job.setJobGroup("mywhiz2") job.setName("mywhizard2") whiz = Whizard2() whiz.setVersion("2.7.0") whiz.setNumberOfEvents(nbevts) whiz.setEvtType("P2f_bB") whiz.setProcessVariables("P2f_bB") # whiz.setRandomSeed(15) whiz.setSinFile("P2f_qqbar.sin") whiz.setOutputFile(outputFile) job.append(whiz) if outputDir != "": print " outputDir = " + outputDir print " outputSE = " + outputSE job.setOutputData([outputFile], OutputPath=outputDir, OutputSE=outputSE) if isLocal: job.submit(dIlc, mode="local") else: job.submit(dIlc)
def getJob(): """ produce a job: it's always the same, so we don't need to put it in the main """ j = UserJob() ma = Marlin() ma.setVersion("v0111Prod") ma.setSteeringFile("clic_ild_cdr_steering.xml") ma.setGearFile("clic_ild_cdr.gear") result = j.append(ma) if not result['OK']: gLogger.error(result["Message"]) dexit(1) j.setCPUTime(10000) j.setOutputSandbox("*.log") return j
def testAndProbeSites(): """submits jobs to test sites""" clip = Params() clip.registerSwitches() Script.parseCommandLine() from DIRAC import gLogger, exit as dexit from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import CheckWNs from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getQueues res = getQueues(siteList=clip.site, ceList=clip.ce) if not res['OK']: gLogger.error("Failed getting the queues", res['Message']) dexit(1) sitedict = res['Value'] CEs = [] for ces in sitedict.values(): CEs.extend(ces.keys()) gLogger.notice("Found %s CEs to look at." % len(CEs)) d = DiracILC(True, "SiteProbe.rep") for CE in CEs: j = UserJob() j.setDestinationCE(CE) c = CheckWNs() res = j.append(c) if not res['OK']: gLogger.error(res['Message']) continue j.setOutputSandbox("*.log") j.setCPUTime(30000) j.dontPromptMe() res = j.submit(d) if not res['OK']: gLogger.error("Failed to submit job, aborting") dexit(1) dexit(0)
class Submit(): #*****************************************# # Function name : __init__ # # role : create a DIRAC job, an instance # # of DiracILC, and filter the output # #*****************************************# def __init__(self): self.dIlc = DiracILC(False) self.job = UserJob() self.job.setJobGroup("FCC") self.job.setName("FCC APP") self.job.setOutputSandbox(["*.log", '*.root']) #self.job.setDestination('LCG.DESY-HH.de') #EOS public location self.EOS_MGM_URL = 'root://eospublic.cern.ch' #EOS environment self.setEOS = 'export EOS_MGM_URL=' + self.EOS_MGM_URL self.myclient = client.FileSystem(self.EOS_MGM_URL + ':1094') #sandbox self.InputSandbox = [] self.folders_to_upload = [] self.filtered_extensions = [] self.excludes_or_includes = [] self.temp_cwd = os.path.join(os.getcwd(), 'fcc_temp_dirac') #********************************************# # Function name : read_from_file # # input : file_name # # role : read a file and return its content # #********************************************# def read_from_file(self, file_name): try: with open(file_name, 'r') as f: content = f.read() return content except: return False #*********************************************************************************# # Function name : create_temp_tree_from_files # # role : given a relative tree path of file to the local FCCSW folder of the user # # it looks for this file, recreates the relative tree in a temporary folder # # containing only this file and not all files of the source folder # # Finally, the folder will be added to the DIRAC sandbox # #*********************************************************************************# def create_temp_tree_from_files(self, files, fccsw_path): if files: for file in files: tree = os.path.dirname(file) tree_full_path = os.path.join(self.temp_cwd, tree) if not os.path.exists(tree_full_path): os.makedirs(tree_full_path) root_folder = tree.split(os.path.sep)[0] root_folder_full_path = os.path.join(self.temp_cwd, root_folder) if root_folder_full_path not in self.folders_to_upload: self.folders_to_upload += [root_folder_full_path] source = os.path.join(fccsw_path, file) destination = os.path.join(self.temp_cwd, file) if not os.path.exists(source): print '\nThe file : ' + source + ' does not exist\n' quit() else: copyfile(source, destination) #***************************************************# # Function name : compress # # role : it compress all folders, the job # # will need once in the grid # #***************************************************# def compress(self, temp_folder, actual_folder, tar_extension, filtered_extension, exclude_or_include): #print filtered_extension,exclude_or_include exclude_func = None if False is not filtered_extension: if exclude_or_include: exclude_func = lambda filename: filename.find( filtered_extension) >= 0 else: exclude_func = lambda filename: os.path.isfile( filename) and filename.find(filtered_extension) < 0 tar = tarfile.open(temp_folder + tar_extension, "w:gz") for name in os.listdir(actual_folder): #print name renamed = os.path.join(actual_folder, name) #print renamed #print os.path.basename(renamed) tar.add(renamed, arcname=os.path.basename(renamed), exclude=exclude_func) tar.close() #*************************************************# # Function name : find_eos_file # # input : file_name # # role : check if file exists on EOS # # before sending the job to DIRAC # #*************************************************# def find_eos_file(self, file_name): #then the file is in EOS eos_file_full_path = self.EOS_MGM_URL + '/' + file_name with client.File() as eosFile: file_status = eosFile.open(eos_file_full_path, OpenFlags.UPDATE) #problem with file created directly on eos #no problem with uploded files with xrdcp cmd #print eos_file_full_path #print file_status status = self.XRootDStatus2Dictionnary(file_status) if 'False' == status[' ok'] or False == status: return file_name, False else: return eos_file_full_path, True #*************************************************# # Function name : find_eos_folder # # input : folder_name # # role : check if folder exists on eos # # before sending the job to the worker # #*************************************************# def find_eos_folder(self, folder_name): #then the file is in eos eos_folder_full_path = self.EOS_MGM_URL + '/' + folder_name status, listing = self.myclient.dirlist(folder_name, DirListFlags.STAT) if None == listing: return folder_name, False else: return eos_folder_full_path, True #*************************************************# # Function name : find_path # # input : file_name # # role : check if file/folder exists on afs # # before checking on eos # #*************************************************# def find_path(self, path, file_or_dir='file'): #we suppose that the user enter absolute or relative afs path #or only absolute eos path if not path.startswith('/eos/'): #afs path are absolute or relative #because software are stored in this filesystem #and users generally submit their job from lxplus #absolute path provided #os.isabs not cross platform #i used startwith... if path.startswith('/afs/') and os.path.exists(path): return os.path.abspath(path), True #if relative (does not start with /afs/) #add cwd elif os.path.exists(os.path.abspath(path)): return os.path.abspath(path), True #maybe local user machine, print upload error message else: return path, False #absolute path elif path.startswith('/eos/'): #print "the file is in eos" #eos path are absolute file_path, is_file_exist = self.find_eos_file(path) folder_path, is_folder_exist = self.find_eos_folder(path) if is_file_exist: return file_path, is_file_exist elif is_folder_exist: return folder_path, is_folder_exist else: return path, False else: #other file system return path, False #********************************************************# # Function name : upload_sandbox_with_application_files # # role : upload all extra folders or files # # specified by the user for an application # # For now, we do not check if file exists on cvmfs # #********************************************************# def upload_sandbox_with_application_files(self, paths): upload_path_message = " does not exist\nPlease ensure that your path exist in an accessible file system (EOS or AFS)\n" for path in paths: if not path.startswith('/cvmfs/'): path, is_exist = self.find_path(path) if False == is_exist: message = "\nThe path '" + path + "'" + upload_path_message else: if path.startswith('/afs/'): print '\nWARNING : STORING FILES ON AFS IS DEPRECATED\n' print '\nYou plan to upload :' + path + ' which is stored on AFS\n' #files are directly added to the sandbox if os.path.isfile(path): self.InputSandbox += [path] else: #folders are compressed before self.folders_to_upload += [path] #********************************************************# # Function name : upload_sandbox_with_fccsw_files # # role : upload files called in FCCSW configuration file # # and needed folders relative to FCCSW # #********************************************************# def upload_sandbox_with_fccsw_files(self, fccsw_path, fcc_conf_file): InstallArea_folder = os.path.join(fccsw_path, 'InstallArea') Detector_folder = os.path.join(fccsw_path, 'Detector') fccsw_folders = [InstallArea_folder, Detector_folder] #explanation #InstallArea_folder : dbg files are excluded #Detector_folder : only xml files are included self.filtered_extensions += ['.dbg', '.xml'] self.excludes_or_includes += [True, False] content = self.read_from_file(fcc_conf_file) if False is content: print "\nError in reading configuration file :\n" + fcc_conf_file quit() #xml_files = re.findall(r'file:(.*.xml)',content) txt_files = re.findall(r'="(.*.txt)', content) cmd_files = re.findall(r'filename="(.*.cmd)', content) #print txt_files #print cmd_files folders = self.create_temp_tree_from_files(txt_files, fccsw_path) if None != folders: fccsw_folders += folders folders = self.create_temp_tree_from_files(cmd_files, fccsw_path) if None != folders: fccsw_folders += folders self.folders_to_upload = fccsw_folders + self.folders_to_upload #********************************************************# # Function name : update_sandbox # # role : check the files and folders required by the job # # and add them to the sandbox # # Indeed, it calls upload_sandbox_with_* functions # #********************************************************# def update_sandbox(self, fccsw_path, paths, fcc_conf_file): #first, it creates a specific local working directory for the sandbox if not os.path.exists(self.temp_cwd): os.makedirs(self.temp_cwd) #update sandbox with application files if '' != paths: self.upload_sandbox_with_application_files(paths) #update sandbox with FCCSW application files if '' != fccsw_path: self.upload_sandbox_with_fccsw_files(fccsw_path, fcc_conf_file) #********************************************************# # Function name : compress_sandbox_subfolders # # role : compress all local folders required by the job # #********************************************************# def compress_sandbox_subfolders(self): compressed_folders = [] for idx, actual_folder in enumerate(self.folders_to_upload): tar_extension = '.tgz' if idx < len(self.filtered_extensions): filtered_extension = self.filtered_extensions[idx] exclude_or_include = self.excludes_or_includes[idx] else: filtered_extension = False exclude_or_include = False temp_folder = os.path.join(self.temp_cwd, os.path.basename(actual_folder)) #DIRAC already compressed the sandbox before submitting the job self.compress(temp_folder, actual_folder, tar_extension, filtered_extension, exclude_or_include) compressed_folders += [temp_folder + tar_extension] self.InputSandbox += compressed_folders #**************************************************# # Function name : submit # # role : create DIRAC generic application for each # # FCC application and submit the DIRAC job # # containing these generic DIRAC applications # #**************************************************# def submit(self, applications, script_to_source): fcc_execution_module = os.path.join(os.getcwd(), "fcc_execution_module.py") fcc_installation_module = os.path.join(os.getcwd(), 'fcc_installation_module.py') fcc_environment = script_to_source #Initialization of the sandbox paths = [fcc_execution_module, fcc_environment] self.upload_sandbox_with_application_files(paths) for application in applications: #application specification job_specification = application.job_specification #application specification details fcc_executable = job_specification['fcc_executable'] fccsw_path = job_specification['fccsw_path'] paths = job_specification['paths'] fcc_conf_file = job_specification['fcc_conf_file'] fcc_input_files = '' if '' == job_specification[ 'fcc_input_files'] else ' --ifiles ' + ' '.join( job_specification['fcc_input_files']) fcc_output_file = '' if '' == job_specification[ 'fcc_output_file'] else ' --ofile ' + job_specification[ 'fcc_output_file'] fccsw = '' if '' == job_specification[ 'fccsw_path'] else ' --fccsw ' + job_specification['fccsw_path'] number_of_events = '' if '' == job_specification[ 'number_of_events'] else ' -N ' + job_specification[ 'number_of_events'] #**************************************************MINIMUM REQUIREMENTS CHECKING*******************************************************# if '' == fcc_executable or '' == fcc_conf_file or '' == fcc_environment: print "\nError in parsing applications :\n" + fcc_executable print "\nYou have to provide at least an executable, a configuration file and a script to source for each application\n" quit() #**************************************************MINIMUM REQUIREMENTS CHECKING*******************************************************# #we update the sandbox according to the specification of the application self.update_sandbox(fccsw_path, paths, fcc_conf_file) #we instanciate a generic DIRAC application generic_dirac_application = GenericApplication() #we set the the installation module as the main script generic_dirac_application.setScript(fcc_installation_module) #we set the arguments of the script arguments = '--source ' + fcc_environment + ' --exec ' + fcc_executable + ' --conf ' + fcc_conf_file + fcc_input_files + fcc_output_file + fccsw + number_of_events generic_dirac_application.setExtraCLIArguments(arguments) #we add the generic DIRAC application to the DIRAC job try: res = self.job.append(generic_dirac_application) if not res['OK']: print res['Message'] quit() except: print "\nPlease, configure your proxy before submitting a job from DIRAC" print "If you do not set up a proxy, refer to the manual or maybe you have to refresh it" print "by typing :" print "dirac-proxy-init\n" quit() #before submitting the DIRAC job, we compress all folders of the sandbox self.compress_sandbox_subfolders() print '\n**********************************HERE THE CONTENT OF YOUR SANDBOX*********************************\n' print self.InputSandbox print '\n**********************************HERE THE CONTENT OF YOUR SANDBOX*********************************\n' #we set the sandbox self.job.setInputSandbox(self.InputSandbox) #-------------------------------now we submit the job containing all applications-------------------------------------------# res = self.job.submit(self.dIlc) if not res['OK']: print 'Please check your application requirements' print res['Message'] quit() else: print "The Job you submited has the following ID : " + str( res['JobID'])
slic = SLIC() slic.setVersion('v3r0p3') #can loop over stdhep files, takes precedence over what is in macro #slic.setInputFile("some_file.stdhep") #need not loop over macro #slic.setInputFile("newDetector.zip") slic.setSteeringFile(macro) slic.setNumberOfEvents(nEvts) #also overwrites macro #two options #1) specify lcdd filename (newDetector.lcdd) local or grid #2) specify detector name (like below). looks up detector in org.lcsim detector page and downloads the tarball slic.setDetectorModel('detector_vtx_matbudghalf_nonsensitivelayer.zip') #same rules as for InputFile EXCEPT NOT automatically added to output data (sandbox too small usually) slic.setOutputFile('slicOut.slcio') res = job.append(slic) lcsim = LCSIM() lcsim.getInputFromApp(slic) lcsim.setVersion('2.5') lcsim.setSteeringFile( "/users/detector/ssetru/SiDSim/detectors/detector_vtx_matbudghalf_nonsensitivelayer/sid_dbd_prePandora_noOverlay2.xml" ) lcsim.setAliasProperties('alias.properties') #lcsim.setTrackingStrategy("/afs/cern.ch/user/s/ssetru/examples/autogen_ttbar_sidloi3.xml") lcsim.setTrackingStrategy( "/users/detector/ssetru/SiDSim/detectors/detector_vtx_matbudghalf_nonsensitivelayer/strategies_vtx_matbudghalf_nonsensitivelayer.xml" ) #lcsim.setAliasProperties('alias.properties') lcsim.setOutputFile(output) #lcsim_prepandora.willRunSLICPandora()
def getJob(dirac, jobid, jobpara): iser=jobid+100 outdir = "/ilc/user/a/amiyamoto/myprod2/test/" outdst = "toto-ovl-%5.5i.dst.slcio"%iser outrec = "toto-ovl-%5.5i.rec.slcio"%iser dstlfn = outdir+"dst/"+outdst reclfn = outdir+"rec/"+outrec outsrm = "CERN-SRM" ###In case one wants a loop: comment the folowing. #for i in range(2): j = UserJob() j.setJobGroup("Tutorial") j.setName("MarlinOverlayParametric%i"%iser) j.setInputSandbox(jobpara["setting_file"]) ## Define the overlay ov = OverlayInput() ov.setMachine("ilc_dbd") ov.setEnergy(energy) ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"])) ov.setBXOverlay(int(jobpara["BXOverlay"])) ov.setGGToHadInt(float(jobpara["GGToHadInt500"])) ov.setBkgEvtType("aa_lowpt") # ov.setBackgroundType("aa_lowpt") ov.setDetectorModel("ILD_o1_v05") res = j.append(ov) if not res['OK']: print res['Message'] exit(1) ## Define Marlin job ma = Marlin() ma.setDebug() ma.setVersion("ILCSoft-01-17-09") ma.setSteeringFile("marlin_stdreco.xml") ma.setGearFile("GearOutput.xml") # ma.setInputFile(simfile) ma.setInputFile(simlists[jobid]) ma.setOutputDstFile(outdst) ma.setOutputRecFile(outrec) res = j.append(ma) if not res['OK']: print res['Message'] exit(1) # Upload files to different directories upload_script="upload%i.sh"%iser upload = GenericApplication() # Create a script to upload files. shfile = open(upload_script,"w") shfile.write("#!/bin/bash\n") shfile.write("/bin/ls -l \n") shfile.write("dirac-dms-add-file -ddd "+dstlfn+" "+outdst+" "+outsrm+" \n") shfile.write("dirac-dms-add-file -ddd "+reclfn+" "+outrec+" "+outsrm+" \n") shfile.close() os.chmod(upload_script,0755) upload.setScript(upload_script) res = j.append(upload) if not res['OK'] : print res['Message'] exit(1) # j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM") j.setInputSandbox([ setting_file, upload_script ] ) j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"]) j.setCPUTime(10000) j.dontPromptMe() res = j.submit(dirac) if not res["OK"] : print "Failed submit job, jobid=%s" %jobid print res os.remove(upload_script) return j
from DIRAC.Core.Base import Script Script.parseCommandLine() from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin from ILCDIRAC.Interfaces.API.DiracILC import DiracILC d = DiracILC() # Provides job checking utilities j = UserJob() # You are running a user job m = Marlin() # Get an application instance m.setVersion("v0116") # Define the version to use m.setSteeringFile("clic_ild_cdr_steering.xml") #What the app should do m.setInputFile("LFN:/ilc/prod/clic/3tev/ee_h_bb/ILD/DST/00000375/000/\ ee_h_bb_dst_375_999.slcio") # Add some input m.setGearFile("clic_ilc_cdr.gear") # Application specific field res = j.append(m) # Add the application to the job if not res['OK']: print res['Message'] # Catch any error j.submit(d) # Submit the job #Not shown here: metadata queries, chaining of applications
def main(): # Take the input arguments from the argument parser, and check they exist... args = parse_args() if not args: print 'Invalid Arguments' sys.exit(1) print args.chain[0] # softVersions = ["v3r0p3", "3.0-SNAPSHOT", "ILC_DBD", "0116"] softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended) # softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working # softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working # softVersions = ["v3r0p3", "HEAD", "ILCSoft-01-17-08", "0116"] check_events_arguments(args.events, args.split) detector = args.detector alias_properties(detector) outputPath, outputBase, repoName = input_output(args.Input, detector, args.chain, args.digiSteering) inputSandbox, outputSandbox = setup_sandboxes(args.macFile) dirac = DiracILC(True, repoName) # Prepares values for the job loop... if args.split < 0: nInputEvents = int(args.events) nOutputEvents = int(args.events) if args.split > 0: nInputEvents = int(args.events) nOutputEvents = int(args.split) # Loop that runs through the required number of jobs to be executed... for startEvent in range(0, nInputEvents, nOutputEvents): ################## Job Initialise ######################################## job = UserJob() job.setName(outputBase) job.setJobGroup('JobGroup') job.setInputSandbox(inputSandbox) fileNumber = startEvent/nOutputEvents print "Job ---> ", fileNumber ################## SLIC ################################################## if 1 in args.chain: slic = SLIC() slic.setVersion(softVersions[0]) slic.setSteeringFile(args.macFile) # slic.setInputFile(lfn) slic.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio')) slic.setDetectorModel(detector) slic.setNumberOfEvents(nOutputEvents) slic.setStartFrom(startEvent) #print slic.listAttributes() result = job.append(slic) if not result['OK']: print result['Message'] sys.exit(2) ################## Overlay ############################################### if 2 in args.chain: ''' #Add the gghad background overlay. gghad = OverlayInput() #gghad.setProdID(1767) gghad.setEnergy(500.0) gghad.setBXOverlay('args.bunches') gghad.setGGToHadInt( 4.1 ) gghad.setNbSigEvtsPerJob(nOutputEvents) gghad.setMachine('ilc_dbd') gghad.setDetectorModel('sidloi3') gghad.setBkgEvtType('aa_lowpt') result = job.append( gghad ) if not result['OK']: print result['Message'] sys.exit(2) #Add the pair background overlay. pairs = OverlayInput() pairs.setProdID(2) pairs.setEnergy(500.0) pairs.setBXOverlay('args.bunches') pairs.setGGToHadInt(1.) pairs.setNbSigEvtsPerJob(nOutputEvents) pairs.setMachine('ilc_dbd') pairs.setDetectorModel('sidloi3') pairs.setBkgEvtType('eepairs') result = job.append( pairs ) if not result['OK']: print result['Message'] sys.exit(2) ''' gghad = OverlayInput() gghad.setPathToFiles('/ilc/user/j/jstrube/gghadron_lowpt/sidloi3/') gghad.setBXOverlay(int(args.bunches)) gghad.setGGToHadInt( 4.1 ) gghad.setNbSigEvtsPerJob(nOutputEvents) gghad.setBkgEvtType('aa_lowpt') result = job.append( gghad ) if not result['OK']: print result['Message'] sys.exit(2) ''' pairs = OverlayInput() pairs.setPathToFiles('/ilc/user/j/jstrube/GuineaPig/sidloi3/') pairs.setBXOverlay(int(args.bunches)) pairs.setGGToHadInt(1.) pairs.setBkgEvtType('eepairs') pairs.setNbSigEvtsPerJob(nOutputEvents) result = job.append( pairs ) if not result['OK']: print result['Message'] sys.exit(2) ''' ################## lcsim (digitization and tracking) ##################### if 3 in args.chain: lcsim = LCSIM() lcsim.setVersion(softVersions[1]) lcsim.setSteeringFile(args.digiSteering) # Another version is included in /steeringFiles if 1 in args.chain: lcsim.getInputFromApp(slic) lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml') lcsim.setAliasProperties('steeringFiles/alias.properties') lcsim.setDetectorModel(detector+".zip") #lcsim.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio')) lcsim.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio')) #NEED TO CHANGE!!! lcsim.setNumberOfEvents(nOutputEvents) #print lcsim.listAttributes() result = job.append(lcsim) if not result['OK']: print result['Message'] sys.exit(2) ################## slicPandora ########################################### if 4 in args.chain: slicPandora = SLICPandora() slicPandora.setVersion(softVersions[2]) slicPandora.setDetectorModel(detector) slicPandora.getInputFromApp(lcsim) slicPandora.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio')) slicPandora.setPandoraSettings('pandoraSettings.xml') slicPandora.setNumberOfEvents(nOutputEvents) #print slicPandora.listAttributes() result = job.append(slicPandora) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus Vertexing ############################ if 5 in args.chain: vertexing = Marlin() vertexing.setVersion(softVersions[3]) vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml') vertexing.setGearFile('steeringFiles/' + detector + '.gear') vertexing.getInputFromApp(slicPandora) vertexing.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_vertexing.slcio')) vertexing.setNumberOfEvents(nOutputEvents) #print vertexing.listAttributes() result = job.append(vertexing) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (DST production) ################################ lcsimDst = LCSIM() lcsimDst.setVersion(softVersions[1]) lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml') lcsimDst.getInputFromApp(vertexing) lcsimDst.setNumberOfEvents(nOutputEvents) lcsimDst.setAliasProperties('steeringFiles/alias.properties') lcsimDst.setDetectorModel(detector+".zip") lcsimDst.setOutputRecFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_Rec.slcio')) lcsimDst.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio')) #print lcsimDst.listAttributes() result = job.append(lcsimDst) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus flavortag ############################ if 6 in args.chain: flavortag = Marlin() flavortag.setVersion(softVersions[3]) flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml') flavortag.setGearFile('steeringFiles/' + detector + '.gear') flavortag.setInputFile(lcsimDstOutput) flavortag.setOutputFile(outputBase.replace('.slcio', '_' + '_flavortag.slcio')) flavortag.setNumberOfEvents(nOutputEvents) #print flavortag.listAttributes() result = job.append(flavortag) if not result['OK']: print result['Message'] sys.exit(2) ################## Job Finalise ########################################## # List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on, # This list is likely to change. job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp', 'OSG.PNNL.us','OSG.UConn.us','OSG.GridUNESP_CENTRAL.br','LCG.SCOTGRIDDURHAM.uk', 'LCG.TECHNIONself.il','LCG.UKI-SOUTHGRID-RALPP.uk','OSG.FNAL_FERMIGRID.us','LCG.UKI-LT2-IC-HEP.uk']) job.setCPUTime(50000) job.setPlatform('x86_64-slc5-gcc43-opt') # Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/ # directory on the grid. outputLevel = max(args.chain) if outputLevel == 1: job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'), outputPath, 'CERN-SRM') if outputLevel == 3: #job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'), outputPath, 'CERN-SRM') job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM') if outputLevel == 4: job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'), outputPath, 'CERN-SRM') if outputLevel == 5: job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM') if outputLevel == 6: job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_flavortag.slcio'), outputPath, 'CERN-SRM') job.setOutputSandbox(outputSandbox) job.setInputData(args.Input) if args.dontPromptMe: job.dontPromptMe() # Submits Job!!! job.submit() return 0;
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin from ILCDIRAC.Interfaces.API.DiracILC import DiracILC d= DiracILC(True,"repo.rep") ###In case one wants a loop: comment the folowing. #for i in range(2): j = UserJob() j.setJobGroup("Tutorial") j.setName("example")#%i) ma = Marlin() ma.setVersion("v0111Prod") ma.setSteeringFile("clic_ild_cdr_steering.xml") ma.setGearFile("clic_ild_cdr.gear") ma.setInputFile("LFN:/ilc/prod/clic/3tev/gghad/ILD/SIM/00000187/000/gghad_sim_187_97.slcio") outdst = "toto.dst.slcio" #% i outrec = "toto.rec.slcio" #% i ma.setOutputDstFile(outdst) ma.setOutputRecFile(outrec) res = j.append(ma) if not res['OK']: print res['Message'] exit(1) j.setOutputData([outdst,outrec],"some/path","KEK-SRM") j.setOutputSandbox("*.log") j.dontPromptMe() j.submit(d)
dirac = DiracILC() # wh.setOutputFile("myfile.stdhep") j = UserJob() wh = Whizard(processlist=dirac.getProcessList()) wh.setEnergy(3000) wh.setEvtType("ee_h_mumu") wh.setNbEvts(1) wh.setEnergy(3000) params = {} params["USERB1"] = "F" wh.setParameterDict(params) wh.setModel("sm") res = j.append(wh) if not res["OK"]: print res["Message"] dexit(1) mo = Mokka() mo.getInputFromApp(wh) mo.setVersion("0706P08") mo.setSteeringFile("clic_ild_cdr.steer") mo.setNbEvts(1) mo.setOutputFile("somefile.slcio") res = j.append(mo) if not res["OK"]: print res["Message"] dexit(1)
j.setName("DDSim-example") #%i) # j.setInputSandbox(setting_file) ddsim = DDSim() ddsim.setVersion("ILCSoft-01-17-10") ddsim.setDetectorModel("ILD_o1_v05") ddsim.setInputFile(genfile) ddsim.setRandomSeed(12345) # ddsim.setStartFrom(1) ddsim.setNumberOfEvents(5) # Number of events should not exceed number of events in file. # Otherwise, G4exception is thrown # ddsim.setDebug() ddsim.setSteeringFile(steeringfile) ddsim.setOutputFile(simfile) res = j.append(ddsim) if not res['OK']: print res['Message'] exit(1) j.setOutputSandbox(["*.log","*.xml","*.sh","*.root"]) j.dontPromptMe() if localjob : j.submit(d, mode="local") else: simdir="testjob" j.setOutputData(simfile,simdir,"PNNL-SRM") res=j.submit(d) if res['OK']:
ovi.setBXOverlay(300) ovi.setGGToHadInt(0.3) ovi.setNbSigEvtsPerJob(10) ovi.setBkgEvtType("gghad") ovi.setDetectorModel("CLIC_ILD_CDR") overlay = [True,False] for ov in overlay: d = DiracILC(True,"repo_overlay_%s.rep"%ov) for lfn in lfns: j = UserJob() steeringf = "clic_ild_cdr_steering.xml" if ov: steeringf = "clic_ild_cdr_steering_overlay.xml" res = j.append(ovi) if not res['OK']: print(res['Message']) continue ma = Marlin() ma.setVersion("v0111Prod") ma.setGearFile("clic_ild_cdr.gear") ma.setSteeringFile(steeringf) ma.setInputFile("LFN:"+lfn) ma.setNbEvts(10) ma.setEnergy(500.) ma.setOutputRecFile("myrec_overlay_%s.slcio"%ov) ma.setOutputDstFile("mydst_overlay_%s.slcio"%ov) res =j.append(ma) if not res['OK']: print(res['Message'])
job.setOutputSandbox(["*.log","*.sh","*.py "]) #job.setOutputData(lcoutputSIM,OutputPath="MyTest/sim1",OutputSE="IN2P3-SRM") job.setOutputData( SIMoutput,"MyProd_" + ILDConfigVer + "/E250-TDR_ws/" + chann + "/" +ireq+ "/sim","IN2P3-SRM") #job.setDestinationCE('lyogrid07.in2p3.fr') job.dontPromptMe() job.setBannedSites(['LCG.Tau.il']) #job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.DESYZN.de','LCG.DESY-HH.de','LCG.KEK.jp','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us']) sim = DDSim() sim.setVersion(ILCSoftVer) sim.setDetectorModel(detModelSim) sim.setInputFile(lcinputSIM) sim.setSteeringFile("ddsim_steer.py") sim.setNumberOfEvents(evtsPrun) sim.setEnergy(energy) sim.setOutputFile(lcoutputSIM) #sim.setRandomSeed(RandSeed) sim.setStartFrom(evtStart) simres = job.append(sim) if not simres['OK']: print simres['Not ok appending ddsim to job'] quit() print SIMoutput print job.submit(dirac)
################################################ j = UserJob() j.setJobGroup("Tutorial") j.setName("GenericExec") j.setInputSandbox(["mypre.sh", "myanal.sh"]) j.setInputData(indata) j._setSoftwareTags(["marlin.ILCSoft-02-00-02_gcc49"]) # j._setSoftwareTags(["lcio.ILCSoft-02-00-01_gcc49"]) j.setILDConfig("v02-00-02") ################################################ appre = GenericApplication() appre.setScript("mypre.sh") appre.setArguments("This is input arguments") res = j.append(appre) if not res['OK']: print res['Message'] exit(1) ################################################ appost = GenericApplication() appost.setScript("myanal.sh") appost.setArguments("This is my analysis step") res = j.append(appost) if not res['OK']: print res['Message'] exit(1) # j.setOutputData( [outfile1, outputfile2],"mydir","dest_se_name") j.setOutputSandbox(["*.log", "*.tar.gz", "*.sh"])
def main(): # Take the input arguments from the argument parser, and check they exist... args = parse_args() if not args: print 'Invalid Arguments' sys.exit(1) #### Software Versions #### softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended) # softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working # softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working # Check the --runs and --split arguments to make sure they are compatible, if not exit... if not check_events_arguments(args.events, args.split): sys.exit(1) # Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning... lfn_check, lfn = check_input_LFN(args.stdhepInput) if not lfn_check: sys.exit(1) # Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included... dirac = DiracILC(True, setup_repository_name(args.stdhepInput, args.detector)) # Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging, # into the input sandbox inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag) # Prepares values for the job loop... if args.split < 0: nInputEvents = int(args.events) nOutputEvents = int(args.events) if args.split > 0: nInputEvents = int(args.events) nOutputEvents = int(args.split) # Loop that runs through the required number of jobs to be executed... for startEvent in range(0, nInputEvents, nOutputEvents): ################## Job Initialise ######################################## job = UserJob() job.setName(path.basename(args.stdhepInput)) job.setJobGroup('JobGroup') job.setInputSandbox(inputSandbox) fileNumber = startEvent/nOutputEvents print "Job ", fileNumber outputFiles = setup_output_dict(args.stdhepInput, args.detector, fileNumber, args.outputPath, softVersions) slicOutput=outputFiles['slicOutput'] prePandoraOutput=outputFiles['prePandoraOutput'] pandoraOutput=outputFiles['pandoraOutput'] vertexingOutput=outputFiles['vertexingOutput'] lcsimRecOutput=outputFiles['lcsimRecOutput'] lcsimDstOutput=outputFiles['lcsimDstOutput'] flavortagOutput=outputFiles['flavortagOutput'] diracOutput=outputFiles['diracOutput'] ################## SLIC ################################################## slic = SLIC() slic.setVersion(softVersions[0]) slic.setSteeringFile(args.macFile) # slic.setInputFile(lfn) slic.setOutputFile(slicOutput) slic.setDetectorModel(args.detector) slic.setNumberOfEvents(nOutputEvents) slic.setStartFrom(startEvent) #print slic.listAttributes() result = job.append(slic) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (digitization and tracking) ##################### lcsim = LCSIM() lcsim.setVersion(softVersions[1]) lcsim.setSteeringFile('steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml') # Another version is included in /steeringFiles lcsim.getInputFromApp(slic) lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml') # lcsim.setAliasProperties('alias.properties') lcsim.setDetectorModel('geometryFiles/sidloi3.zip') lcsim.setOutputFile(prePandoraOutput) lcsim.setNumberOfEvents(nOutputEvents) #print lcsim.listAttributes() result = job.append(lcsim) if not result['OK']: print result['Message'] sys.exit(2) ################## slicPandora ########################################### slicPandora = SLICPandora() slicPandora.setVersion(softVersions[2]) slicPandora.setDetectorModel(args.detector) slicPandora.getInputFromApp(lcsim) slicPandora.setOutputFile(pandoraOutput) slicPandora.setPandoraSettings('pandoraSettings.xml') slicPandora.setNumberOfEvents(nOutputEvents) #print slicPandora.listAttributes() result = job.append(slicPandora) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus Vertexing ############################ vertexing = Marlin() vertexing.setVersion(softVersions[3]) vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml') vertexing.setGearFile('steeringFiles/sidloi3.gear') vertexing.getInputFromApp(slicPandora) vertexing.setOutputFile(vertexingOutput) vertexing.setNumberOfEvents(nOutputEvents) #print vertexing.listAttributes() result = job.append(vertexing) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (DST production) ################################ lcsimDst = LCSIM() lcsimDst.setVersion(softVersions[1]) lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml') lcsimDst.getInputFromApp(vertexing) lcsimDst.setNumberOfEvents(nOutputEvents) # lcsimDst.setAliasProperties('alias.properties') lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip') lcsimDst.setOutputRecFile(lcsimRecOutput) lcsimDst.setOutputDstFile(lcsimDstOutput) #print lcsimDst.listAttributes() result = job.append(lcsimDst) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus flavortag ############################ if args.flavortag: flavortag = Marlin() flavortag.setVersion(softVersions[3]) flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml') flavortag.setGearFile('steeringFiles/sidloi3.gear') flavortag.setInputFile(lcsimDstOutput) flavortag.setOutputFile(flavortagOutput) flavortag.setNumberOfEvents(nOutputEvents) #print flavortag.listAttributes() result = job.append(flavortag) if not result['OK']: print result['Message'] sys.exit(2) ################## Job Finalise ########################################## # List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on, # This list is likely to change. job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp', 'OSG.PNNL.us',]) job.setCPUTime(50000) job.setPlatform('x86_64-slc5-gcc43-opt') # Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/ # directory on the grid. if args.flavortag: job.setOutputData(flavortagOutput, diracOutput, args.SE) else: job.setOutputData(lcsimDstOutput, diracOutput, args.SE) job.setOutputSandbox(outputSandbox) job.setInputData(lfn) if args.dontPromptMe: job.dontPromptMe() # Submits Job!!! job.submit() return 0;
dirac =DiracILC() #wh.setOutputFile("myfile.stdhep") j = UserJob() wh = Whizard(processlist=dirac.getProcessList()) wh.setEnergy(3000) wh.setEvtType("ee_h_mumu") wh.setNbEvts(1) wh.setEnergy(3000) params = {} params['USERB1']='F' wh.setParameterDict(params) wh.setModel("sm") res = j.append(wh) if not res['OK']: print res['Message'] dexit(1) mo = Mokka() mo.getInputFromApp(wh) mo.setVersion("0706P08") mo.setSteeringFile("clic_ild_cdr.steer") mo.setNbEvts(1) mo.setOutputFile("somefile.slcio") res = j.append(mo) if not res['OK']: print res['Message'] dexit(1)
outrec = "toto-7.rec.slcio" #% i d= DiracILC(True,"repo.rep") ###In case one wants a loop: comment the folowing. #for i in range(2): j = UserJob() j.setJobGroup("Tutorial") j.setName("MarlinExample")#%i) j.setInputSandbox([setting_file,"mypre.sh", "mypost.sh"]) ################################################ appre = GenericApplication() appre.setScript("mypre.sh") appre.setArguments("This is input") res=j.append(appre) if not res['OK']: print res['Message'] exit(1) ################################################ ma = Marlin() ma.setDebug() ma.setVersion("ILCSoft-01-17-09") ma.setSteeringFile("marlin_stdreco.xml") ma.setGearFile("GearOutput.xml") ma.setInputFile(simfile) ma.setOutputDstFile(outdst) ma.setOutputRecFile(outrec) res = j.append(ma)
n_evts = 500 n_evts_per_job = 100 n_jobs = n_evts / n_evts_per_job for i in range(n_jobs): j = UserJob() mo = Mokka() mo.setEnergy(3000) mo.setVersion("0706P08") mo.setSteeringFile("clic_ild_cdr.steer") mo.setMacFile("particlegun_electron.mac") mo.setOutputFile("MyFile.slcio") mo.setNbEvts(n_evts_per_job) res = j.append(mo) if not res['OK']: print res['Message'] break ma = Marlin() ma.setVersion("v0111Prod") ma.setSteeringFile("clic_ild_cdr_steering.xml") ma.getInputFromApp(mo) ma.setOutputDstFile("mydst_no_ov_%s.slcio" % i) res = j.append(ma) if not res['OK']: print res['Message'] break ov = OverlayInput() ov.setBXOverlay(60) ov.setGGToHadInt(3.2)
n_evts = 500 n_evts_per_job= 100 n_jobs = n_evts/n_evts_per_job for i in range(n_jobs): j = UserJob() mo = Mokka() mo.setEnergy(3000) mo.setVersion("0706P08") mo.setSteeringFile("clic_ild_cdr.steer") mo.setMacFile("particlegun_electron.mac") mo.setOutputFile("MyFile.slcio") mo.setNbEvts(n_evts_per_job) res = j.append(mo) if not res['OK']: print res['Message'] break ma = Marlin() ma.setVersion("v0111Prod") ma.setSteeringFile("clic_ild_cdr_steering.xml") ma.getInputFromApp(mo) ma.setOutputDstFile("mydst_no_ov_%s.slcio"%i) res = j.append(ma) if not res['OK']: print res['Message'] break ov = OverlayInput() ov.setBXOverlay(60) ov.setGGToHadInt(3.2)
def all_jobs(name): d = DiracILC(True, "repo.rep") ################################################ j = UserJob() j.setJobGroup("PM1") j.setName("Exec1") banned_sites = [ "OSG.BNL.us", "LCG.UKI-NORTHGRID-LIV-HEP.uk", "OSG.UCSDT2.us", "LCG.SCOTGRIDDURHAM.uk", "LCG.NIKHEF.nl", "LCG.UKI-SOUTHGRID-RALPP.uk", "LCG.GRIF.fr", "LCG.Manchester.uk", "LCG.UKI-LT2-IC-HEP.uk", "LCG.Weizmann.il" ] j.setBannedSites(banned_sites) caindir = name #print('Cain directory is ',caindir) indata = [ 'LFN:/ilc/user/a/amustahid/cain.exe', str(caindir), 'LFN:/ilc/user/a/amustahid/runcain.sh', 'LFN:/ilc/user/a/amustahid/convert_pairs_lcio.py', 'LFN:/ilc/user/a/amustahid/pyLCIO.tar.gz', '/home/belle2/mustahid/useful/my.sh', './splitInput.py', './subddsim.py', './ddsim_steer_July26.py', './ILD_l5_v05.xml', './my2.sh', './dbd_500GeV.nung_1.xml', 'LFN:/ilc/user/a/amustahid/myProcessors.tar.gz', './create_dir.py', './conf.py', './util.py', './testcain.sh', './beam_250.i' ] j.setInputSandbox(indata) ################################################ #app = GenericApplication() #app.setScript("create_dir.py") #app.setInputFile("testcain.sh") #logf = 'create_dir.log' #app.setLogFile(logf) #app.setDebug(debug=True) #create_dirname = 'create_dir' #app.setName(create_dirname) #res=j.append(app) #if not res['OK']: # print res['Message'] # exit(1) ################################################ appre = GenericApplication() name = name.split('/') #print(name) cain_name = name[-1] subdir = name[-2] dirname = name[-3] #print('Cain file name ', cain_name) appre.setScript("LFN:/ilc/user/a/amustahid/runcain.sh") #appre.setScript("testcain.sh") ifile = cain_name.split('.') ifile = ifile[0] + '.' + ifile[1] + '.' + ifile[2] #print('ifile ',ifile) appre.setArguments(ifile) #direc = 'LFN:/ilc/user/a/amustahid/' #appre.setInputFile(ifile+".i") #appre.setArguments("This is input arguments") logf = ifile + '_' + subdir + '.log' appre.setLogFile(logf) appre.setDebug(debug=True) name = 'CAIN' appre.setName(name) res = j.append(appre) if not res['OK']: print res['Message'] exit(1) ################################################ ################################################ #appost = GenericApplication() #appost.setScript("myanal.sh") #appost.setArguments("This is my analysis step") #res=j.append(appost) #if not res['OK']: # print res['Message'] # exit(1) ap = GenericApplication() ap.setScript('my.sh') logf = 'my.log' ap.setLogFile(logf) ap.setDebug(debug=True) name = 'my' ap.setName(name) res = j.append(ap) if not res['OK']: print res['Message'] exit(1) outfile = 'incoherent_pair.dat' appre.setOutputFile(outfile) ################################################ direc = 'incoherent_pair' inputFile = direc + '/' + 'inco_pair_split.slcio' # global variables to hold command line parameters # ###################################### base = '.' #outdir=base+'/'+dirname+'/slcio_test_2ndrun' outdir = base + '/' + dirname + '/Run_7' #print('outdir'+' '+str(outdir)) geant_name = ifile outputFile = geant_name + '_' + subdir + '.slcio' #_clip = _Params(False,1,inputFile,outputFile,outdir) nbevents = 100 clip = _Params(nbevents, inputFile, outputFile, outdir) ddsim = subDDSim(clip) ################################################ res = j.append(ddsim) if not res['OK']: print res['Message'] exit(1) j.setOutputData(outputFile, outdir, "KEK-SRM") j.setOutputSandbox(["*.log", "*.dat", "*.slcio"]) j.dontPromptMe() res = j.submit(d) #res = j.submit(d, mode='local') if res['OK']: print str(res["Value"]) #print "Dirac job, "+str(res["Value"])+", was submitted." else: print "Failed to submit Dirac job. return message was as follows." pprint.pprint(res)
RECoutput.append(lcoutputREC) job = UserJob() job.setName(jobname) job.setJobGroup(jobGrName) job.setILDConfig(ILDConfigVer) job.setCPUTime(86400) job.setInputData([lcinputREC]) job.setInputSandbox(["runRecoSplit_all_Tmp.py"]) job.setOutputSandbox(["*.log","*.sh","MarlinStdRecoParsed.xml","marlin*.xml","*.py "]) #job.setOutputSandbox(["*.log","*.sh","MarlinStdRecoParsed.xml","marlin*.xml","*.py ","*.root"]) #job.setDestinationCE('lyogrid07.in2p3.fr') job.dontPromptMe() job.setBannedSites(['LCG.QMUL.uk']) #job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.DESYZN.de','LCG.DESY-HH.de','LCG.KEK.jp','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us']) # run Malrin reco jobs mares = job.append(ma) if not mares['OK']: print mares['Not ok appending Marlin to job'] quit() #job.setOutputData( RECoutput,"MyProd_" + ILDConfigVer + "/E250-TDR_ws/" + chann + "/" +ireq+ "/rec","CERN-SRM") job.setOutputData( RECoutput,"MyProd_" + ILDConfigVer + "/E250-TDR_ws/" + chann + "/" +ireq+ "/rec","IN2P3-SRM") print RECoutput #print job.submit(dirac)
job.setOutputSandbox(["*.log"]) #files that should be brought back when retrieving the job outputs job.setOutputData([rootFile],nameDir,"CERN-DST-EOS") ##################################################################### ##################################################################### #ddsim ddsim = DDSim() ddsim.setVersion(ddsimVersion) ddsim.setDetectorModel(detectorModel) ddsim.setOutputFile(outputFile) ddsim.setSteeringFile("/afs/cern.ch/work/v/viazlo/gridSubmission/FCCee_o5/files/clic_steer.py") ddsim.setNumberOfEvents(nEvts) res = job.append(ddsim) if not res['OK']: print res['Message'] sys.exit(2) ##################################################################### # ##################################################################### #marlin ma = Marlin() ma.setVersion(marlinVersion) ma.setNbEvts(nEvts) ma.setDetectorModel(detectorModel) ma.getInputFromApp(ddsim)
j = UserJob() j.setJobGroup("Tutorial") j.setName("MarlinOverlayExample")#%i) j.setInputSandbox([setting_file]) ## Define the overlay ov = OverlayInput() ov.setMachine("ilc_dbd") ov.setEnergy(energy) ov.setNumberOfSignalEventsPerJob(n_events_per_job) ov.setBXOverlay(BXOverlay) ov.setGGToHadInt(GGToHadInt500) ov.setBkgEvtType("aa_lowpt") # ov.setBackgroundType("aa_lowpt") ov.setDetectorModel("ILD_o1_v05") res = j.append(ov) if not res['OK']: print res['Message'] exit(1) ## Define Marlin job ma = Marlin() ma.setDebug() ma.setVersion("ILCSoft-01-17-09") ma.setSteeringFile("marlin_ovl_stdreco.xml") ma.setGearFile("GearOutput.xml") ma.setInputFile(simfile) ma.setOutputDstFile(outdst) ma.setOutputRecFile(outrec) res = j.append(ma) if not res['OK']:
def execute(self): """ First we update the site list and banned site """ res = getProxyInfo(False, False) if not res['OK']: self.log.error("submitTasks: Failed to determine credentials for submission", res['Message']) return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] self.log.info("submitTasks: Jobs will be submitted with the credentials %s:%s" % (owner, ownerGroup)) sites = self.diracadmin.getSiteMask()['Value'] for site in sites: res = self.ppc.changeSiteStatus( {'SiteName' : site, 'Status' : 'OK'} ) if not res['OK']: self.log.error('Cannot add or update site %s' % site) banned_sites = self.diracadmin.getBannedSites()['Value'] for banned_site in banned_sites: self.ppc.changeSiteStatus( {'SiteName' : banned_site, 'Status' : 'Banned'} ) if not res['OK']: self.log.error('Cannot mark as banned site %s' % banned_site) ##Then we need to get new installation tasks res = self.ppc.getInstallSoftwareTask() if not res['OK']: self.log.error('Failed to obtain task') task_dict = res['Value'] for softdict in task_dict.values(): self.log.info('Will install %s %s at %s' % (softdict['AppName'], softdict['AppVersion'], softdict['Sites'])) for site in softdict['Sites']: j = UserJob() j.setPlatform(softdict['Platform']) j.dontPromptMe() j.setDestination(site) j.setJobGroup("Installation") j.setName('install_%s' % site) j._addSoftware(softdict['AppName'], softdict['AppVersion']) #Add the application here somehow. res = j.append(SoftwareInstall()) if not res['OK']: self.log.error(res['Message']) continue res = j.submit(self.dirac) #res = self.dirac.submit(j) if not res['OK']: self.log.error('Could not create the job') continue jobdict = {} jobdict['AppName'] = softdict['AppName'] jobdict['AppVersion'] = softdict['AppVersion'] jobdict['Platform'] = softdict['Platform'] jobdict['JobID'] = res['Value'] jobdict['Status'] = 'Waiting' jobdict['Site'] = site res = self.ppc.addOrUpdateJob(jobdict) if not res['OK']: self.log.error('Could not add job %s: %s' % (jobdict['JobID'], res['Message'])) ##Monitor jobs jobs = {} res = self.ppc.getJobs() if not res['OK']: self.log.error('Could not retrieve jobs') else: jobs = res['Value'] for job in jobs: res = self.dirac.status(job['JobID']) if res['OK']: jobstatuses = res['Value'] job['Status'] = jobstatuses['JobID']['Status'] res = self.ppc.addOrUpdateJob(job) if not res['OK']: self.log.error("Failed to updated job %s: %s" % (job['JobID'], res['Message'])) else: self.log.error("Failed to update job %s status" % job['JobID']) return S_OK()
jobname="m" + idin + "_" + str(mrgix1) print jobname job = UserJob() job.setName(jobname) job.setJobGroup(jobGrName) job.setILDConfig(ILDConfigVer) job.setCPUTime(6400) job.setInputSandbox(["runDSTmerge_Tmp.py"]) job.setOutputSandbox( ["*.log","*.sh","*.py"] ) job.setInputData(mergeList) #job.setInputData(mergeList[0]) #job.setOutputData( lcoutputMRG, outpath, "CERN-SRM" ) job.setOutputData( lcoutputMRG, outpath, "IN2P3-SRM" ) job.dontPromptMe() slcioconcat = SLCIOConcatenate() slcioconcat.setInputFile(mergeList) #slcioconcat.setInputFile(mergeList[0]) slcioconcat.setNumberOfEvents(evtsPerMrg) slcioconcat.setOutputFile( lcoutputMRG, outpath) slcioconcat.setLogFile("merge.log") mergejob = job.append(slcioconcat) if not mergejob['OK']: print mergejob['Not ok appending slcioconcat to job'] quit() print job.submit(dirac)
###In case one wants a loop: comment the folowing. #for i in range(2): j = UserJob() j.setJobGroup("Tutorial") j.setName("MarlinExample")#%i) ma = Marlin() ma.setDebug() # ma.setLogLevel("verbose") # ma.setILDConfig("v01-16-p05_500") ma.setVersion("v01-16-02") ma.setSteeringFile("marlin_stdreco.xml") ma.setGearFile(gearfile) ma.setInputFile([simfile, pandoraLikelihoodData, bg_aver]) ma.setOutputDstFile(outdst) ma.setOutputRecFile(outrec) res = j.append(ma) if not res['OK']: print res['Message'] exit(1) j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM") j.setOutputSandbox(["*.log","*.xml","*.sh"]) j.dontPromptMe() j.submit(d)
###In case one wants a loop: comment the folowing. #for i in range(2): j = UserJob() j.setJobGroup("Tutorial") j.setName("MarlinExample")#%i) ma = Marlin() ma.setDebug() ma.setVersion("ILCSoft-01-17-10") ma.setSteeringFile("marlin_stdreco.xml") ma.setGearFile("GearOutput.xml") ma.setInputFile(simfile) ma.setOutputDstFile(outdst) ma.setOutputRecFile(outrec) res = j.append(ma) if not res['OK']: print res['Message'] exit(1) # Upload files to different directories upload_script="upload.sh" upload = GenericApplication() # Create a script to upload files. shfile = open(upload_script,"w") shfile.write("#!/bin/bash\n") shfile.write("/bin/ls -l \n") shfile.write("dirac-dms-add-file -ddd "+outdir+"dst/"+outdst+" "+outdst+" CERN-SRM\n") shfile.write("dirac-dms-add-file -ddd "+outdir+"rec/"+outrec+" "+outrec+" CERN-SRM\n") shfile.close() os.chmod(upload_script,0755)
def subOverlay(): # Decide parameters for a job outputSE = "KEK-SRM" isLocal = _clip.isLocal nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents nbevts = 0 # To analize all input events outputFilePrefix = "overlay_example" if _clip.outputFilePrefix == "" else _clip.outputFilePrefix outputDir = _clip.outputDir inputFile = _clip.inputFile if inputFile == "": gLogger.error("Input file for ddsim does not given.") exit(-1) recfile = outputFilePrefix + ".rec.slcio" dstfile = outputFilePrefix + ".dst.slcio" detector_model = "ILD_l5_o1_v02" key = detector_model.split('_') sim_detectorModel = "_".join([key[0], key[1], key[3]]) # Create DIRAC objects for job submission dIlc = DiracILC() job = UserJob() job.setJobGroup("myoverlayjob") job.setName("myoverlay") job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml']) job.setILDConfig("v02-00-02") # job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"]) # job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination # job.setBannedSites([]) # a list of sites not to submit job # job.setCPUTime( cputime_limit_in_seconds_by_dirac_units ) # Create Overlay application ovldata = [{ "ProcessorName": "BgOverlayWW", "evttype": "aa_lowpt_WW", "ProdID": 10237, "expBG": 0.211, "subdir": "000" }, { "ProcessorName": "BgOverlayWB", "evttype": "aa_lowpt_WB", "ProdID": 10241, "expBG": 0.24605, "subdir": "000" }, { "ProcessorName": "BgOverlayBW", "evttype": "aa_lowpt_BW", "ProdID": 10239, "expBG": 0.243873, "subdir": "000" }, { "ProcessorName": "BgOverlayBB", "evttype": "aa_lowpt_BB", "ProdID": 10235, "expBG": 0.35063, "subdir": "000" }, { "ProcessorName": "PairBgOverlay", "evttype": "seeablepairs", "ProdID": 10233, "expBG": 1.0, "subdir": "100" }] BXOverlay = 1 NbSigEvtsPerJob = 100 numberOfSignalEvents = NbSigEvtsPerJob basebkgpath = "/ilc/prod/ilc/mc-opt-3/ild/sim/500-TDR_ws" energy = "500" for ovl in ovldata: print "### OverlayInput ... " + ovl["ProcessorName"] ovlapp = OverlayInput() ovlpath = "%s/%s/%s/v02-00-01/%8.8d/%s" % \ ( basebkgpath, ovl["evttype"], sim_detectorModel, ovl["ProdID"] , ovl["subdir"] ) print " OverlayPath ... " + ovlpath ovlapp.setMachine("ilc_dbd") # ovlapp.setEnergy(energy) # ovlapp.setDetectorModel(sim_detectorModel) ovlapp.setProcessorName(ovl["ProcessorName"]) ovlapp.setBkgEvtType(ovl["evttype"]) ovlapp.setPathToFiles(ovlpath) ovlapp.setGGToHadInt(ovl["expBG"]) ovlapp.setBXOverlay(BXOverlay) ovlapp.setNbSigEvtsPerJob(NbSigEvtsPerJob) ovlapp.setNumberOfSignalEventsPerJob(numberOfSignalEvents) res = job.append(ovlapp) if not res['OK']: print res['Message'] exit(1) # Create Marlin application marlin = Marlin() marlin.setVersion("ILCSoft-02-00-02_gcc49") marlin.setDetectorModel(detector_model) marlin.setSteeringFile("MarlinStdReco.xml") marlin.setInputFile(inputFile) marlin.setNumberOfEvents(nbevts) marlin.setOutputDstFile(dstfile) marlin.setOutputRecFile(recfile) extraCLIArguments = " --constant.DetectorModel=%s " % detector_model extraCLIArguments += " --constant.RunOverlay=true --constant.CMSEnergy=%s " % str( energy) extraCLIArguments += " --global.Verbosity=MESSAGE " marlin.setExtraCLIArguments(extraCLIArguments) job.append(marlin) if outputDir != "": job.setOutputData([dstfile, recfile], OutputPath=outputDir, OutputSE=outputSE) if isLocal: job.submit(dIlc, mode="local") else: job.submit(dIlc)
def main(): # Take the input arguments from the argument parser, and check they exist... args = parse_args() if not args: print 'Invalid Arguments' sys.exit(1) #### Software Versions #### softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended) # softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working # softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working # Check the --runs and --split arguments to make sure they are compatible, if not exit... if not check_events_arguments(args.events, args.split): sys.exit(1) # Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning... lfn_check, lfn = check_input_LFN(args.stdhepInput) if not lfn_check: sys.exit(1) # Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included... dirac = DiracILC(True, setup_repository_name(args.stdhepInput, args.detector)) # Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging, # into the input sandbox inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag) # Prepares values for the job loop... if args.split < 0: nInputEvents = int(args.events) nOutputEvents = int(args.events) if args.split > 0: nInputEvents = int(args.events) nOutputEvents = int(args.split) # Loop that runs through the required number of jobs to be executed... for startEvent in range(0, nInputEvents, nOutputEvents): ################## Job Initialise ######################################## job = UserJob() job.setName(path.basename(args.stdhepInput)) job.setJobGroup('JobGroup') job.setInputSandbox(inputSandbox) fileNumber = startEvent / nOutputEvents print "Job ", fileNumber outputFiles = setup_output_dict(args.stdhepInput, args.detector, fileNumber, args.outputPath, softVersions) slicOutput = outputFiles['slicOutput'] prePandoraOutput = outputFiles['prePandoraOutput'] pandoraOutput = outputFiles['pandoraOutput'] vertexingOutput = outputFiles['vertexingOutput'] lcsimRecOutput = outputFiles['lcsimRecOutput'] lcsimDstOutput = outputFiles['lcsimDstOutput'] flavortagOutput = outputFiles['flavortagOutput'] diracOutput = outputFiles['diracOutput'] ################## SLIC ################################################## slic = SLIC() slic.setVersion(softVersions[0]) slic.setSteeringFile(args.macFile) # slic.setInputFile(lfn) slic.setOutputFile(slicOutput) slic.setDetectorModel(args.detector) slic.setNumberOfEvents(nOutputEvents) slic.setStartFrom(startEvent) #print slic.listAttributes() result = job.append(slic) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (digitization and tracking) ##################### lcsim = LCSIM() lcsim.setVersion(softVersions[1]) lcsim.setSteeringFile( 'steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml' ) # Another version is included in /steeringFiles lcsim.getInputFromApp(slic) lcsim.setTrackingStrategy( 'steeringFiles/sidloi3_trackingStrategies_default.xml') # lcsim.setAliasProperties('alias.properties') lcsim.setDetectorModel('geometryFiles/sidloi3.zip') lcsim.setOutputFile(prePandoraOutput) lcsim.setNumberOfEvents(nOutputEvents) #print lcsim.listAttributes() result = job.append(lcsim) if not result['OK']: print result['Message'] sys.exit(2) ################## slicPandora ########################################### slicPandora = SLICPandora() slicPandora.setVersion(softVersions[2]) slicPandora.setDetectorModel(args.detector) slicPandora.getInputFromApp(lcsim) slicPandora.setOutputFile(pandoraOutput) slicPandora.setPandoraSettings('pandoraSettings.xml') slicPandora.setNumberOfEvents(nOutputEvents) #print slicPandora.listAttributes() result = job.append(slicPandora) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus Vertexing ############################ vertexing = Marlin() vertexing.setVersion(softVersions[3]) vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml') vertexing.setGearFile('steeringFiles/sidloi3.gear') vertexing.getInputFromApp(slicPandora) vertexing.setOutputFile(vertexingOutput) vertexing.setNumberOfEvents(nOutputEvents) #print vertexing.listAttributes() result = job.append(vertexing) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (DST production) ################################ lcsimDst = LCSIM() lcsimDst.setVersion(softVersions[1]) lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml') lcsimDst.getInputFromApp(vertexing) lcsimDst.setNumberOfEvents(nOutputEvents) # lcsimDst.setAliasProperties('alias.properties') lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip') lcsimDst.setOutputRecFile(lcsimRecOutput) lcsimDst.setOutputDstFile(lcsimDstOutput) #print lcsimDst.listAttributes() result = job.append(lcsimDst) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus flavortag ############################ if args.flavortag: flavortag = Marlin() flavortag.setVersion(softVersions[3]) flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml') flavortag.setGearFile('steeringFiles/sidloi3.gear') flavortag.setInputFile(lcsimDstOutput) flavortag.setOutputFile(flavortagOutput) flavortag.setNumberOfEvents(nOutputEvents) #print flavortag.listAttributes() result = job.append(flavortag) if not result['OK']: print result['Message'] sys.exit(2) ################## Job Finalise ########################################## # List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on, # This list is likely to change. job.setBannedSites([ 'LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp', 'OSG.PNNL.us', ]) job.setCPUTime(50000) job.setPlatform('x86_64-slc5-gcc43-opt') # Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/ # directory on the grid. if args.flavortag: job.setOutputData(flavortagOutput, diracOutput, args.SE) else: job.setOutputData(lcsimDstOutput, diracOutput, args.SE) job.setOutputSandbox(outputSandbox) job.setInputData(lfn) if args.dontPromptMe: job.dontPromptMe() # Submits Job!!! job.submit() return 0
ovi.setBXOverlay(300) ovi.setGGToHadInt(0.3) ovi.setNbSigEvtsPerJob(10) ovi.setBkgEvtType("gghad") ovi.setDetectorModel("CLIC_ILD_CDR") overlay = [True, False] for ov in overlay: d = DiracILC(True, "repo_overlay_%s.rep" % ov) for lfn in lfns: j = UserJob() steeringf = "clic_ild_cdr_steering.xml" if ov: steeringf = "clic_ild_cdr_steering_overlay.xml" res = j.append(ovi) if not res['OK']: print res['Message'] continue ma = Marlin() ma.setVersion("v0111Prod") ma.setGearFile("clic_ild_cdr.gear") ma.setSteeringFile(steeringf) ma.setInputFile("LFN:" + lfn) ma.setNbEvts(10) ma.setEnergy(500.) ma.setOutputRecFile("myrec_overlay_%s.slcio" % ov) ma.setOutputDstFile("mydst_overlay_%s.slcio" % ov) res = j.append(ma) if not res['OK']: print res['Message']
def main(argv): # Input arguments ildconfig_version = "$ILDCONFIGVER" ilcsoft_version = "$ILCSOFTVER" evts_per_run = $EVTSPERRUN detector_model = "$DETECTOR" sim_input = "$SIMINPUT" process_name = "$PROCESS" index = $IND sim_input = diracpath_from_pnfspath( sim_input ) sim_detector_model = detector_model_wo_option( detector_model ) job_group = ilcsoft_version + "_" + ildconfig_version + "_" + process_name + "_" + detector_model dirac = DiracILC(True,job_group+".rep") # outputs to be saved onto grid SE RECoutput = [] # DDSim evtStart = (index-1)*evts_per_run evtEnd = index*evts_per_run - 1 RandSeed = random.randrange(11623, 99999) lcinputSIM = "LFN:" + sim_input lcoutputSIM = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_SIM.slcio"%(str(evtStart),(str)(evtEnd)) sim = DDSim() sim.setVersion(ilcsoft_version) sim.setDetectorModel(sim_detector_model) sim.setInputFile(lcinputSIM) sim.setSteeringFile("ddsim_steer.py") sim.setNumberOfEvents(evts_per_run) sim.setRandomSeed(RandSeed) sim.setEnergy(1000) sim.setStartFrom(evtStart) sim.setOutputFile(lcoutputSIM) # Marlin lcoutputDST = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_DST.slcio"%(str(evtStart),(str)(evtEnd)) ma = Marlin() ma.setVersion(ilcsoft_version) ma.setDetectorModel(detector_model) ma.setSteeringFile("MarlinStdReco.xml") ma.setExtraCLIArguments( "--constant.lcgeo_DIR=$lcgeo_DIR --constant.DetectorModel={} --global.MaxRecordNumber=0".format(detector_model) ) ma.setLogFile("marlin.log") ma.getInputFromApp(sim) ma.setEnergy(1000) ma.setOutputDstFile(lcoutputDST) RECoutput.append(lcoutputDST) # ILCDirac user job job = UserJob() job.setName("user_sim_reco") job.setJobGroup(job_group) job.setILDConfig(ildconfig_version) job.setCPUTime(86400) tmp_file_name = process_name + "_sim_reco_job_tmp.py" job.setInputSandbox([tmp_file_name]) job.setOutputSandbox(["*.log","MarlinStdRecoParsed.xml","marlin*.xml","*.py "]) job.dontPromptMe() job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us']) # run simulation job simres = job.append(sim) if not simres['OK']: print simres['Not ok appending ddsim to job'] quit() # run Malrin reco jobs mares = job.append(ma) if not mares['OK']: print mares['Not ok appending Marlin to job'] quit() job.setOutputData(RECoutput,"ILDPerformance/WWZZSeparation/{}_ILDConfig_{}_{}".format(ilcsoft_version,ildconfig_version,detector_model),"DESY-SRM") print RECoutput submit_output = job.submit(dirac) print submit_output