def basicTest(self): j = Job() j.setCPUTime(50000) j.setExecutable( '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py' ) # j.setExecutable('/bin/echo hello') j.setOwner('paterson') j.setType('test') j.setName('MyJobName') #j.setAncestorDepth(1) j.setInputSandbox([ '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts', '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts' ]) j.setOutputSandbox(['firstfile.txt', 'anotherfile.root']) j.setInputData([ '/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst', '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst' ]) j.setOutputData(['my.dst', 'myfile.log']) j.setDestination('LCG.CERN.ch') j.setPlatform('LCG') j.setSystemConfig('x86_64-slc5-gcc43-opt') j.setSoftwareTags(['VO-lhcb-Brunel-v30r17', 'VO-lhcb-Boole-v12r10']) #print j._toJDL() #print j.printObj() xml = j._toXML() testFile = 'jobDescription.xml' if os.path.exists(testFile): os.remove(testFile) xmlfile = open(testFile, 'w') xmlfile.write(xml) xmlfile.close() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow' print j.createCode() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow' j.execute() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file' workflow = fromXMLFile(testFile) code = workflow.createCode() print code workflow.execute()
def submit_job(self, request_id, executable, macro, starting_seed=8000000, njobs=10, platform='ANY', output_data_site='UKI-LT2-IC-HEP-disk', output_log='lzproduction_output.log'): """ Submit LZProduction job to DIRAC. Args: request_id (int): The id number of the associated request executable (str): The full path to the executable job script macro (str): The full path to the macro for this job starting_seed (int): The random seed for the first of the parametric jobs njobs (int): The number of parametric jobs to create platform (str): The required platform output_data_site (str): The name of the grid site to store the output data at output_log (str): The file name for the output log file Returns: list: The list of created parametric job DIRAC ids """ j = Job() j.setName(os.path.splitext(os.path.basename(macro))[0] + '%(args)s') j.setExecutable(os.path.basename(executable), os.path.basename(macro) + ' %(args)s', output_log) j.setInputSandbox([executable, macro]) j.setOutputData('*.root', output_data_site, str(request_id)) j.setParameterSequence( "args", [str(i) for i in xrange(starting_seed, starting_seed + njobs)], addToWorkflow=True) j.setPlatform(platform) return self.status(self._dirac_api.submit(j).get("Value", []))
def basicTest(self): j = Job() j.setCPUTime(50000) j.setExecutable('/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py') # j.setExecutable('/bin/echo hello') j.setOwner('paterson') j.setType('test') j.setName('MyJobName') #j.setAncestorDepth(1) j.setInputSandbox(['/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts','/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts']) j.setOutputSandbox(['firstfile.txt','anotherfile.root']) j.setInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst', '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst']) j.setOutputData(['my.dst','myfile.log']) j.setDestination('LCG.CERN.ch') j.setPlatform('LCG') j.setSystemConfig('x86_64-slc5-gcc43-opt') j.setSoftwareTags(['VO-lhcb-Brunel-v30r17','VO-lhcb-Boole-v12r10']) #print j._toJDL() #print j.printObj() xml = j._toXML() testFile = 'jobDescription.xml' if os.path.exists(testFile): os.remove(testFile) xmlfile = open(testFile,'w') xmlfile.write(xml) xmlfile.close() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow' print j.createCode() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow' j.execute() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file' workflow = fromXMLFile(testFile) code = workflow.createCode() print code workflow.execute()
args = visit + ' ' + insidename + ' ' + str(startsensor) + ' ' + str( numsensors) + ' ' + str(idx) outputname = 'fits_' + visit + '_' + str(idx) + '.tar' j.setCPUTime(1209600) j.setExecutable('runimsim2.1.sh', arguments=args) j.stderr = "std.err" j.stdout = "std.out" #!!! May need the 2.1i directory here depending on visit number !!! j.setInputSandbox([ "runimsim2.1.sh", "run_imsim_nersc.py", "LFN:/lsst/user/j/james.perry/instcats/2.1i/" + instcatname ]) j.setOutputSandbox(["std.out", "std.err"]) j.setTag(["8Processors"]) #j.setOutputData([visit + "/" + outputname], outputPath="", outputSE=["IN2P3-CC-disk"]) j.setOutputData([visit + "/" + outputname], outputPath="", outputSE=["UKI-NORTHGRID-LANCS-HEP-disk"]) j.setPlatform("AnyPlatform") j.setDestination(site) jobID = dirac.submitJob(j) print("Submitted job to " + site + " as ID " + str(jobID)) print "Status is:", dirac.status(jobID['JobID']) joblistfile.write(str(jobID['JobID']) + '\n') joblistfile.close()
args = visit + ' ' + insidename + ' ' + str(startsensor) + ' ' + str( numsensors) + ' ' + str(idx) outputname = 'fits_' + visit + '_' + str(idx) + '.tar' j.setCPUTime(1209600) j.setExecutable('launch_container.sh', arguments=args) j.stderr = "std.err" j.stdout = "std.out" #!!! May need the 2.1i directory here depending on visit number !!! j.setInputSandbox([ "launch_container.sh", "docker_run.sh", "run_imsim_nersc.py", "parsl_imsim_configs", "finals2000A.all", "LFN:/lsst/user/j/james.perry/2.2i_test/" + instcatname ]) j.setOutputSandbox(["std.out", "std.err"]) j.setTag(["8Processors"]) j.setOutputData([outputname], outputPath="", outputSE=["UKI-NORTHGRID-LANCS-HEP-disk"]) j.setPlatform("EL7") j.setDestination(site) jobID = dirac.submitJob(j) print("Submitted job as ID " + str(jobID)) print "Status is:", dirac.status(jobID['JobID']) joblistfile.write(str(jobID['JobID']) + '\n') joblistfile.close()
inputdata_list.append(lfn + 'second/HPC_data.tar.gz.0' + str(i)) else: inputdata_list.append(lfn + 'second/HPC_data.tar.gz.' + str(i)) #SitesList = ['LCG.UKI-NORTHGRID-MAN-HEP.uk', 'LCG.UKI-LT2-IC-HEP.uk', 'LCG.UKI-LT2-QMUL.uk', 'LCG.UKI-NORTHGRID-LANCS-HEP.uk'] #SEList = ['UKI-NORTHGRID-MAN-HEP-disk', 'UKI-LT2-IC-HEP-disk', 'UKI-LT2-QMUL2-disk', 'UKI-NORTHGRID-LANCS-HEP-disk'] SitesList = ['LCG.UKI-NORTHGRID-MAN-HEP.uk'] SEList = ['UKI-NORTHGRID-MAN-HEP-disk'] print("Input data list:\n") print(inputdata_list) for i in range(0, total_pixels, chunks): id_start = i id_end = i + chunks dirac = Dirac() j.setName('CS Faraday Rotation Measurement Reconstruction - Pixels from ' + str(id_start) + ' to ' + str(id_end - 1)) j.setPlatform('EL7') j.setTag([str(nprocs) + 'Processors']) j.setDestination(SitesList) j.setExecutable('RMSynthesis2.sh', arguments=str(nprocs) + ' ' + str(id_start) + ' ' + str(id_end) + ' ' + str(expmnt)) # Input data j.setInputData(inputdata_list) j.setInputSandbox([ 'RMSynthesis2.sh', 'run2.sh', 'prmon_1.0.1_x86_64-static-gnu72-opt.tar.gz' ]) # Output data j.setOutputSandbox([ 'StdOut', 'StdErr', 'outputtxt_' + str(id_start) + '_' + str(id_end - 1) + '.txt',