# create the single job job = JobObject(si) # better to set the application to use explicitely because in that case we don't need to use mds (faster) job.setCommandline('cat ' + pathToInputFiles + 'commonJobFile.txt ' + 'singleJobFile.txt') # adding a job-specific input file job.addInputFileUrl("/home/markus/tmp/singleJobFile.txt") # adding the job to the multijob batch_job.addJob(job) # now we are adding a file that can be used by all of the child jobs. it needs to be referenced via the pathToInputFiles() method shown above batch_job.addInputFile('/home/markus/tmp/commonJobFile.txt') batch_job.setDefaultNoCpus(1) batch_job.setDefaultWalltimeInSeconds(60) batch_job.setLocationsToExclude(["gt5test:ng1.canterbury.ac.nz"]) try: print "Creating jobs on the backend and staging files..." # by specifying "True" we tell the backend to automatically distribute the jobs to all available submission locations # this can be finetuned by exluding or including sites. another option would be to specifying the submission location # for every single job and setting "False" below (this would make job submission faster since jobs don't need to be re-distributed/moved on the backend). batch_job.prepareAndCreateJobs(True) except (JobsException), error: for job in error.getFailures().keySet(): print "Job: " + job.getJobname() + ", Error: " + error.getFailures( ).get(job).getLocalizedMessage() sys.exit() print "Job distribution:"
# Set some job settings application = "python" version = "2.4" print "INFO: Creating a Batch Job Object called " + batch_job_name batch_jobs = BatchJobObject(service_interface, batch_job_name, group, application, version) batch_jobs.setConcurrentInputFileUploadThreads(5) # Set the number of concurrent uploads batch_jobs.setConcurrentJobCreationThreads(5) # Set the number of concurrent jobs batch_jobs.setDefaultNoCpus(1) # Set the number of CPUs required batch_jobs.setDefaultWalltimeInSeconds(300) # Set the maximum walltime to 5 minutes exclude_sites = list() exclude_sites.append("AUT") batch_jobs.setLocationsToExclude(exclude_sites) # Create a blacklist of sites to exclude # Currently the AUT location is not behaving, so always exclude it print "INFO: Adding common files to Batch Job Object " + batch_job_name batch_jobs.addInputFile(os.path.join(current_dir, dictionary_path)) batch_jobs.addInputFile(os.path.join(current_dir, "countbacon.py")) print "INFO: Defining jobs from input directory" job_count = 0 for file_name in os.listdir(input_path): print "INFO: Defining job for " + file_name job_name = base_job_name + "-" + file_name job = JobObject(service_interface) job.setJobname(job_name) job.setApplication("python") # Set the application being run job.setApplicationVersion("2.4") # Set the application version, note this is an exact match
# create the single job job = JobObject(si) # better to set the application to use explicitely because in that case we don't need to use mds (faster) job.setCommandline('cat ' + pathToInputFiles+'commonJobFile.txt ' + 'singleJobFile.txt') # adding a job-specific input file job.addInputFileUrl("/home/markus/tmp/singleJobFile.txt") # adding the job to the multijob batch_job.addJob(job) # now we are adding a file that can be used by all of the child jobs. it needs to be referenced via the pathToInputFiles() method shown above batch_job.addInputFile('/home/markus/tmp/commonJobFile.txt') batch_job.setDefaultNoCpus(1); batch_job.setDefaultWalltimeInSeconds(60); batch_job.setLocationsToExclude(["gt5test:ng1.canterbury.ac.nz"]) try: print "Creating jobs on the backend and staging files..." # by specifying "True" we tell the backend to automatically distribute the jobs to all available submission locations # this can be finetuned by exluding or including sites. another option would be to specifying the submission location # for every single job and setting "False" below (this would make job submission faster since jobs don't need to be re-distributed/moved on the backend). batch_job.prepareAndCreateJobs(True) except (JobsException), error: for job in error.getFailures().keySet(): print "Job: "+job.getJobname()+", Error: "+error.getFailures().get(job).getLocalizedMessage() sys.exit() print "Job distribution:" print batch_job.getOptimizationResult()
print "INFO: Base job name is " + base_job_name print "INFO: Batch job name is " + batch_job_name # Set some job settings application = "python" version = "2.4" print "INFO: Creating a Batch Job Object called " + batch_job_name batch_jobs = BatchJobObject(service_interface, batch_job_name, group, application, version) batch_jobs.setConcurrentInputFileUploadThreads(5) # Set the number of concurrent uploads batch_jobs.setConcurrentJobCreationThreads(5) # Set the number of concurrent jobs batch_jobs.setDefaultNoCpus(1) # Set the number of CPUs required batch_jobs.setDefaultWalltimeInSeconds(300) # Set the maximum walltime to 5 minutes batch_jobs.setLocationsToExclude(["AUT"]) # Create a blacklist of sites to exclude # Currently the AUT location is not behaving, so always exclude it print "INFO: Adding common files to Batch Job Object " + batch_job_name batch_jobs.addInputFile(os.path.join(current_dir, dictionary_path)) batch_jobs.addInputFile(os.path.join(current_dir, "countbacon.py")) print "INFO: Defining jobs from input directory" job_count = 0 for file_name in os.listdir(input_path): print "INFO: Defining job for " + file_name job_name = base_job_name + "-" + file_name job = JobObject(service_interface) job.setJobname(job_name) job.setApplication("python") # Set the application being run job.setApplicationVersion("2.4") # Set the application version, note this is an exact match
# Set some job settings application = "python" version = "2.4" print "INFO: Creating a Batch Job Object called " + batch_job_name batch_jobs = BatchJobObject(service_interface, batch_job_name, group, application, version) batch_jobs.setConcurrentInputFileUploadThreads( 5) # Set the number of concurrent uploads batch_jobs.setConcurrentJobCreationThreads( 5) # Set the number of concurrent jobs batch_jobs.setDefaultNoCpus(1) # Set the number of CPUs required batch_jobs.setDefaultWalltimeInSeconds(300) # Set the maximum walltime to 5 minutes batch_jobs.setLocationsToExclude(["AUT" ]) # Create a blacklist of sites to exclude # Currently the AUT location is not behaving, so always exclude it print "INFO: Adding common files to Batch Job Object " + batch_job_name batch_jobs.addInputFile(os.path.join(current_dir, dictionary_path)) batch_jobs.addInputFile(os.path.join(current_dir, "countbacon.py")) print "INFO: Defining jobs from input directory" job_count = 0 for file_name in os.listdir(input_path): print "INFO: Defining job for " + file_name job_name = base_job_name + "-" + file_name job = JobObject(service_interface) job.setJobname(job_name) job.setApplication("python") # Set the application being run job.setApplicationVersion(