job = JobObject(si) job.setEmail_address(email) job.setEmail_on_job_finish(True) job.setCommandline('R --no-readline --no-restore --no-save -f '+path_to_inputfile) batch_job.addJob(job) batch_job.addInputFile('/home/markus/Desktop/R/'+inputfilename) batch_job.setDefaultNoCpus(1) batch_job.setDefaultWalltimeInSeconds(walltime) print 'preparing jobs on backend...' batch_job.prepareAndCreateJobs(redistribute) if redistribute: print 'job distribution:' print batch_job.getOptimizationResult() print 'submitting jobs to grid...' batch_job.submit(True) print 'submission finished...'
job_name = base_job_name + "-" + file_name job = JobObject(service_interface) job.setJobname(job_name) job.setApplication("python") # Set the application being run job.setApplicationVersion("2.4") # Set the application version, note this is an exact match job.addInputFileUrl(os.path.join(current_dir, input_path, file_name)) job.setCommandline("python ../countbacon.py ../" + dictionary_path + " " + file_name) print "INFO: " + job.getJobname() + " defined" batch_jobs.addJob(job) print "INFO: " + job.getJobname() + " added to batch " + batch_jobs.getJobname() job_count += 1 print "INFO: " + str(job_count) + " jobs defined" print "INFO: Sending batch " + batch_jobs.getJobname() + " to " + backend + " and staging files..." try: batch_jobs.prepareAndCreateJobs(False) except (JobsException), error: print ("HALT: Exception submitting jobs from BatchJobObject " + batch_jobs.getJobname() + "!") for job in error.getFailures().keySet(): print "Job: " + job.getJobname() + ", Error: " + error.getFailures().get(job).getLocalizedMessage() sys.exit(1) except (BackendException), error: print ("HALT: Exception from grisu backend " + backend + "!") print (error.getLocalizedMessage()) print ("========================") time.sleep(3) error.printStackTrace() sys.exit(1) time.sleep(3) print "INFO: Submitting jobs in batch " + batch_jobs.getJobname()
print 'jobname on backend: ' + batch_job_name path_to_inputfile = batch_job.pathToInputFiles() + inputfilename for i in range(1, gen_jobs + 1): job = JobObject(si) job.setEmail_address(email) job.setEmail_on_job_finish(True) job.setCommandline('R --no-readline --no-restore --no-save -f ' + path_to_inputfile) batch_job.addJob(job) batch_job.addInputFile('/home/markus/Desktop/R/' + inputfilename) batch_job.setDefaultNoCpus(1) batch_job.setDefaultWalltimeInSeconds(walltime) print 'preparing jobs on backend...' batch_job.prepareAndCreateJobs(redistribute) if redistribute: print 'job distribution:' print batch_job.getOptimizationResult() print 'submitting jobs to grid...' batch_job.submit(True) print 'submission finished...'
# adding the job to the multijob batch_job.addJob(job) # now we are adding a file that can be used by all of the child jobs. it needs to be referenced via the pathToInputFiles() method shown above batch_job.addInputFile('/home/markus/tmp/commonJobFile.txt') batch_job.setDefaultNoCpus(1); batch_job.setDefaultWalltimeInSeconds(60); batch_job.setLocationsToExclude(["gt5test:ng1.canterbury.ac.nz"]) try: print "Creating jobs on the backend and staging files..." # by specifying "True" we tell the backend to automatically distribute the jobs to all available submission locations # this can be finetuned by exluding or including sites. another option would be to specifying the submission location # for every single job and setting "False" below (this would make job submission faster since jobs don't need to be re-distributed/moved on the backend). batch_job.prepareAndCreateJobs(True) except (JobsException), error: for job in error.getFailures().keySet(): print "Job: "+job.getJobname()+", Error: "+error.getFailures().get(job).getLocalizedMessage() sys.exit() print "Job distribution:" print batch_job.getOptimizationResult() print "Submitting jobs..." batch_job.submit() # now we wait for all jobs to be finished, checking for updates every 10 seconds. in real life we would set a much higher check intervall since we don't want to overload
job.setApplicationVersion( "2.4") # Set the application version, note this is an exact match job.addInputFileUrl(os.path.join(current_dir, input_path, file_name)) job.setCommandline("python ../countbacon.py ../" + dictionary_path + " " + file_name) print "INFO: " + job.getJobname() + " defined" batch_jobs.addJob(job) print "INFO: " + job.getJobname( ) + " added to batch " + batch_jobs.getJobname() job_count += 1 print "INFO: " + str(job_count) + " jobs defined" print "INFO: Sending batch " + batch_jobs.getJobname( ) + " to " + backend + " and staging files..." try: batch_jobs.prepareAndCreateJobs(False) except (JobsException), error: print("HALT: Exception submitting jobs from BatchJobObject " + batch_jobs.getJobname() + "!") for job in error.getFailures().keySet(): print "Job: " + job.getJobname() + ", Error: " + error.getFailures( ).get(job).getLocalizedMessage() sys.exit(1) except (BackendException), error: print("HALT: Exception from grisu backend " + backend + "!") print(error.getLocalizedMessage()) print("========================") time.sleep(3) error.printStackTrace() sys.exit(1) time.sleep(3)