batch_job = BatchJobObject(si, basename, group, 'R', Constants.NO_VERSION_INDICATOR_STRING) batch_job_name = batch_job.getJobname() print 'jobname on backend: '+batch_job_name path_to_inputfile = batch_job.pathToInputFiles()+inputfilename for i in range(1,gen_jobs+1): job = JobObject(si) job.setEmail_address(email) job.setEmail_on_job_finish(True) job.setCommandline('R --no-readline --no-restore --no-save -f '+path_to_inputfile) batch_job.addJob(job) batch_job.addInputFile('/home/markus/Desktop/R/'+inputfilename) batch_job.setDefaultNoCpus(1) batch_job.setDefaultWalltimeInSeconds(walltime) print 'preparing jobs on backend...' batch_job.prepareAndCreateJobs(redistribute) if redistribute: print 'job distribution:' print batch_job.getOptimizationResult() print 'submitting jobs to grid...'
start = 30 end = 40 pathToInputFiles = batchJob.pathToInputFiles() inputFile1relPath = pathToInputFiles+'inputFile1.txt ' inputFile2relPath = pathToInputFiles+'inputFile2.txt' for i in range(start, end): # create a unique jobname for every job jobname = batchJobName+"_"+ str(i) print 'Creating job: '+jobname # create the single job job = JobObject(si) job.setJobname(jobname) # better to set the application to use explicitely because in that case we don't need to use mds (faster) job.setApplication('UnixCommands') job.setCommandline('cat '+ inputFile1relPath + ' ' + inputFile2relPath) job.setWalltimeInSeconds(60) # adding the job to the multijob batchJob.addJob(job) # only start the newly added jobs and wait for the restart to finish batchJob.restart(False, False, True, True) # don't forget to exit properly. this cleans up possible existing threads/executors sys.exit()
# now we can calculate the relative path (from every job directory) to the common input file folder pathToInputFiles = batchJob.pathToInputFiles() for i in range(0, numberOfJobs): # create the single job job = JobObject(si) # better to set the application to use explicitely because in that case we don't need to use mds (faster) job.setCommandline('cat ' + pathToInputFiles + 'commonFile.txt ' + 'singleJobFile.txt') # adding a job-specific input file job.addInputFileUrl("/home/markus/tmp/singleJobFile.txt") # adding the job to the multijob batch_job.addJob(job) # now we are adding a file that can be used by all of the child jobs. it needs to be referenced via the pathToInputFiles() method shown above batch_job.addInputFile('/home/markus/tmp/commonJobFile.txt') batch_job.setDefaultNoCpus(1) batch_job.setDefaultWalltimeInSeconds(60) try: print "Creating jobs on the backend and staging files..." # by specifying "True" we tell the backend to automatically distribute the jobs to all available submission locations # this can be finetuned by exluding or including sites. another option would be to specifying the submission location # for every single job and setting "False" below (this would make job submission faster since jobs don't need to be re-distributed/moved on the backend). batch_job.prepareAndCreateJobs(True) except (JobsException), error: for job in error.getFailures().keySet(): print "Job: " + job.getJobname() + ", Error: " + error.getFailures(
# create the single job job = JobObject(si) job.setJobname(jobname) # better to set the application to use explicitely because in that case we don't need to use mds (faster) job.setApplication('UnixCommands') if i == 3 or i == 13: # this is just to demonstrate how to restart a failed job later on job.setCommandline('cat ' + inputFile3relPath) else: job.setCommandline('cat ' + inputFile1relPath + ' ' + inputFile2relPath) job.setWalltimeInSeconds(60) # adding the job to the multijob batchJob.addJob(job) # this should be set because it's used for the matchmaking/metascheduling batchJob.setDefaultNoCpus(1) batchJob.setDefaultWalltimeInSeconds(60) # now we add an input file that is common to all jobs batchJob.addInputFile(inputFile1Url) batchJob.addInputFile(inputFile2Url) batchJob.addInputFile(inputFile3Url) # we don't want to submit to tpac because it doesn't work #multiPartJob.setSitesToExclude(["uq", "hpsc", "auckland", "canterbury"]); try: print "Creating jobs on the backend and staging files..." batchJob.prepareAndCreateJobs(True)
batch_jobs.addInputFile(os.path.join(current_dir, dictionary_path)) batch_jobs.addInputFile(os.path.join(current_dir, "countbacon.py")) print "INFO: Defining jobs from input directory" job_count = 0 for file_name in os.listdir(input_path): print "INFO: Defining job for " + file_name job_name = base_job_name + "-" + file_name job = JobObject(service_interface) job.setJobname(job_name) job.setApplication("python") # Set the application being run job.setApplicationVersion("2.4") # Set the application version, note this is an exact match job.addInputFileUrl(os.path.join(current_dir, input_path, file_name)) job.setCommandline("python ../countbacon.py ../" + dictionary_path + " " + file_name) print "INFO: " + job.getJobname() + " defined" batch_jobs.addJob(job) print "INFO: " + job.getJobname() + " added to batch " + batch_jobs.getJobname() job_count += 1 print "INFO: " + str(job_count) + " jobs defined" print "INFO: Sending batch " + batch_jobs.getJobname() + " to " + backend + " and staging files..." try: batch_jobs.prepareAndCreateJobs(False) except (JobsException), error: print ("HALT: Exception submitting jobs from BatchJobObject " + batch_jobs.getJobname() + "!") for job in error.getFailures().keySet(): print "Job: " + job.getJobname() + ", Error: " + error.getFailures().get(job).getLocalizedMessage() sys.exit(1) except (BackendException), error: print ("HALT: Exception from grisu backend " + backend + "!") print (error.getLocalizedMessage())
print "INFO: Defining jobs from input directory" job_count = 0 for file_name in os.listdir(input_path): print "INFO: Defining job for " + file_name job_name = base_job_name + "-" + file_name job = JobObject(service_interface) job.setJobname(job_name) job.setApplication("python") # Set the application being run job.setApplicationVersion( "2.4") # Set the application version, note this is an exact match job.addInputFileUrl(os.path.join(current_dir, input_path, file_name)) job.setCommandline("python ../countbacon.py ../" + dictionary_path + " " + file_name) print "INFO: " + job.getJobname() + " defined" batch_jobs.addJob(job) print "INFO: " + job.getJobname( ) + " added to batch " + batch_jobs.getJobname() job_count += 1 print "INFO: " + str(job_count) + " jobs defined" print "INFO: Sending batch " + batch_jobs.getJobname( ) + " to " + backend + " and staging files..." try: batch_jobs.prepareAndCreateJobs(False) except (JobsException), error: print("HALT: Exception submitting jobs from BatchJobObject " + batch_jobs.getJobname() + "!") for job in error.getFailures().keySet(): print "Job: " + job.getJobname() + ", Error: " + error.getFailures( ).get(job).getLocalizedMessage()