try: print "Creating jobs on the backend and staging files..." multiPartJob.prepareAndCreateJobs(True) except (JobsException), error: for job in error.getFailures().keySet(): print "Job: "+job.getJobname()+", Error: "+error.getFailures().get(job).getLocalizedMessage() sys.exit() print "Job distribution:" for subLoc in multiPartJob.getOptimizationResult().keySet(): print subLoc + " : " +multiPartJob.getOptimizationResult().get(subLoc) print "Submitting jobs..." multiPartJob.submit() restarted = False # now we wait for all jobs to finish. Actually, we probably should test whether the job was successful as well... while not multiPartJob.isFinished(True): # printing some stats print multiPartJob.getProgress() # restart failed jobs everytime failedpolicy = DefaultResubmitPolicy() # to only resubmit failed jobs, we have to remove the waiting jobs resubmission that is set by default multiPartJob.restart(failedpolicy, True) # restart once after the jobsubmission is finished to optimize job distributions to queues where the job actually runs if not restarted:
# now we add an input file that is common to all jobs batchJob.addInputFile(inputFile1Url); batchJob.addInputFile(inputFile2Url); batchJob.addInputFile(inputFile3Url); # we don't want to submit to tpac because it doesn't work #multiPartJob.setSitesToExclude(["uq", "hpsc", "auckland", "canterbury"]); try: print "Creating jobs on the backend and staging files..." batchJob.prepareAndCreateJobs(True) except (JobsException), error: for job in error.getFailures().keySet(): print "Job: "+job.getJobname()+", Error: "+error.getFailures().get(job).getLocalizedMessage() sys.exit() # this is not really needed print "Job distribution:" for subLoc in batchJob.getOptimizationResult().keySet(): print subLoc + " : " +batchJob.getOptimizationResult().get(subLoc) print "Submitting jobs..." batchJob.submit(True) print 'Submission finished.' print 'Name of submitted batchjob: '+batchJobName # don't forget to exit properly. this cleans up possible existing threads/executors sys.exit()