job = JobObject(si)
job.setEmail_address(email)
job.setEmail_on_job_finish(True)
job.setCommandline('R --no-readline --no-restore --no-save -f '+path_to_inputfile)
batch_job.addJob(job)
batch_job.addInputFile('/home/markus/Desktop/R/'+inputfilename)
batch_job.setDefaultNoCpus(1)
batch_job.setDefaultWalltimeInSeconds(walltime)
print 'preparing jobs on backend...'
batch_job.prepareAndCreateJobs(redistribute)
if redistribute:
print 'job distribution:'
print batch_job.getOptimizationResult()
print 'submitting jobs to grid...'
batch_job.submit(True)
print 'submission finished...'
job_name = base_job_name + "-" + file_name
job = JobObject(service_interface)
job.setJobname(job_name)
job.setApplication("python") # Set the application being run
job.setApplicationVersion("2.4") # Set the application version, note this is an exact match
job.addInputFileUrl(os.path.join(current_dir, input_path, file_name))
job.setCommandline("python ../countbacon.py ../" + dictionary_path + " " + file_name)
print "INFO: " + job.getJobname() + " defined"
batch_jobs.addJob(job)
print "INFO: " + job.getJobname() + " added to batch " + batch_jobs.getJobname()
job_count += 1
print "INFO: " + str(job_count) + " jobs defined"
print "INFO: Sending batch " + batch_jobs.getJobname() + " to " + backend + " and staging files..."
try:
batch_jobs.prepareAndCreateJobs(False)
except (JobsException), error:
print ("HALT: Exception submitting jobs from BatchJobObject " + batch_jobs.getJobname() + "!")
for job in error.getFailures().keySet():
print "Job: " + job.getJobname() + ", Error: " + error.getFailures().get(job).getLocalizedMessage()
sys.exit(1)
except (BackendException), error:
print ("HALT: Exception from grisu backend " + backend + "!")
print (error.getLocalizedMessage())
print ("========================")
time.sleep(3)
error.printStackTrace()
sys.exit(1)
time.sleep(3)
print "INFO: Submitting jobs in batch " + batch_jobs.getJobname()
print 'jobname on backend: ' + batch_job_name
path_to_inputfile = batch_job.pathToInputFiles() + inputfilename
for i in range(1, gen_jobs + 1):
job = JobObject(si)
job.setEmail_address(email)
job.setEmail_on_job_finish(True)
job.setCommandline('R --no-readline --no-restore --no-save -f ' +
path_to_inputfile)
batch_job.addJob(job)
batch_job.addInputFile('/home/markus/Desktop/R/' + inputfilename)
batch_job.setDefaultNoCpus(1)
batch_job.setDefaultWalltimeInSeconds(walltime)
print 'preparing jobs on backend...'
batch_job.prepareAndCreateJobs(redistribute)
if redistribute:
print 'job distribution:'
print batch_job.getOptimizationResult()
print 'submitting jobs to grid...'
batch_job.submit(True)
print 'submission finished...'
# adding the job to the multijob
batch_job.addJob(job)
# now we are adding a file that can be used by all of the child jobs. it needs to be referenced via the pathToInputFiles() method shown above
batch_job.addInputFile('/home/markus/tmp/commonJobFile.txt')
batch_job.setDefaultNoCpus(1);
batch_job.setDefaultWalltimeInSeconds(60);
batch_job.setLocationsToExclude(["gt5test:ng1.canterbury.ac.nz"])
try:
print "Creating jobs on the backend and staging files..."
# by specifying "True" we tell the backend to automatically distribute the jobs to all available submission locations
# this can be finetuned by exluding or including sites. another option would be to specifying the submission location
# for every single job and setting "False" below (this would make job submission faster since jobs don't need to be re-distributed/moved on the backend).
batch_job.prepareAndCreateJobs(True)
except (JobsException), error:
for job in error.getFailures().keySet():
print "Job: "+job.getJobname()+", Error: "+error.getFailures().get(job).getLocalizedMessage()
sys.exit()
print "Job distribution:"
print batch_job.getOptimizationResult()
print "Submitting jobs..."
batch_job.submit()
# now we wait for all jobs to be finished, checking for updates every 10 seconds. in real life we would set a much higher check intervall since we don't want to overload
job.setApplicationVersion(
"2.4") # Set the application version, note this is an exact match
job.addInputFileUrl(os.path.join(current_dir, input_path, file_name))
job.setCommandline("python ../countbacon.py ../" + dictionary_path + " " +
file_name)
print "INFO: " + job.getJobname() + " defined"
batch_jobs.addJob(job)
print "INFO: " + job.getJobname(
) + " added to batch " + batch_jobs.getJobname()
job_count += 1
print "INFO: " + str(job_count) + " jobs defined"
print "INFO: Sending batch " + batch_jobs.getJobname(
) + " to " + backend + " and staging files..."
try:
batch_jobs.prepareAndCreateJobs(False)
except (JobsException), error:
print("HALT: Exception submitting jobs from BatchJobObject " +
batch_jobs.getJobname() + "!")
for job in error.getFailures().keySet():
print "Job: " + job.getJobname() + ", Error: " + error.getFailures(
).get(job).getLocalizedMessage()
sys.exit(1)
except (BackendException), error:
print("HALT: Exception from grisu backend " + backend + "!")
print(error.getLocalizedMessage())
print("========================")
time.sleep(3)
error.printStackTrace()
sys.exit(1)
time.sleep(3)
