batch_job = BatchJobObject(si, basename, group, 'R', Constants.NO_VERSION_INDICATOR_STRING)
batch_job_name = batch_job.getJobname()
print 'jobname on backend: '+batch_job_name
path_to_inputfile = batch_job.pathToInputFiles()+inputfilename
for i in range(1,gen_jobs+1):
job = JobObject(si)
job.setEmail_address(email)
job.setEmail_on_job_finish(True)
job.setCommandline('R --no-readline --no-restore --no-save -f '+path_to_inputfile)
batch_job.addJob(job)
batch_job.addInputFile('/home/markus/Desktop/R/'+inputfilename)
batch_job.setDefaultNoCpus(1)
batch_job.setDefaultWalltimeInSeconds(walltime)
print 'preparing jobs on backend...'
batch_job.prepareAndCreateJobs(redistribute)
if redistribute:
print 'job distribution:'
print batch_job.getOptimizationResult()
print 'submitting jobs to grid...'
start = 30
end = 40
pathToInputFiles = batchJob.pathToInputFiles()
inputFile1relPath = pathToInputFiles+'inputFile1.txt '
inputFile2relPath = pathToInputFiles+'inputFile2.txt'
for i in range(start, end):
# create a unique jobname for every job
jobname = batchJobName+"_"+ str(i)
print 'Creating job: '+jobname
# create the single job
job = JobObject(si)
job.setJobname(jobname)
# better to set the application to use explicitely because in that case we don't need to use mds (faster)
job.setApplication('UnixCommands')
job.setCommandline('cat '+ inputFile1relPath + ' ' + inputFile2relPath)
job.setWalltimeInSeconds(60)
# adding the job to the multijob
batchJob.addJob(job)
# only start the newly added jobs and wait for the restart to finish
batchJob.restart(False, False, True, True)
# don't forget to exit properly. this cleans up possible existing threads/executors
sys.exit()
# now we can calculate the relative path (from every job directory) to the common input file folder
pathToInputFiles = batchJob.pathToInputFiles()
for i in range(0, numberOfJobs):
# create the single job
job = JobObject(si)
# better to set the application to use explicitely because in that case we don't need to use mds (faster)
job.setCommandline('cat ' + pathToInputFiles + 'commonFile.txt ' +
'singleJobFile.txt')
# adding a job-specific input file
job.addInputFileUrl("/home/markus/tmp/singleJobFile.txt")
# adding the job to the multijob
batch_job.addJob(job)
# now we are adding a file that can be used by all of the child jobs. it needs to be referenced via the pathToInputFiles() method shown above
batch_job.addInputFile('/home/markus/tmp/commonJobFile.txt')
batch_job.setDefaultNoCpus(1)
batch_job.setDefaultWalltimeInSeconds(60)
try:
print "Creating jobs on the backend and staging files..."
# by specifying "True" we tell the backend to automatically distribute the jobs to all available submission locations
# this can be finetuned by exluding or including sites. another option would be to specifying the submission location
# for every single job and setting "False" below (this would make job submission faster since jobs don't need to be re-distributed/moved on the backend).
batch_job.prepareAndCreateJobs(True)
except (JobsException), error:
for job in error.getFailures().keySet():
print "Job: " + job.getJobname() + ", Error: " + error.getFailures(
# create the single job
job = JobObject(si)
job.setJobname(jobname)
# better to set the application to use explicitely because in that case we don't need to use mds (faster)
job.setApplication('UnixCommands')
if i == 3 or i == 13:
# this is just to demonstrate how to restart a failed job later on
job.setCommandline('cat ' + inputFile3relPath)
else:
job.setCommandline('cat ' + inputFile1relPath + ' ' +
inputFile2relPath)
job.setWalltimeInSeconds(60)
# adding the job to the multijob
batchJob.addJob(job)
# this should be set because it's used for the matchmaking/metascheduling
batchJob.setDefaultNoCpus(1)
batchJob.setDefaultWalltimeInSeconds(60)
# now we add an input file that is common to all jobs
batchJob.addInputFile(inputFile1Url)
batchJob.addInputFile(inputFile2Url)
batchJob.addInputFile(inputFile3Url)
# we don't want to submit to tpac because it doesn't work
#multiPartJob.setSitesToExclude(["uq", "hpsc", "auckland", "canterbury"]);
try:
print "Creating jobs on the backend and staging files..."
batchJob.prepareAndCreateJobs(True)
batch_jobs.addInputFile(os.path.join(current_dir, dictionary_path))
batch_jobs.addInputFile(os.path.join(current_dir, "countbacon.py"))
print "INFO: Defining jobs from input directory"
job_count = 0
for file_name in os.listdir(input_path):
print "INFO: Defining job for " + file_name
job_name = base_job_name + "-" + file_name
job = JobObject(service_interface)
job.setJobname(job_name)
job.setApplication("python") # Set the application being run
job.setApplicationVersion("2.4") # Set the application version, note this is an exact match
job.addInputFileUrl(os.path.join(current_dir, input_path, file_name))
job.setCommandline("python ../countbacon.py ../" + dictionary_path + " " + file_name)
print "INFO: " + job.getJobname() + " defined"
batch_jobs.addJob(job)
print "INFO: " + job.getJobname() + " added to batch " + batch_jobs.getJobname()
job_count += 1
print "INFO: " + str(job_count) + " jobs defined"
print "INFO: Sending batch " + batch_jobs.getJobname() + " to " + backend + " and staging files..."
try:
batch_jobs.prepareAndCreateJobs(False)
except (JobsException), error:
print ("HALT: Exception submitting jobs from BatchJobObject " + batch_jobs.getJobname() + "!")
for job in error.getFailures().keySet():
print "Job: " + job.getJobname() + ", Error: " + error.getFailures().get(job).getLocalizedMessage()
sys.exit(1)
except (BackendException), error:
print ("HALT: Exception from grisu backend " + backend + "!")
print (error.getLocalizedMessage())
print "INFO: Defining jobs from input directory"
job_count = 0
for file_name in os.listdir(input_path):
print "INFO: Defining job for " + file_name
job_name = base_job_name + "-" + file_name
job = JobObject(service_interface)
job.setJobname(job_name)
job.setApplication("python") # Set the application being run
job.setApplicationVersion(
"2.4") # Set the application version, note this is an exact match
job.addInputFileUrl(os.path.join(current_dir, input_path, file_name))
job.setCommandline("python ../countbacon.py ../" + dictionary_path + " " +
file_name)
print "INFO: " + job.getJobname() + " defined"
batch_jobs.addJob(job)
print "INFO: " + job.getJobname(
) + " added to batch " + batch_jobs.getJobname()
job_count += 1
print "INFO: " + str(job_count) + " jobs defined"
print "INFO: Sending batch " + batch_jobs.getJobname(
) + " to " + backend + " and staging files..."
try:
batch_jobs.prepareAndCreateJobs(False)
except (JobsException), error:
print("HALT: Exception submitting jobs from BatchJobObject " +
batch_jobs.getJobname() + "!")
for job in error.getFailures().keySet():
print "Job: " + job.getJobname() + ", Error: " + error.getFailures(
).get(job).getLocalizedMessage()
