print "INFO: Adding common files to Batch Job Object " + batch_job_name
batch_jobs.addInputFile(os.path.join(current_dir, dictionary_path))
batch_jobs.addInputFile(os.path.join(current_dir, "countbacon.py"))
print "INFO: Defining jobs from input directory"
job_count = 0
for file_name in os.listdir(input_path):
print "INFO: Defining job for " + file_name
job_name = base_job_name + "-" + file_name
job = JobObject(service_interface)
job.setJobname(job_name)
job.setApplication("python") # Set the application being run
job.setApplicationVersion("2.4") # Set the application version, note this is an exact match
job.addInputFileUrl(os.path.join(current_dir, input_path, file_name))
job.setCommandline("python ../countbacon.py ../" + dictionary_path + " " + file_name)
print "INFO: " + job.getJobname() + " defined"
batch_jobs.addJob(job)
print "INFO: " + job.getJobname() + " added to batch " + batch_jobs.getJobname()
job_count += 1
print "INFO: " + str(job_count) + " jobs defined"
print "INFO: Sending batch " + batch_jobs.getJobname() + " to " + backend + " and staging files..."
try:
batch_jobs.prepareAndCreateJobs(False)
except (JobsException), error:
print ("HALT: Exception submitting jobs from BatchJobObject " + batch_jobs.getJobname() + "!")
for job in error.getFailures().keySet():
print "Job: " + job.getJobname() + ", Error: " + error.getFailures().get(job).getLocalizedMessage()
sys.exit(1)
except (BackendException), error:
print ("HALT: Exception from grisu backend " + backend + "!")
submitted_jobs = []
# better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though)
jobname_base = 'staging_test'
for total in range(1, amount_of_jobs_total + 1):
job = JobObject(si)
job.setJobname(jobname_base + '_' + str(total))
# always good to set the application if you know it, processing the job will be faster
job.setApplication('UnixCommands')
# also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster
job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz')
# create a random sized outfile
mbsize = 1024 * 1024
random_size = random.randrange(10, 100)
size_string = str(mbsize * random_size)
job.setCommandline('dd if=/dev/zero of=outfile bs=' + size_string +
' count=1')
job.createJob('/nz/nesi')
job.submitJob()
print 'created and submitted job: ' + job.getJobname() + ' (size: ' + str(
random_size) + 'mb)'
submitted_jobs.append(job)
print 'Submission finished'
sys.exit(0)
# better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though)
jobname_base = 'workflow_test'
for total in range(1, amount_of_jobs_total+1):
job = JobObject(si)
job.setJobname(jobname_base+'_'+str(total))
# always good to set the application if you know it, processing the job will be faster
job.setApplication('UnixCommands')
# also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster
job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz')
# job sleeps for a random time
random_sleep = random.randrange(5, 75)
job.setCommandline('sleep '+str(random_sleep))
job.createJob('/nz/nesi')
print 'created job: '+ job.getJobname()+' (sleeptime: '+str(random_sleep)+')'
created_jobs.append(job)
finished = False
while not finished:
# submit another bunch of jobs while there are some
if len(created_jobs) > 0:
print 'still '+str(len(created_jobs))+' jobs to submit...'
while len(submitted_jobs) < amount_of_jobs_concurrent:
if len(created_jobs) <= 0:
break
job = created_jobs[0]
# now we are adding a file that can be used by all of the child jobs. it needs to be referenced via the pathToInputFiles() method shown above
batch_job.addInputFile('/home/markus/tmp/commonJobFile.txt')
batch_job.setDefaultNoCpus(1);
batch_job.setDefaultWalltimeInSeconds(60);
batch_job.setLocationsToExclude(["gt5test:ng1.canterbury.ac.nz"])
try:
print "Creating jobs on the backend and staging files..."
# by specifying "True" we tell the backend to automatically distribute the jobs to all available submission locations
# this can be finetuned by exluding or including sites. another option would be to specifying the submission location
# for every single job and setting "False" below (this would make job submission faster since jobs don't need to be re-distributed/moved on the backend).
batch_job.prepareAndCreateJobs(True)
except (JobsException), error:
for job in error.getFailures().keySet():
print "Job: "+job.getJobname()+", Error: "+error.getFailures().get(job).getLocalizedMessage()
sys.exit()
print "Job distribution:"
print batch_job.getOptimizationResult()
print "Submitting jobs..."
batch_job.submit()
# now we wait for all jobs to be finished, checking for updates every 10 seconds. in real life we would set a much higher check intervall since we don't want to overload
# the backend and also it's not really necessary
batch_job.waitForJobToFinish(10)
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
# create a service interface to the BeSTGRID backend
service_interface = LoginManager.loginCommandline("BeSTGRID")
print 'Creating job object...'
job = JobObject(service_interface)
job.setJobname("echo_job-1") # job name must be unique
print 'Set jobname to: ' + job.getJobname()
# set the name of the application as it is published in MDS.
# "generic" means not to use MDS for the lookup.
job.setApplication("generic")
# "generic" jobs require a submission location to be specified
job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE")
# set the command that needs to be executed
job.setCommandline("echo \"Hello World\"")
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/BeSTGRID")
print 'Submitting job...'
# submit the job
job.submitJob()
print 'Waiting for the job to finish...'
# this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)
finished = job.waitForJobToFinish(10)
batch_job.addJob(job)
# now we are adding a file that can be used by all of the child jobs. it needs to be referenced via the pathToInputFiles() method shown above
batch_job.addInputFile('/home/markus/tmp/commonJobFile.txt')
batch_job.setDefaultNoCpus(1)
batch_job.setDefaultWalltimeInSeconds(60)
try:
print "Creating jobs on the backend and staging files..."
# by specifying "True" we tell the backend to automatically distribute the jobs to all available submission locations
# this can be finetuned by exluding or including sites. another option would be to specifying the submission location
# for every single job and setting "False" below (this would make job submission faster since jobs don't need to be re-distributed/moved on the backend).
batch_job.prepareAndCreateJobs(True)
except (JobsException), error:
for job in error.getFailures().keySet():
print "Job: " + job.getJobname() + ", Error: " + error.getFailures(
).get(job).getLocalizedMessage()
sys.exit()
print "Job distribution:"
for subLoc in batch_job.getOptimizationResult().keySet():
print subLoc + " : " + batch_job.getOptimizationResult().get(subLoc)
print "Submitting jobs..."
batch_job.submit()
restarted = False
# now we wait for all jobs to finish. Actually, we probably should test whether the job was successful as well...
while not batch_job.isFinished(True):
batch_jobs.addInputFile(os.path.join(current_dir, "countbacon.py"))
print "INFO: Defining jobs from input directory"
job_count = 0
for file_name in os.listdir(input_path):
print "INFO: Defining job for " + file_name
job_name = base_job_name + "-" + file_name
job = JobObject(service_interface)
job.setJobname(job_name)
job.setApplication("python") # Set the application being run
job.setApplicationVersion(
"2.4") # Set the application version, note this is an exact match
job.addInputFileUrl(os.path.join(current_dir, input_path, file_name))
job.setCommandline("python ../countbacon.py ../" + dictionary_path + " " +
file_name)
print "INFO: " + job.getJobname() + " defined"
batch_jobs.addJob(job)
print "INFO: " + job.getJobname(
) + " added to batch " + batch_jobs.getJobname()
job_count += 1
print "INFO: " + str(job_count) + " jobs defined"
print "INFO: Sending batch " + batch_jobs.getJobname(
) + " to " + backend + " and staging files..."
try:
batch_jobs.prepareAndCreateJobs(False)
except (JobsException), error:
print("HALT: Exception submitting jobs from BatchJobObject " +
batch_jobs.getJobname() + "!")
for job in error.getFailures().keySet():
print "Job: " + job.getJobname() + ", Error: " + error.getFailures(
jobnames = []
for i in range(10):
file = folder + "test" + str(i) + ".txt"
job = JobObject(si)
job.setUniqueJobname("diff_" + str(i))
job.setApplication("UnixCommands")
job.setCommandline("diff original.txt test" + str(i) + ".txt")
job.createJob("/ARCS/BeSTGRID")
job.addInputFileUrl(file)
job.addInputFileUrl(original)
job.submitJob()
jobnames.append(job.getJobname())
for jobname in jobnames:
finished = job.waitForJobToFinish(10)
print "Job: " + jobname
print
print "Stdout:"
print job.getStdOutContent()
print
print "Stderr:"
print job.getStdErrContent()
print
print
si = LoginManager.loginCommandline()
print "Parsing commandline arguments..."
file1url = sys.argv[1]
file1Name = FileManager.getFilename(file1url)
file2url = sys.argv[2]
file2Name = FileManager.getFilename(file2url)
print "Creating job..."
# create the job object
job = JobObject(si)
# set a unique jobname
job.setTimestampJobname("diff_job")
print "Set jobname to: " + job.getJobname()
# setting the application. this means that grisu can figure out the submission location and
# you don't have to do that manually
job.setApplication("UnixCommands")
# set the commandline that needs to be executed
job.setCommandline("diff " + file1Name + " " + file2Name)
job.addInputFileUrl(file1url)
job.addInputFileUrl(file2url)
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/StartUp")
print "Submitting job..."
# submit the job
job.submitJob()
# Creating a list of jobs
jobs = []
print "INFO: Defining " + str(job_count) + " helloworld jobs"
for i in range(1, job_count + 1):
print "INFO: Defining job " + str(i) + " of " + str(job_count)
#The next lines define the actual job's parameters
job = JobObject(service_interface) # Create a job
job.setJobname(base_job_name + str(i)) # Give it a (hopefully) unique name
job.setApplication("python") # Set the application being run
job.setApplicationVersion(
"2.4") # Set the application version, note this is an exact match
# job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE") # Set the location the job will be submitted to
job.addInputFileUrl(os.path.join(current_dir,
"helloworld.py")) # Add a file
job.setCommandline("python helloworld.py") # Set the command to be run
print "INFO: job " + job.getJobname() + " defined"
jobs.append(job)
# Submit the jobs to be run
# Note the exception catching to give more information about a job failing
for job in jobs:
time_start = time.time()
try:
print "INFO: Creating job " + job.getJobname(
) + " on " + backend + " backend, with " + group + " group"
job.createJob(group)
print "INFO: Submitting job " + job.getJobname()
job.submitJob()
except (JobsException), error:
print "HALT: Exception submitting job!"
print "Job: " + job.getJobname() + ", Error: " + error.getFailures(
job.setWalltimeInSeconds(DEFAULT_WALLTIME)
# stop annoying stats from being written to stderr
job.addEnvironmentVariable("SUPPRESS_STATS", "true")
#create the job script#
except:
print "Cannot setup the job environment"
sys.exit(-4)
#create nesi job_script
try:
# save jobname for job
njn = open(jobname_file, "w")
njn.write(job.getJobname())
njn.close()
except:
print "Cannot write jobname to file"
sys.exit(-2)
command_arguments = command.split()
print input_files
new_commandline = ""
file = open("/home/jamesboocock/blah.txt", 'a')
for arg in command_arguments:
file.write(arg + '\n')
arg=arg.replace('"','')
print("arg: " + arg)
if ((os.path.exists(arg)) or (os.path.isfile(arg)==True)) and (arg not in input_files) and ("_file" not in arg):
# There are three stages, creating the jobs, submitting the jobs, then after they have finished, retrieving the job outputs
# Creating a list of jobs
jobs = []
print "INFO: Defining " + str(job_count) + " helloworld jobs"
for i in range(1, job_count + 1):
print "INFO: Defining job " + str(i) + " of " + str(job_count)
#The next lines define the actual job's parameters
job = JobObject(service_interface) # Create a job
job.setJobname(base_job_name + str(i)) # Give it a (hopefully) unique name
job.setApplication("python") # Set the application being run
job.setApplicationVersion("2.4") # Set the application version, note this is an exact match
# job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE") # Set the location the job will be submitted to
job.addInputFileUrl(os.path.join(current_dir, "helloworld.py")) # Add a file
job.setCommandline("python helloworld.py") # Set the command to be run
print"INFO: job " + job.getJobname() + " defined"
jobs.append(job)
# Submit the jobs to be run
# Note the exception catching to give more information about a job failing
for job in jobs:
time_start = time.time()
try:
print "INFO: Creating job " + job.getJobname() + " on " + backend + " backend, with " + group + " group"
job.createJob(group)
print "INFO: Submitting job " + job.getJobname()
job.submitJob()
except (JobsException), error:
print "HALT: Exception submitting job!"
print "Job: " + job.getJobname() + ", Error: " + error.getFailures().get(job).getLocalizedMessage()
print"========================"
submitted_jobs = []
# better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though)
jobname_base = 'staging_test'
for total in range(1, amount_of_jobs_total+1):
job = JobObject(si)
job.setJobname(jobname_base+'_'+str(total))
# always good to set the application if you know it, processing the job will be faster
job.setApplication('UnixCommands')
# also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster
job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz')
# create a random sized outfile
mbsize = 1024*1024
random_size = random.randrange(10, 100)
size_string = str(mbsize*random_size)
job.setCommandline('dd if=/dev/zero of=outfile bs='+size_string+' count=1')
job.createJob('/nz/nesi')
job.submitJob()
print 'created and submitted job: '+ job.getJobname()+' (size: '+str(random_size)+'mb)'
submitted_jobs.append(job)
print 'Submission finished'
sys.exit(0)
batchJob.setDefaultNoCpus(1);
batchJob.setDefaultWalltimeInSeconds(60);
# now we add an input file that is common to all jobs
batchJob.addInputFile(inputFile1Url);
batchJob.addInputFile(inputFile2Url);
batchJob.addInputFile(inputFile3Url);
# we don't want to submit to tpac because it doesn't work
#multiPartJob.setSitesToExclude(["uq", "hpsc", "auckland", "canterbury"]);
try:
print "Creating jobs on the backend and staging files..."
batchJob.prepareAndCreateJobs(True)
except (JobsException), error:
for job in error.getFailures().keySet():
print "Job: "+job.getJobname()+", Error: "+error.getFailures().get(job).getLocalizedMessage()
sys.exit()
# this is not really needed
print "Job distribution:"
for subLoc in batchJob.getOptimizationResult().keySet():
print subLoc + " : " +batchJob.getOptimizationResult().get(subLoc)
print "Submitting jobs..."
batchJob.submit(True)
print 'Submission finished.'
print 'Name of submitted batchjob: '+batchJobName
batchJob.setDefaultNoCpus(1)
batchJob.setDefaultWalltimeInSeconds(60)
# now we add an input file that is common to all jobs
batchJob.addInputFile(inputFile1Url)
batchJob.addInputFile(inputFile2Url)
batchJob.addInputFile(inputFile3Url)
# we don't want to submit to tpac because it doesn't work
#multiPartJob.setSitesToExclude(["uq", "hpsc", "auckland", "canterbury"]);
try:
print "Creating jobs on the backend and staging files..."
batchJob.prepareAndCreateJobs(True)
except (JobsException), error:
for job in error.getFailures().keySet():
print "Job: " + job.getJobname() + ", Error: " + error.getFailures(
).get(job).getLocalizedMessage()
sys.exit()
# this is not really needed
print "Job distribution:"
for subLoc in batchJob.getOptimizationResult().keySet():
print subLoc + " : " + batchJob.getOptimizationResult().get(subLoc)
print "Submitting jobs..."
batchJob.submit(True)
print 'Submission finished.'
print 'Name of submitted batchjob: ' + batchJobName
jobnames = []
for i in range(10):
file = folder+'test'+str(i)+'.txt'
job = JobObject(si);
job.setUniqueJobname('diff_'+str(i))
job.setApplication('UnixCommands')
job.setCommandline('diff original.txt test'+str(i)+'.txt')
job.createJob("/ARCS/BeSTGRID")
job.addInputFileUrl(file)
job.addInputFileUrl(original)
job.submitJob()
jobnames.append(job.getJobname())
for jobname in jobnames:
finished = job.waitForJobToFinish(10)
print 'Job: '+jobname
print
print 'Stdout:'
print job.getStdOutContent()
print
print 'Stderr:'
print job.getStdErrContent()
print
print
@author: markus
'''
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
si = LoginManager.loginCommandline()
print 'Creating job...'
# create the job object
job = JobObject(si);
# set a unique jobname
job.setUniqueJobname("echo_job1")
print 'Set jobname to: '+ job.getJobname()
# set the name of the application like it is published in mds. "generic" means not to use mds for the lookup.
job.setApplication("generic")
# since we are using a "generic" job, we need to specify a submission location. I'll make that easier later on...
job.setSubmissionLocation("dque@edda-m:ng2.vpac.org")
# set the commandline that needs to be executed
job.setCommandline("echo \"Hello World\"")
job.addInputFileUrl('/home/markus/test/singleJobFile_0.txt');
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/NGAdmin")
print 'Submitting job...'
# submit the job
job.submitJob()
script = folder + 'wrap.sh'
job = JobObject(si);
job.setUniqueJobname('diff_wrap')
job.setApplication('UnixCommands')
job.setCommandline('sh wrap.sh 0 9')
job.addInputFileUrl(original)
job.addInputFileUrl(script)
for i in range(10):
file = folder+'test'+str(i)+'.txt'
job.addInputFileUrl(file)
job.createJob("/ARCS/BeSTGRID")
job.submitJob()
finished = job.waitForJobToFinish(10)
print 'Job: '+job.getJobname()
print
print 'Stdout:'
print job.getStdOutContent()
print
print 'Stderr:'
print job.getStdErrContent()
print
print
sys.exit()
# better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though)
jobname_base = 'workflow_test'
for total in range(1, amount_of_jobs_total + 1):
job = JobObject(si)
job.setJobname(jobname_base + '_' + str(total))
# always good to set the application if you know it, processing the job will be faster
job.setApplication('UnixCommands')
# also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster
job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz')
# job sleeps for a random time
random_sleep = random.randrange(5, 75)
job.setCommandline('sleep ' + str(random_sleep))
job.createJob('/nz/nesi')
print 'created job: ' + job.getJobname() + ' (sleeptime: ' + str(
random_sleep) + ')'
created_jobs.append(job)
finished = False
while not finished:
# submit another bunch of jobs while there are some
if len(created_jobs) > 0:
print 'still ' + str(len(created_jobs)) + ' jobs to submit...'
while len(submitted_jobs) < amount_of_jobs_concurrent:
if len(created_jobs) <= 0:
break
original = folder + 'original.txt'
script = folder + 'wrap.sh'
job = JobObject(si)
job.setUniqueJobname('diff_wrap')
job.setApplication('UnixCommands')
job.setCommandline('sh wrap.sh 0 9')
job.addInputFileUrl(original)
job.addInputFileUrl(script)
for i in range(10):
file = folder + 'test' + str(i) + '.txt'
job.addInputFileUrl(file)
job.createJob("/ARCS/BeSTGRID")
job.submitJob()
finished = job.waitForJobToFinish(10)
print 'Job: ' + job.getJobname()
print
print 'Stdout:'
print job.getStdOutContent()
print
print 'Stderr:'
print job.getStdErrContent()
print
print
sys.exit()
