def __init__(self, si, subLoc=defaultSubLoc):
super(simpleStageJob, self).__init__(si, subLoc)
self.job = JobObject(self.si)
self.job.setCommandline("echo \"hello world\"")
self.job.setApplication(Constants.GENERIC_APPLICATION_NAME)
self.job.setSubmissionLocation(self.subLoc)
def prepare(self):
job = JobObject(self.si)
job.setUniqueJobname("echo_job1")
job.setCommandline("echo \"Hello World\"")
job.addInputFileUrl("/home/markus/tmp/text0.txt")
job.createJob()
self.job = job
def prepare(self):
job = JobObject(self.si)
job.setUniqueJobname("echo_job1")
job.setCommandline('echo "Hello World"')
job.addInputFileUrl("/home/markus/tmp/text0.txt")
job.createJob()
self.job = job
def prepare(self):
job = JobObject(self.si)
job.setUniqueJobname("echo_job1")
job.setCommandline("echo \"Hello World\"")
job.createJob()
self.job = job
def __init__(self, si, subLoc=defaultSubLoc):
super(simpleStageJob, self).__init__(si, subLoc)
self.job = JobObject(self.si)
self.job.setCommandline('echo "hello world"')
self.job.setApplication(Constants.GENERIC_APPLICATION_NAME)
self.job.setSubmissionLocation(self.subLoc)
def prepare(self):
job = JobObject(self.si)
job.setUniqueJobname("echo_job1")
job.setCommandline('echo "Hello World"')
job.createJob()
self.job = job
numberOfJobs = 10
# the (unique) name of the batchjob
batch_job_name = "test_batch"
# create the batchjob
batch_job = BatchJobObject(si, batch_job_name, "/nz/nesi", "UnixCommands",
Constants.NO_VERSION_INDICATOR_STRING)
# now we can calculate the relative path (from every job directory) to the common input file folder
pathToInputFiles = batch_job.pathToInputFiles()
for i in range(0, numberOfJobs):
# create the single job
job = JobObject(si)
# better to set the application to use explicitely because in that case we don't need to use mds (faster)
job.setCommandline('cat ' + pathToInputFiles + 'commonJobFile.txt ' +
'singleJobFile.txt')
# adding a job-specific input file
job.addInputFileUrl("/home/markus/tmp/singleJobFile.txt")
# adding the job to the multijob
batch_job.addJob(job)
# now we are adding a file that can be used by all of the child jobs. it needs to be referenced via the pathToInputFiles() method shown above
batch_job.addInputFile('/home/markus/tmp/commonJobFile.txt')
batch_job.setDefaultNoCpus(1)
batch_job.setDefaultWalltimeInSeconds(60)
batch_job.setLocationsToExclude(["gt5test:ng1.canterbury.ac.nz"])
batch_jobs.setDefaultWalltimeInSeconds(300)
# Set the maximum walltime to 5 minutes
batch_jobs.setLocationsToExclude(["AUT"
]) # Create a blacklist of sites to exclude
# Currently the AUT location is not behaving, so always exclude it
print "INFO: Adding common files to Batch Job Object " + batch_job_name
batch_jobs.addInputFile(os.path.join(current_dir, dictionary_path))
batch_jobs.addInputFile(os.path.join(current_dir, "countbacon.py"))
print "INFO: Defining jobs from input directory"
job_count = 0
for file_name in os.listdir(input_path):
print "INFO: Defining job for " + file_name
job_name = base_job_name + "-" + file_name
job = JobObject(service_interface)
job.setJobname(job_name)
job.setApplication("python") # Set the application being run
job.setApplicationVersion(
"2.4") # Set the application version, note this is an exact match
job.addInputFileUrl(os.path.join(current_dir, input_path, file_name))
job.setCommandline("python ../countbacon.py ../" + dictionary_path + " " +
file_name)
print "INFO: " + job.getJobname() + " defined"
batch_jobs.addJob(job)
print "INFO: " + job.getJobname(
) + " added to batch " + batch_jobs.getJobname()
job_count += 1
print "INFO: " + str(job_count) + " jobs defined"
print "INFO: Sending batch " + batch_jobs.getJobname(
# Set the maximum walltime to 5 minutes
exclude_sites = list()
exclude_sites.append("AUT")
batch_jobs.setLocationsToExclude(exclude_sites) # Create a blacklist of sites to exclude
# Currently the AUT location is not behaving, so always exclude it
print "INFO: Adding common files to Batch Job Object " + batch_job_name
batch_jobs.addInputFile(os.path.join(current_dir, dictionary_path))
batch_jobs.addInputFile(os.path.join(current_dir, "countbacon.py"))
print "INFO: Defining jobs from input directory"
job_count = 0
for file_name in os.listdir(input_path):
print "INFO: Defining job for " + file_name
job_name = base_job_name + "-" + file_name
job = JobObject(service_interface)
job.setJobname(job_name)
job.setApplication("python") # Set the application being run
job.setApplicationVersion("2.4") # Set the application version, note this is an exact match
job.addInputFileUrl(os.path.join(current_dir, input_path, file_name))
job.setCommandline("python ../countbacon.py ../" + dictionary_path + " " + file_name)
print "INFO: " + job.getJobname() + " defined"
batch_jobs.addJob(job)
print "INFO: " + job.getJobname() + " added to batch " + batch_jobs.getJobname()
job_count += 1
print "INFO: " + str(job_count) + " jobs defined"
print "INFO: Sending batch " + batch_jobs.getJobname() + " to " + backend + " and staging files..."
try:
batch_jobs.prepareAndCreateJobs(False)
except (JobsException), error:
def prepare(self):
job = JobObject(self.si)
job.setUniqueJobname("echo_job1")
job.setCommandline("echo \"Hello World\"")
job.addInputFileUrl("/home/markus/tmp/text0.txt")
job.addInputFileUrl("/home/markus/tmp/text1.txt")
job.addInputFileUrl("/home/markus/tmp/text2.txt")
job.addInputFileUrl("/home/markus/tmp/text3.txt")
job.addInputFileUrl("/home/markus/tmp/text4.txt")
job.setApplication(Constants.GENERIC_APPLICATION_NAME)
job.setSubmissionLocation(self.subLoc)
job.createJob()
self.job = job
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
from grisu.model import FileManager
import sys
si = LoginManager.loginCommandline()
print 'Parsing commandline arguments...'
file1url = sys.argv[1]
file1Name = FileManager.getFilename(file1url)
file2url = sys.argv[2]
file2Name = FileManager.getFilename(file2url)
print 'Creating job...'
# create the job object
job = JobObject(si)
# set a unique jobname
job.setTimestampJobname("diff_job")
print 'Set jobname to: ' + job.getJobname()
# setting the application. this means that grisu can figure out the submission location and
# you don't have to do that manually
job.setApplication("UnixCommands")
# set the commandline that needs to be executed
job.setCommandline('diff ' + file1Name + ' ' + file2Name)
job.addInputFileUrl(file1url)
job.addInputFileUrl(file2url)
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/StartUp")
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
from grisu.model import GrisuRegistryManager
si = LoginManager.loginCommandline("LOCAL")
appInfo = GrisuRegistryManager.getDefault(si).getApplicationInformation(
"ImageJ")
print 'app: ' + appInfo.getApplicationName()
for subloc in appInfo.getAvailableAllSubmissionLocations():
print subloc
job = JobObject(si)
job.setTimestampJobname("imageJ")
job.setApplication("java")
job.setApplication("ImageJ")
job.setCommandline("echo Hello")
job.setSubmissionLocation("normal:ng2.ivec.org")
job.createJob("/ARCS/StartUp")
job.submitJob()
job.waitForJobToFinish(3)
print "Stdout: " + job.getStdOutContent()
print "Stderr: " + job.getStdErrContent()
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
# create a service interface to the BeSTGRID backend
service_interface = LoginManager.loginCommandline("BeSTGRID")
print 'Creating job object...'
job = JobObject(service_interface);
job.setJobname("echo_job-1") # job name must be unique
print 'Set jobname to: '+ job.getJobname()
# set the name of the application as it is published in MDS.
# "generic" means not to use MDS for the lookup.
job.setApplication("generic")
# "generic" jobs require a submission location to be specified
job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE")
# set the command that needs to be executed
job.setCommandline("echo \"Hello World\"")
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/BeSTGRID")
print 'Submitting job...'
# submit the job
job.submitJob()
print 'Waiting for the job to finish...'
# this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)
# to see whats going on we add a simple event listener. Hm. This doesn't seem to work reliably in jython.
#SystemOutMultiJobLogger(multiJobName)
# create the multipart job
batch_job = BatchJobObject(si, batch_job_name, "/nz/nesi", "cat", Constants.NO_VERSION_INDICATOR_STRING);
# now we can calculate the relative path (from every job directory) to the common input file folder
pathToInputFiles = batchJob.pathToInputFiles()
for i in range(0, numberOfJobs):
# create the single job
job = JobObject(si)
# better to set the application to use explicitely because in that case we don't need to use mds (faster)
job.setCommandline('cat ' + pathToInputFiles+'commonFile.txt ' + 'singleJobFile.txt')
# adding a job-specific input file
job.addInputFileUrl("/home/markus/tmp/singleJobFile.txt")
# adding the job to the multijob
batch_job.addJob(job)
# now we are adding a file that can be used by all of the child jobs. it needs to be referenced via the pathToInputFiles() method shown above
batch_job.addInputFile('/home/markus/tmp/commonJobFile.txt')
batch_job.setDefaultNoCpus(1);
batch_job.setDefaultWalltimeInSeconds(60);
try:
working_directory = sys.argv[7]
input_files = list()
job_header="""#!/bin/sh
%s
"""
if group == '':
group = DEFAULT_GROUP
if queue == '':
queue = DEFAULT_QUEUE
for f in sys.argv[8:]:
input_files.append(f)
try:
job = JobObject(si)
job.setSubmissionLocation(queue)
job.setTimestampJobname("galaxy_" + galaxy_job_id)
job.setMemory(DEFAULT_MEMORY)
job.setWalltimeInSeconds(DEFAULT_WALLTIME)
# stop annoying stats from being written to stderr
job.addEnvironmentVariable("SUPPRESS_STATS", "true")
#create the job script#
except:
print "Cannot setup the job environment"
sys.exit(-4)
# Setting the number of jobs to be submitted
job_count = 5
# Since there may be many jobs submitted in the workshop, lets make them a bit unique to avoid issues.
base_job_name = str(random.randint(10000, 99999)) + '-hello-'
print "INFO: Base job name is " + base_job_name
# There are three stages, creating the jobs, submitting the jobs, then after they have finished, retrieving the job outputs
# Creating a list of jobs
jobs = []
print "INFO: Defining " + str(job_count) + " helloworld jobs"
for i in range(1, job_count + 1):
print "INFO: Defining job " + str(i) + " of " + str(job_count)
#The next lines define the actual job's parameters
job = JobObject(service_interface) # Create a job
job.setJobname(base_job_name + str(i)) # Give it a (hopefully) unique name
job.setApplication("python") # Set the application being run
job.setApplicationVersion(
"2.4") # Set the application version, note this is an exact match
# job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE") # Set the location the job will be submitted to
job.addInputFileUrl(os.path.join(current_dir,
"helloworld.py")) # Add a file
job.setCommandline("python helloworld.py") # Set the command to be run
print "INFO: job " + job.getJobname() + " defined"
jobs.append(job)
# Submit the jobs to be run
# Note the exception catching to give more information about a job failing
for job in jobs:
time_start = time.time()
'''
Created on 17/11/2009
@author: markus
'''
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
HttpProxyManager.useSystemHttpProxy()
si = LoginManager.loginCommandline("bestgrid")
# create the job object
job = JobObject(si)
# set a unique jobname
job.setUniqueJobname("echo_job1")
# set the commandline that needs to be executed
job.setCommandline("echo \"Hello World\"")
# create the job on the backend and specify the VO to use
job.createJob("/nz/nesi")
# submit the job
job.submitJob()
# this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)
finished = job.waitForJobToFinish(10)
if not finished:
from grisu.Grython import serviceInterface as si
from grisu.frontend.model.job import JobObject
from grisu.model import FileManager
import sys
print 'Parsing commandline arguments...'
file1url = sys.argv[1]
file1Name = FileManager.getFilename(file1url)
file2url = sys.argv[2]
file2Name = FileManager.getFilename(file2url);
print 'Creating job...'
# create the job object
job = JobObject(si);
# set a unique jobname
job.setTimestampJobname("diff_job")
print 'Set jobname to: '+ job.getJobname()
# set the commandline that needs to be executed
job.setCommandline('diff ' + file1Name+ ' ' + file2Name)
job.addInputFileUrl(file1url);
job.addInputFileUrl(file2url);
# create the job on the backend
job.createJob()
print 'Submitting job...'
# submit the job
job.submitJob()
def submit(self):
if self.jobs:
print 'There is already a run with the name "'+self.jobname+'". Exiting...'
sys.exit(1)
# uploading input file once, so we don't need to do it for every job again and again
fm.cp(self.filepath, 'gsiftp://pan.nesi.org.nz/~/inputfiles/'+self.jobname, True)
for i in range(0,self.runs):
# create the job object
job = JobObject(si);
# set a unique jobname
number = str(i+1).zfill(4)
job.setUniqueJobname(self.jobname+"_run_"+number)
# set the commandline that needs to be executed
job.setCommandline(megabeast_path+' '+remote_home_dir+'/inputfiles/'+self.jobname+'/'+self.filename)
job.setSubmissionLocation('pan:pan.nesi.org.nz')
job.setCpus(self.cpus)
job.setWalltime(self.walltime)
job.setMemory(self.memory)
job.setApplication('UltraBEAST')
job.setApplicationVersion('0.1')
#job.addInputFileUrl(self.filepath)
# create the job on the backend and specify the VO to use
temp_jobname = job.createJob("/nz/nesi")
print "created job: '"+temp_jobname+"', submitting..."
# submit the job
job.submitJob()
print "submission finished: " + temp_jobname
'''
Created on 17/11/2009
@author: markus
'''
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
si = LoginManager.loginCommandline()
print 'Creating job...'
# create the job object
job = JobObject(si)
# set a unique jobname
job.setUniqueJobname("echo_job1")
print 'Set jobname to: ' + job.getJobname()
# set the name of the application like it is published in mds. "generic" means not to use mds for the lookup.
job.setApplication("generic")
# since we are using a "generic" job, we need to specify a submission location. I'll make that easier later on...
job.setSubmissionLocation("dque@edda-m:ng2.vpac.org")
# set the commandline that needs to be executed
job.setCommandline("echo \"Hello World\"")
job.addInputFileUrl('/home/markus/test/singleJobFile_0.txt')
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/NGAdmin")
print 'Submitting job...'
batch_job_name = "test_batch"
# to see whats going on we add a simple event listener. Hm. This doesn't seem to work reliably in jython.
#SystemOutMultiJobLogger(multiJobName)
# create the multipart job
batch_job = BatchJobObject(si, batch_job_name, "/nz/nesi", "cat",
Constants.NO_VERSION_INDICATOR_STRING)
# now we can calculate the relative path (from every job directory) to the common input file folder
pathToInputFiles = batchJob.pathToInputFiles()
for i in range(0, numberOfJobs):
# create the single job
job = JobObject(si)
# better to set the application to use explicitely because in that case we don't need to use mds (faster)
job.setCommandline('cat ' + pathToInputFiles + 'commonFile.txt ' +
'singleJobFile.txt')
# adding a job-specific input file
job.addInputFileUrl("/home/markus/tmp/singleJobFile.txt")
# adding the job to the multijob
batch_job.addJob(job)
# now we are adding a file that can be used by all of the child jobs. it needs to be referenced via the pathToInputFiles() method shown above
batch_job.addInputFile('/home/markus/tmp/commonJobFile.txt')
batch_job.setDefaultNoCpus(1)
batch_job.setDefaultWalltimeInSeconds(60)
try:
import random
import sys
# si stands for serviceInterface and holds all session information
si = LoginManager.loginCommandline('bestgrid')
amount_of_jobs_total = 10
amount_of_jobs_concurrent = 4
submitted_jobs = []
# better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though)
jobname_base = 'staging_test'
for total in range(1, amount_of_jobs_total+1):
job = JobObject(si)
job.setJobname(jobname_base+'_'+str(total))
# always good to set the application if you know it, processing the job will be faster
job.setApplication('UnixCommands')
# also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster
job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz')
# create a random sized outfile
mbsize = 1024*1024
random_size = random.randrange(10, 100)
size_string = str(mbsize*random_size)
job.setCommandline('dd if=/dev/zero of=outfile bs='+size_string+' count=1')
job.createJob('/nz/nesi')
job.submitJob()
print 'created and submitted job: '+ job.getJobname()+' (size: '+str(random_size)+'mb)'
# si stands for serviceInterface and holds all session information
si = LoginManager.loginCommandline('bestgrid-test')
amount_of_jobs_total = 10
amount_of_jobs_concurrent = 4
created_jobs = []
submitted_jobs = []
finished_jobs = []
# better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though)
jobname_base = 'workflow_test'
for total in range(1, amount_of_jobs_total + 1):
job = JobObject(si)
job.setJobname(jobname_base + '_' + str(total))
# always good to set the application if you know it, processing the job will be faster
job.setApplication('UnixCommands')
# also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster
job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz')
# job sleeps for a random time
random_sleep = random.randrange(5, 75)
job.setCommandline('sleep ' + str(random_sleep))
job.createJob('/nz/nesi')
print 'created job: ' + job.getJobname() + ' (sleeptime: ' + str(
random_sleep) + ')'
created_jobs.append(job)
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
from grisu.model import GrisuRegistryManager
si = LoginManager.loginCommandline("LOCAL")
appInfo = GrisuRegistryManager.getDefault(si).getApplicationInformation("ImageJ")
print 'app: '+appInfo.getApplicationName()
for subloc in appInfo.getAvailableAllSubmissionLocations():
print subloc
job = JobObject(si);
job.setTimestampJobname("imageJ");
job.setApplication("java");
job.setApplication("ImageJ");
job.setCommandline("echo Hello");
job.setSubmissionLocation("normal:ng2.ivec.org");
job.createJob("/ARCS/StartUp");
job.submitJob();
job.waitForJobToFinish(3);
print "Stdout: "+job.getStdOutContent()
print "Stderr: "+job.getStdErrContent()
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
si = LoginManager.loginCommandline("BeSTGRID-DEV")
print "Logged in."
job = JobObject(si)
job.setUniqueJobname("cat_job", si)
job.setCommandline("cat text0.txt")
job.addInputFileUrl("/home/markus/tmp/text0.txt")
job.createJob("/nz/nesi")
# job.setSubmissionLocation('[email protected]:ng2.auckland.ac.nz')
job.submitJob()
print "Job submitted."
job.waitForJobToFinish(10)
print "Job finished. Status: " + job.getStatusString(False)
print "Stdout: " + job.getStdOutContent()
print "Stderr: " + job.getStdErrContent()
job.kill(True)
def prepare(self):
job = JobObject(self.si)
job.setUniqueJobname("echo_job1")
job.setCommandline('echo "Hello World"')
job.addInputFileUrl("/home/markus/tmp/text0.txt")
job.addInputFileUrl("/home/markus/tmp/text1.txt")
job.addInputFileUrl("/home/markus/tmp/text2.txt")
job.addInputFileUrl("/home/markus/tmp/text3.txt")
job.addInputFileUrl("/home/markus/tmp/text4.txt")
job.setApplication(Constants.GENERIC_APPLICATION_NAME)
job.setSubmissionLocation(self.subLoc)
job.createJob()
self.job = job
si = LoginManager.loginCommandline('bestgrid')
uem = GrisuRegistryManager.getDefault(si).getUserEnvironmentManager()
fm = GrisuRegistryManager.getDefault(si).getFileManager()
allJobs = uem.getCurrentJobs(False)
myJobs = []
finishedJobs = []
# getting all the jobs for the run we are interested in
for job in allJobs:
name = job.jobname()
if name.startswith('staging_test'):
tempJob = JobObject(si, name)
myJobs.append(tempJob)
fileUrls = []
# we could just wait for all the jobs, but looping makes more sense
# since we can stage out finished jobs while some other jobs of the batch
# are still running
while len(myJobs) > 0:
for job in myJobs:
if job.isFinished():
if job.isFailed(False):
print 'Job '+job.getJobname()+' failed. Not doing anything...'
myJobs.remove(job)
inputFile1relPath = pathToInputFiles+'inputFile1.txt '
inputFile2Url = 'gsiftp://ng2.vpac.org/home/grid-vpac/DC_au_DC_org_DC_arcs_DC_slcs_O_VPAC_CN_Markus_Binsteiner_qTrDzHY7L1aKo3WSy8623-7bjgM/inputFile2.txt'
inputFile2relPath = pathToInputFiles+'inputFile2.txt'
inputFile3Url = '/home/markus/test/errorFile.txt'
inputFile3relPath = pathToInputFiles + 'errorFile.txt'
for i in range(0, numberOfJobs):
# create a unique jobname for every job
jobname = batchJobName+"_"+ str(i)
print 'Creating job: '+jobname
# create the single job
job = JobObject(si)
job.setJobname(jobname)
# better to set the application to use explicitely because in that case we don't need to use mds (faster)
job.setApplication('UnixCommands')
if i == 3 or i == 13:
# this is just to demonstrate how to restart a failed job later on
job.setCommandline('cat '+inputFile3relPath)
else:
job.setCommandline('cat '+ inputFile1relPath + ' ' + inputFile2relPath)
job.setWalltimeInSeconds(60)
# adding the job to the multijob
batchJob.addJob(job)
# this should be set because it's used for the matchmaking/metascheduling
batchJob.setDefaultNoCpus(1);
# get the files that i should have
outfile = sys.argv[1]
errfile = sys.argv[2]
error_codefile = sys.argv[3]
job_name = sys.argv[4]
working_directory = sys.argv[5]
output_files = list()
# get list of output files for this job
for f in sys.argv[6:]:
output_files.append(f)
print output_files
job = JobObject(si, job_name)
# Save stdout and stderr to files to be read by galaxy
try:
out = open(outfile, "w")
out.write(job.getStdOutContent())
out.close()
except:
print "Cannot open files to write results to"
sys.exit(-2)
try:
err = open(errfile, "w")
err.write(job.getStdErrContent())
except:
# There is no stderr so just write blank file
print "No stderr So just writing blakn file"
# Setting the number of jobs to be submitted
job_count = 5
# Since there may be many jobs submitted in the workshop, lets make them a bit unique to avoid issues.
base_job_name = str(random.randint(10000, 99999)) + '-hello-'
print "INFO: Base job name is " + base_job_name
# There are three stages, creating the jobs, submitting the jobs, then after they have finished, retrieving the job outputs
# Creating a list of jobs
jobs = []
print "INFO: Defining " + str(job_count) + " helloworld jobs"
for i in range(1, job_count + 1):
print "INFO: Defining job " + str(i) + " of " + str(job_count)
#The next lines define the actual job's parameters
job = JobObject(service_interface) # Create a job
job.setJobname(base_job_name + str(i)) # Give it a (hopefully) unique name
job.setApplication("python") # Set the application being run
job.setApplicationVersion("2.4") # Set the application version, note this is an exact match
# job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE") # Set the location the job will be submitted to
job.addInputFileUrl(os.path.join(current_dir, "helloworld.py")) # Add a file
job.setCommandline("python helloworld.py") # Set the command to be run
print"INFO: job " + job.getJobname() + " defined"
jobs.append(job)
# Submit the jobs to be run
# Note the exception catching to give more information about a job failing
for job in jobs:
time_start = time.time()
try:
print "INFO: Creating job " + job.getJobname() + " on " + backend + " backend, with " + group + " group"
class simpleStageJob(action):
def __init__(self, si, subLoc=defaultSubLoc):
super(simpleStageJob, self).__init__(si, subLoc)
self.job = JobObject(self.si)
self.job.setCommandline("echo \"hello world\"")
self.job.setApplication(Constants.GENERIC_APPLICATION_NAME)
self.job.setSubmissionLocation(self.subLoc)
def prepare(self):
self.job.setUniqueJobname(self.name())
self.job.createJob()
def execute(self):
self.job.submitJob()
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
si = LoginManager.loginCommandline("Local")
folder = "/home/markus/test/batch/"
original = folder + "original.txt"
jobnames = []
for i in range(10):
file = folder + "test" + str(i) + ".txt"
job = JobObject(si)
job.setUniqueJobname("diff_" + str(i))
job.setApplication("UnixCommands")
job.setCommandline("diff original.txt test" + str(i) + ".txt")
job.createJob("/ARCS/BeSTGRID")
job.addInputFileUrl(file)
job.addInputFileUrl(original)
job.submitJob()
jobnames.append(job.getJobname())
for jobname in jobnames:
finished = job.waitForJobToFinish(10)
# si stands for serviceInterface and holds all session information
si = LoginManager.loginCommandline('bestgrid-test')
amount_of_jobs_total = 10
amount_of_jobs_concurrent = 4
created_jobs = []
submitted_jobs = []
finished_jobs = []
# better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though)
jobname_base = 'workflow_test'
for total in range(1, amount_of_jobs_total+1):
job = JobObject(si)
job.setJobname(jobname_base+'_'+str(total))
# always good to set the application if you know it, processing the job will be faster
job.setApplication('UnixCommands')
# also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster
job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz')
# job sleeps for a random time
random_sleep = random.randrange(5, 75)
job.setCommandline('sleep '+str(random_sleep))
job.createJob('/nz/nesi')
print 'created job: '+ job.getJobname()+' (sleeptime: '+str(random_sleep)+')'
created_jobs.append(job)
finished = False
def prepare(self):
job = JobObject(self.si);
job.setUniqueJobname("echo_job1")
job.setCommandline("echo \"Hello World\"")
job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text0.txt");
job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text1.txt");
job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text2.txt");
job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text3.txt");
job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text4.txt");
job.setApplication(Constants.GENERIC_APPLICATION_NAME)
job.setSubmissionLocation(self.subLoc)
job.createJob()
self.job = job
from grisu.frontend.model.job import JobObject
from grisu.model import FileManager
import sys
si = LoginManager.loginCommandline()
print "Parsing commandline arguments..."
file1url = sys.argv[1]
file1Name = FileManager.getFilename(file1url)
file2url = sys.argv[2]
file2Name = FileManager.getFilename(file2url)
print "Creating job..."
# create the job object
job = JobObject(si)
# set a unique jobname
job.setTimestampJobname("diff_job")
print "Set jobname to: " + job.getJobname()
# setting the application. this means that grisu can figure out the submission location and
# you don't have to do that manually
job.setApplication("UnixCommands")
# set the commandline that needs to be executed
job.setCommandline("diff " + file1Name + " " + file2Name)
job.addInputFileUrl(file1url)
job.addInputFileUrl(file2url)
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/StartUp")
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
si = LoginManager.loginCommandline("Local")
folder = '/home/markus/test/batch/'
original = folder + 'original.txt'
jobnames = []
for i in range(10):
file = folder+'test'+str(i)+'.txt'
job = JobObject(si);
job.setUniqueJobname('diff_'+str(i))
job.setApplication('UnixCommands')
job.setCommandline('diff original.txt test'+str(i)+'.txt')
job.createJob("/ARCS/BeSTGRID")
job.addInputFileUrl(file)
job.addInputFileUrl(original)
job.submitJob()
jobnames.append(job.getJobname())
for jobname in jobnames:
finished = job.waitForJobToFinish(10)
class simpleStageJob(action):
def __init__(self, si, subLoc=defaultSubLoc):
super(simpleStageJob, self).__init__(si, subLoc)
self.job = JobObject(self.si)
self.job.setCommandline('echo "hello world"')
self.job.setApplication(Constants.GENERIC_APPLICATION_NAME)
self.job.setSubmissionLocation(self.subLoc)
def prepare(self):
self.job.setUniqueJobname(self.name())
self.job.createJob()
def execute(self):
self.job.submitJob()
'''
@author: markus
'''
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
si = LoginManager.loginCommandline("Local")
folder = '/home/markus/test/batchWrap/'
original = folder + 'original.txt'
script = folder + 'wrap.sh'
job = JobObject(si);
job.setUniqueJobname('diff_wrap')
job.setApplication('UnixCommands')
job.setCommandline('sh wrap.sh 0 9')
job.addInputFileUrl(original)
job.addInputFileUrl(script)
for i in range(10):
file = folder+'test'+str(i)+'.txt'
job.addInputFileUrl(file)
job.createJob("/ARCS/BeSTGRID")
job.submitJob()
finished = job.waitForJobToFinish(10)
Created on 17/11/2009
@author: markus
"""
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
HttpProxyManager.useSystemHttpProxy()
si = LoginManager.loginCommandline("bestgrid")
# create the job object
job = JobObject(si)
# set a unique jobname
job.setUniqueJobname("echo_job1")
# set the commandline that needs to be executed
job.setCommandline('echo "Hello World"')
# create the job on the backend and specify the VO to use
job.createJob("/nz/nesi")
# submit the job
job.submitJob()
# this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)
finished = job.waitForJobToFinish(10)
if not finished:
print 'starting job creation...' group = '/nz/nesi' #sub_loc = '[email protected]:ng2.auckland.ac.nz' batch_job = BatchJobObject(si, basename, group, 'R', Constants.NO_VERSION_INDICATOR_STRING) batch_job_name = batch_job.getJobname() print 'jobname on backend: ' + batch_job_name path_to_inputfile = batch_job.pathToInputFiles() + inputfilename for i in range(1, gen_jobs + 1): job = JobObject(si) job.setEmail_address(email) job.setEmail_on_job_finish(True) job.setCommandline('R --no-readline --no-restore --no-save -f ' + path_to_inputfile) batch_job.addJob(job) batch_job.addInputFile('/home/markus/Desktop/R/' + inputfilename) batch_job.setDefaultNoCpus(1) batch_job.setDefaultWalltimeInSeconds(walltime) print 'preparing jobs on backend...' batch_job.prepareAndCreateJobs(redistribute)
inputFile1relPath = pathToInputFiles + 'inputFile1.txt '
inputFile2Url = 'gsiftp://ng2.vpac.org/home/grid-vpac/DC_au_DC_org_DC_arcs_DC_slcs_O_VPAC_CN_Markus_Binsteiner_qTrDzHY7L1aKo3WSy8623-7bjgM/inputFile2.txt'
inputFile2relPath = pathToInputFiles + 'inputFile2.txt'
inputFile3Url = '/home/markus/test/errorFile.txt'
inputFile3relPath = pathToInputFiles + 'errorFile.txt'
for i in range(0, numberOfJobs):
# create a unique jobname for every job
jobname = batchJobName + "_" + str(i)
print 'Creating job: ' + jobname
# create the single job
job = JobObject(si)
job.setJobname(jobname)
# better to set the application to use explicitely because in that case we don't need to use mds (faster)
job.setApplication('UnixCommands')
if i == 3 or i == 13:
# this is just to demonstrate how to restart a failed job later on
job.setCommandline('cat ' + inputFile3relPath)
else:
job.setCommandline('cat ' + inputFile1relPath + ' ' +
inputFile2relPath)
job.setWalltimeInSeconds(60)
# adding the job to the multijob
batchJob.addJob(job)
# this should be set because it's used for the matchmaking/metascheduling
start = 30
end = 40
pathToInputFiles = batchJob.pathToInputFiles()
inputFile1relPath = pathToInputFiles + 'inputFile1.txt '
inputFile2relPath = pathToInputFiles + 'inputFile2.txt'
for i in range(start, end):
# create a unique jobname for every job
jobname = batchJobName + "_" + str(i)
print 'Creating job: ' + jobname
# create the single job
job = JobObject(si)
job.setJobname(jobname)
# better to set the application to use explicitely because in that case we don't need to use mds (faster)
job.setApplication('UnixCommands')
job.setCommandline('cat ' + inputFile1relPath + ' ' + inputFile2relPath)
job.setWalltimeInSeconds(60)
# adding the job to the multijob
batchJob.addJob(job)
# only start the newly added jobs and wait for the restart to finish
batchJob.restart(False, False, True, True)
# don't forget to exit properly. this cleans up possible existing threads/executors
sys.exit()
'''
Created on 17/11/2009
@author: markus
'''
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
si = LoginManager.loginCommandline()
print 'Creating job...'
# create the job object
job = JobObject(si);
# set a unique jobname
job.setUniqueJobname("echo_job1")
print 'Set jobname to: '+ job.getJobname()
# set the name of the application like it is published in mds. "generic" means not to use mds for the lookup.
job.setApplication("generic")
# since we are using a "generic" job, we need to specify a submission location. I'll make that easier later on...
job.setSubmissionLocation("dque@edda-m:ng2.vpac.org")
# set the commandline that needs to be executed
job.setCommandline("echo \"Hello World\"")
job.addInputFileUrl('/home/markus/test/singleJobFile_0.txt');
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/NGAdmin")
print 'Submitting job...'
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
si = LoginManager.loginCommandline("BeSTGRID-DEV")
print 'Logged in.'
job = JobObject(si)
job.setUniqueJobname("cat_job", si)
job.setCommandline("cat text0.txt")
job.addInputFileUrl('/home/markus/tmp/text0.txt')
job.createJob("/nz/nesi")
#job.setSubmissionLocation('[email protected]:ng2.auckland.ac.nz')
job.submitJob()
print 'Job submitted.'
job.waitForJobToFinish(10)
print 'Job finished. Status: ' + job.getStatusString(False)
print "Stdout: " + job.getStdOutContent()
print "Stderr: " + job.getStdErrContent()
job.kill(True)
start = 30
end = 40
pathToInputFiles = batchJob.pathToInputFiles()
inputFile1relPath = pathToInputFiles+'inputFile1.txt '
inputFile2relPath = pathToInputFiles+'inputFile2.txt'
for i in range(start, end):
# create a unique jobname for every job
jobname = batchJobName+"_"+ str(i)
print 'Creating job: '+jobname
# create the single job
job = JobObject(si)
job.setJobname(jobname)
# better to set the application to use explicitely because in that case we don't need to use mds (faster)
job.setApplication('UnixCommands')
job.setCommandline('cat '+ inputFile1relPath + ' ' + inputFile2relPath)
job.setWalltimeInSeconds(60)
# adding the job to the multijob
batchJob.addJob(job)
# only start the newly added jobs and wait for the restart to finish
batchJob.restart(False, False, True, True)
# don't forget to exit properly. this cleans up possible existing threads/executors
sys.exit()
# how many jobs do we want
numberOfJobs = 10
# the (unique) name of the batchjob
batch_job_name = "test_batch";
# create the batchjob
batch_job = BatchJobObject(si, batch_job_name, "/nz/nesi", "UnixCommands", Constants.NO_VERSION_INDICATOR_STRING);
# now we can calculate the relative path (from every job directory) to the common input file folder
pathToInputFiles = batch_job.pathToInputFiles()
for i in range(0, numberOfJobs):
# create the single job
job = JobObject(si)
# better to set the application to use explicitely because in that case we don't need to use mds (faster)
job.setCommandline('cat ' + pathToInputFiles+'commonJobFile.txt ' + 'singleJobFile.txt')
# adding a job-specific input file
job.addInputFileUrl("/home/markus/tmp/singleJobFile.txt")
# adding the job to the multijob
batch_job.addJob(job)
# now we are adding a file that can be used by all of the child jobs. it needs to be referenced via the pathToInputFiles() method shown above
batch_job.addInputFile('/home/markus/tmp/commonJobFile.txt')
batch_job.setDefaultNoCpus(1);
batch_job.setDefaultWalltimeInSeconds(60);
batch_job.setLocationsToExclude(["gt5test:ng1.canterbury.ac.nz"])
try:
if isfile(line):
files_to_upload.append(line)
else:
print 'Not a file: '+line
sys.exit(1)
return files_to_upload
files_to_upload = read_files(files_file)
if not si:
LoginManager.initEnvironment()
si = LoginManager.login('bestgrid', True)
filemanager = GrisuRegistryManager.getDefault(si).getFileManager()
job = JobObject(si)
job.setSubmissionLocation('pan:pan.nesi.org.nz')
job.setTimestampJobname(jobname_template)
job.setCommandline(commandline)
# add input files
for file in files_to_upload:
job.addInputFileUrl(file)
jobname = job.createJob('/nz/nesi')
print 'Submitting job...'
job.submitJob()
print 'Jobname: '+jobname
print 'Waiting for job to finish...'
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
# create a service interface to the BeSTGRID backend
service_interface = LoginManager.loginCommandline("BeSTGRID")
print 'Creating job object...'
job = JobObject(service_interface)
job.setJobname("echo_job-1") # job name must be unique
print 'Set jobname to: ' + job.getJobname()
# set the name of the application as it is published in MDS.
# "generic" means not to use MDS for the lookup.
job.setApplication("generic")
# "generic" jobs require a submission location to be specified
job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE")
# set the command that needs to be executed
job.setCommandline("echo \"Hello World\"")
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/BeSTGRID")
print 'Submitting job...'
# submit the job
job.submitJob()
print 'Waiting for the job to finish...'
# this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)
finished = job.waitForJobToFinish(10)
print 'starting job creation...' group = '/nz/nesi' #sub_loc = '[email protected]:ng2.auckland.ac.nz' batch_job = BatchJobObject(si, basename, group, 'R', Constants.NO_VERSION_INDICATOR_STRING) batch_job_name = batch_job.getJobname() print 'jobname on backend: '+batch_job_name path_to_inputfile = batch_job.pathToInputFiles()+inputfilename for i in range(1,gen_jobs+1): job = JobObject(si) job.setEmail_address(email) job.setEmail_on_job_finish(True) job.setCommandline('R --no-readline --no-restore --no-save -f '+path_to_inputfile) batch_job.addJob(job) batch_job.addInputFile('/home/markus/Desktop/R/'+inputfilename) batch_job.setDefaultNoCpus(1) batch_job.setDefaultWalltimeInSeconds(walltime) print 'preparing jobs on backend...' batch_job.prepareAndCreateJobs(redistribute)
import random
import sys
# si stands for serviceInterface and holds all session information
si = LoginManager.loginCommandline('bestgrid')
amount_of_jobs_total = 10
amount_of_jobs_concurrent = 4
submitted_jobs = []
# better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though)
jobname_base = 'staging_test'
for total in range(1, amount_of_jobs_total + 1):
job = JobObject(si)
job.setJobname(jobname_base + '_' + str(total))
# always good to set the application if you know it, processing the job will be faster
job.setApplication('UnixCommands')
# also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster
job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz')
# create a random sized outfile
mbsize = 1024 * 1024
random_size = random.randrange(10, 100)
size_string = str(mbsize * random_size)
job.setCommandline('dd if=/dev/zero of=outfile bs=' + size_string +
' count=1')
job.createJob('/nz/nesi')
job.submitJob()
print 'created and submitted job: ' + job.getJobname() + ' (size: ' + str(
files_to_upload.append(line)
else:
print 'Not a file: ' + line
sys.exit(1)
return files_to_upload
files_to_upload = read_files(files_file)
if not si:
LoginManager.initEnvironment()
si = LoginManager.login('bestgrid', True)
filemanager = GrisuRegistryManager.getDefault(si).getFileManager()
job = JobObject(si)
job.setSubmissionLocation('pan:pan.nesi.org.nz')
job.setTimestampJobname(jobname_template)
job.setCommandline(commandline)
# add input files
for file in files_to_upload:
job.addInputFileUrl(file)
jobname = job.createJob('/nz/nesi')
print 'Submitting job...'
job.submitJob()
print 'Jobname: ' + jobname
print 'Waiting for job to finish...'
job.waitForJobToFinish(jobstate_check_intervall)
'''
@author: markus
'''
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
si = LoginManager.loginCommandline("Local")
folder = '/home/markus/test/batchWrap/'
original = folder + 'original.txt'
script = folder + 'wrap.sh'
job = JobObject(si)
job.setUniqueJobname('diff_wrap')
job.setApplication('UnixCommands')
job.setCommandline('sh wrap.sh 0 9')
job.addInputFileUrl(original)
job.addInputFileUrl(script)
for i in range(10):
file = folder + 'test' + str(i) + '.txt'
job.addInputFileUrl(file)
job.createJob("/ARCS/BeSTGRID")
job.submitJob()
finished = job.waitForJobToFinish(10)
print 'Job: ' + job.getJobname()
