def prepare(self):
job = JobObject(self.si)
job.setUniqueJobname("echo_job1")
job.setCommandline('echo "Hello World"')
job.addInputFileUrl("/home/markus/tmp/46mbInput0.bin")
job.setApplication(Constants.GENERIC_APPLICATION_NAME)
job.setSubmissionLocation(self.subLoc)
job.createJob()
self.job = job
def prepare(self):
job = JobObject(self.si)
job.setUniqueJobname("echo_job1")
job.setCommandline("echo \"Hello World\"")
job.addInputFileUrl("/home/markus/tmp/46mbInput0.bin")
job.setApplication(Constants.GENERIC_APPLICATION_NAME)
job.setSubmissionLocation(self.subLoc)
job.createJob()
self.job = job
class simpleStageJob(action):
def __init__(self, si, subLoc=defaultSubLoc):
super(simpleStageJob, self).__init__(si, subLoc)
self.job = JobObject(self.si)
self.job.setCommandline('echo "hello world"')
self.job.setApplication(Constants.GENERIC_APPLICATION_NAME)
self.job.setSubmissionLocation(self.subLoc)
def prepare(self):
self.job.setUniqueJobname(self.name())
self.job.createJob()
def execute(self):
self.job.submitJob()
class simpleStageJob(action):
def __init__(self, si, subLoc=defaultSubLoc):
super(simpleStageJob, self).__init__(si, subLoc)
self.job = JobObject(self.si)
self.job.setCommandline("echo \"hello world\"")
self.job.setApplication(Constants.GENERIC_APPLICATION_NAME)
self.job.setSubmissionLocation(self.subLoc)
def prepare(self):
self.job.setUniqueJobname(self.name())
self.job.createJob()
def execute(self):
self.job.submitJob()
def prepare(self):
job = JobObject(self.si);
job.setUniqueJobname("echo_job1")
job.setCommandline("echo \"Hello World\"")
job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text0.txt");
job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text1.txt");
job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text2.txt");
job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text3.txt");
job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text4.txt");
job.setApplication(Constants.GENERIC_APPLICATION_NAME)
job.setSubmissionLocation(self.subLoc)
job.createJob()
self.job = job
def prepare(self):
job = JobObject(self.si)
job.setUniqueJobname("echo_job1")
job.setCommandline("echo \"Hello World\"")
job.addInputFileUrl(
"gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text0.txt")
job.addInputFileUrl(
"gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text1.txt")
job.addInputFileUrl(
"gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text2.txt")
job.addInputFileUrl(
"gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text3.txt")
job.addInputFileUrl(
"gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text4.txt")
job.setApplication(Constants.GENERIC_APPLICATION_NAME)
job.setSubmissionLocation(self.subLoc)
job.createJob()
self.job = job
def submit(self):
if self.jobs:
print 'There is already a run with the name "'+self.jobname+'". Exiting...'
sys.exit(1)
# uploading input file once, so we don't need to do it for every job again and again
fm.cp(self.filepath, 'gsiftp://pan.nesi.org.nz/~/inputfiles/'+self.jobname, True)
for i in range(0,self.runs):
# create the job object
job = JobObject(si);
# set a unique jobname
number = str(i+1).zfill(4)
job.setUniqueJobname(self.jobname+"_run_"+number)
# set the commandline that needs to be executed
job.setCommandline(megabeast_path+' '+remote_home_dir+'/inputfiles/'+self.jobname+'/'+self.filename)
job.setSubmissionLocation('pan:pan.nesi.org.nz')
job.setCpus(self.cpus)
job.setWalltime(self.walltime)
job.setMemory(self.memory)
job.setApplication('UltraBEAST')
job.setApplicationVersion('0.1')
#job.addInputFileUrl(self.filepath)
# create the job on the backend and specify the VO to use
temp_jobname = job.createJob("/nz/nesi")
print "created job: '"+temp_jobname+"', submitting..."
# submit the job
job.submitJob()
print "submission finished: " + temp_jobname
files_to_upload.append(line)
else:
print 'Not a file: '+line
sys.exit(1)
return files_to_upload
files_to_upload = read_files(files_file)
if not si:
LoginManager.initEnvironment()
si = LoginManager.login('bestgrid', True)
filemanager = GrisuRegistryManager.getDefault(si).getFileManager()
job = JobObject(si)
job.setSubmissionLocation('pan:pan.nesi.org.nz')
job.setTimestampJobname(jobname_template)
job.setCommandline(commandline)
# add input files
for file in files_to_upload:
job.addInputFileUrl(file)
jobname = job.createJob('/nz/nesi')
print 'Submitting job...'
job.submitJob()
print 'Jobname: '+jobname
print 'Waiting for job to finish...'
job.waitForJobToFinish(jobstate_check_intervall)
amount_of_jobs_concurrent = 4
created_jobs = []
submitted_jobs = []
finished_jobs = []
# better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though)
jobname_base = 'workflow_test'
for total in range(1, amount_of_jobs_total + 1):
job = JobObject(si)
job.setJobname(jobname_base + '_' + str(total))
# always good to set the application if you know it, processing the job will be faster
job.setApplication('UnixCommands')
# also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster
job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz')
# job sleeps for a random time
random_sleep = random.randrange(5, 75)
job.setCommandline('sleep ' + str(random_sleep))
job.createJob('/nz/nesi')
print 'created job: ' + job.getJobname() + ' (sleeptime: ' + str(
random_sleep) + ')'
created_jobs.append(job)
finished = False
while not finished:
# submit another bunch of jobs while there are some
if len(created_jobs) > 0:
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
si = LoginManager.loginCommandline()
print 'Creating job...'
# create the job object
job = JobObject(si);
# set a unique jobname
job.setUniqueJobname("echo_job1")
print 'Set jobname to: '+ job.getJobname()
# set the name of the application like it is published in mds. "generic" means not to use mds for the lookup.
job.setApplication("generic")
# since we are using a "generic" job, we need to specify a submission location. I'll make that easier later on...
job.setSubmissionLocation("dque@edda-m:ng2.vpac.org")
# set the commandline that needs to be executed
job.setCommandline("echo \"Hello World\"")
job.addInputFileUrl('/home/markus/test/singleJobFile_0.txt');
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/NGAdmin")
print 'Submitting job...'
# submit the job
job.submitJob()
print 'Waiting for the job to finish...'
# this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)
finished = job.waitForJobToFinish(10)
amount_of_jobs_concurrent = 4
created_jobs = []
submitted_jobs = []
finished_jobs = []
# better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though)
jobname_base = 'workflow_test'
for total in range(1, amount_of_jobs_total+1):
job = JobObject(si)
job.setJobname(jobname_base+'_'+str(total))
# always good to set the application if you know it, processing the job will be faster
job.setApplication('UnixCommands')
# also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster
job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz')
# job sleeps for a random time
random_sleep = random.randrange(5, 75)
job.setCommandline('sleep '+str(random_sleep))
job.createJob('/nz/nesi')
print 'created job: '+ job.getJobname()+' (sleeptime: '+str(random_sleep)+')'
created_jobs.append(job)
finished = False
while not finished:
# submit another bunch of jobs while there are some
if len(created_jobs) > 0:
print 'still '+str(len(created_jobs))+' jobs to submit...'
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
from grisu.model import GrisuRegistryManager
si = LoginManager.loginCommandline("LOCAL")
appInfo = GrisuRegistryManager.getDefault(si).getApplicationInformation("ImageJ")
print 'app: '+appInfo.getApplicationName()
for subloc in appInfo.getAvailableAllSubmissionLocations():
print subloc
job = JobObject(si);
job.setTimestampJobname("imageJ");
job.setApplication("java");
job.setApplication("ImageJ");
job.setCommandline("echo Hello");
job.setSubmissionLocation("normal:ng2.ivec.org");
job.createJob("/ARCS/StartUp");
job.submitJob();
job.waitForJobToFinish(3);
print "Stdout: "+job.getStdOutContent()
print "Stderr: "+job.getStdErrContent()
import sys
# create a service interface to the BeSTGRID backend
service_interface = LoginManager.loginCommandline("BeSTGRID")
print 'Creating job object...'
job = JobObject(service_interface)
job.setJobname("echo_job-1") # job name must be unique
print 'Set jobname to: ' + job.getJobname()
# set the name of the application as it is published in MDS.
# "generic" means not to use MDS for the lookup.
job.setApplication("generic")
# "generic" jobs require a submission location to be specified
job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE")
# set the command that needs to be executed
job.setCommandline("echo \"Hello World\"")
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/BeSTGRID")
print 'Submitting job...'
# submit the job
job.submitJob()
print 'Waiting for the job to finish...'
# this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)
finished = job.waitForJobToFinish(10)
print 'Job finished. Status: ' + job.getStatusString(False)
input_files = list()
job_header="""#!/bin/sh
%s
"""
if group == '':
group = DEFAULT_GROUP
if queue == '':
queue = DEFAULT_QUEUE
for f in sys.argv[8:]:
input_files.append(f)
try:
job = JobObject(si)
job.setSubmissionLocation(queue)
job.setTimestampJobname("galaxy_" + galaxy_job_id)
job.setMemory(DEFAULT_MEMORY)
job.setWalltimeInSeconds(DEFAULT_WALLTIME)
# stop annoying stats from being written to stderr
job.addEnvironmentVariable("SUPPRESS_STATS", "true")
#create the job script#
except:
print "Cannot setup the job environment"
sys.exit(-4)
#create nesi job_script
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
si = LoginManager.loginCommandline()
print 'Creating job...'
# create the job object
job = JobObject(si)
# set a unique jobname
job.setUniqueJobname("echo_job1")
print 'Set jobname to: ' + job.getJobname()
# set the name of the application like it is published in mds. "generic" means not to use mds for the lookup.
job.setApplication("generic")
# since we are using a "generic" job, we need to specify a submission location. I'll make that easier later on...
job.setSubmissionLocation("dque@edda-m:ng2.vpac.org")
# set the commandline that needs to be executed
job.setCommandline("echo \"Hello World\"")
job.addInputFileUrl('/home/markus/test/singleJobFile_0.txt')
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/NGAdmin")
print 'Submitting job...'
# submit the job
job.submitJob()
print 'Waiting for the job to finish...'
# this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)
finished = job.waitForJobToFinish(10)
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
from grisu.model import GrisuRegistryManager
si = LoginManager.loginCommandline("LOCAL")
appInfo = GrisuRegistryManager.getDefault(si).getApplicationInformation(
"ImageJ")
print 'app: ' + appInfo.getApplicationName()
for subloc in appInfo.getAvailableAllSubmissionLocations():
print subloc
job = JobObject(si)
job.setTimestampJobname("imageJ")
job.setApplication("java")
job.setApplication("ImageJ")
job.setCommandline("echo Hello")
job.setSubmissionLocation("normal:ng2.ivec.org")
job.createJob("/ARCS/StartUp")
job.submitJob()
job.waitForJobToFinish(3)
print "Stdout: " + job.getStdOutContent()
print "Stderr: " + job.getStdErrContent()
else:
print 'Not a file: ' + line
sys.exit(1)
return files_to_upload
files_to_upload = read_files(files_file)
if not si:
LoginManager.initEnvironment()
si = LoginManager.login('bestgrid', True)
filemanager = GrisuRegistryManager.getDefault(si).getFileManager()
job = JobObject(si)
job.setSubmissionLocation('pan:pan.nesi.org.nz')
job.setTimestampJobname(jobname_template)
job.setCommandline(commandline)
# add input files
for file in files_to_upload:
job.addInputFileUrl(file)
jobname = job.createJob('/nz/nesi')
print 'Submitting job...'
job.submitJob()
print 'Jobname: ' + jobname
print 'Waiting for job to finish...'
job.waitForJobToFinish(jobstate_check_intervall)
import sys
# create a service interface to the BeSTGRID backend
service_interface = LoginManager.loginCommandline("BeSTGRID")
print 'Creating job object...'
job = JobObject(service_interface);
job.setJobname("echo_job-1") # job name must be unique
print 'Set jobname to: '+ job.getJobname()
# set the name of the application as it is published in MDS.
# "generic" means not to use MDS for the lookup.
job.setApplication("generic")
# "generic" jobs require a submission location to be specified
job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE")
# set the command that needs to be executed
job.setCommandline("echo \"Hello World\"")
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/BeSTGRID")
print 'Submitting job...'
# submit the job
job.submitJob()
print 'Waiting for the job to finish...'
# this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)
finished = job.waitForJobToFinish(10)