def prepare(self): job = JobObject(self.si) job.setUniqueJobname("echo_job1") job.setCommandline('echo "Hello World"') job.addInputFileUrl("/home/markus/tmp/46mbInput0.bin") job.setApplication(Constants.GENERIC_APPLICATION_NAME) job.setSubmissionLocation(self.subLoc) job.createJob() self.job = job
def prepare(self): job = JobObject(self.si) job.setUniqueJobname("echo_job1") job.setCommandline("echo \"Hello World\"") job.addInputFileUrl("/home/markus/tmp/46mbInput0.bin") job.setApplication(Constants.GENERIC_APPLICATION_NAME) job.setSubmissionLocation(self.subLoc) job.createJob() self.job = job
class simpleStageJob(action): def __init__(self, si, subLoc=defaultSubLoc): super(simpleStageJob, self).__init__(si, subLoc) self.job = JobObject(self.si) self.job.setCommandline('echo "hello world"') self.job.setApplication(Constants.GENERIC_APPLICATION_NAME) self.job.setSubmissionLocation(self.subLoc) def prepare(self): self.job.setUniqueJobname(self.name()) self.job.createJob() def execute(self): self.job.submitJob()
class simpleStageJob(action): def __init__(self, si, subLoc=defaultSubLoc): super(simpleStageJob, self).__init__(si, subLoc) self.job = JobObject(self.si) self.job.setCommandline("echo \"hello world\"") self.job.setApplication(Constants.GENERIC_APPLICATION_NAME) self.job.setSubmissionLocation(self.subLoc) def prepare(self): self.job.setUniqueJobname(self.name()) self.job.createJob() def execute(self): self.job.submitJob()
def prepare(self): job = JobObject(self.si); job.setUniqueJobname("echo_job1") job.setCommandline("echo \"Hello World\"") job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text0.txt"); job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text1.txt"); job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text2.txt"); job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text3.txt"); job.addInputFileUrl("gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text4.txt"); job.setApplication(Constants.GENERIC_APPLICATION_NAME) job.setSubmissionLocation(self.subLoc) job.createJob() self.job = job
def prepare(self): job = JobObject(self.si) job.setUniqueJobname("echo_job1") job.setCommandline("echo \"Hello World\"") job.addInputFileUrl( "gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text0.txt") job.addInputFileUrl( "gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text1.txt") job.addInputFileUrl( "gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text2.txt") job.addInputFileUrl( "gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text3.txt") job.addInputFileUrl( "gsiftp://ng2.canterbury.ac.nz/home/gridcloud061/tmp/text4.txt") job.setApplication(Constants.GENERIC_APPLICATION_NAME) job.setSubmissionLocation(self.subLoc) job.createJob() self.job = job
def submit(self): if self.jobs: print 'There is already a run with the name "'+self.jobname+'". Exiting...' sys.exit(1) # uploading input file once, so we don't need to do it for every job again and again fm.cp(self.filepath, 'gsiftp://pan.nesi.org.nz/~/inputfiles/'+self.jobname, True) for i in range(0,self.runs): # create the job object job = JobObject(si); # set a unique jobname number = str(i+1).zfill(4) job.setUniqueJobname(self.jobname+"_run_"+number) # set the commandline that needs to be executed job.setCommandline(megabeast_path+' '+remote_home_dir+'/inputfiles/'+self.jobname+'/'+self.filename) job.setSubmissionLocation('pan:pan.nesi.org.nz') job.setCpus(self.cpus) job.setWalltime(self.walltime) job.setMemory(self.memory) job.setApplication('UltraBEAST') job.setApplicationVersion('0.1') #job.addInputFileUrl(self.filepath) # create the job on the backend and specify the VO to use temp_jobname = job.createJob("/nz/nesi") print "created job: '"+temp_jobname+"', submitting..." # submit the job job.submitJob() print "submission finished: " + temp_jobname
files_to_upload.append(line) else: print 'Not a file: '+line sys.exit(1) return files_to_upload files_to_upload = read_files(files_file) if not si: LoginManager.initEnvironment() si = LoginManager.login('bestgrid', True) filemanager = GrisuRegistryManager.getDefault(si).getFileManager() job = JobObject(si) job.setSubmissionLocation('pan:pan.nesi.org.nz') job.setTimestampJobname(jobname_template) job.setCommandline(commandline) # add input files for file in files_to_upload: job.addInputFileUrl(file) jobname = job.createJob('/nz/nesi') print 'Submitting job...' job.submitJob() print 'Jobname: '+jobname print 'Waiting for job to finish...' job.waitForJobToFinish(jobstate_check_intervall)
amount_of_jobs_concurrent = 4 created_jobs = [] submitted_jobs = [] finished_jobs = [] # better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though) jobname_base = 'workflow_test' for total in range(1, amount_of_jobs_total + 1): job = JobObject(si) job.setJobname(jobname_base + '_' + str(total)) # always good to set the application if you know it, processing the job will be faster job.setApplication('UnixCommands') # also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz') # job sleeps for a random time random_sleep = random.randrange(5, 75) job.setCommandline('sleep ' + str(random_sleep)) job.createJob('/nz/nesi') print 'created job: ' + job.getJobname() + ' (sleeptime: ' + str( random_sleep) + ')' created_jobs.append(job) finished = False while not finished: # submit another bunch of jobs while there are some if len(created_jobs) > 0:
from grisu.frontend.control.login import LoginManager from grisu.frontend.model.job import JobObject import sys si = LoginManager.loginCommandline() print 'Creating job...' # create the job object job = JobObject(si); # set a unique jobname job.setUniqueJobname("echo_job1") print 'Set jobname to: '+ job.getJobname() # set the name of the application like it is published in mds. "generic" means not to use mds for the lookup. job.setApplication("generic") # since we are using a "generic" job, we need to specify a submission location. I'll make that easier later on... job.setSubmissionLocation("dque@edda-m:ng2.vpac.org") # set the commandline that needs to be executed job.setCommandline("echo \"Hello World\"") job.addInputFileUrl('/home/markus/test/singleJobFile_0.txt'); # create the job on the backend and specify the VO to use job.createJob("/ARCS/NGAdmin") print 'Submitting job...' # submit the job job.submitJob() print 'Waiting for the job to finish...' # this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job) finished = job.waitForJobToFinish(10)
amount_of_jobs_concurrent = 4 created_jobs = [] submitted_jobs = [] finished_jobs = [] # better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though) jobname_base = 'workflow_test' for total in range(1, amount_of_jobs_total+1): job = JobObject(si) job.setJobname(jobname_base+'_'+str(total)) # always good to set the application if you know it, processing the job will be faster job.setApplication('UnixCommands') # also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz') # job sleeps for a random time random_sleep = random.randrange(5, 75) job.setCommandline('sleep '+str(random_sleep)) job.createJob('/nz/nesi') print 'created job: '+ job.getJobname()+' (sleeptime: '+str(random_sleep)+')' created_jobs.append(job) finished = False while not finished: # submit another bunch of jobs while there are some if len(created_jobs) > 0: print 'still '+str(len(created_jobs))+' jobs to submit...'
from grisu.frontend.control.login import LoginManager from grisu.frontend.model.job import JobObject from grisu.model import GrisuRegistryManager si = LoginManager.loginCommandline("LOCAL") appInfo = GrisuRegistryManager.getDefault(si).getApplicationInformation("ImageJ") print 'app: '+appInfo.getApplicationName() for subloc in appInfo.getAvailableAllSubmissionLocations(): print subloc job = JobObject(si); job.setTimestampJobname("imageJ"); job.setApplication("java"); job.setApplication("ImageJ"); job.setCommandline("echo Hello"); job.setSubmissionLocation("normal:ng2.ivec.org"); job.createJob("/ARCS/StartUp"); job.submitJob(); job.waitForJobToFinish(3); print "Stdout: "+job.getStdOutContent() print "Stderr: "+job.getStdErrContent()
import sys # create a service interface to the BeSTGRID backend service_interface = LoginManager.loginCommandline("BeSTGRID") print 'Creating job object...' job = JobObject(service_interface) job.setJobname("echo_job-1") # job name must be unique print 'Set jobname to: ' + job.getJobname() # set the name of the application as it is published in MDS. # "generic" means not to use MDS for the lookup. job.setApplication("generic") # "generic" jobs require a submission location to be specified job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE") # set the command that needs to be executed job.setCommandline("echo \"Hello World\"") # create the job on the backend and specify the VO to use job.createJob("/ARCS/BeSTGRID") print 'Submitting job...' # submit the job job.submitJob() print 'Waiting for the job to finish...' # this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job) finished = job.waitForJobToFinish(10) print 'Job finished. Status: ' + job.getStatusString(False)
input_files = list() job_header="""#!/bin/sh %s """ if group == '': group = DEFAULT_GROUP if queue == '': queue = DEFAULT_QUEUE for f in sys.argv[8:]: input_files.append(f) try: job = JobObject(si) job.setSubmissionLocation(queue) job.setTimestampJobname("galaxy_" + galaxy_job_id) job.setMemory(DEFAULT_MEMORY) job.setWalltimeInSeconds(DEFAULT_WALLTIME) # stop annoying stats from being written to stderr job.addEnvironmentVariable("SUPPRESS_STATS", "true") #create the job script# except: print "Cannot setup the job environment" sys.exit(-4) #create nesi job_script
from grisu.frontend.control.login import LoginManager from grisu.frontend.model.job import JobObject import sys si = LoginManager.loginCommandline() print 'Creating job...' # create the job object job = JobObject(si) # set a unique jobname job.setUniqueJobname("echo_job1") print 'Set jobname to: ' + job.getJobname() # set the name of the application like it is published in mds. "generic" means not to use mds for the lookup. job.setApplication("generic") # since we are using a "generic" job, we need to specify a submission location. I'll make that easier later on... job.setSubmissionLocation("dque@edda-m:ng2.vpac.org") # set the commandline that needs to be executed job.setCommandline("echo \"Hello World\"") job.addInputFileUrl('/home/markus/test/singleJobFile_0.txt') # create the job on the backend and specify the VO to use job.createJob("/ARCS/NGAdmin") print 'Submitting job...' # submit the job job.submitJob() print 'Waiting for the job to finish...' # this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job) finished = job.waitForJobToFinish(10)
from grisu.frontend.control.login import LoginManager from grisu.frontend.model.job import JobObject from grisu.model import GrisuRegistryManager si = LoginManager.loginCommandline("LOCAL") appInfo = GrisuRegistryManager.getDefault(si).getApplicationInformation( "ImageJ") print 'app: ' + appInfo.getApplicationName() for subloc in appInfo.getAvailableAllSubmissionLocations(): print subloc job = JobObject(si) job.setTimestampJobname("imageJ") job.setApplication("java") job.setApplication("ImageJ") job.setCommandline("echo Hello") job.setSubmissionLocation("normal:ng2.ivec.org") job.createJob("/ARCS/StartUp") job.submitJob() job.waitForJobToFinish(3) print "Stdout: " + job.getStdOutContent() print "Stderr: " + job.getStdErrContent()
else: print 'Not a file: ' + line sys.exit(1) return files_to_upload files_to_upload = read_files(files_file) if not si: LoginManager.initEnvironment() si = LoginManager.login('bestgrid', True) filemanager = GrisuRegistryManager.getDefault(si).getFileManager() job = JobObject(si) job.setSubmissionLocation('pan:pan.nesi.org.nz') job.setTimestampJobname(jobname_template) job.setCommandline(commandline) # add input files for file in files_to_upload: job.addInputFileUrl(file) jobname = job.createJob('/nz/nesi') print 'Submitting job...' job.submitJob() print 'Jobname: ' + jobname print 'Waiting for job to finish...' job.waitForJobToFinish(jobstate_check_intervall)
import sys # create a service interface to the BeSTGRID backend service_interface = LoginManager.loginCommandline("BeSTGRID") print 'Creating job object...' job = JobObject(service_interface); job.setJobname("echo_job-1") # job name must be unique print 'Set jobname to: '+ job.getJobname() # set the name of the application as it is published in MDS. # "generic" means not to use MDS for the lookup. job.setApplication("generic") # "generic" jobs require a submission location to be specified job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE") # set the command that needs to be executed job.setCommandline("echo \"Hello World\"") # create the job on the backend and specify the VO to use job.createJob("/ARCS/BeSTGRID") print 'Submitting job...' # submit the job job.submitJob() print 'Waiting for the job to finish...' # this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job) finished = job.waitForJobToFinish(10)