'''
Created on 16/06/2010
@author: markus
'''
from exampleJobs import *
from grisu.frontend.control.login import LoginManager
from grisu.model import GrisuRegistryManager
import sys
# setup benchmark stuff
submitter = submitter()
si = LoginManager.loginCommandline("Local")
registry = GrisuRegistryManager.getDefault(si)
uem = registry.getUserEnvironmentManager()
uem.setCurrentFqan('/ARCS/NGAdmin')
#actionsGT4 = (simpleSubmitJob(si), simpleSubmitJobWith1SmallInputFile(si), simpleMdsSubmitJob(si), simpleMdsSubmitJobWith1SmallInputFile(si),
# simpleSubmitJobWith5SmallInputFiles(si), simpleSubmitJobWith1MediumInputFile(si),simpleSubmitJobWith5MediumInputFiles(si), simpleSubmitJobWith1_46mb_InputFile(si), simpleSubmitJobWith5_46mb_InputFiles(si) )
#actionsGT5 = (simpleSubmitJob(si, 'gt5test:ng1.canterbury.ac.nz'), simpleSubmitJobWith1SmallInputFile(si, 'gt5test:ng1.canterbury.ac.nz'), simpleMdsSubmitJob(si, 'gt5test:ng1.canterbury.ac.nz'), simpleMdsSubmitJobWith1SmallInputFile(si, 'gt5test:ng1.canterbury.ac.nz'),
# simpleSubmitJobWith5SmallInputFiles(si, 'gt5test:ng1.canterbury.ac.nz'), simpleSubmitJobWith1MediumInputFile(si, 'gt5test:ng1.canterbury.ac.nz'),simpleSubmitJobWith5MediumInputFiles(si, 'gt5test:ng1.canterbury.ac.nz'), simpleSubmitJobWith1_46mb_InputFile(si, 'gt5test:ng1.canterbury.ac.nz'), simpleSubmitJobWith5_46mb_InputFiles(si, 'gt5test:ng1.canterbury.ac.nz') )
actionsGT4 = (nonMdsJob(si), nonMdsJob5SmallInputFiles(si))
actionsGT5 = (nonMdsJob(si, 'gt5test:ng1.canterbury.ac.nz'),
nonMdsJob5SmallInputFiles(si, 'gt5test:ng1.canterbury.ac.nz'))
for index in range(len(actionsGT4)):
action = actionsGT4[index]
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import BatchJobObject, JobObject
import sys
batchJobName = sys.argv[1]
# display commandline login menu if no local proxy exists
si = LoginManager.loginCommandline()
batchJob = BatchJobObject(si, batchJobName, False)
start = 30
end = 40
pathToInputFiles = batchJob.pathToInputFiles()
inputFile1relPath = pathToInputFiles+'inputFile1.txt '
inputFile2relPath = pathToInputFiles+'inputFile2.txt'
for i in range(start, end):
# create a unique jobname for every job
jobname = batchJobName+"_"+ str(i)
print 'Creating job: '+jobname
# create the single job
job = JobObject(si)
job.setJobname(jobname)
# better to set the application to use explicitely because in that case we don't need to use mds (faster)
job.setApplication('UnixCommands')
job.setCommandline('cat '+ inputFile1relPath + ' ' + inputFile2relPath)
Simple example of a workflow that submits x amount of jobs, always y at a time.
Documentation: http://grisu.github.com/grisu/javadoc/
Created on 16/11/2011
@author: markus
'''
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import random
import sys
import time
# si stands for serviceInterface and holds all session information
si = LoginManager.loginCommandline('bestgrid-test')
amount_of_jobs_total = 10
amount_of_jobs_concurrent = 4
created_jobs = []
submitted_jobs = []
finished_jobs = []
# better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though)
jobname_base = 'workflow_test'
for total in range(1, amount_of_jobs_total + 1):
job = JobObject(si)
job.setJobname(jobname_base + '_' + str(total))
# always good to set the application if you know it, processing the job will be faster
import sys
import time
from exampleJobs import *
# setup benchmark stuff
submitter = submitter()
#siNames = ["ARCS", "BeSTGRID"]
siNames = ["Local"]
serviceInterfaces = []
for siName in siNames:
with timer:
temp = LoginManager.loginCommandline(siName)
print 'Login ' + siName + ': ' + str(timer.duration_in_seconds())
serviceInterfaces.append(temp)
GrisuRegistryManager.getDefault(
temp).getUserEnvironmentManager().setCurrentFqan("/ARCS/NGAdmin")
print
print
#actionClasses = (simpleSubmitJob, simpleSubmitJobWith1SmallInputFile, simpleSubmitJobWith5SmallInputFiles,
# simpleSubmitJobWith1MediumInputFile, simpleSubmitJobWith5MediumInputFiles, simpleSubmitJobWith1_46mb_InputFile, simpleSubmitJobWith5_46mb_InputFiles)
actionClasses = {
dynamicStageJob: [True, 1, 0, 0],
dynamicStageJob: [True, 5, 0, 0],
'''
from grisu.jcommons.constants import Constants
from grisu.control import ResubmitPolicy, ResubmitPolicy, \
DefaultResubmitPolicy
from grisu.control.exceptions import NoSuchJobException
from grisu.frontend.control.login import LoginManager, LoginParams
from grisu.frontend.model.job import JobObject, BatchJobObject, \
JobsException
from grisu.frontend.view.swing.jobmonitoring.batch import \
BatchJobDialog
from grisu.model import GrisuRegistryManager
import sys
import time
si = LoginManager.loginCommandline("BeSTGRID-DEV")
# how many jobs do we want
numberOfJobs = 10
# the (unique) name of the multijob
batch_job_name = "test_batch"
# to see whats going on we add a simple event listener. Hm. This doesn't seem to work reliably in jython.
#SystemOutMultiJobLogger(multiJobName)
# create the multipart job
batch_job = BatchJobObject(si, batch_job_name, "/nz/nesi", "cat",
Constants.NO_VERSION_INDICATOR_STRING)
# now we can calculate the relative path (from every job directory) to the common input file folder
'''
Created on 16/06/2010
@author: markus
'''
from exampleJobs import *
from grisu.frontend.control.login import LoginManager
from grisu.model import GrisuRegistryManager
import sys
# setup benchmark stuff
submitter = submitter()
si = LoginManager.loginCommandline("Local")
registry = GrisuRegistryManager.getDefault(si)
uem = registry.getUserEnvironmentManager()
uem.setCurrentFqan('/ARCS/NGAdmin')
#actionsGT4 = (simpleSubmitJob(si), simpleSubmitJobWith1SmallInputFile(si), simpleMdsSubmitJob(si), simpleMdsSubmitJobWith1SmallInputFile(si),
# simpleSubmitJobWith5SmallInputFiles(si), simpleSubmitJobWith1MediumInputFile(si),simpleSubmitJobWith5MediumInputFiles(si), simpleSubmitJobWith1_46mb_InputFile(si), simpleSubmitJobWith5_46mb_InputFiles(si) )
#actionsGT5 = (simpleSubmitJob(si, 'gt5test:ng1.canterbury.ac.nz'), simpleSubmitJobWith1SmallInputFile(si, 'gt5test:ng1.canterbury.ac.nz'), simpleMdsSubmitJob(si, 'gt5test:ng1.canterbury.ac.nz'), simpleMdsSubmitJobWith1SmallInputFile(si, 'gt5test:ng1.canterbury.ac.nz'),
# simpleSubmitJobWith5SmallInputFiles(si, 'gt5test:ng1.canterbury.ac.nz'), simpleSubmitJobWith1MediumInputFile(si, 'gt5test:ng1.canterbury.ac.nz'),simpleSubmitJobWith5MediumInputFiles(si, 'gt5test:ng1.canterbury.ac.nz'), simpleSubmitJobWith1_46mb_InputFile(si, 'gt5test:ng1.canterbury.ac.nz'), simpleSubmitJobWith5_46mb_InputFiles(si, 'gt5test:ng1.canterbury.ac.nz') )
actionsGT4 = (nonMdsJob(si), nonMdsJob5SmallInputFiles(si))
actionsGT5 = (nonMdsJob(si, 'gt5test:ng1.canterbury.ac.nz'), nonMdsJob5SmallInputFiles(si, 'gt5test:ng1.canterbury.ac.nz'))
for index in range(len(actionsGT4)):
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
from grisu.model import GrisuRegistryManager
si = LoginManager.loginCommandline("LOCAL")
appInfo = GrisuRegistryManager.getDefault(si).getApplicationInformation("ImageJ")
print 'app: '+appInfo.getApplicationName()
for subloc in appInfo.getAvailableAllSubmissionLocations():
print subloc
job = JobObject(si);
job.setTimestampJobname("imageJ");
job.setApplication("java");
job.setApplication("ImageJ");
job.setCommandline("echo Hello");
job.setSubmissionLocation("normal:ng2.ivec.org");
job.createJob("/ARCS/StartUp");
job.submitJob();
job.waitForJobToFinish(3);
print "Stdout: "+job.getStdOutContent()
print "Stderr: "+job.getStdErrContent()
Simple example of a workflow that submits x amount of jobs, always y at a time.
Documentation: http://grisu.github.com/grisu/javadoc/
Created on 16/11/2011
@author: markus
'''
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import random
import sys
import time
# si stands for serviceInterface and holds all session information
si = LoginManager.loginCommandline('bestgrid-test')
amount_of_jobs_total = 10
amount_of_jobs_concurrent = 4
created_jobs = []
submitted_jobs = []
finished_jobs = []
# better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though)
jobname_base = 'workflow_test'
for total in range(1, amount_of_jobs_total+1):
job = JobObject(si)
job.setJobname(jobname_base+'_'+str(total))
# always good to set the application if you know it, processing the job will be faster
"""
Created on 17/11/2009
@author: markus
"""
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
HttpProxyManager.useSystemHttpProxy()
si = LoginManager.loginCommandline("bestgrid")
# create the job object
job = JobObject(si)
# set a unique jobname
job.setUniqueJobname("echo_job1")
# set the commandline that needs to be executed
job.setCommandline('echo "Hello World"')
# create the job on the backend and specify the VO to use
job.createJob("/nz/nesi")
# submit the job
job.submitJob()
# this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)
finished = job.waitForJobToFinish(10)
# constants backend = 'BeSTGRID-DEV' backend = 'Local' redistribute = False walltime = 1800 email = '*****@*****.**' basename = 'r-batch' gen_jobs = 40 inputdir = '/home/markus/Desktop/R/' #inputfilename = 'Evaluation_Markov-ADF-Test-2011-05-09-mc50.r' inputfilename = 'Evaluation_Markov-ADF-Test-2011-05-09-mc50-test.r' print 'logging in...' si = LoginManager.loginCommandline(backend) print 'starting job creation...' group = '/nz/nesi' #sub_loc = '[email protected]:ng2.auckland.ac.nz' batch_job = BatchJobObject(si, basename, group, 'R', Constants.NO_VERSION_INDICATOR_STRING) batch_job_name = batch_job.getJobname() print 'jobname on backend: ' + batch_job_name path_to_inputfile = batch_job.pathToInputFiles() + inputfilename
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
# create a service interface to the BeSTGRID backend
service_interface = LoginManager.loginCommandline("BeSTGRID")
print 'Creating job object...'
job = JobObject(service_interface)
job.setJobname("echo_job-1") # job name must be unique
print 'Set jobname to: ' + job.getJobname()
# set the name of the application as it is published in MDS.
# "generic" means not to use MDS for the lookup.
job.setApplication("generic")
# "generic" jobs require a submission location to be specified
job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE")
# set the command that needs to be executed
job.setCommandline("echo \"Hello World\"")
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/BeSTGRID")
print 'Submitting job...'
# submit the job
job.submitJob()
print 'Waiting for the job to finish...'
# this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)
finished = job.waitForJobToFinish(10)
# setup benchmark stuff
submitter = submitter()
#siNames = ["ARCS", "BeSTGRID"]
siNames = ["Local"]
serviceInterfaces = []
for siName in siNames:
with timer:
temp = LoginManager.loginCommandline(siName)
print 'Login '+siName+': '+str(timer.duration_in_seconds())
serviceInterfaces.append(temp)
GrisuRegistryManager.getDefault(temp).getUserEnvironmentManager().setCurrentFqan("/ARCS/NGAdmin")
print
print
#actionClasses = (simpleSubmitJob, simpleSubmitJobWith1SmallInputFile, simpleSubmitJobWith5SmallInputFiles,
# simpleSubmitJobWith1MediumInputFile, simpleSubmitJobWith5MediumInputFiles, simpleSubmitJobWith1_46mb_InputFile, simpleSubmitJobWith5_46mb_InputFiles)
actionClasses = {dynamicStageJob : [True, 1,0,0], dynamicStageJob : [True, 5,0,0], dynamicStageJob : [True, 0,1,0], dynamicStageJob : [True, 0,5,0]}
actions = []
'''
Created on 17/11/2009
@author: markus
'''
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
HttpProxyManager.useSystemHttpProxy()
si = LoginManager.loginCommandline("bestgrid")
# create the job object
job = JobObject(si)
# set a unique jobname
job.setUniqueJobname("echo_job1")
# set the commandline that needs to be executed
job.setCommandline("echo \"Hello World\"")
# create the job on the backend and specify the VO to use
job.createJob("/nz/nesi")
# submit the job
job.submitJob()
# this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)
finished = job.waitForJobToFinish(10)
if not finished:
# Use BeSTGRID-DEV for testing and development # Use BeSTGRID for live runs backend = "BeSTGRID" # Set the group under which this job is submitted # these are also called "Virtual Organisation" or a VO group = "/ARCS/BeSTGRID" # We need an absolute path to the local directory current_dir = os.path.abspath(os.path.curdir) # NOTE: from this point on, some comments will be in the form of notification output # as this is an example script, it's pretty verbose print "INFO: Creating service inteface to " + backend service_interface = LoginManager.loginCommandline(backend) print "INFO: Service interface to " + backend + " Created." print "INFO: Service Interface connected as: " + service_interface.getDN() # Setting the number of jobs to be submitted job_count = 5 # Since there may be many jobs submitted in the workshop, lets make them a bit unique to avoid issues. base_job_name = str(random.randint(10000, 99999)) + '-hello-' print "INFO: Base job name is " + base_job_name # There are three stages, creating the jobs, submitting the jobs, then after they have finished, retrieving the job outputs # Creating a list of jobs jobs = [] print "INFO: Defining " + str(job_count) + " helloworld jobs"
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
from grisu.model import GrisuRegistryManager
si = LoginManager.loginCommandline("LOCAL")
appInfo = GrisuRegistryManager.getDefault(si).getApplicationInformation(
"ImageJ")
print 'app: ' + appInfo.getApplicationName()
for subloc in appInfo.getAvailableAllSubmissionLocations():
print subloc
job = JobObject(si)
job.setTimestampJobname("imageJ")
job.setApplication("java")
job.setApplication("ImageJ")
job.setCommandline("echo Hello")
job.setSubmissionLocation("normal:ng2.ivec.org")
job.createJob("/ARCS/StartUp")
job.submitJob()
job.waitForJobToFinish(3)
print "Stdout: " + job.getStdOutContent()
print "Stderr: " + job.getStdErrContent()
if not os.path.isfile(dictionary_path):
print "HALT: Dictionary file " + dictionary_path + " is not a file"
sys.exit(1)
if not os.path.isdir(input_path):
print "HALT: Input path " + input_path + " is not a directory"
sys.exit(1)
elif not os.listdir(input_path):
print "FINISHED: Input directory is empty, no files to process"
sys.exit(0)
# NOTE: from this point on, some comments will be in the form of notification output
# as this is an example script, it's pretty verbose
print "INFO: Creating service inteface to " + backend
service_interface = LoginManager.loginCommandline(backend)
print "INFO: Service interface to " + backend + " Created."
print "INFO: Service Interface connected as: " + service_interface.getDN()
# Create some base strings to build jobs with
base_job_name = "bacon"
batch_job_name = str(random.randint(10000, 99999)) + "-" + base_job_name
print "INFO: Base job name is " + base_job_name
print "INFO: Batch job name is " + batch_job_name
# Set some job settings
application = "python"
version = "2.4"
print "INFO: Creating a Batch Job Object called " + batch_job_name
batch_jobs = BatchJobObject(service_interface, batch_job_name, group, application, version)
# constants backend = 'BeSTGRID-DEV' backend = 'Local' redistribute = False walltime = 1800 email = '*****@*****.**' basename = 'r-batch' gen_jobs = 40 inputdir = '/home/markus/Desktop/R/' #inputfilename = 'Evaluation_Markov-ADF-Test-2011-05-09-mc50.r' inputfilename = 'Evaluation_Markov-ADF-Test-2011-05-09-mc50-test.r' print 'logging in...' si = LoginManager.loginCommandline(backend) print 'starting job creation...' group = '/nz/nesi' #sub_loc = '[email protected]:ng2.auckland.ac.nz' batch_job = BatchJobObject(si, basename, group, 'R', Constants.NO_VERSION_INDICATOR_STRING) batch_job_name = batch_job.getJobname() print 'jobname on backend: '+batch_job_name path_to_inputfile = batch_job.pathToInputFiles()+inputfilename for i in range(1,gen_jobs+1):
from grisu.jcommons.constants import Constants
from grisu.control import DefaultResubmitPolicy, JobnameHelpers
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import BatchJobObject, JobObject, \
JobsException
from grisu.frontend.view.swing.jobmonitoring.batch import \
BatchJobDialog
import sys
import time
# display commandline login menu if no local proxy exists
si = LoginManager.loginCommandline()
# how many jobs do we want
numberOfJobs = 20
# the (unique) name of the multijob
batchJobName = JobnameHelpers.calculateTimestampedJobname('exampleBatchJob')
print 'Creating batchjob ' + batchJobName
# create the multipart job
batchJob = BatchJobObject(si, batchJobName, '/ARCS/NGAdmin', 'UnixCommands',
Constants.NO_VERSION_INDICATOR_STRING)
# now we can calculate the relative path (from every job directory) to the common input file folder
pathToInputFiles = batchJob.pathToInputFiles()
inputFile1Url = '/home/markus/test/inputFile1.txt'
inputFile1relPath = pathToInputFiles + 'inputFile1.txt '
inputFile2Url = 'gsiftp://ng2.vpac.org/home/grid-vpac/DC_au_DC_org_DC_arcs_DC_slcs_O_VPAC_CN_Markus_Binsteiner_qTrDzHY7L1aKo3WSy8623-7bjgM/inputFile2.txt'
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
si = LoginManager.loginCommandline("BeSTGRID-DEV")
print "Logged in."
job = JobObject(si)
job.setUniqueJobname("cat_job", si)
job.setCommandline("cat text0.txt")
job.addInputFileUrl("/home/markus/tmp/text0.txt")
job.createJob("/nz/nesi")
# job.setSubmissionLocation('[email protected]:ng2.auckland.ac.nz')
job.submitJob()
print "Job submitted."
job.waitForJobToFinish(10)
print "Job finished. Status: " + job.getStatusString(False)
print "Stdout: " + job.getStdOutContent()
print "Stderr: " + job.getStdErrContent()
job.kill(True)
from grisu.frontend.control.login import LoginManager
from grisu.frontend.model.job import JobObject
import sys
# create a service interface to the BeSTGRID backend
service_interface = LoginManager.loginCommandline("BeSTGRID")
print 'Creating job object...'
job = JobObject(service_interface);
job.setJobname("echo_job-1") # job name must be unique
print 'Set jobname to: '+ job.getJobname()
# set the name of the application as it is published in MDS.
# "generic" means not to use MDS for the lookup.
job.setApplication("generic")
# "generic" jobs require a submission location to be specified
job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE")
# set the command that needs to be executed
job.setCommandline("echo \"Hello World\"")
# create the job on the backend and specify the VO to use
job.createJob("/ARCS/BeSTGRID")
print 'Submitting job...'
# submit the job
job.submitJob()
print 'Waiting for the job to finish...'
# this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)
