# si stands for serviceInterface and holds all session information si = LoginManager.loginCommandline('bestgrid-test') amount_of_jobs_total = 10 amount_of_jobs_concurrent = 4 created_jobs = [] submitted_jobs = [] finished_jobs = [] # better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though) jobname_base = 'workflow_test' for total in range(1, amount_of_jobs_total + 1): job = JobObject(si) job.setJobname(jobname_base + '_' + str(total)) # always good to set the application if you know it, processing the job will be faster job.setApplication('UnixCommands') # also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz') # job sleeps for a random time random_sleep = random.randrange(5, 75) job.setCommandline('sleep ' + str(random_sleep)) job.createJob('/nz/nesi') print 'created job: ' + job.getJobname() + ' (sleeptime: ' + str( random_sleep) + ')' created_jobs.append(job) finished = False
inputFile2Url = 'gsiftp://ng2.vpac.org/home/grid-vpac/DC_au_DC_org_DC_arcs_DC_slcs_O_VPAC_CN_Markus_Binsteiner_qTrDzHY7L1aKo3WSy8623-7bjgM/inputFile2.txt' inputFile2relPath = pathToInputFiles + 'inputFile2.txt' inputFile3Url = '/home/markus/test/errorFile.txt' inputFile3relPath = pathToInputFiles + 'errorFile.txt' for i in range(0, numberOfJobs): # create a unique jobname for every job jobname = batchJobName + "_" + str(i) print 'Creating job: ' + jobname # create the single job job = JobObject(si) job.setJobname(jobname) # better to set the application to use explicitely because in that case we don't need to use mds (faster) job.setApplication('UnixCommands') if i == 3 or i == 13: # this is just to demonstrate how to restart a failed job later on job.setCommandline('cat ' + inputFile3relPath) else: job.setCommandline('cat ' + inputFile1relPath + ' ' + inputFile2relPath) job.setWalltimeInSeconds(60) # adding the job to the multijob batchJob.addJob(job) # this should be set because it's used for the matchmaking/metascheduling batchJob.setDefaultNoCpus(1)
start = 30 end = 40 pathToInputFiles = batchJob.pathToInputFiles() inputFile1relPath = pathToInputFiles+'inputFile1.txt ' inputFile2relPath = pathToInputFiles+'inputFile2.txt' for i in range(start, end): # create a unique jobname for every job jobname = batchJobName+"_"+ str(i) print 'Creating job: '+jobname # create the single job job = JobObject(si) job.setJobname(jobname) # better to set the application to use explicitely because in that case we don't need to use mds (faster) job.setApplication('UnixCommands') job.setCommandline('cat '+ inputFile1relPath + ' ' + inputFile2relPath) job.setWalltimeInSeconds(60) # adding the job to the multijob batchJob.addJob(job) # only start the newly added jobs and wait for the restart to finish batchJob.restart(False, False, True, True) # don't forget to exit properly. this cleans up possible existing threads/executors sys.exit()
# si stands for serviceInterface and holds all session information si = LoginManager.loginCommandline('bestgrid-test') amount_of_jobs_total = 10 amount_of_jobs_concurrent = 4 created_jobs = [] submitted_jobs = [] finished_jobs = [] # better make that unique for each run, so we can resume workflows easier if necessary (this introduces quite a bit more complexity though) jobname_base = 'workflow_test' for total in range(1, amount_of_jobs_total+1): job = JobObject(si) job.setJobname(jobname_base+'_'+str(total)) # always good to set the application if you know it, processing the job will be faster job.setApplication('UnixCommands') # also good to set queue if you know where you want to submit your job, not necessary, but processing of the job will be faster job.setSubmissionLocation('default:gram5.ceres.auckland.ac.nz') # job sleeps for a random time random_sleep = random.randrange(5, 75) job.setCommandline('sleep '+str(random_sleep)) job.createJob('/nz/nesi') print 'created job: '+ job.getJobname()+' (sleeptime: '+str(random_sleep)+')' created_jobs.append(job) finished = False while not finished:
from grisu.frontend.control.login import LoginManager from grisu.frontend.model.job import JobObject import sys # create a service interface to the BeSTGRID backend service_interface = LoginManager.loginCommandline("BeSTGRID") print 'Creating job object...' job = JobObject(service_interface) job.setJobname("echo_job-1") # job name must be unique print 'Set jobname to: ' + job.getJobname() # set the name of the application as it is published in MDS. # "generic" means not to use MDS for the lookup. job.setApplication("generic") # "generic" jobs require a submission location to be specified job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE") # set the command that needs to be executed job.setCommandline("echo \"Hello World\"") # create the job on the backend and specify the VO to use job.createJob("/ARCS/BeSTGRID") print 'Submitting job...' # submit the job job.submitJob() print 'Waiting for the job to finish...' # this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job) finished = job.waitForJobToFinish(10)
job_count = 5 # Since there may be many jobs submitted in the workshop, lets make them a bit unique to avoid issues. base_job_name = str(random.randint(10000, 99999)) + '-hello-' print "INFO: Base job name is " + base_job_name # There are three stages, creating the jobs, submitting the jobs, then after they have finished, retrieving the job outputs # Creating a list of jobs jobs = [] print "INFO: Defining " + str(job_count) + " helloworld jobs" for i in range(1, job_count + 1): print "INFO: Defining job " + str(i) + " of " + str(job_count) #The next lines define the actual job's parameters job = JobObject(service_interface) # Create a job job.setJobname(base_job_name + str(i)) # Give it a (hopefully) unique name job.setApplication("python") # Set the application being run job.setApplicationVersion( "2.4") # Set the application version, note this is an exact match # job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE") # Set the location the job will be submitted to job.addInputFileUrl(os.path.join(current_dir, "helloworld.py")) # Add a file job.setCommandline("python helloworld.py") # Set the command to be run print "INFO: job " + job.getJobname() + " defined" jobs.append(job) # Submit the jobs to be run # Note the exception catching to give more information about a job failing for job in jobs: time_start = time.time() try:
job_count = 5 # Since there may be many jobs submitted in the workshop, lets make them a bit unique to avoid issues. base_job_name = str(random.randint(10000, 99999)) + '-hello-' print "INFO: Base job name is " + base_job_name # There are three stages, creating the jobs, submitting the jobs, then after they have finished, retrieving the job outputs # Creating a list of jobs jobs = [] print "INFO: Defining " + str(job_count) + " helloworld jobs" for i in range(1, job_count + 1): print "INFO: Defining job " + str(i) + " of " + str(job_count) #The next lines define the actual job's parameters job = JobObject(service_interface) # Create a job job.setJobname(base_job_name + str(i)) # Give it a (hopefully) unique name job.setApplication("python") # Set the application being run job.setApplicationVersion("2.4") # Set the application version, note this is an exact match # job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE") # Set the location the job will be submitted to job.addInputFileUrl(os.path.join(current_dir, "helloworld.py")) # Add a file job.setCommandline("python helloworld.py") # Set the command to be run print"INFO: job " + job.getJobname() + " defined" jobs.append(job) # Submit the jobs to be run # Note the exception catching to give more information about a job failing for job in jobs: time_start = time.time() try: print "INFO: Creating job " + job.getJobname() + " on " + backend + " backend, with " + group + " group" job.createJob(group)
from grisu.frontend.control.login import LoginManager from grisu.frontend.model.job import JobObject import sys # create a service interface to the BeSTGRID backend service_interface = LoginManager.loginCommandline("BeSTGRID") print 'Creating job object...' job = JobObject(service_interface); job.setJobname("echo_job-1") # job name must be unique print 'Set jobname to: '+ job.getJobname() # set the name of the application as it is published in MDS. # "generic" means not to use MDS for the lookup. job.setApplication("generic") # "generic" jobs require a submission location to be specified job.setSubmissionLocation("all.q:ng2.scenzgrid.org#SGE") # set the command that needs to be executed job.setCommandline("echo \"Hello World\"") # create the job on the backend and specify the VO to use job.createJob("/ARCS/BeSTGRID") print 'Submitting job...' # submit the job job.submitJob() print 'Waiting for the job to finish...' # this waits until the job is finished. Checks every 10 seconds (which would be too often for a real job)