def main(): """ Run an mpi job on a grid resource. To run in local mode please install mpi. """ # mig.debug_mode_on() # uncomment to enable debug print outs # mig.local_mode_on() # uncomment to enable local mode execution mig.test_connection() # Check if we can connect to the MiG server mpi_file = "example.c" # mpi program source file # The shell command to execute on the grid resource using 4 processes. We need to it compile on the resource first. cmds = ["mpicc -O2 example.c -o example", "$MPI_WRAP mpirun -np 4 ./example Hello"] # specify that we need require MPI as a runtime env and use the DIKU vgrid cluster specifications = {"RUNTIMEENVIRONMENT": "MPI-WRAP-2.0", "VGRID": "DIKU"} # Create and submit the grid job job_id = mig.create_job(cmds, input_files=mpi_file, resource_specifications=specifications) print "\nJob (ID : %s) submitted. \n\n" % job_id # Wait for the job to finish while monitoring the status polling_frequency = 10 # seconds while not mig.job_finished(job_id): job_info = mig.job_info(job_id) # get an info dictionary print "Grid job : %(ID)s \t %(STATUS)s " % job_info time.sleep(polling_frequency) # wait a while before polling again print mig.job_output(job_id)
def main(): """ Run an mpi job on a grid resource. To run in local mode please install mpi. """ # mig.debug_mode_on() # uncomment to enable debug print outs # mig.local_mode_on() # uncomment to enable local mode execution mig.test_connection() # Check if we can connect to the MiG server mpi_file = "example.c" # mpi program source file # The shell command to execute on the grid resource using 4 processes. We need to it compile on the resource first. cmds = ["mpicc -O2 example.c -o example", "$MPI_WRAP mpirun -np 4 ./example Hello"] # specify that we need require MPI as a runtime env and use the DIKU vgrid cluster specifications = {"RUNTIMEENVIRONMENT":"MPI-WRAP-2.0", "VGRID":"DIKU"} # Create and submit the grid job job_id = mig.create_job(cmds, input_files=mpi_file, resource_specifications=specifications) print "\nJob (ID : %s) submitted. \n\n" % job_id # Wait for the job to finish while monitoring the status polling_frequency = 10 # seconds while not mig.job_finished(job_id): job_info = mig.job_info(job_id) # get an info dictionary print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info time.sleep(polling_frequency) # wait a while before polling again print mig.job_output(job_id)
def main(): """ Run an pvm job on a grid resource. To run in local mode please install pvm """ # mig.debug_mode_on() # uncomment to enable debug print outs # mig.local_mode_on() # uncomment to enable local mode execution mig.test_connection() # Check if we can connect to the MiG server proc = subprocess.Popen("gcc example.c -o pvm_example -lpvm3 -lgpvm3", shell= True, stdout=subprocess.PIPE,stderr=subprocess.PIPE) out, err = proc.communicate() pvm_program = "pvm_example" # The PVM executable cmd = "$PVM_WRAP ./pvm_example 4 Hello" # The shell command to execute on the grid resource # specify to the job that we want PVM as RTE and we want to use the DIKU VGRID specifications = {"RUNTIMEENVIRONMENT":"PVM-WRAP-1.0", "VGRID":"DIKU"} # Create and submit the grid job job_id = mig.create_job(cmd, executables=pvm_program, resource_specifications=specifications) print "\nJob (ID : %s) submitted. \n\n" % job_id # Wait for the job to finish while monitoring the status polling_frequency = 10 # seconds while not mig.job_finished(job_id): job_info = mig.job_info(job_id) # get an info dictionary print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info time.sleep(polling_frequency) # wait a while before polling again print mig.job_output(job_id)
def main(): """ Executes the bash file test_executable.sh in a grid job. Afterwards, the result is downloaded and printed to screen. """ # mig.debug_mode_on() # uncomment to enable debug print outs # mig.local_mode_on() # uncomment to enable local mode execution mig.test_connection() # Check if we can connect to the MiG server # The program we want to execute on the grid executable_file = "test_executable.sh" # The shell command to execute on the grid resource cmd = "./test_executable.sh > out.txt" # Create and submit the grid job job_id = mig.create_job(cmd, output_files=["out.txt"], executables=[executable_file]) print "\nJob (ID : %s) submitted. \n\n" % job_id # Wait for the job to finish while monitoring the status polling_frequency = 10 # seconds while not mig.job_finished(job_id): job_info = mig.job_info(job_id) # get an info dictionary print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info time.sleep(polling_frequency) # wait a while before polling again # Download the result file and print output_file = mig.get_file("out.txt") f = open(output_file) print "Output file (%s) contains :\n %s \n\n" % (output_file, str(f.readlines())) f.close() # Clean up os.remove(output_file) # remove locally mig.remove(output_file) # remove on the MiG server print "Output (" + output_file + ") deleted."
def update_solver_data(name, status="", state=""): """ Write the current status to the status file. """ job_data_dir = os.path.join(config.jobdata_directory, name) solver_data_path = os.path.join(job_data_dir, config.solver_data_file) data_file = open(solver_data_path) fcntl.flock(data_file, fcntl.LOCK_EX) # lock the file while updating solver_data = cPickle.load(data_file) data_file.close() retries = 3 if solver_data.has_key("grid_enabled") and not solver_data["grid_enabled"]: mig.local_mode_on() for job in solver_data["timesteps"][-1]["jobs"]: # Go through the jobs in the current time step (indexed last: -1) # there seems to be incidents where MiG does not recognize the job id even though it should. # in such a case we let it pass unless the error is consistent across 3 retries. if not retries: break try : job_info = mig.job_info(job["job_id"]) except migerror.MigUnknownJobIdError, e: log(str(e)) retries -= 1 continue for (key, value) in job_info.items(): job_info.pop(key) job_info[key.lower()] = value job.update(job_info)
def main(): """ Execute a simple grid job and print the output. """ # mig.debug_mode_on() # uncomment to enable debug print outs # mig.local_mode_on() # uncomment to enable local mode execution # Check if we can connect to the MiG server mig.test_connection() # Create and submit the grid job job_id = mig.create_job("echo HELLO GRID") print "\nJob (ID : %s) submitted. \n\n" % job_id # Wait for the job to finish while monitoring the status polling_frequency = 10 # seconds while not mig.job_finished(job_id): job_info = mig.job_info(job_id) # get an info dictionary print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info time.sleep(polling_frequency) # wait a while before polling again print mig.job_output(job_id)
def main(): """ Executes the bash file test_executable.sh in a grid job. Afterwards, the result is downloaded and printed to screen. """ # mig.debug_mode_on() # uncomment to enable debug print outs # mig.local_mode_on() # uncomment to enable local mode execution mig.test_connection() # Check if we can connect to the MiG server # The program we want to execute on the grid executable_file = "test_executable.sh" # The shell command to execute on the grid resource cmd = "./test_executable.sh > out.txt" # Create and submit the grid job job_id = mig.create_job(cmd, output_files=["out.txt"], executables=[executable_file]) print "\nJob (ID : %s) submitted. \n\n" % job_id # Wait for the job to finish while monitoring the status polling_frequency = 10 # seconds while not mig.job_finished(job_id): job_info = mig.job_info(job_id) # get an info dictionary print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info time.sleep(polling_frequency) # wait a while before polling again # Download the result file and print output_file = mig.get_file("out.txt") f = open(output_file) print "Output file (%s) contains :\n %s \n\n" % (output_file, str(f.readlines())) f.close() # Clean up os.remove(output_file) # remove locally mig.remove(output_file) # remove on the MiG server print "Output ("+output_file+") deleted."
def main(): """ Run five grid jobs executing the bash file parameter_sweet_script.sh with different input arguments. When a job has finished executing, the corresponding output file is downloaded. Finally, the output contents are printed. """ # mig.debug_mode_on() # uncomment to enable debug print outs # mig.local_mode_on() # uncomment to enable local mode execution mig.test_connection() # Check if we can connect to the MiG server input_values = range(5) # Input parameters # The program we want to execute on grid resources executable_file = "parameter_sweep_script.sh" print "\nStarting grid jobs:\n" jobs = [] for i in input_values: # Start a job for each input output_file = "output%s.txt" % i # The output file name # The shell command to start the script on the resource cmd = "./parameter_sweep_script.sh %i > %s" % (i, output_file) # Run the job resources on any vgrid resource_requirements = {"VGRID": "ANY"} # Start the grid job job_id = mig.create_job(cmd, output_files=[output_file], executables=[executable_file], resource_specifications=resource_requirements) jobs.append((job_id, output_file)) print "Job (ID : %s) submitted." % job_id print "\n\n" print "Monitor job status...\n" # Now we wait for results finished_jobs = [] while len(finished_jobs) < len(jobs): for id, output_file in jobs: job_info = mig.job_info(id) # get an info dictionary print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info if mig.job_finished(id) and id not in finished_jobs: # Download the output file from the server mig.get_file(output_file) finished_jobs.append(id) mig.remove( output_file) # clean up the result file on the server time.sleep(10) # Wait a few seconds before trying again print "\n\n" print "All jobs finished." # Clean up the result files and print out the contents print "Cleaning up." output_lines = [] for _, output_file in jobs: fh = open(output_file) output_lines.append(" ".join(fh.readlines())) fh.close() os.remove(output_file) print "Output file (" + output_file + ") deleted." print "\n\nOutput contents : \n\n%s\n" % "\n".join(output_lines)