def monitor_timestep(jobs):
finished_jobs = []
while True:
for j in jobs:
if j in finished_jobs:
continue
try:
j_done = mig.job_finished(j["job_id"])
except migerror.MigInterfaceError, e:
log(str(e))
j_done = False
if j_done:
output_files = download_result(j)
if verify_job_output(output_files, j):
finished_jobs.append(j)
clean_up_mig_home(output_files)
else:
return 1
update(status="waiting for jobs", state=STATE_RUNNING)
if len(jobs) == len(finished_jobs):
update(status="iteration finished", state=STATE_RUNNING)
return 0 # success
time.sleep(config.POLLING_INTERVAL)
def main():
"""
Run an mpi job on a grid resource. To run in local mode please install mpi.
"""
# mig.debug_mode_on() # uncomment to enable debug print outs
# mig.local_mode_on() # uncomment to enable local mode execution
mig.test_connection() # Check if we can connect to the MiG server
mpi_file = "example.c" # mpi program source file
# The shell command to execute on the grid resource using 4 processes. We need to it compile on the resource first.
cmds = ["mpicc -O2 example.c -o example", "$MPI_WRAP mpirun -np 4 ./example Hello"]
# specify that we need require MPI as a runtime env and use the DIKU vgrid cluster
specifications = {"RUNTIMEENVIRONMENT":"MPI-WRAP-2.0", "VGRID":"DIKU"}
# Create and submit the grid job
job_id = mig.create_job(cmds, input_files=mpi_file, resource_specifications=specifications)
print "\nJob (ID : %s) submitted. \n\n" % job_id
# Wait for the job to finish while monitoring the status
polling_frequency = 10 # seconds
while not mig.job_finished(job_id):
job_info = mig.job_info(job_id) # get an info dictionary
print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
time.sleep(polling_frequency) # wait a while before polling again
print mig.job_output(job_id)
def main():
"""
Run an mpi job on a grid resource. To run in local mode please install mpi.
"""
# mig.debug_mode_on() # uncomment to enable debug print outs
# mig.local_mode_on() # uncomment to enable local mode execution
mig.test_connection() # Check if we can connect to the MiG server
mpi_file = "example.c" # mpi program source file
# The shell command to execute on the grid resource using 4 processes. We need to it compile on the resource first.
cmds = ["mpicc -O2 example.c -o example", "$MPI_WRAP mpirun -np 4 ./example Hello"]
# specify that we need require MPI as a runtime env and use the DIKU vgrid cluster
specifications = {"RUNTIMEENVIRONMENT": "MPI-WRAP-2.0", "VGRID": "DIKU"}
# Create and submit the grid job
job_id = mig.create_job(cmds, input_files=mpi_file, resource_specifications=specifications)
print "\nJob (ID : %s) submitted. \n\n" % job_id
# Wait for the job to finish while monitoring the status
polling_frequency = 10 # seconds
while not mig.job_finished(job_id):
job_info = mig.job_info(job_id) # get an info dictionary
print "Grid job : %(ID)s \t %(STATUS)s " % job_info
time.sleep(polling_frequency) # wait a while before polling again
print mig.job_output(job_id)
def monitor_timestep(jobs):
finished_jobs = []
while True:
for j in jobs:
if j in finished_jobs:
continue
try:
j_done = mig.job_finished(j["job_id"])
except migerror.MigInterfaceError, e:
log(str(e))
j_done = False
if j_done:
output_files = download_result(j)
if verify_job_output(output_files, j):
finished_jobs.append(j)
clean_up_mig_home(output_files)
else:
return 1
update(status="waiting for jobs", state=STATE_RUNNING)
if len(jobs) == len(finished_jobs):
update(status="iteration finished", state=STATE_RUNNING)
return 0 # success
time.sleep(config.POLLING_INTERVAL)
def main():
"""
Run an pvm job on a grid resource. To run in local mode please install pvm
"""
# mig.debug_mode_on() # uncomment to enable debug print outs
# mig.local_mode_on() # uncomment to enable local mode execution
mig.test_connection() # Check if we can connect to the MiG server
proc = subprocess.Popen("gcc example.c -o pvm_example -lpvm3 -lgpvm3", shell= True, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
out, err = proc.communicate()
pvm_program = "pvm_example" # The PVM executable
cmd = "$PVM_WRAP ./pvm_example 4 Hello" # The shell command to execute on the grid resource
# specify to the job that we want PVM as RTE and we want to use the DIKU VGRID
specifications = {"RUNTIMEENVIRONMENT":"PVM-WRAP-1.0", "VGRID":"DIKU"}
# Create and submit the grid job
job_id = mig.create_job(cmd, executables=pvm_program, resource_specifications=specifications)
print "\nJob (ID : %s) submitted. \n\n" % job_id
# Wait for the job to finish while monitoring the status
polling_frequency = 10 # seconds
while not mig.job_finished(job_id):
job_info = mig.job_info(job_id) # get an info dictionary
print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
time.sleep(polling_frequency) # wait a while before polling again
print mig.job_output(job_id)
def main():
"""
Find edit distance value between entries in a reference file. First, divide the file into smaller blocks
and create a grid job for each. levenshtein.py is used to process each input block.
When a job has finished executing, the corresponding output file is downloaded.
"""
# mig.debug_mode_on() # uncomment to enable debug print outs
# mig.local_mode_on() # uncomment to enable local mode execution
reference_file = "ref1000.txt"
# The reference blocks. One for each job we want to run.
block_files = create_blocks(reference_file, block_size=200)
# These are static input files for each job.
levenshtein_files = ["Levenshtein_ucs4.so", "Levenshtein_ucs2.so", "levenshtein.py", "Levenshtein_i686.so"]
resource_requirements = {}
resource_requirements["RUNTIMEENVIRONMENT"] = "PYTHON-2" # we need python on the resource
resource_requirements["VGRID"] = "ANY"
jobs = []
# Start a grid job for each block file
for block_file in block_files:
output_file = block_file.strip(".txt")+"_output.txt"
cmd = "$PYTHON levenshtein.py %s > %s" % (block_file, output_file)
input_files = []
input_files.extend(levenshtein_files)
input_files.append(block_file)
job_id = mig.create_job(cmd, input_files=input_files, output_files=output_file,resource_specifications=resource_requirements)
jobs.append((job_id, output_file))
print "started job %s" % job_id
jobs_done = 0
# now wait for the results
while len(jobs):
print "Checking job status. jobs done : %i." % jobs_done
try:
for job_id, result_file in jobs:
print "Checking job %s." % job_id
if mig.job_finished(job_id):
if not os.path.exists("output"):
os.mkdir("output")
mig.get_file(result_file, "output/"+result_file)
jobs.remove((job_id, result_file))
jobs_done += 1
print "Job done. Downloaded result file %s." % result_file
time.sleep(10) # wait a little before polling
except KeyboardInterrupt:
job_ids = [x[0] for x in jobs]
mig.cancel_jobs(job_ids)
print "Cancelled jobs."
break
def update_jobs(self,jobs):
new_status = False
job_ids = map(lambda x : x["id"], jobs)
job_info_list = mig.jobs_status(job_ids)
if len(job_info_list) != len(job_ids):
print str(job_info_list)
print str(jobs)
raise Exception("Critical job management error. Job list lengths.")
for i in range(len(jobs)):
jobs[i]["status"] = job_info_list[i]
if mig.job_finished(jobs[i]["id"]):
jobs[i]["finished"] = self.get_time()
print "update jobs", jobs
return jobs
def main():
"""
Executes the bash file test_executable.sh in a grid job.
Afterwards, the result is downloaded and printed to screen.
"""
# mig.debug_mode_on() # uncomment to enable debug print outs
# mig.local_mode_on() # uncomment to enable local mode execution
mig.test_connection() # Check if we can connect to the MiG server
# The program we want to execute on the grid
executable_file = "test_executable.sh"
# The shell command to execute on the grid resource
cmd = "./test_executable.sh > out.txt"
# Create and submit the grid job
job_id = mig.create_job(cmd,
output_files=["out.txt"],
executables=[executable_file])
print "\nJob (ID : %s) submitted. \n\n" % job_id
# Wait for the job to finish while monitoring the status
polling_frequency = 10 # seconds
while not mig.job_finished(job_id):
job_info = mig.job_info(job_id) # get an info dictionary
print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
time.sleep(polling_frequency) # wait a while before polling again
# Download the result file and print
output_file = mig.get_file("out.txt")
f = open(output_file)
print "Output file (%s) contains :\n %s \n\n" % (output_file,
str(f.readlines()))
f.close()
# Clean up
os.remove(output_file) # remove locally
mig.remove(output_file) # remove on the MiG server
print "Output (" + output_file + ") deleted."
def main():
"""
Execute a simple grid job and print the output.
"""
# mig.debug_mode_on() # uncomment to enable debug print outs
# mig.local_mode_on() # uncomment to enable local mode execution
# Check if we can connect to the MiG server
mig.test_connection()
# Create and submit the grid job
job_id = mig.create_job("echo HELLO GRID")
print "\nJob (ID : %s) submitted. \n\n" % job_id
# Wait for the job to finish while monitoring the status
polling_frequency = 10 # seconds
while not mig.job_finished(job_id):
job_info = mig.job_info(job_id) # get an info dictionary
print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
time.sleep(polling_frequency) # wait a while before polling again
print mig.job_output(job_id)
def main():
"""
Execute a simple grid job and print the output.
"""
# mig.debug_mode_on() # uncomment to enable debug print outs
# mig.local_mode_on() # uncomment to enable local mode execution
# Check if we can connect to the MiG server
mig.test_connection()
# Create and submit the grid job
job_id = mig.create_job("echo HELLO GRID")
print "\nJob (ID : %s) submitted. \n\n" % job_id
# Wait for the job to finish while monitoring the status
polling_frequency = 10 # seconds
while not mig.job_finished(job_id):
job_info = mig.job_info(job_id) # get an info dictionary
print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
time.sleep(polling_frequency) # wait a while before polling again
print mig.job_output(job_id)
def main():
"""
Executes the bash file test_executable.sh in a grid job.
Afterwards, the result is downloaded and printed to screen.
"""
# mig.debug_mode_on() # uncomment to enable debug print outs
# mig.local_mode_on() # uncomment to enable local mode execution
mig.test_connection() # Check if we can connect to the MiG server
# The program we want to execute on the grid
executable_file = "test_executable.sh"
# The shell command to execute on the grid resource
cmd = "./test_executable.sh > out.txt"
# Create and submit the grid job
job_id = mig.create_job(cmd, output_files=["out.txt"], executables=[executable_file])
print "\nJob (ID : %s) submitted. \n\n" % job_id
# Wait for the job to finish while monitoring the status
polling_frequency = 10 # seconds
while not mig.job_finished(job_id):
job_info = mig.job_info(job_id) # get an info dictionary
print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
time.sleep(polling_frequency) # wait a while before polling again
# Download the result file and print
output_file = mig.get_file("out.txt")
f = open(output_file)
print "Output file (%s) contains :\n %s \n\n" % (output_file, str(f.readlines()))
f.close()
# Clean up
os.remove(output_file) # remove locally
mig.remove(output_file) # remove on the MiG server
print "Output ("+output_file+") deleted."
def main():
"""
Run five grid jobs executing the bash file parameter_sweet_script.sh with different input arguments.
When a job has finished executing, the corresponding output file is downloaded.
Finally, the output contents are printed.
"""
# mig.debug_mode_on() # uncomment to enable debug print outs
# mig.local_mode_on() # uncomment to enable local mode execution
mig.test_connection() # Check if we can connect to the MiG server
input_values = range(5) # Input parameters
# The program we want to execute on grid resources
executable_file = "parameter_sweep_script.sh"
print "\nStarting grid jobs:\n"
jobs = []
for i in input_values: # Start a job for each input
output_file = "output%s.txt" % i # The output file name
# The shell command to start the script on the resource
cmd = "./parameter_sweep_script.sh %i > %s" % (i, output_file)
# Run the job resources on any vgrid
resource_requirements = {"VGRID": "ANY"}
# Start the grid job
job_id = mig.create_job(cmd,
output_files=[output_file],
executables=[executable_file],
resource_specifications=resource_requirements)
jobs.append((job_id, output_file))
print "Job (ID : %s) submitted." % job_id
print "\n\n"
print "Monitor job status...\n" # Now we wait for results
finished_jobs = []
while len(finished_jobs) < len(jobs):
for id, output_file in jobs:
job_info = mig.job_info(id) # get an info dictionary
print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
if mig.job_finished(id) and id not in finished_jobs:
# Download the output file from the server
mig.get_file(output_file)
finished_jobs.append(id)
mig.remove(
output_file) # clean up the result file on the server
time.sleep(10) # Wait a few seconds before trying again
print "\n\n"
print "All jobs finished."
# Clean up the result files and print out the contents
print "Cleaning up."
output_lines = []
for _, output_file in jobs:
fh = open(output_file)
output_lines.append(" ".join(fh.readlines()))
fh.close()
os.remove(output_file)
print "Output file (" + output_file + ") deleted."
print "\n\nOutput contents : \n\n%s\n" % "\n".join(output_lines)
def main():
"""
Find edit distance value between entries in a reference file. First, divide the file into smaller blocks
and create a grid job for each. levenshtein.py is used to process each input block.
When a job has finished executing, the corresponding output file is downloaded.
"""
# mig.debug_mode_on() # uncomment to enable debug print outs
# mig.local_mode_on() # uncomment to enable local mode execution
reference_file = "ref1000.txt"
# The reference blocks. One for each job we want to run.
block_files = create_blocks(reference_file, block_size=200)
# These are static input files for each job.
levenshtein_files = [
"Levenshtein_ucs4.so", "Levenshtein_ucs2.so", "levenshtein.py",
"Levenshtein_i686.so"
]
resource_requirements = {}
resource_requirements[
"RUNTIMEENVIRONMENT"] = "PYTHON-2" # we need python on the resource
resource_requirements["VGRID"] = "ANY"
jobs = []
# Start a grid job for each block file
for block_file in block_files:
output_file = block_file.strip(".txt") + "_output.txt"
cmd = "$PYTHON levenshtein.py %s > %s" % (block_file, output_file)
input_files = []
input_files.extend(levenshtein_files)
input_files.append(block_file)
job_id = mig.create_job(cmd,
input_files=input_files,
output_files=output_file,
resource_specifications=resource_requirements)
jobs.append((job_id, output_file))
print "started job %s" % job_id
jobs_done = 0
# now wait for the results
while len(jobs):
print "Checking job status. jobs done : %i." % jobs_done
try:
for job_id, result_file in jobs:
print "Checking job %s." % job_id
if mig.job_finished(job_id):
if not os.path.exists("output"):
os.mkdir("output")
mig.get_file(result_file, "output/" + result_file)
jobs.remove((job_id, result_file))
jobs_done += 1
print "Job done. Downloaded result file %s." % result_file
time.sleep(10) # wait a little before polling
except KeyboardInterrupt:
job_ids = [x[0] for x in jobs]
mig.cancel_jobs(job_ids)
print "Cancelled jobs."
break