Пример #1
0
def monitor_timestep(jobs):
      
    finished_jobs = []
    while True:
     
        for j in jobs:
            if j in finished_jobs:
                continue
            try: 
                j_done = mig.job_finished(j["job_id"])
            
            except migerror.MigInterfaceError, e:
                log(str(e))
                j_done = False
                
            if j_done:
                output_files = download_result(j)
                if verify_job_output(output_files, j):
                    finished_jobs.append(j)            
                    clean_up_mig_home(output_files)
                else:
                    return 1                    
            
        update(status="waiting for jobs", state=STATE_RUNNING)
        if len(jobs) == len(finished_jobs):
            update(status="iteration finished", state=STATE_RUNNING)
            return 0 # success
        time.sleep(config.POLLING_INTERVAL)
Пример #2
0
def main():
    """
    Run an mpi job on a grid resource. To run in local mode please install mpi.
    """

    # mig.debug_mode_on() # uncomment to enable debug print outs
    # mig.local_mode_on() # uncomment to enable local mode execution
     
    mig.test_connection() # Check if we can connect to the MiG server
    mpi_file = "example.c" # mpi program source file
    
    # The shell command to execute on the grid resource using 4 processes. We need to it compile on the resource first.
    cmds = ["mpicc -O2 example.c -o example", "$MPI_WRAP mpirun -np 4 ./example Hello"]

    # specify that we need require MPI as a runtime env and use the DIKU vgrid cluster
    specifications = {"RUNTIMEENVIRONMENT":"MPI-WRAP-2.0", "VGRID":"DIKU"}
    # Create and submit the grid job
    job_id = mig.create_job(cmds, input_files=mpi_file, resource_specifications=specifications)
    print "\nJob (ID : %s) submitted. \n\n" % job_id

    # Wait for the job to finish while monitoring the status
    polling_frequency = 10 # seconds
    while not mig.job_finished(job_id):
        job_info = mig.job_info(job_id) # get an info dictionary
        print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
        time.sleep(polling_frequency) # wait a while before polling again

    print mig.job_output(job_id)
Пример #3
0
def main():
    """
    Run an mpi job on a grid resource. To run in local mode please install mpi.
    """

    # mig.debug_mode_on() # uncomment to enable debug print outs
    # mig.local_mode_on() # uncomment to enable local mode execution

    mig.test_connection()  # Check if we can connect to the MiG server
    mpi_file = "example.c"  # mpi program source file

    # The shell command to execute on the grid resource using 4 processes. We need to it compile on the resource first.
    cmds = ["mpicc -O2 example.c -o example", "$MPI_WRAP mpirun -np 4 ./example Hello"]

    # specify that we need require MPI as a runtime env and use the DIKU vgrid cluster
    specifications = {"RUNTIMEENVIRONMENT": "MPI-WRAP-2.0", "VGRID": "DIKU"}
    # Create and submit the grid job
    job_id = mig.create_job(cmds, input_files=mpi_file, resource_specifications=specifications)
    print "\nJob (ID : %s) submitted. \n\n" % job_id

    # Wait for the job to finish while monitoring the status
    polling_frequency = 10  # seconds
    while not mig.job_finished(job_id):
        job_info = mig.job_info(job_id)  # get an info dictionary
        print "Grid job : %(ID)s \t %(STATUS)s " % job_info
        time.sleep(polling_frequency)  # wait a while before polling again

    print mig.job_output(job_id)
Пример #4
0
def monitor_timestep(jobs):

    finished_jobs = []
    while True:

        for j in jobs:
            if j in finished_jobs:
                continue
            try:
                j_done = mig.job_finished(j["job_id"])

            except migerror.MigInterfaceError, e:
                log(str(e))
                j_done = False

            if j_done:
                output_files = download_result(j)
                if verify_job_output(output_files, j):
                    finished_jobs.append(j)
                    clean_up_mig_home(output_files)
                else:
                    return 1

        update(status="waiting for jobs", state=STATE_RUNNING)
        if len(jobs) == len(finished_jobs):
            update(status="iteration finished", state=STATE_RUNNING)
            return 0  # success
        time.sleep(config.POLLING_INTERVAL)
Пример #5
0
def main():
    """
    Run an pvm job on a grid resource. To run in local mode please install pvm
    """

    # mig.debug_mode_on() # uncomment to enable debug print outs
    # mig.local_mode_on() # uncomment to enable local mode execution
    
    
    mig.test_connection() # Check if we can connect to the MiG server
    

    proc = subprocess.Popen("gcc example.c -o pvm_example -lpvm3 -lgpvm3", shell= True, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    out, err = proc.communicate()
    pvm_program = "pvm_example" # The PVM executable
    cmd = "$PVM_WRAP ./pvm_example 4 Hello" # The shell command to execute on the grid resource

    # specify to the job that we want PVM as RTE and we want to use the DIKU VGRID
    specifications = {"RUNTIMEENVIRONMENT":"PVM-WRAP-1.0", "VGRID":"DIKU"}
    
    # Create and submit the grid job
    job_id = mig.create_job(cmd, executables=pvm_program, resource_specifications=specifications)
    print "\nJob (ID : %s) submitted. \n\n" % job_id

    # Wait for the job to finish while monitoring the status
    polling_frequency = 10 # seconds
    while not mig.job_finished(job_id):
        job_info = mig.job_info(job_id) # get an info dictionary
        print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
        time.sleep(polling_frequency) # wait a while before polling again

    print mig.job_output(job_id)
Пример #6
0
def main():
    """
    Find edit distance value between entries in a reference file. First, divide the file into smaller blocks 
    and create a grid job for each. levenshtein.py is used to process each input block.
    When a job has finished executing, the corresponding output file is downloaded.
    """
    
    # mig.debug_mode_on() # uncomment to enable debug print outs
    # mig.local_mode_on() # uncomment to enable local mode execution 

    reference_file = "ref1000.txt"
    # The reference blocks. One for each job we want to run.
    block_files = create_blocks(reference_file, block_size=200)
    
    # These are static input files for each job. 
    levenshtein_files = ["Levenshtein_ucs4.so", "Levenshtein_ucs2.so", "levenshtein.py", "Levenshtein_i686.so"]
        
    resource_requirements = {}
    resource_requirements["RUNTIMEENVIRONMENT"] = "PYTHON-2" # we need python on the resource
    resource_requirements["VGRID"] = "ANY"
    
    jobs = []
    
    # Start a grid job for each block file
    for block_file in block_files:
        output_file = block_file.strip(".txt")+"_output.txt"
        cmd = "$PYTHON levenshtein.py %s > %s" % (block_file, output_file)
        input_files = []
        input_files.extend(levenshtein_files)
        input_files.append(block_file)
        job_id = mig.create_job(cmd, input_files=input_files, output_files=output_file,resource_specifications=resource_requirements)
        jobs.append((job_id, output_file))
        print "started job %s" % job_id
    
    jobs_done = 0
    # now wait for the results 
    while len(jobs):
        print "Checking job status. jobs done : %i." % jobs_done 
        try:  
            for job_id, result_file in jobs:
                print "Checking job %s." % job_id
                if mig.job_finished(job_id):
                    if not os.path.exists("output"):
                        os.mkdir("output")
                    mig.get_file(result_file, "output/"+result_file)
                    jobs.remove((job_id, result_file))
                    jobs_done += 1
                    
                    print "Job done. Downloaded result file %s." % result_file
                    
    
            time.sleep(10) # wait a little before polling
        except KeyboardInterrupt:
            job_ids = [x[0] for x in jobs]
            mig.cancel_jobs(job_ids)
            print "Cancelled jobs."
            break
Пример #7
0
    def update_jobs(self,jobs):
        new_status = False
        job_ids = map(lambda x : x["id"], jobs)
        job_info_list = mig.jobs_status(job_ids)
        if len(job_info_list) != len(job_ids):
            print str(job_info_list)
            print str(jobs)
            raise Exception("Critical job management error.  Job list lengths.")

        for i in range(len(jobs)):
            jobs[i]["status"] = job_info_list[i]

            if mig.job_finished(jobs[i]["id"]):
                jobs[i]["finished"] = self.get_time()
        
        print "update jobs", jobs
        
        return jobs
Пример #8
0
def main():
    """
    Executes the bash file test_executable.sh in a grid job. 
    Afterwards, the result is downloaded and printed to screen.
    """

    # mig.debug_mode_on() # uncomment to enable debug print outs
    # mig.local_mode_on() # uncomment to enable local mode execution

    mig.test_connection()  # Check if we can connect to the MiG server

    # The program we want to execute on the grid
    executable_file = "test_executable.sh"
    # The shell command to execute on the grid resource
    cmd = "./test_executable.sh > out.txt"
    # Create and submit the grid job
    job_id = mig.create_job(cmd,
                            output_files=["out.txt"],
                            executables=[executable_file])
    print "\nJob (ID : %s) submitted. \n\n" % job_id

    # Wait for the job to finish while monitoring the status
    polling_frequency = 10  # seconds
    while not mig.job_finished(job_id):
        job_info = mig.job_info(job_id)  # get an info dictionary
        print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
        time.sleep(polling_frequency)  # wait a while before polling again

    # Download the result file and print
    output_file = mig.get_file("out.txt")
    f = open(output_file)
    print "Output file (%s) contains :\n %s \n\n" % (output_file,
                                                     str(f.readlines()))
    f.close()

    # Clean up
    os.remove(output_file)  # remove locally
    mig.remove(output_file)  # remove on the MiG server
    print "Output (" + output_file + ") deleted."
Пример #9
0
def main():
    """
    Execute a simple grid job and print the output.
    """
    
    # mig.debug_mode_on() # uncomment to enable debug print outs
    # mig.local_mode_on() # uncomment to enable local mode execution

    # Check if we can connect to the MiG server
    mig.test_connection()

    # Create and submit the grid job
    job_id = mig.create_job("echo HELLO GRID")
    print "\nJob (ID : %s) submitted. \n\n" % job_id

    # Wait for the job to finish while monitoring the status
    polling_frequency = 10 # seconds
    while not mig.job_finished(job_id):
        job_info = mig.job_info(job_id) # get an info dictionary
        print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
        time.sleep(polling_frequency) # wait a while before polling again
    
    print mig.job_output(job_id)
Пример #10
0
def main():
    """
    Execute a simple grid job and print the output.
    """

    # mig.debug_mode_on() # uncomment to enable debug print outs
    # mig.local_mode_on() # uncomment to enable local mode execution

    # Check if we can connect to the MiG server
    mig.test_connection()

    # Create and submit the grid job
    job_id = mig.create_job("echo HELLO GRID")
    print "\nJob (ID : %s) submitted. \n\n" % job_id

    # Wait for the job to finish while monitoring the status
    polling_frequency = 10  # seconds
    while not mig.job_finished(job_id):
        job_info = mig.job_info(job_id)  # get an info dictionary
        print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
        time.sleep(polling_frequency)  # wait a while before polling again

    print mig.job_output(job_id)
Пример #11
0
def main():
    """
    Executes the bash file test_executable.sh in a grid job. 
    Afterwards, the result is downloaded and printed to screen.
    """

    # mig.debug_mode_on() # uncomment to enable debug print outs
    # mig.local_mode_on() # uncomment to enable local mode execution
    
    mig.test_connection() # Check if we can connect to the MiG server

    # The program we want to execute on the grid
    executable_file = "test_executable.sh"
    # The shell command to execute on the grid resource
    cmd = "./test_executable.sh > out.txt"
    # Create and submit the grid job
    job_id = mig.create_job(cmd, output_files=["out.txt"], executables=[executable_file])
    print "\nJob (ID : %s) submitted. \n\n" % job_id
    
    # Wait for the job to finish while monitoring the status
    polling_frequency = 10 # seconds
    while not mig.job_finished(job_id):
        job_info = mig.job_info(job_id) # get an info dictionary
        print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
        time.sleep(polling_frequency) # wait a while before polling again

    # Download the result file and print
    output_file = mig.get_file("out.txt")
    f = open(output_file)
    print "Output file (%s) contains :\n %s \n\n" % (output_file, str(f.readlines()))
    f.close()

    # Clean up
    os.remove(output_file) # remove locally
    mig.remove(output_file) # remove on the MiG server
    print "Output ("+output_file+") deleted."
Пример #12
0
def main():
    """
    Run five grid jobs executing the bash file parameter_sweet_script.sh with different input arguments.
    When a job has finished executing, the corresponding output file is downloaded.
    Finally, the output contents are printed.
    """

    # mig.debug_mode_on() # uncomment to enable debug print outs
    # mig.local_mode_on() # uncomment to enable local mode execution
    mig.test_connection()  # Check if we can connect to the MiG server

    input_values = range(5)  # Input parameters
    # The program we want to execute on grid resources
    executable_file = "parameter_sweep_script.sh"

    print "\nStarting grid jobs:\n"

    jobs = []
    for i in input_values:  # Start a job for each input
        output_file = "output%s.txt" % i  # The output file name
        # The shell command to start the script on the resource
        cmd = "./parameter_sweep_script.sh %i > %s" % (i, output_file)
        # Run the job resources on any vgrid
        resource_requirements = {"VGRID": "ANY"}
        # Start the grid job
        job_id = mig.create_job(cmd,
                                output_files=[output_file],
                                executables=[executable_file],
                                resource_specifications=resource_requirements)
        jobs.append((job_id, output_file))
        print "Job (ID : %s) submitted." % job_id
    print "\n\n"

    print "Monitor job status...\n"  # Now we wait for results

    finished_jobs = []
    while len(finished_jobs) < len(jobs):
        for id, output_file in jobs:
            job_info = mig.job_info(id)  # get an info dictionary
            print 'Grid job : %(ID)s \t %(STATUS)s ' % job_info
            if mig.job_finished(id) and id not in finished_jobs:
                # Download the output file from the server
                mig.get_file(output_file)
                finished_jobs.append(id)
                mig.remove(
                    output_file)  # clean up the result file on the server

        time.sleep(10)  # Wait a few seconds before trying again
        print "\n\n"

    print "All jobs finished."
    # Clean up the result files and print out the contents
    print "Cleaning up."
    output_lines = []
    for _, output_file in jobs:
        fh = open(output_file)
        output_lines.append(" ".join(fh.readlines()))
        fh.close()
        os.remove(output_file)
        print "Output file (" + output_file + ") deleted."

    print "\n\nOutput contents : \n\n%s\n" % "\n".join(output_lines)
Пример #13
0
def main():
    """
    Find edit distance value between entries in a reference file. First, divide the file into smaller blocks 
    and create a grid job for each. levenshtein.py is used to process each input block.
    When a job has finished executing, the corresponding output file is downloaded.
    """

    # mig.debug_mode_on() # uncomment to enable debug print outs
    # mig.local_mode_on() # uncomment to enable local mode execution

    reference_file = "ref1000.txt"
    # The reference blocks. One for each job we want to run.
    block_files = create_blocks(reference_file, block_size=200)

    # These are static input files for each job.
    levenshtein_files = [
        "Levenshtein_ucs4.so", "Levenshtein_ucs2.so", "levenshtein.py",
        "Levenshtein_i686.so"
    ]

    resource_requirements = {}
    resource_requirements[
        "RUNTIMEENVIRONMENT"] = "PYTHON-2"  # we need python on the resource
    resource_requirements["VGRID"] = "ANY"

    jobs = []

    # Start a grid job for each block file
    for block_file in block_files:
        output_file = block_file.strip(".txt") + "_output.txt"
        cmd = "$PYTHON levenshtein.py %s > %s" % (block_file, output_file)
        input_files = []
        input_files.extend(levenshtein_files)
        input_files.append(block_file)
        job_id = mig.create_job(cmd,
                                input_files=input_files,
                                output_files=output_file,
                                resource_specifications=resource_requirements)
        jobs.append((job_id, output_file))
        print "started job %s" % job_id

    jobs_done = 0
    # now wait for the results
    while len(jobs):
        print "Checking job status. jobs done : %i." % jobs_done
        try:
            for job_id, result_file in jobs:
                print "Checking job %s." % job_id
                if mig.job_finished(job_id):
                    if not os.path.exists("output"):
                        os.mkdir("output")
                    mig.get_file(result_file, "output/" + result_file)
                    jobs.remove((job_id, result_file))
                    jobs_done += 1

                    print "Job done. Downloaded result file %s." % result_file

            time.sleep(10)  # wait a little before polling
        except KeyboardInterrupt:
            job_ids = [x[0] for x in jobs]
            mig.cancel_jobs(job_ids)
            print "Cancelled jobs."
            break