Exemple #1
0
 def connect():
     try:
         db = JobDB()
         db.connect()
     except Exception, e:
         print(e)
         raise Exception("connection error")
Exemple #2
0
 def connect():
     try:
         db = JobDB()
         db.connect()
     except Exception, e:
         print(e)
         raise Exception("connection error")
Exemple #3
0
    def do_job(self, args, arguments):
        """
        ::

            Usage:
                job server start
                job server stop
                job server clean
                job server kill
                job server deploy
                job server ps
                job stat
                job list
                job add JOBLIST [--host=HOST] [--options=OPTIONS] [--inputs=INPUTS] [--outputs=OUTPUTS]
                job add --file=filename
                job write --file=filename
                job find --name=NAME
                job find --attribute=ATTRIBUTE --value=VALUE
                job delete JOBLIST

            Arguments:

                NAME       the name of the job
                HOST       the host on which the job should run
                OPTIONS    options passed to the command
                INPUTS     input files
                OUTPUTS    output files
                ATTRIBUTE  an attribute
                VALUE      a value
                JOBLIST    the job list

            Description:

                manages a job catalog to submit them to a computation cloud
                or Grid.

                job server start

                    starts the job server

                job server stop

                    stops the job server

                job server clean

                    removes all data in the job server and does a graceful clean, e.g deletes all scheduled jobs

                job server kill

                    kills just the job server, but does not delete the jobs from the schedulers.
                    this command should not be called in normal circumstances.

                job set GROUP

                    sets the default job group

                job add  GROUP TODO

                    adds a job to a group

                job server start

                    starts the server

                job server stop

                    stops the server

                job stat

                    prints a simple statistics of the jobs

                job add NAMES [--host=HOST] [--option=OPTIONS] [--inputs=INPUTS] [--outputs=OUTPUTS]

                    adds a number of jobs

                job add --file=filename

                    adds the job from the file. The postfix of the file deterimnes which
                    format it is. The formats supported are .csv, .yaml, .json

                job write --file=filename

                    writes the jobs to a file. The postfix of the file deterimnes which
                    format it is. The formats supported are .csv, .yaml, .json


                job list [--output=OUTPUT]

                    lists the jobs in the format specified

                job insert NAME [HOST] [OPTIONS] [INPUT_FILES] [OUTPUT_FILES]

                    inserts the job with the name into the job database. Options,
                    input and output files could be specified

                job find --name=NAME

                    find the job with the given name

                job find --attribute=ATTRIBUTE --value=VALUE

                    find jobs that match the given attribute.

                job delete JOBLIST

                    delete the job with the specified names in the joblist.

                THE FOLLOWING IS NOT YET DEFINED OR MAY CHANGE

                job add TODO

                    ... not yet sure what in the rest of the command

                    adds a job to the job server and returns its id

                job last

                    returns the last job added to the server

                job delete ID

                    deletes the job from the job server and cancels it if it is scheduled for execution.

                job info ID

                    give the info of a job

                job submit ID HOST

                    submits the job with the given ID to the host

                job list GROUP

                    lists the jobs in the group

                job status [ID | GROUP]

                    list the status of a single job or the status of all jobs in the group

                job status statistics

                    list the statistics of the jobs in the job server (e.g. for the states)



        """
        # pprint(arguments)

        if arguments.get("server"):

            if arguments["start"]:

                db = JobDB()
                db.start()

                Console.ok("job server start")

            elif arguments["stop"]:

                db = JobDB()
                db.stop()

                Console.ok("job server stop")

            elif arguments["ps"]:

                db = JobDB()
                db.ps()

            elif arguments["clean"]:

                Console.ok("job server clean")

            elif arguments["kill"]:

                Console.ok("job server kill")

            elif arguments["deploy"]:

                Console.ok("job server deploy")

        elif arguments["delete"] and arguments["JOBLIST"]:

            joblist = hostlist.expand_hostlist(arguments["JOBLIST"])

            # debug msg
            print(joblist)

            db = JobDB()
            db.connect()

            for job in joblist:
                # if job exists:
                Console.ok("delete job {:}".format(job))

                db.delete_jobs(attribute="name", value=job)


        elif arguments["add"]:

            joblist = hostlist.expand_hostlist(arguments["JOBLIST"])
            host = arguments["--host"]

            inputs = [None]
            outputs = [None]
            options = [None]

            if arguments["--inputs"]:
                inputs = hostlist.expand_hostlist(arguments["--inputs"])
            if arguments["--outputs"]:
                outputs = hostlist.expand_hostlist(arguments["--outputs"])
            if arguments["--options"]:
                options = hostlist.expand_hostlist(arguments["--options"])
            # check if inputs are either 0, 1 or the length of joblist

            def expand_parameter(parameter, label):
                """

                :param parameter:
                :param label:
                :return: list of strings
                """
                _parameter = parameter
                if len(_parameter) == 1:
                    _parameter = _parameter * len(joblist)
                elif len(_parameter) == len(joblist):
                    pass
                else:
                    Console.error("the number of input files do not match the hostlist")
                    print("joblist count:", len(joblist))
                    print(label, "count: ", len(_parameter))
                return _parameter

            options = expand_parameter(options, "options")
            inputs = expand_parameter(inputs, "inputs")
            outputs = expand_parameter(inputs, "outputs")

            pprint(joblist)
            pprint(inputs)
            pprint(outputs)

            # dependent on if 0, 1, or length of joblist handle that

            # debug msg
            print(joblist)

            for i in range(len(joblist)):
                banner(i)

                Console.ok("add job : {:} ".format(joblist[i]))
                Console.ok("  input : {:} ".format(inputs[i]))
                Console.ok("  output: {:} ".format(outputs[i]))




        elif arguments["stat"]:

            Console.ok("job stat")

        elif arguments["list"]:

            output = arguments["--output"]

            Console.ok("lists the jobs in the format specified")

        elif arguments["insert"]:

            name = arguments["NAME"]
            host = arguments["HOST"]
            options = arguments["OPTIONS"]
            input_files = arguments["INPUT_FILES"]
            output_file = arguments["OUTPUT_FILES"]

            Console.ok("insert")

        elif arguments["find"] and arguments["--name"]:

            name = arguments["NAME"]

            Console.ok("find the job with the given name")

        elif arguments["find"] and arguments["--attribute"] and arguments["--value"]:

            name = arguments["NAME"]
            attribute = arguments["--attribute"]
            value = arguments["--value"]

            Console.ok("job find --attribute=ATTRIBUTE --value=VALUE")



        pass
Exemple #4
0
def main():
    db = JobDB()

    banner("info")
    db.info()

    db.start()

    # db.ps()

    db.connect()

    db.clear()

    # Insert two jobs - one with only a name and one with input and output files
    job0_id = db.insert("job0")
    job1_id = db.insert("job1", "input1", "output1")

    # Print job IDs of both added jobs
    print job0_id
    print job1_id

    # Get all jobs with given parameters
    # Available key names at this time are:
    #   _id
    #   job_name
    #   input_filename
    #   output_filename
    jobs = db.find_jobs("job_name", "job1")

    # Print out all returned jobs
    for job in jobs:
        print job

    # Query by job ID to return a single job
    singleJob = list(db.find_jobs("_id", job0_id))

    # Print out the first job in the list
    print singleJob[0]

    # Print out count of all jobs
    print db.count()

    # Print out count of jobs given query parameters
    print db.count("job_name", "job1")

    # Show updating a job attribute
    print "\nORIGINAL JOB:"
    singleJob = list(db.find_jobs("_id", job1_id))
    print singleJob[0]

    # Update the input filename
    db.update_job_attribute(job1_id, "input_filename", "new_input_file")

    # Print out the updated job
    print "\nUPDATED JOB:"
    singleJob = list(db.find_jobs("_id", job1_id))
    print singleJob[0]

    # Add a job using add()
    job = {"job_name": "job25"}

    db.add(job)

    # Show modify() functionality
    job = {"job_name": "job30", "input_filename": "file1"}

    db.modify(job)

    print "\nORIGINAL JOB"
    for job in db.find_jobs():
        print job

    job = {"job_name": "job30", "input_filename": "file2"}

    db.modify(job)

    print "MODIFIED JOB"
    for job in db.find_jobs():
        print job

    # Show job statuses functionality
    print "\nJOB STATUSES:"
    db.job_status_stats()

    print "\nJOB STATUSES WITH JOBS PRINTED:"
    db.job_status_stats(True)

    # SHOW FIND_JOBS_WITH_FILE
    job100_id = db.insert("job100")
    job101_id = db.insert("job101", "file100", "file200")
    job102_id = db.insert("job102", "file200", "file300")

    inputs, outputs = db.find_jobs_with_file("file200")

    print("\nJobs with matching file in input:")
    for job in inputs:
        print job

    print("\nJobs with matching file in output:")
    for job in outputs:
        print job

    # Delete all jobs
    db.clear()

    # SHOW ADD_FROM_YAML
    db.add_from_yaml("job_example.yaml")

    for job in db.find_jobs():
        print job

    # Delete all jobs
    db.clear()
    print "Database cleared."
    print "Job count: " + str(db.count())

    db.stop()
Exemple #5
0
class TestJobDB:
    def setup(self):
        # HEADING()
        self.db = JobDB()

    def teardown(self):
        # HEADING()
        pass

    def test_001_start(self):
        """
        tests if the mongo db can be started
        :return:
        """
        HEADING()
        self.db.start()
        up = self.db.isup()
        result = up
        assert result

    def test_002_up(self):
        """
        tests if the mongo db can be started
        :return:
        """
        HEADING()
        up = self.db.isup()
        print(up)
        self.db.stop()
        down = not self.db.isup()
        self.db.start()
        assert up and down

    def test_003_pid(self):
        """
        tests if the mongo db can be started
        :return:
        """
        HEADING()
        pid = self.db.pid()
        print(pid)
        assert True

    def test_004_connect(self):
        """
        tests if a mongo db can be connected to
        :return:
        """
        HEADING()
        self.db.connect()

        result = True
        assert result

    def test_005_clear(self):
        """
        tests clearing all jobs from the db
        :return:
        """
        HEADING()
        db = self.db
        db.connect()

        db.clear()

        # assert not os.path.isfile(path_expand("~/.cloudmesh/pbs/pbs.db"))
        assert (len(db) == 0)

    def test_006_add(self):
        """
        tests adding jobs to the db
        :return:
        """
        HEADING()
        db = self.db

        count = 5

        db.connect()

        db.delete_jobs()

        for id in range(0, count):
            job = db.insert("job" + str(id))

        assert len(db) == count

    def test_007_delete(self):
        """
        tests deleting a single job from the db
        :return:
        """
        HEADING()
        db = self.db

        db.connect()
        print("AAA")
        before_count = len(db)
        print("CCC", len(db))
        job = db.insert("deleteme")
        print("DDD", len(db))

        job = db.delete_jobs("job_name", "deleteme")
        print("EEE")
        after_count = len(db)
        print("FFF", len(db))
        assert (before_count - after_count == 0)

    def test_008_modify(self):
        """
        tests modifying a single job in the db
        :return:
        """
        HEADING()
        db = self.db

        db.connect()

        job = {"job_name": "modifyme", "input_filename": "file1"}

        db.add(job)

        originalFilename = self.db.find_jobs("job_name",
                                             "modifyme")[0]["input_filename"]

        job = {"job_name": "modifyme", "input_filename": "file2"}

        db.modify(job)

        newFilename = self.db.find_jobs("job_name",
                                        "modifyme")[0]["input_filename"]

        assert (originalFilename != newFilename)

    def test_010_info(self):
        """
        prints the info about the db
        :return:
        """
        HEADING()
        db = self.db

        db.connect()
        db.info()
        pass

    def test_011_len(self):
        """
        tests modifying a single job in the db
        :return:
        """
        HEADING()
        db = self.db

        db.connect()
        count = len(db)
        print(count)
        assert count == 6

    def test_012_yaml_load(self):
        """
        tests adding jobs from a YAML file
        :return:
        """
        HEADING()
        db = self.db
        db.connect()

        # Clear all jobs currently in the database to ensure a correct final assertion
        db.clear()

        # Add the jobs outlined in the YAML file
        db.add_from_yaml("etc/jobs.yaml")

        count_fgrep = len(Shell.fgrep("input:", "etc/jobs.yaml").split("\n"))

        # Assert that the correct number jobs have been added
        assert (db.count() == count_fgrep)

    def test_013_find_files(self):
        """
        tests searching for a file
            the file being searched for exists in 3 files: twice as input and twice as output
        :return:
        """
        HEADING()
        db = self.db

        db.connect()

        # Clear all jobs currently in the database to ensure a correct final assertion
        db.clear()

        # Add the jobs outlined in the YAML file
        db.add_from_yaml("etc/jobs.yaml")
        inputs, outputs = db.find_jobs_with_file("in1.txt")

        # Assert that the lengths of the inputs and outputs arrays are correct
        count_fgrep = len(
            Shell.fgrep("in1.txt", "etc/jobs.yaml").strip().split("\n"))
        assert (len(inputs) == count_fgrep)

    def test_14_jobid(self):
        HEADING()
        db = self.db

        db.connect()

        db.set_jobid(11)

        a = db.get_jobid()
        db.incr_jobid()
        b = db.get_jobid()

        print(a, b)
        assert b == a + 1

    def test_15_jobscript(self):
        HEADING()
        db = self.db
        db.connect()

        name = "test1"
        contents = "hallo"

        db.add_script(name, contents)
        db.add_script("test2", "other")
        script = db.get_script(name)

        print("Script:", script)
        print("Content:", contents)

        db.write_script(name, "/tmp/script.txt")
        what = Shell.cat("/tmp/script.txt")

        assert contents == what
        assert contents == script

    def test_999_stop(self):
        """
        tests if the mongo db can be shutdown
        :return:
        """
        HEADING()
        self.db.stop()
        result = True
        assert result
Exemple #6
0
def main():
    db = JobDB()

    banner("info")
    db.info()

    db.start()

    # db.ps()

    db.connect()

    db.clear()

    # Insert two jobs - one with only a name and one with input and output files
    job0_id = db.insert("job0")
    job1_id = db.insert("job1", "input1", "output1")

    # Print job IDs of both added jobs
    print job0_id
    print job1_id

    # Get all jobs with given parameters
    # Available key names at this time are:
    #   _id
    #   job_name
    #   input_filename
    #   output_filename
    jobs = db.find_jobs("job_name", "job1")

    # Print out all returned jobs
    for job in jobs:
        print job

    # Query by job ID to return a single job
    singleJob = list(db.find_jobs("_id", job0_id))

    # Print out the first job in the list
    print singleJob[0]

    # Print out count of all jobs
    print db.count()

    # Print out count of jobs given query parameters
    print db.count("job_name", "job1")

    # Show updating a job attribute
    print "\nORIGINAL JOB:"
    singleJob = list(db.find_jobs("_id", job1_id))
    print singleJob[0]

    # Update the input filename
    db.update_job_attribute(job1_id, "input_filename", "new_input_file")

    # Print out the updated job
    print "\nUPDATED JOB:"
    singleJob = list(db.find_jobs("_id", job1_id))
    print singleJob[0]

    # Add a job using add()
    job = {"job_name": "job25"}

    db.add(job)

    # Show modify() functionality
    job = {"job_name": "job30", "input_filename": "file1"}

    db.modify(job)

    print "\nORIGINAL JOB"
    for job in db.find_jobs():
        print job

    job = {"job_name": "job30", "input_filename": "file2"}

    db.modify(job)

    print "MODIFIED JOB"
    for job in db.find_jobs():
        print job

    # Show job statuses functionality
    print "\nJOB STATUSES:"
    db.job_status_stats()

    print "\nJOB STATUSES WITH JOBS PRINTED:"
    db.job_status_stats(True)

    # SHOW FIND_JOBS_WITH_FILE
    job100_id = db.insert("job100")
    job101_id = db.insert("job101", "file100", "file200")
    job102_id = db.insert("job102", "file200", "file300")

    inputs, outputs = db.find_jobs_with_file("file200")

    print("\nJobs with matching file in input:")
    for job in inputs:
        print job

    print("\nJobs with matching file in output:")
    for job in outputs:
        print job

    # Delete all jobs
    db.clear()

    # SHOW ADD_FROM_YAML
    db.add_from_yaml("job_example.yaml")

    for job in db.find_jobs():
        print job

    # Delete all jobs
    db.clear()
    print "Database cleared."
    print "Job count: " + str(db.count())

    db.stop()
Exemple #7
0
class TestJobDB:

    def setup(self):
        # HEADING()
        self.db = JobDB()

    def teardown(self):
        # HEADING()
        pass

    def test_001_start(self):
        """
        tests if the mongo db can be started
        :return:
        """
        HEADING()
        self.db.start()
        up = self.db.isup()
        result = up
        assert result
        
    def test_002_up(self):
        """
        tests if the mongo db can be started
        :return:
        """
        HEADING()
        up = self.db.isup()
        print (up)
        self.db.stop()
        down = not self.db.isup()
        self.db.start()
        assert up and down

    def test_003_pid(self):
        """
        tests if the mongo db can be started
        :return:
        """
        HEADING()
        pid = self.db.pid()
        print (pid)
        assert True

    def test_004_connect(self):
        """
        tests if a mongo db can be connected to
        :return:
        """
        HEADING()
        self.db.connect()

        result = True
        assert result

    def test_005_clear(self):
        """
        tests clearing all jobs from the db
        :return:
        """
        HEADING()
        db = self.db
        db.connect()

        db.clear()

        # assert not os.path.isfile(path_expand("~/.cloudmesh/pbs/pbs.db"))
        assert(len(db) == 0)

    def test_006_add(self):
        """
        tests adding jobs to the db
        :return:
        """
        HEADING()
        db = self.db

        count = 5

        db.connect()

        db.delete_jobs()

        for id in range(0,count):
            job = db.insert("job" + str(id))

        assert len(db) == count

    def test_007_delete(self):
        """
        tests deleting a single job from the db
        :return:
        """
        HEADING()
        db = self.db

        db.connect()
        print ("AAA")
        before_count = len(db)
        print ("CCC", len(db))
        job = db.insert("deleteme")
        print ("DDD", len(db))

        job = db.delete_jobs("job_name", "deleteme")
        print ("EEE")
        after_count = len(db)
        print ("FFF", len(db))
        assert(before_count - after_count == 0)

    def test_008_modify(self):
        """
        tests modifying a single job in the db
        :return:
        """
        HEADING()
        db = self.db

        db.connect()

        job = {"job_name": "modifyme", "input_filename":"file1"}

        db.add(job)

        originalFilename = self.db.find_jobs("job_name", "modifyme")[0]["input_filename"]

        job = {"job_name": "modifyme", "input_filename":"file2"}

        db.modify(job)

        newFilename = self.db.find_jobs("job_name", "modifyme")[0]["input_filename"]

        assert(originalFilename != newFilename)

    def test_010_info(self):
        """
        prints the info about the db
        :return:
        """
        HEADING()
        db = self.db

        db.connect()
        db.info()
        pass

    def test_011_len(self):
        """
        tests modifying a single job in the db
        :return:
        """
        HEADING()
        db = self.db

        db.connect()
        count = len(db)
        print (count)
        assert count == 6

    def test_012_yaml_load(self):
        """
        tests adding jobs from a YAML file
        :return:
        """
        HEADING()
        db = self.db
        db.connect()

        # Clear all jobs currently in the database to ensure a correct final assertion
        db.clear()

        # Add the jobs outlined in the YAML file
        db.add_from_yaml("etc/jobs.yaml")

        count_fgrep = len(Shell.fgrep("input:", "etc/jobs.yaml").split("\n"))

        # Assert that the correct number jobs have been added
        assert(db.count() == count_fgrep)

    def test_013_find_files(self):
        """
        tests searching for a file
            the file being searched for exists in 3 files: twice as input and twice as output
        :return:
        """
        HEADING()
        db = self.db

        db.connect()

        # Clear all jobs currently in the database to ensure a correct final assertion
        db.clear()

        # Add the jobs outlined in the YAML file
        db.add_from_yaml("etc/jobs.yaml")
        inputs, outputs = db.find_jobs_with_file("in1.txt")

        # Assert that the lengths of the inputs and outputs arrays are correct
        count_fgrep = len(Shell.fgrep("in1.txt", "etc/jobs.yaml").strip().split("\n"))
        assert(len(inputs) == count_fgrep)


    def test_14_jobid(self):
        HEADING()
        db = self.db

        db.connect()

        db.set_jobid(11)

        a = db.get_jobid()
        db.incr_jobid()
        b = db.get_jobid()

        print (a, b)
        assert b == a + 1

    def test_15_jobscript(self):
        HEADING()
        db = self.db
        db.connect()

        name = "test1"
        contents = "hallo"

        db.add_script(name, contents)
        db.add_script("test2", "other")
        script = db.get_script(name)

        print ("Script:", script)
        print ("Content:", contents)

        db.write_script(name, "/tmp/script.txt")
        what = Shell.cat("/tmp/script.txt")

        assert contents == what
        assert contents == script



    def test_999_stop(self):
        """
        tests if the mongo db can be shutdown
        :return:
        """
        HEADING()
        self.db.stop()
        result = True
        assert result