Example #1
0
    def __init__(self, source=None):  # Open file or path to file with limit set
        """Construct a limit processor.

        The 'source' argument can be one of three types:

        string - Path to a file to open and read.

        file - Open file handle to read.

        None - The limit processor becomes inert and passes all limits
        unconditionally.
        """

        # If we were given no source, put the processor into an inert
        # mode where everything passes.
        self.inert = source is None
        if self.inert:
            return

        #
        # Load the validation data
        #

        validation_path = os.path.join(os.path.dirname(__file__), "pscheduler-limits-validate.json")
        # TODO: Throw something nicer than IOError if this fails.
        validation_file = open(validation_path, "r")
        validation = pscheduler.json_load(validation_file)
        validation_file.close()

        #
        # Inhale the source and validate it
        #

        if type(source) is str or type(source) is unicode:
            source = open(source, "r")
        elif type(source) is file:
            pass  # We're good with this.
        else:
            raise ValueError("Source must be a file or path")

        # At this point, source is a file.

        assert type(source) is file
        limit_config = pscheduler.json_load(source)

        valid, message = pscheduler.json_validate(limit_config, validation)

        if not valid:
            raise ValueError("Invalid limit file: %s" % message)

        #
        # Set up all of the stages
        #

        self.identifiers = IdentifierSet(limit_config["identifiers"])
        self.classifiers = ClassifierSet(limit_config["classifiers"], self.identifiers)
        self.limits = LimitSet(limit_config["limits"])
        self.applications = ApplicationSet(limit_config["applications"], self.classifiers, self.limits)
Example #2
0
    def __init__(self,
                 source=None  # Open file or path to file with limit set
    ):
        """Construct a limit processor.

        The 'source' argument can be one of three types:

        string - Path to a file to open and read.

        file - Open file handle to read.

        None - The limit processor becomes inert and passes all limits
        unconditionally.
        """

        # If we were given no source, put the processor into an inert
        # mode where everything passes.
        self.inert = source is None
        if self.inert:
            return


        #
        # Load the validation data
        #

        validation_path = os.path.join(os.path.dirname(__file__),
                                       'pscheduler-limits-validate.json')
        # TODO: Throw something nicer than IOError if this fails.
        validation_file = open(validation_path, 'r')
        # NOTE: Don't max_schema this.  The limit validation file is
        # tied to this module.
        try:
            validation = pscheduler.json_load(validation_file)
        except Exception as ex:
            raise ValueError("Invalid validation file: %s" % (str(ex)))
        finally:
            validation_file.close()

        #
        # Inhale the source and validate it
        #

        if type(source) is str or type(source) is unicode:
            source = open(source, 'r')
        elif type(source) is file:
            pass  # We're good with this.
        else:
            raise ValueError("Source must be a file or path")

        # At this point, source is a file.

        assert type(source) is file
        limit_config = pscheduler.json_load(source)

        valid, message = pscheduler.json_validate(limit_config, validation)

        if not valid:
            raise ValueError("Invalid limit file: %s" % message)

        #
        # Set up all of the stages
        #

        self.identifiers  = IdentifierSet(limit_config['identifiers'])
        self.classifiers  = ClassifierSet(limit_config['classifiers'],
                                          self.identifiers)
        self.rewriter     = Rewriter(limit_config['rewrite']) \
                            if 'rewrite' in limit_config else None
        self.limits       = LimitSet(limit_config['limits'])
        self.applications = ApplicationSet(limit_config['applications'],
                                           self.classifiers, self.limits)

        try:
            self.prioritizer = Prioritizer(limit_config["priority"])
        except KeyError:
            self.prioritizer = None
Example #3
0
class LimitProcessor():

    """
    pScheduler limit processing class
    """


    def __init__(self,
                 source=None  # Open file or path to file with limit set
    ):
        """Construct a limit processor.

        The 'source' argument can be one of three types:

        string - Path to a file to open and read.

        file - Open file handle to read.

        None - The limit processor becomes inert and passes all limits
        unconditionally.
        """

        # If we were given no source, put the processor into an inert
        # mode where everything passes.
        self.inert = source is None
        if self.inert:
            return


        #
        # Load the validation data
        #

        validation_path = os.path.join(os.path.dirname(__file__),
                                       'pscheduler-limits-validate.json')
        # TODO: Throw something nicer than IOError if this fails.
        validation_file = open(validation_path, 'r')
        # NOTE: Don't max_schema this.  The limit validation file is
        # tied to this module.
        try:
            validation = pscheduler.json_load(validation_file)
        except Exception as ex:
            raise ValueError("Invalid validation file: %s" % (str(ex)))
        finally:
            validation_file.close()

        #
        # Inhale the source and validate it
        #

        if type(source) is str or type(source) is unicode:
            source = open(source, 'r')
        elif type(source) is file:
            pass  # We're good with this.
        else:
            raise ValueError("Source must be a file or path")

        # At this point, source is a file.

        assert type(source) is file
        limit_config = pscheduler.json_load(source)

        valid, message = pscheduler.json_validate(limit_config, validation)

        if not valid:
            raise ValueError("Invalid limit file: %s" % message)

        #
        # Set up all of the stages
        #

        self.identifiers  = IdentifierSet(limit_config['identifiers'])
        self.classifiers  = ClassifierSet(limit_config['classifiers'],
                                          self.identifiers)
        self.rewriter     = Rewriter(limit_config['rewrite']) \
                            if 'rewrite' in limit_config else None
        self.limits       = LimitSet(limit_config['limits'])
        self.applications = ApplicationSet(limit_config['applications'],
                                           self.classifiers, self.limits)

        try:
            self.prioritizer = Prioritizer(limit_config["priority"])
        except KeyError:
            self.prioritizer = None


    def process(self, task, hints, rewrite=True, prioritize=False):
        """Evaluate a proposed task against the full limit set.

        If the task has no 'run_schedule' section it will be assumed that
        is being evaluated on all other parameters except when it
        runs.  (This will be used for restricting submission of new
        tasks.)

        Arguments:
            task - The proposed task, with run_schedule if there's a run time
            hints - A hash of the hints to be used in identification
            rewrite - True if the rewriter should be applied
            prioritize - True to prioritize the task

        Returns a tuple containing:
            passed - True if the proposed task passed the limits applied
            limits - A list of the limits that passed
            diags - A textual summary of how the conclusion was reached
            task - The task, after rewriting or None if unchanged
            priority - Integer priority or None of not calculated
        """

        if self.inert:
            return True, [], "No limits were applied", None, None

        # TODO: Should this be JSON, or is text sufficient?
        diags = []

        if hints is not None and len(hints) > 0:
            diags.append("Hints:")
            diags.extend([
                pscheduler.indent("%s: %s" % (item, str(hints[item])))
                for item in sorted(hints)
            ])

        # Everything we know about what's being proposed
        proposal = {
            "hints": hints,
            "task": task,
        }

        #
        # Identification
        #

        identifications = self.identifiers.identities(hints)
        if not identifications:
            diags.append("Made no identifications.")
            return False, [], '\n'.join(diags), None, None
        diags.append("Identified as %s" % (', '.join(identifications)))

        #
        # Classification
        #

        classifications = self.classifiers.classifications(identifications)
        if not classifications:
            diags.append("Made no classifications.")
            return False, [], '\n'.join(diags), None, None
        diags.append("Classified as %s" % (', '.join(classifications)))

        check_schedule='run_schedule' in task

        re_new_task = None

        #
        # Rewriting
        #

        if self.rewriter is not None and rewrite:

            try:
                re_changed, re_new_task, re_diags \
                    = self.rewriter(proposal, classifications)
            except Exception as ex:
                return False, [], "Error while rewriting: %s" % (str(ex)), None, None

            if re_changed:
                diags.append("Rewriter made changes:")
                if len(re_diags):
                    diags += map(lambda s: "  " + s, re_diags)
                else:
                    diags.append("  (Not enumerated)")
                task = re_new_task


        #
        # Applications
        #

        passed, app_limits_passed, app_diags \
            = self.applications.check(proposal, classifications, check_schedule)

        diags.append(app_diags)
        diags.append("Proposal %s limits" % ("meets" if passed else "does not meet"))

        # If any of the passed applications had no task limits, there
        # should be no limits placed on the run.

        unlimited = len(app_limits_passed) == 0 \
                    or min([ len(item) for item in app_limits_passed ]) == 0


        #
        # Priorities
        #

        if prioritize and passed and self.prioritizer is not None:
            try:
                priority, pri_diags = self.prioritizer(task, classifications)
            except Exception as ex:
                return False, [], "Error determining priority: %s" % (str(ex)), None, None

            if priority is None:
                return False, [], "Prioritizer produced no result", None, None

            requested_priority = task.get("priority", None)
            if requested_priority is not None:
                requested_message = " %d requested," % (requested_priority)
            else:
                requested_message =""

            diags.append("Priority%s set at %d%s" % (
                requested_message, priority, ":" if len(pri_diags) else "."))
            if len(pri_diags):
                diags += map(lambda s: "  " + s, pri_diags)
        else:
            priority = 0
            diags.append("Priority set to default of %d" % (priority))



        return passed, \
            [] if (unlimited or not passed) else app_limits_passed, \
            '\n'.join(diags), \
            re_new_task, \
            priority
Example #4
0
class LimitProcessor:

    """
    pScheduler limit processing class
    """

    def __init__(self, source=None):  # Open file or path to file with limit set
        """Construct a limit processor.

        The 'source' argument can be one of three types:

        string - Path to a file to open and read.

        file - Open file handle to read.

        None - The limit processor becomes inert and passes all limits
        unconditionally.
        """

        # If we were given no source, put the processor into an inert
        # mode where everything passes.
        self.inert = source is None
        if self.inert:
            return

        #
        # Load the validation data
        #

        validation_path = os.path.join(os.path.dirname(__file__), "pscheduler-limits-validate.json")
        # TODO: Throw something nicer than IOError if this fails.
        validation_file = open(validation_path, "r")
        validation = pscheduler.json_load(validation_file)
        validation_file.close()

        #
        # Inhale the source and validate it
        #

        if type(source) is str or type(source) is unicode:
            source = open(source, "r")
        elif type(source) is file:
            pass  # We're good with this.
        else:
            raise ValueError("Source must be a file or path")

        # At this point, source is a file.

        assert type(source) is file
        limit_config = pscheduler.json_load(source)

        valid, message = pscheduler.json_validate(limit_config, validation)

        if not valid:
            raise ValueError("Invalid limit file: %s" % message)

        #
        # Set up all of the stages
        #

        self.identifiers = IdentifierSet(limit_config["identifiers"])
        self.classifiers = ClassifierSet(limit_config["classifiers"], self.identifiers)
        self.limits = LimitSet(limit_config["limits"])
        self.applications = ApplicationSet(limit_config["applications"], self.classifiers, self.limits)

    def process(self, task, hints):
        """Evaluate a proposed task against the full limit set.

        If the task has no 'schedule' section it will be assumed that
        is being evaluated on all other parameters except when it
        runs.  (This will be used for restricting submission of new
        tasks.)

        Arguments:
            task - The proposed task
            hints - A hash of the hints to be used in identification

        Returns a tuple containing:
            passed - True if the proposed task passed the limits applied
            diags - A textual summary of how the conclusion was reached
        """

        if self.inert:
            return True, "No limits were applied"

        # TODO: Consider checking to see if the source file has
        # changed and reloading it if it has.

        # TODO: Should this be JSON, or is text sufficient?
        diags = []

        identifications = self.identifiers.identities(hints)
        if not identifications:
            diags.append("Made no identifications.")
            return False, "\n".join(diags)
        diags.append("Identified as %s" % (", ".join(identifications)))

        classifications = self.classifiers.classifications(identifications)
        if not classifications:
            diags.append("Made no classifications.")
            return False, "\n".join(diags)
        diags.append("Classified as %s" % (", ".join(classifications)))

        check_schedule = "schedule" in task

        passed, app_diags = self.applications.check(task, classifications, check_schedule)

        diags.append(app_diags)

        diags.append("Proposal %s limits" % ("meets" if passed else "does not meet"))

        return passed, "\n".join(diags)
class LimitProcessor():

    """
    pScheduler limit processing class
    """


    def __init__(self,
                 source=None  # Open file or path to file with limit set
    ):
        """Construct a limit processor.

        The 'source' argument can be one of three types:

        string - Path to a file to open and read.

        file - Open file handle to read.

        None - The limit processor becomes inert and passes all limits
        unconditionally.
        """

        # If we were given no source, put the processor into an inert
        # mode where everything passes.
        self.inert = source is None
        if self.inert:
            return


        #
        # Load the validation data
        #

        validation_path = os.path.join(os.path.dirname(__file__),
                                       'pscheduler-limits-validate.json')
        # TODO: Throw something nicer than IOError if this fails.
        validation_file = open(validation_path, 'r')
        # NOTE: Don't max_schema this.  The limit validation file is
        # tied to this module.
        validation = pscheduler.json_load(validation_file)
        validation_file.close()

        #
        # Inhale the source and validate it
        #

        if type(source) is str or type(source) is unicode:
            source = open(source, 'r')
        elif type(source) is file:
            pass  # We're good with this.
        else:
            raise ValueError("Source must be a file or path")

        # At this point, source is a file.

        assert type(source) is file
        limit_config = pscheduler.json_load(source, max_schema=1)

        valid, message = pscheduler.json_validate(limit_config, validation)

        if not valid:
            raise ValueError("Invalid limit file: %s" % message)

        #
        # Set up all of the stages
        #

        self.identifiers  = IdentifierSet(limit_config['identifiers'])
        self.classifiers  = ClassifierSet(limit_config['classifiers'],
                                          self.identifiers)
        self.limits       = LimitSet(limit_config['limits'])
        self.applications = ApplicationSet(limit_config['applications'],
                                           self.classifiers, self.limits)



    def process(self, task, hints):
        """Evaluate a proposed task against the full limit set.

        If the task has no 'schedule' section it will be assumed that
        is being evaluated on all other parameters except when it
        runs.  (This will be used for restricting submission of new
        tasks.)

        Arguments:
            task - The proposed task
            hints - A hash of the hints to be used in identification

        Returns a tuple containing:
            passed - True if the proposed task passed the limits applied
            limits - A list of the limits that passed
            diags - A textual summary of how the conclusion was reached
        """

        if self.inert:
            return True, [], "No limits were applied"

        # TODO: Should this be JSON, or is text sufficient?
        diags = []

        if hints is not None and len(hints) > 0:
            diags.append("Hints:")
            diags.extend([
                pscheduler.indent("%s: %s" % (item, str(hints[item])))
                for item in sorted(hints)
            ])

        identifications = self.identifiers.identities(hints)
        if not identifications:
            diags.append("Made no identifications.")
            return False, [], '\n'.join(diags)
        diags.append("Identified as %s" % (', '.join(identifications)))

        classifications = self.classifiers.classifications(identifications)
        if not classifications:
            diags.append("Made no classifications.")
            return False, [], '\n'.join(diags)
        diags.append("Classified as %s" % (', '.join(classifications)))

        check_schedule='schedule' in task

        passed, app_limits_passed, app_diags \
            = self.applications.check(task, classifications, check_schedule)

        diags.append(app_diags)
        diags.append("Proposal %s limits" % ("meets" if passed else "does not meet"))

        # If any of the passed applications had no task limits, there
        # should be no limits placed on the run.

        unlimited = len(app_limits_passed) == 0 \
                    or min([ len(item) for item in app_limits_passed ]) == 0


        return passed, \
            [] if (unlimited or not passed) else app_limits_passed, \
            '\n'.join(diags)