示例#1
0
文件: broker.py 项目: piojo/aquilon
    def __init__(self):
        """ Provides some convenient variables for commands.

        Also sets requires_* parameters for some sets of commands.
        All of the command objects are singletons (or Borg).

        """
        self.dbf = DbFactory()
        self.config = Config()
        self.az = AuthorizationBroker()
        self.formatter = ResponseFormatter()
        self.catalog = StatusCatalog()
        # Force the instance to have local copies of the class defaults...
        # This allows resources.py to modify instances without worrying
        # about inheritance issues (classes sharing required or optional
        # parameters).
        self.required_parameters = self.required_parameters[:]
        self.optional_parameters = self.optional_parameters[:]

        # Parameter checks are filled in automatically based on input.xml. This
        # lets us do some rudimentary checks before the actual command is
        # invoked.
        self.parameter_checks = {}

        self.action = self.__module__
        package_prefix = "aquilon.worker.commands."
        if self.action.startswith(package_prefix):
            self.action = self.action[len(package_prefix):]
        # self.command is set correctly in resources.py after parsing input.xml
        self.command = self.action
        # The readonly and format flags are done here for convenience
        # and simplicity.  They could be overridden by the __init__
        # method of any show/search/cat commands that do not want these
        # defaults.  Some 'one-off' commands (like ping and status)
        # just set the variables themselves.
        if self.action.startswith("show") or self.action.startswith("search"):
            self.requires_readonly = True
            self.requires_format = True
        if self.action.startswith("cat"):
            self.requires_format = True
            self.requires_readonly = True
            self._is_lock_free = True
        if not self.requires_readonly \
           and self.config.get('broker', 'mode') == 'readonly':
            self.badmode = 'readonly'
        else:
            self.badmode = False
        self._update_render(self.render)
        if not self.defer_to_thread:
            if self.requires_azcheck or self.requires_transaction:
                self.defer_to_thread = True
                log.msg("Forcing defer_to_thread to True because of "
                        "required authorization or transaction for %s" %
                        self.command)
            # Not sure how to handle formatting with deferred...
            self.requires_format = False
示例#2
0
    def __init__(self, config):
        formatter = ResponseFormatter()
        ResponsePage.__init__(self, '', formatter)
        self.config = config

        # Regular Expression for matching variables in a path definition.
        # Currently only supports stuffing a single variable in a path
        # component.
        varmatch = re.compile(r'^%\((.*)\)s$')

        BINDIR = os.path.dirname(os.path.realpath(sys.argv[0]))
        tree = ET.parse(os.path.join(BINDIR, '..', 'etc', 'input.xml'))

        for command in tree.getiterator("command"):
            for transport in command.getiterator("transport"):
                if "name" not in command.attrib \
                        or "method" not in transport.attrib \
                        or "path" not in transport.attrib:
                    continue
                name = command.attrib["name"]
                method = transport.attrib["method"]
                path = transport.attrib["path"]
                trigger = transport.attrib.get("trigger")
                container = self
                relative = ""
                # Traverse down the resource tree, container will
                # end up pointing to the correct spot.
                # Create branches and leaves as necessary, continueing to
                # traverse downward.
                for component in path.split("/"):
                    relative = relative + "/" + component
                    #log.msg("Working with component '" + component + "' of '" + relative + "'.")
                    m = varmatch.match(component)
                    # Is this piece of the path dynamic?
                    if not m:
                        #log.msg("Component '" + component + "' is static.")
                        child = container.getStaticEntity(component)
                        if child is None:
                            #log.msg("Creating new static component '" + component + "'.")
                            child = ResponsePage(relative, formatter)
                            container.putChild(component, child)
                        container = child
                    else:
                        #log.msg("Component '" + component + "' is dynamic.")
                        path_variable = m.group(1)
                        if container.dynamic_child is not None:
                            #log.msg("Dynamic component '" + component + "' already exists.")
                            current_variable = container.dynamic_child.\
                                    path_variable
                            if current_variable != path_variable:
                                log.msg("Warning: Could not use variable '" +
                                        path_variable + "', already have " +
                                        "dynamic variable '" +
                                        current_variable + "'.")
                                # XXX: Raise an error if they don't match
                                container = container.dynamic_child
                            else:
                                #log.msg("Dynamic component '" + component + "' had correct variable.")
                                container = container.dynamic_child
                        else:
                            #log.msg("Creating new dynamic component '" + component + "'.")
                            child = ResponsePage(relative,
                                                 formatter,
                                                 path_variable=path_variable)
                            container.dynamic_child = child
                            container = child

                fullcommand = name
                if trigger:
                    fullcommand = fullcommand + "_" + trigger
                mymodule = getattr(commands, fullcommand, None)
                if not mymodule:
                    log.msg("No module available in aquilon.worker.commands " +
                            "for %s" % fullcommand)
                # See commands/__init__.py for more info here...
                myinstance = getattr(mymodule, "broker_command", None)
                if not myinstance:
                    log.msg("No class instance available for %s" % fullcommand)
                    myinstance = BrokerCommand()
                myinstance.command = name
                rendermethod = method.upper()
                if container.handlers.get(rendermethod, None):
                    log.msg("Warning: Already have a %s here at %s..." %
                            (rendermethod, container.path))
                #log.msg("Setting 'command_" + fullcommand + "' as '" + rendermethod + "' for container '" + container.path + "'.")
                container.handlers[rendermethod] = myinstance

                # Since we are parsing input.xml anyway, record the possible
                # parameters...
                for option in command.getiterator("option"):
                    if "name" not in option.attrib:
                        continue
                    option_name = option.attrib["name"]
                    if option_name not in myinstance.optional_parameters:
                        myinstance.optional_parameters.append(option_name)
                    if "type" in option.attrib:
                        paramtype = option.attrib["type"]
                        myinstance.parameter_types[option_name] = paramtype
                        if paramtype == "int":
                            myinstance.parameter_checks[
                                option_name] = force_int
                        elif paramtype == "float":
                            myinstance.parameter_checks[
                                option_name] = force_float
                        elif paramtype == "boolean" or paramtype == "flag":
                            myinstance.parameter_checks[
                                option_name] = force_boolean
                        elif paramtype == "ipv4":
                            myinstance.parameter_checks[
                                option_name] = force_ipv4
                        elif paramtype == "mac":
                            myinstance.parameter_checks[
                                option_name] = force_mac
                        elif paramtype == "json":
                            myinstance.parameter_checks[
                                option_name] = force_json_dict
                        elif paramtype == "string" or paramtype == "file":
                            # Current twisted can't handle unicode output, so
                            # do not allow non-ASCII input either
                            myinstance.parameter_checks[
                                option_name] = force_ascii
                        elif paramtype == "list":
                            myinstance.parameter_checks[
                                option_name] = force_list
                        else:  # pragma: no cover
                            log.msg("Warning: unknown option type %s" %
                                    paramtype)
                    else:  # pragma: no cover
                        log.msg("Warning: argument type not known for %s.%s" %
                                (myinstance.command, option_name))
                    pbt = myinstance.parameters_by_type
                    for option_name, paramtype \
                            in myinstance.parameter_types.items():
                        if paramtype in pbt:
                            pbt[paramtype].append(option_name)
                        else:
                            pbt[paramtype] = [option_name]

        cache_version(config)
        log.msg("Starting aqd version %s" % config.get("broker", "version"))
        self.make_required_dirs()

        def _logChildren(level, container):
            for (key, child) in container.listStaticEntities():
                log.msg("Resource at level %d for %s [key:%s]" %
                        (level, child.path, key))
                _logChildren(level + 1, child)
            if getattr(container, "dynamic_child", None):
                log.msg("Resource at level %d for %s [dynamic]" %
                        (level, container.dynamic_child.path))
                _logChildren(level + 1, container.dynamic_child)
示例#3
0
文件: broker.py 项目: jrha/aquilon
class BrokerCommand(object):
    """ The basis for each command module under commands.

    Several class-level lists and flags are defined here that can be
    overridden on a per-command basis.  Some can only be overridden
    in __init__, though, so check the docstrings.

    """

    required_parameters = []
    """ This will generally be overridden in the command class.

    It could theoretically be parsed out of input.xml, but that is
    tricky in some cases and possibly error-prone.

    """

    optional_parameters = []
    """ Optional parameters are filled in automatically from input.xml.

    This may contain entries that are also in the required_parameters list.
    If so, the required entry "wins".

    """

    requires_azcheck = True
    """ Opt out of authorization checks by setting this flag to False."""

    requires_transaction = True
    """ This sets up a session and cleans up when finished.

    Currently, setting azcheck to True will force this value to be True,
    because the azcheck requires a session that will need to be cleaned
    up.

    """

    requires_format = False
    """ Run command results through the formatter.

    It is automatically set to True for all cat, search, and show
    commands, but could be reversed back to False by overriding __init__
    for the command.

    """

    requires_readonly = False
    """ Require read only isolation level for the render session.

    It is automatically set to True for all search and show commands,
    but could be reversed back to False by overriding __init__ for the
    command.

    """

    # Override to indicate whether the command will generally take a
    # lock during execution.
    #
    # Any command with this flag set to True will use the separate
    # NLSession thread pool and should not have to wait on commands
    # that are potentially all blocking on the same lock.
    #
    # If set to None (the default), the is_lock_free property will
    # examine the command's module to try to determine if a lock
    # may be required and then cache the value.
    _is_lock_free = None

    # Run the render method on a separate thread.  This will be forced
    # to True if requires_azcheck or requires_transaction.
    defer_to_thread = True

    def __init__(self):
        """ Provides some convenient variables for commands.

        Also sets requires_* parameters for some sets of commands.
        All of the command objects are singletons (or Borg).

        """
        self.dbf = DbFactory()
        self.config = Config()
        self.az = AuthorizationBroker()
        self.formatter = ResponseFormatter()
        self.catalog = StatusCatalog()
        # Force the instance to have local copies of the class defaults...
        # This allows resources.py to modify instances without worrying
        # about inheritance issues (classes sharing required or optional
        # parameters).
        self.required_parameters = self.required_parameters[:]
        self.optional_parameters = self.optional_parameters[:]

        # Parameter checks are filled in automatically based on input.xml. This
        # lets us do some rudimentary checks before the actual command is
        # invoked.
        self.parameter_checks = {}

        # The parameter types are filled in automatically based on input.xml.
        self.parameter_types = {}
        # This is the pivot of the above, filled in at the same time.  It is a
        # dictionary of type names to lists of parameters.
        self.parameters_by_type = {}

        self.action = self.__module__
        package_prefix = "aquilon.worker.commands."
        if self.action.startswith(package_prefix):
            self.action = self.action[len(package_prefix):]
        # self.command is set correctly in resources.py after parsing input.xml
        self.command = self.action
        # The readonly and format flags are done here for convenience
        # and simplicity.  They could be overridden by the __init__
        # method of any show/search/cat commands that do not want these
        # defaults.  Some 'one-off' commands (like ping and status)
        # just set the variables themselves.
        if self.action.startswith("show") or self.action.startswith("search"):
            self.requires_readonly = True
            self.requires_format = True
        if self.action.startswith("cat"):
            self.requires_format = True
            self.requires_readonly = True
            self._is_lock_free = True
        if not self.requires_readonly \
           and self.config.get('broker', 'mode') == 'readonly':
            self.badmode = 'readonly'
        else:
            self.badmode = False
        self._update_render(self.render)
        if not self.defer_to_thread:
            if self.requires_azcheck or self.requires_transaction:
                self.defer_to_thread = True
                log.msg("Forcing defer_to_thread to True because of "
                        "required authorization or transaction for %s" %
                        self.command)
            # Not sure how to handle formatting with deferred...
            self.requires_format = False
        #free = "True " if self.is_lock_free else "False"
        #log.msg("is_lock_free = %s [%s]" % (free, self.command))

    def audit_result(self, session, key, value, **arguments):
        # We need a place to store the result somewhere until we can finish the
        # audit record. Use the request object for now.
        request = arguments["request"]
        if not hasattr(request, "_audit_result"):
            request._audit_result = []

        request._audit_result.append((key, value))

    def render(self, **arguments):  # pragma: no cover
        """ Implement this method to create a functional broker command.

        The base __init__ method wraps all implementations using
        _update_render() to enforce the class requires_* flags.

        """
        if self.__class__.__module__ == 'aquilon.worker.broker':
            # Default class... no useful command info to repeat back...
            raise UnimplementedError("Command has not been implemented.")
        raise UnimplementedError("%s has not been implemented" %
                self.__class__.__module__)

    def _update_render(self, command):
        """ Wrap the render method using the requires_* attributes.

        An alternate implementation would be to just have a
        wrap_rendor() method or similar that got called instead
        of rendor().

        """

        def updated_render(self, *args, **kwargs):
            principal = kwargs["user"]
            request = kwargs["request"]
            logger = kwargs["logger"]
            raising_exception = None
            rollback_failed = False
            try:
                if self.requires_transaction or self.requires_azcheck:
                    # Set up a session...
                    if not "session" in kwargs:
                        if self.is_lock_free:
                            kwargs["session"] = self.dbf.NLSession()
                        else:
                            kwargs["session"] = self.dbf.Session()
                    session = kwargs["session"]

                    if session.bind.dialect.name == "oracle":
                        # Make the name of the command and the request ID
                        # available in v$session. Trying to set a value longer
                        # than the allowed length will generate ORA-24960, so
                        # do an explicit truncation.
                        conn = session.connection()
                        dbapi_con = conn.connection.connection
                        dbapi_con.action = str(self.action)[:32]
                        # TODO: we should include the command number as well,
                        # since that is easier to find in the logs
                        dbapi_con.clientinfo = str(kwargs["requestid"])[:64]

                    # This does a COMMIT, which in turn invalidates the session.
                    # We should therefore avoid looking up anything in the DB
                    # before this point which might be used later.
                    self._record_xtn(session, logger.get_status())

                    dbuser = get_or_create_user_principal(session, principal,
                                                          commitoncreate=True)
                    kwargs["dbuser"] = dbuser

                    if self.requires_azcheck:
                        self.az.check(principal=principal, dbuser=dbuser,
                                      action=self.action,
                                      resource=request.path)

                    if self.requires_readonly:
                        self._set_readonly(session)
                    # begin() is only required if session transactional=False
                    #session.begin()
                if self.badmode:  # pragma: no cover
                    raise UnimplementedError("Command %s not available on "
                                             "a %s broker." %
                                             (self.command, self.badmode))
                for key in kwargs.keys():
                    if key in self.parameter_checks:
                        kwargs[key] = self.parameter_checks[key]("--" + key,
                                                                 kwargs[key])
                # Command is an instance method already having self...
                retval = command(*args, **kwargs)
                if self.requires_format:
                    style = kwargs.get("style", None)
                    retval = self.formatter.format(style, retval, request)
                if "session" in kwargs:
                    session.commit()
                return retval
            except Exception, e:
                raising_exception = e
                # Need to close after the rollback, or the next time session
                # is accessed it tries to commit the transaction... (?)
                if "session" in kwargs:
                    try:
                        session.rollback()
                    except:  # pragma: no cover
                        rollback_failed = True
                        raise
                    session.close()
                raise
            finally:
示例#4
0
文件: broker.py 项目: ned21/aquilon
class BrokerCommand(object):
    """ The basis for each command module under commands.

    Several class-level lists and flags are defined here that can be
    overridden on a per-command basis.  Some can only be overridden
    in __init__, though, so check the docstrings.

    """

    required_parameters = []
    """ This will generally be overridden in the command class.

    It could theoretically be parsed out of input.xml, but that is
    tricky in some cases and possibly error-prone.

    """

    optional_parameters = []
    """ Optional parameters are filled in automatically from input.xml.

    This may contain entries that are also in the required_parameters list.
    If so, the required entry "wins".

    """

    requires_azcheck = True
    """ Opt out of authorization checks by setting this flag to False."""

    requires_transaction = True
    """ This sets up a session and cleans up when finished.

    Currently, setting azcheck to True will force this value to be True,
    because the azcheck requires a session that will need to be cleaned
    up.

    """

    requires_format = False
    """ Run command results through the formatter.

    It is automatically set to True for all cat, search, and show
    commands, but could be reversed back to False by overriding __init__
    for the command.

    """

    requires_readonly = False
    """ Require read only isolation level for the render session.

    It is automatically set to True for all search and show commands,
    but could be reversed back to False by overriding __init__ for the
    command.

    """

    # Override to indicate whether the command will generally take a
    # lock during execution.
    #
    # Any command with this flag set to True will use the separate
    # NLSession thread pool and should not have to wait on commands
    # that are potentially all blocking on the same lock.
    #
    # If set to None (the default), the is_lock_free property will
    # examine the command's module to try to determine if a lock
    # may be required and then cache the value.
    _is_lock_free = None

    # Run the render method on a separate thread.  This will be forced
    # to True if requires_azcheck or requires_transaction.
    defer_to_thread = True

    def __init__(self):
        """ Provides some convenient variables for commands.

        Also sets requires_* parameters for some sets of commands.
        All of the command objects are singletons (or Borg).

        """
        self.dbf = DbFactory()
        self.config = Config()
        self.az = AuthorizationBroker()
        self.formatter = ResponseFormatter()
        self.catalog = StatusCatalog()
        # Force the instance to have local copies of the class defaults...
        # This allows resources.py to modify instances without worrying
        # about inheritance issues (classes sharing required or optional
        # parameters).
        self.required_parameters = self.required_parameters[:]
        self.optional_parameters = self.optional_parameters[:]

        # Parameter checks are filled in automatically based on input.xml. This
        # lets us do some rudimentary checks before the actual command is
        # invoked.
        self.parameter_checks = {}

        # The parameter types are filled in automatically based on input.xml.
        self.parameter_types = {}
        # This is the pivot of the above, filled in at the same time.  It is a
        # dictionary of type names to lists of parameters.
        self.parameters_by_type = {}

        self.action = self.__module__
        package_prefix = "aquilon.worker.commands."
        if self.action.startswith(package_prefix):
            self.action = self.action[len(package_prefix):]
        # self.command is set correctly in resources.py after parsing input.xml
        self.command = self.action
        # The readonly and format flags are done here for convenience
        # and simplicity.  They could be overridden by the __init__
        # method of any show/search/cat commands that do not want these
        # defaults.  Some 'one-off' commands (like ping and status)
        # just set the variables themselves.
        if self.action.startswith("show") or self.action.startswith("search"):
            self.requires_readonly = True
            self.requires_format = True
        if self.action.startswith("cat"):
            self.requires_format = True
            self.requires_readonly = True
            self._is_lock_free = True
        if not self.requires_readonly \
           and self.config.get('broker', 'mode') == 'readonly':
            self.badmode = 'readonly'
        else:
            self.badmode = False
        self._update_render(self.render)
        if not self.defer_to_thread:
            if self.requires_azcheck or self.requires_transaction:
                self.defer_to_thread = True
                log.msg("Forcing defer_to_thread to True because of "
                        "required authorization or transaction for %s" %
                        self.command)
            # Not sure how to handle formatting with deferred...
            self.requires_format = False
        #free = "True " if self.is_lock_free else "False"
        #log.msg("is_lock_free = %s [%s]" % (free, self.command))

    def audit_result(self, session, key, value, **arguments):
        # We need a place to store the result somewhere until we can finish the
        # audit record. Use the request object for now.
        request = arguments["request"]
        if not hasattr(request, "_audit_result"):
            request._audit_result = []

        request._audit_result.append((key, value))

    def render(self, **arguments):  # pragma: no cover
        """ Implement this method to create a functional broker command.

        The base __init__ method wraps all implementations using
        _update_render() to enforce the class requires_* flags.

        """
        if self.__class__.__module__ == 'aquilon.worker.broker':
            # Default class... no useful command info to repeat back...
            raise UnimplementedError("Command has not been implemented.")
        raise UnimplementedError("%s has not been implemented" %
                                 self.__class__.__module__)

    def _update_render(self, command):
        """ Wrap the render method using the requires_* attributes.

        An alternate implementation would be to just have a
        wrap_rendor() method or similar that got called instead
        of rendor().

        """
        def updated_render(self, *args, **kwargs):
            principal = kwargs["user"]
            request = kwargs["request"]
            logger = kwargs["logger"]
            raising_exception = None
            rollback_failed = False
            try:
                if self.requires_transaction or self.requires_azcheck:
                    # Set up a session...
                    if not "session" in kwargs:
                        if self.is_lock_free:
                            kwargs["session"] = self.dbf.NLSession()
                        else:
                            kwargs["session"] = self.dbf.Session()
                    session = kwargs["session"]

                    if session.bind.dialect.name == "oracle":
                        # Make the name of the command and the request ID
                        # available in v$session. Trying to set a value longer
                        # than the allowed length will generate ORA-24960, so
                        # do an explicit truncation.
                        conn = session.connection()
                        dbapi_con = conn.connection.connection
                        dbapi_con.action = str(self.action)[:32]
                        # TODO: we should include the command number as well,
                        # since that is easier to find in the logs
                        dbapi_con.clientinfo = str(kwargs["requestid"])[:64]

                    # This does a COMMIT, which in turn invalidates the session.
                    # We should therefore avoid looking up anything in the DB
                    # before this point which might be used later.
                    self._record_xtn(session, logger.get_status())

                    dbuser = get_or_create_user_principal(session,
                                                          principal,
                                                          commitoncreate=True)
                    kwargs["dbuser"] = dbuser

                    if self.requires_azcheck:
                        self.az.check(principal=principal,
                                      dbuser=dbuser,
                                      action=self.action,
                                      resource=request.path)

                    if self.requires_readonly:
                        self._set_readonly(session)
                    # begin() is only required if session transactional=False
                    #session.begin()
                if self.badmode:  # pragma: no cover
                    raise UnimplementedError("Command %s not available on "
                                             "a %s broker." %
                                             (self.command, self.badmode))
                for key in kwargs.keys():
                    if key in self.parameter_checks:
                        kwargs[key] = self.parameter_checks[key]("--" + key,
                                                                 kwargs[key])
                # Command is an instance method already having self...
                retval = command(*args, **kwargs)
                if self.requires_format:
                    style = kwargs.get("style", None)
                    retval = self.formatter.format(style, retval, request)
                if "session" in kwargs:
                    session.commit()
                return retval
            except Exception, e:
                raising_exception = e
                # Need to close after the rollback, or the next time session
                # is accessed it tries to commit the transaction... (?)
                if "session" in kwargs:
                    try:
                        session.rollback()
                    except:  # pragma: no cover
                        rollback_failed = True
                        raise
                    session.close()
                raise
            finally:
示例#5
0
文件: broker.py 项目: ned21/aquilon
    def __init__(self):
        """ Provides some convenient variables for commands.

        Also sets requires_* parameters for some sets of commands.
        All of the command objects are singletons (or Borg).

        """
        self.dbf = DbFactory()
        self.config = Config()
        self.az = AuthorizationBroker()
        self.formatter = ResponseFormatter()
        self.catalog = StatusCatalog()
        # Force the instance to have local copies of the class defaults...
        # This allows resources.py to modify instances without worrying
        # about inheritance issues (classes sharing required or optional
        # parameters).
        self.required_parameters = self.required_parameters[:]
        self.optional_parameters = self.optional_parameters[:]

        # Parameter checks are filled in automatically based on input.xml. This
        # lets us do some rudimentary checks before the actual command is
        # invoked.
        self.parameter_checks = {}

        # The parameter types are filled in automatically based on input.xml.
        self.parameter_types = {}
        # This is the pivot of the above, filled in at the same time.  It is a
        # dictionary of type names to lists of parameters.
        self.parameters_by_type = {}

        self.action = self.__module__
        package_prefix = "aquilon.worker.commands."
        if self.action.startswith(package_prefix):
            self.action = self.action[len(package_prefix):]
        # self.command is set correctly in resources.py after parsing input.xml
        self.command = self.action
        # The readonly and format flags are done here for convenience
        # and simplicity.  They could be overridden by the __init__
        # method of any show/search/cat commands that do not want these
        # defaults.  Some 'one-off' commands (like ping and status)
        # just set the variables themselves.
        if self.action.startswith("show") or self.action.startswith("search"):
            self.requires_readonly = True
            self.requires_format = True
        if self.action.startswith("cat"):
            self.requires_format = True
            self.requires_readonly = True
            self._is_lock_free = True
        if not self.requires_readonly \
           and self.config.get('broker', 'mode') == 'readonly':
            self.badmode = 'readonly'
        else:
            self.badmode = False
        self._update_render(self.render)
        if not self.defer_to_thread:
            if self.requires_azcheck or self.requires_transaction:
                self.defer_to_thread = True
                log.msg("Forcing defer_to_thread to True because of "
                        "required authorization or transaction for %s" %
                        self.command)
            # Not sure how to handle formatting with deferred...
            self.requires_format = False