def __init__(self): """ Provides some convenient variables for commands. Also sets requires_* parameters for some sets of commands. All of the command objects are singletons (or Borg). """ self.dbf = DbFactory() self.config = Config() self.az = AuthorizationBroker() self.formatter = ResponseFormatter() self.catalog = StatusCatalog() # Force the instance to have local copies of the class defaults... # This allows resources.py to modify instances without worrying # about inheritance issues (classes sharing required or optional # parameters). self.required_parameters = self.required_parameters[:] self.optional_parameters = self.optional_parameters[:] # Parameter checks are filled in automatically based on input.xml. This # lets us do some rudimentary checks before the actual command is # invoked. self.parameter_checks = {} self.action = self.__module__ package_prefix = "aquilon.worker.commands." if self.action.startswith(package_prefix): self.action = self.action[len(package_prefix):] # self.command is set correctly in resources.py after parsing input.xml self.command = self.action # The readonly and format flags are done here for convenience # and simplicity. They could be overridden by the __init__ # method of any show/search/cat commands that do not want these # defaults. Some 'one-off' commands (like ping and status) # just set the variables themselves. if self.action.startswith("show") or self.action.startswith("search"): self.requires_readonly = True self.requires_format = True if self.action.startswith("cat"): self.requires_format = True self.requires_readonly = True self._is_lock_free = True if not self.requires_readonly \ and self.config.get('broker', 'mode') == 'readonly': self.badmode = 'readonly' else: self.badmode = False self._update_render(self.render) if not self.defer_to_thread: if self.requires_azcheck or self.requires_transaction: self.defer_to_thread = True log.msg("Forcing defer_to_thread to True because of " "required authorization or transaction for %s" % self.command) # Not sure how to handle formatting with deferred... self.requires_format = False
def __init__(self, config): formatter = ResponseFormatter() ResponsePage.__init__(self, '', formatter) self.config = config # Regular Expression for matching variables in a path definition. # Currently only supports stuffing a single variable in a path # component. varmatch = re.compile(r'^%\((.*)\)s$') BINDIR = os.path.dirname(os.path.realpath(sys.argv[0])) tree = ET.parse(os.path.join(BINDIR, '..', 'etc', 'input.xml')) for command in tree.getiterator("command"): for transport in command.getiterator("transport"): if "name" not in command.attrib \ or "method" not in transport.attrib \ or "path" not in transport.attrib: continue name = command.attrib["name"] method = transport.attrib["method"] path = transport.attrib["path"] trigger = transport.attrib.get("trigger") container = self relative = "" # Traverse down the resource tree, container will # end up pointing to the correct spot. # Create branches and leaves as necessary, continueing to # traverse downward. for component in path.split("/"): relative = relative + "/" + component #log.msg("Working with component '" + component + "' of '" + relative + "'.") m = varmatch.match(component) # Is this piece of the path dynamic? if not m: #log.msg("Component '" + component + "' is static.") child = container.getStaticEntity(component) if child is None: #log.msg("Creating new static component '" + component + "'.") child = ResponsePage(relative, formatter) container.putChild(component, child) container = child else: #log.msg("Component '" + component + "' is dynamic.") path_variable = m.group(1) if container.dynamic_child is not None: #log.msg("Dynamic component '" + component + "' already exists.") current_variable = container.dynamic_child.\ path_variable if current_variable != path_variable: log.msg("Warning: Could not use variable '" + path_variable + "', already have " + "dynamic variable '" + current_variable + "'.") # XXX: Raise an error if they don't match container = container.dynamic_child else: #log.msg("Dynamic component '" + component + "' had correct variable.") container = container.dynamic_child else: #log.msg("Creating new dynamic component '" + component + "'.") child = ResponsePage(relative, formatter, path_variable=path_variable) container.dynamic_child = child container = child fullcommand = name if trigger: fullcommand = fullcommand + "_" + trigger mymodule = getattr(commands, fullcommand, None) if not mymodule: log.msg("No module available in aquilon.worker.commands " + "for %s" % fullcommand) # See commands/__init__.py for more info here... myinstance = getattr(mymodule, "broker_command", None) if not myinstance: log.msg("No class instance available for %s" % fullcommand) myinstance = BrokerCommand() myinstance.command = name rendermethod = method.upper() if container.handlers.get(rendermethod, None): log.msg("Warning: Already have a %s here at %s..." % (rendermethod, container.path)) #log.msg("Setting 'command_" + fullcommand + "' as '" + rendermethod + "' for container '" + container.path + "'.") container.handlers[rendermethod] = myinstance # Since we are parsing input.xml anyway, record the possible # parameters... for option in command.getiterator("option"): if "name" not in option.attrib: continue option_name = option.attrib["name"] if option_name not in myinstance.optional_parameters: myinstance.optional_parameters.append(option_name) if "type" in option.attrib: paramtype = option.attrib["type"] myinstance.parameter_types[option_name] = paramtype if paramtype == "int": myinstance.parameter_checks[ option_name] = force_int elif paramtype == "float": myinstance.parameter_checks[ option_name] = force_float elif paramtype == "boolean" or paramtype == "flag": myinstance.parameter_checks[ option_name] = force_boolean elif paramtype == "ipv4": myinstance.parameter_checks[ option_name] = force_ipv4 elif paramtype == "mac": myinstance.parameter_checks[ option_name] = force_mac elif paramtype == "json": myinstance.parameter_checks[ option_name] = force_json_dict elif paramtype == "string" or paramtype == "file": # Current twisted can't handle unicode output, so # do not allow non-ASCII input either myinstance.parameter_checks[ option_name] = force_ascii elif paramtype == "list": myinstance.parameter_checks[ option_name] = force_list else: # pragma: no cover log.msg("Warning: unknown option type %s" % paramtype) else: # pragma: no cover log.msg("Warning: argument type not known for %s.%s" % (myinstance.command, option_name)) pbt = myinstance.parameters_by_type for option_name, paramtype \ in myinstance.parameter_types.items(): if paramtype in pbt: pbt[paramtype].append(option_name) else: pbt[paramtype] = [option_name] cache_version(config) log.msg("Starting aqd version %s" % config.get("broker", "version")) self.make_required_dirs() def _logChildren(level, container): for (key, child) in container.listStaticEntities(): log.msg("Resource at level %d for %s [key:%s]" % (level, child.path, key)) _logChildren(level + 1, child) if getattr(container, "dynamic_child", None): log.msg("Resource at level %d for %s [dynamic]" % (level, container.dynamic_child.path)) _logChildren(level + 1, container.dynamic_child)
class BrokerCommand(object): """ The basis for each command module under commands. Several class-level lists and flags are defined here that can be overridden on a per-command basis. Some can only be overridden in __init__, though, so check the docstrings. """ required_parameters = [] """ This will generally be overridden in the command class. It could theoretically be parsed out of input.xml, but that is tricky in some cases and possibly error-prone. """ optional_parameters = [] """ Optional parameters are filled in automatically from input.xml. This may contain entries that are also in the required_parameters list. If so, the required entry "wins". """ requires_azcheck = True """ Opt out of authorization checks by setting this flag to False.""" requires_transaction = True """ This sets up a session and cleans up when finished. Currently, setting azcheck to True will force this value to be True, because the azcheck requires a session that will need to be cleaned up. """ requires_format = False """ Run command results through the formatter. It is automatically set to True for all cat, search, and show commands, but could be reversed back to False by overriding __init__ for the command. """ requires_readonly = False """ Require read only isolation level for the render session. It is automatically set to True for all search and show commands, but could be reversed back to False by overriding __init__ for the command. """ # Override to indicate whether the command will generally take a # lock during execution. # # Any command with this flag set to True will use the separate # NLSession thread pool and should not have to wait on commands # that are potentially all blocking on the same lock. # # If set to None (the default), the is_lock_free property will # examine the command's module to try to determine if a lock # may be required and then cache the value. _is_lock_free = None # Run the render method on a separate thread. This will be forced # to True if requires_azcheck or requires_transaction. defer_to_thread = True def __init__(self): """ Provides some convenient variables for commands. Also sets requires_* parameters for some sets of commands. All of the command objects are singletons (or Borg). """ self.dbf = DbFactory() self.config = Config() self.az = AuthorizationBroker() self.formatter = ResponseFormatter() self.catalog = StatusCatalog() # Force the instance to have local copies of the class defaults... # This allows resources.py to modify instances without worrying # about inheritance issues (classes sharing required or optional # parameters). self.required_parameters = self.required_parameters[:] self.optional_parameters = self.optional_parameters[:] # Parameter checks are filled in automatically based on input.xml. This # lets us do some rudimentary checks before the actual command is # invoked. self.parameter_checks = {} # The parameter types are filled in automatically based on input.xml. self.parameter_types = {} # This is the pivot of the above, filled in at the same time. It is a # dictionary of type names to lists of parameters. self.parameters_by_type = {} self.action = self.__module__ package_prefix = "aquilon.worker.commands." if self.action.startswith(package_prefix): self.action = self.action[len(package_prefix):] # self.command is set correctly in resources.py after parsing input.xml self.command = self.action # The readonly and format flags are done here for convenience # and simplicity. They could be overridden by the __init__ # method of any show/search/cat commands that do not want these # defaults. Some 'one-off' commands (like ping and status) # just set the variables themselves. if self.action.startswith("show") or self.action.startswith("search"): self.requires_readonly = True self.requires_format = True if self.action.startswith("cat"): self.requires_format = True self.requires_readonly = True self._is_lock_free = True if not self.requires_readonly \ and self.config.get('broker', 'mode') == 'readonly': self.badmode = 'readonly' else: self.badmode = False self._update_render(self.render) if not self.defer_to_thread: if self.requires_azcheck or self.requires_transaction: self.defer_to_thread = True log.msg("Forcing defer_to_thread to True because of " "required authorization or transaction for %s" % self.command) # Not sure how to handle formatting with deferred... self.requires_format = False #free = "True " if self.is_lock_free else "False" #log.msg("is_lock_free = %s [%s]" % (free, self.command)) def audit_result(self, session, key, value, **arguments): # We need a place to store the result somewhere until we can finish the # audit record. Use the request object for now. request = arguments["request"] if not hasattr(request, "_audit_result"): request._audit_result = [] request._audit_result.append((key, value)) def render(self, **arguments): # pragma: no cover """ Implement this method to create a functional broker command. The base __init__ method wraps all implementations using _update_render() to enforce the class requires_* flags. """ if self.__class__.__module__ == 'aquilon.worker.broker': # Default class... no useful command info to repeat back... raise UnimplementedError("Command has not been implemented.") raise UnimplementedError("%s has not been implemented" % self.__class__.__module__) def _update_render(self, command): """ Wrap the render method using the requires_* attributes. An alternate implementation would be to just have a wrap_rendor() method or similar that got called instead of rendor(). """ def updated_render(self, *args, **kwargs): principal = kwargs["user"] request = kwargs["request"] logger = kwargs["logger"] raising_exception = None rollback_failed = False try: if self.requires_transaction or self.requires_azcheck: # Set up a session... if not "session" in kwargs: if self.is_lock_free: kwargs["session"] = self.dbf.NLSession() else: kwargs["session"] = self.dbf.Session() session = kwargs["session"] if session.bind.dialect.name == "oracle": # Make the name of the command and the request ID # available in v$session. Trying to set a value longer # than the allowed length will generate ORA-24960, so # do an explicit truncation. conn = session.connection() dbapi_con = conn.connection.connection dbapi_con.action = str(self.action)[:32] # TODO: we should include the command number as well, # since that is easier to find in the logs dbapi_con.clientinfo = str(kwargs["requestid"])[:64] # This does a COMMIT, which in turn invalidates the session. # We should therefore avoid looking up anything in the DB # before this point which might be used later. self._record_xtn(session, logger.get_status()) dbuser = get_or_create_user_principal(session, principal, commitoncreate=True) kwargs["dbuser"] = dbuser if self.requires_azcheck: self.az.check(principal=principal, dbuser=dbuser, action=self.action, resource=request.path) if self.requires_readonly: self._set_readonly(session) # begin() is only required if session transactional=False #session.begin() if self.badmode: # pragma: no cover raise UnimplementedError("Command %s not available on " "a %s broker." % (self.command, self.badmode)) for key in kwargs.keys(): if key in self.parameter_checks: kwargs[key] = self.parameter_checks[key]("--" + key, kwargs[key]) # Command is an instance method already having self... retval = command(*args, **kwargs) if self.requires_format: style = kwargs.get("style", None) retval = self.formatter.format(style, retval, request) if "session" in kwargs: session.commit() return retval except Exception, e: raising_exception = e # Need to close after the rollback, or the next time session # is accessed it tries to commit the transaction... (?) if "session" in kwargs: try: session.rollback() except: # pragma: no cover rollback_failed = True raise session.close() raise finally:
def __init__(self): """ Provides some convenient variables for commands. Also sets requires_* parameters for some sets of commands. All of the command objects are singletons (or Borg). """ self.dbf = DbFactory() self.config = Config() self.az = AuthorizationBroker() self.formatter = ResponseFormatter() self.catalog = StatusCatalog() # Force the instance to have local copies of the class defaults... # This allows resources.py to modify instances without worrying # about inheritance issues (classes sharing required or optional # parameters). self.required_parameters = self.required_parameters[:] self.optional_parameters = self.optional_parameters[:] # Parameter checks are filled in automatically based on input.xml. This # lets us do some rudimentary checks before the actual command is # invoked. self.parameter_checks = {} # The parameter types are filled in automatically based on input.xml. self.parameter_types = {} # This is the pivot of the above, filled in at the same time. It is a # dictionary of type names to lists of parameters. self.parameters_by_type = {} self.action = self.__module__ package_prefix = "aquilon.worker.commands." if self.action.startswith(package_prefix): self.action = self.action[len(package_prefix):] # self.command is set correctly in resources.py after parsing input.xml self.command = self.action # The readonly and format flags are done here for convenience # and simplicity. They could be overridden by the __init__ # method of any show/search/cat commands that do not want these # defaults. Some 'one-off' commands (like ping and status) # just set the variables themselves. if self.action.startswith("show") or self.action.startswith("search"): self.requires_readonly = True self.requires_format = True if self.action.startswith("cat"): self.requires_format = True self.requires_readonly = True self._is_lock_free = True if not self.requires_readonly \ and self.config.get('broker', 'mode') == 'readonly': self.badmode = 'readonly' else: self.badmode = False self._update_render(self.render) if not self.defer_to_thread: if self.requires_azcheck or self.requires_transaction: self.defer_to_thread = True log.msg("Forcing defer_to_thread to True because of " "required authorization or transaction for %s" % self.command) # Not sure how to handle formatting with deferred... self.requires_format = False