def process_role_internal(self, host, policy, role, local=True, sender=None): if not local: from opsmop.callbacks.event_stream import EventStreamStreamCallbacks from opsmop.callbacks.common import CommonCallbacks Context.set_callbacks([ EventStreamCallbacks(sender=sender), CommonCallbacks() ]) role.pre() # set up the variable scope - this is done later by walk_handlers for lower-level objects in the tree policy.attach_child_scope_for(role) # tell the callbacks we are in validate mode - this may alter or quiet their output Callbacks.on_validate() # always validate the role in every mode (VALIDATE, CHECK ,or APPLY) self.validate_role(role) # skip the role if we need to if not role.conditions_true(): Callbacks.on_skipped(role) return # process the tree for real for non-validate modes if not Context.is_validate(): self.execute_role_resources(host, role) self.execute_role_handlers(host, role) # run any user hooks role.post()
def go(self): if len(self.args) < 3 or sys.argv[1] == "--help": print(USAGE) sys.exit(1) mode = self.args[1] path = sys.argv[2] callbacks = None parser = argparse.ArgumentParser() parser.add_argument('--validate', help='policy file to validate') parser.add_argument('--apply', help="policy file to apply") parser.add_argument('--check', help="policy file to check") parser.add_argument('--tags', help='optional comma seperated list of tags') parser.add_argument('--event-stream', action='store_true', help=argparse.SUPPRESS) args = parser.parse_args(self.args[1:]) all_modes = [ args.validate, args.apply, args.check ] selected_modes = [ x for x in all_modes if x is not None ] if len(selected_modes) != 1: print(USAGE) sys.exit(1) path = args.validate or args.apply or args.check if not args.event_stream: Callbacks.set_callbacks([ LocalCliCallbacks(), CommonCallbacks() ]) else: Callbacks.set_callbacks([ EventStreamCallbacks(), CommonCallbacks() ]) tags = None if args.tags is not None: tags = args.tags.strip().split(",") api = Api.from_file(path=path, tags=tags) try: if args.validate: # just check for missing files and invalid types api.validate() elif args.check: # operate in dry-run mode api.check() elif args.apply: # configure everything api.apply() else: print(USAGE) sys.exit(1) except OpsMopError as ome: print("") print(str(ome)) print("") sys.exit(1) print("") sys.exit(0)
def execute_resource(self, host, resource, handlers=False): """ This handles the plan/apply intercharge for a given resource in the resource tree. It is called recursively via walk_children to run against all resources. """ assert host is not None # we only care about processing leaf node objects if issubclass(type(resource), Collection): return # if in handler mode we do not process the handler unless it was signaled if handlers and not Context().has_seen_any_signal( host, resource.all_handles()): Callbacks().on_skipped(resource, is_handler=handlers) return # tell the callbacks we are about to process a resource # they may use this to print information about the resource Callbacks().on_resource(resource, handlers) # plan always, apply() only if not in check mode, else assume # the plan was executed. provider = self.do_plan(resource) assert provider is not None if Context().is_apply(): self.do_apply(host, provider, handlers) else: # is_check self.do_simulate(host, provider) # if anything has changed, let the callbacks know about it self.signal_changes(host=host, provider=provider, resource=resource)
def do_apply(self, host, provider, handlers): """ Once a provider has a plan generated, see if we need to run the plan. If so, also run any actions associated witht he apply step, which mostly means registering variables from apply results. """ # some simple providers - like Echo, do not have a planning step # we will always run the apply step for them. For others, we will only # run the apply step if we have computed a plan if (not provider.skip_plan_stage()) and ( not provider.has_planned_actions()): return False # indicate we are about take some actions Callbacks().on_apply(provider) # take them result = provider.apply() if not handlers: # let the callbacks now we have taken some actions Callbacks().on_taken_actions(provider, provider.actions_taken) # all Provider apply() methods need to return Result objects or raise # exceptions assert issubclass(type(result), Result) # the 'register' feature saves results into variable scope resource = provider.resource if resource.register is not None: provider.handle_registration(result) # determine if there was a failure fatal = False cond = resource.failed_when if cond is not None: if issubclass(type(cond), Lookup): fatal = cond.evaluate(resource) result.reason = cond else: fatal = cond elif result.fatal: fatal = True result.fatal = fatal result.changed = provider.has_changed() # TODO: eliminate the actions class if result.changed: result.actions = [x.do for x in provider.actions_taken] else: result.actions = [] # tell the callbacks about the result Callbacks().on_result(provider, result) # if there was a failure, handle it # (common callbacks should abort execution) #if fatal: # Callbacks().on_fatal(provider, result) return True
def signal_changes(self, host=None, provider=None, resource=None): """ If any events were signaled, add them to the context here. """ assert host is not None if not provider.has_changed(): return if resource.signals: # record the list of all events signaled while processing this role Context.add_signal(host, resource.signals) # tell the callbacks that a signal occurred Callbacks.on_signaled(resource, resource.signals)
def execute_role_handlers(self, host, role): """ Processes handler resources for one role for CHECK or APPLY mode """ # see comments for prior method for details Callbacks.on_begin_handlers() def execute_handler(handler): handler.pre() result = self.execute_resource(host=host, resource=handler, handlers=True) handler.post() return result role.walk_children(items=role.get_children('handlers'), which='handlers', fn=execute_handler, tags=self._tags)
def run_policy(self, policy=None): """ Runs one specific policy in VALIDATE, CHECK, or APPLY mode """ # assign a new top scope to the policy object. policy.init_scope() roles = policy.get_roles() for role in roles.items: if self._local: self.process_role(policy, role) else: self.process_role_push(policy, role) Callbacks.on_complete(policy)
def do_apply(self, provider): """ Once a provider has a plan generated, see if we need to run the plan. """ from opsmop.callbacks.callbacks import Callbacks # some simple providers - like Echo, do not have a planning step # we will always run the apply step for them. For others, we will only # run the apply step if we have computed a plan if not provider.has_planned_actions(): if not provider.skip_plan_stage(): return Result(provider=provider, changed=False, data=None) # indicate we are about take some actions Callbacks().on_apply(provider) # take them result = provider.apply() Callbacks().on_taken_actions(provider, provider.actions_taken) # all Provider apply() methods need to return Result objects or raise # exceptions assert issubclass(type(result), Result) # the 'register' feature saves results into variable scope resource = provider.resource # determine if there was a failure fatal = False cond = resource.failed_when if cond is not None: if issubclass(type(cond), Lookup): fatal = cond.evaluate(resource) result.reason = cond elif callable(cond): fatal = cond(result) else: fatal = cond elif result.fatal: fatal = True result.fatal = fatal result.changed = provider.has_changed() # TODO: eliminate the actions class if result.changed: result.actions = [x.do for x in provider.actions_taken] else: result.actions = [] Callbacks().on_result(provider, result) return result
def execute_role_resources(self, host, role): """ Processes non-handler resources for one role for CHECK or APPLY mode """ # tell the context we are processing resources now, which may change their behavior # of certain methods like on_resource() Callbacks.on_begin_role(role) def execute_resource(resource): # execute each resource through plan() and if needed apply() stages, but before and after # doing so, run any user pre() or post() hooks implemented on that object. resource.pre() result = self.execute_resource(host=host, resource=resource) resource.post() return result role.walk_children(items=role.get_children('resources'), which='resources', fn=execute_resource, tags=self._tags)
def handle_registration(self, result): assert result is not None va = dict() va[self.register] = result Callbacks().on_update_variables(va) self.resource.update_variables(va) self.resource.update_parent_variables(va)
def compute_max_hostname_length(self, hosts): hostname_length = 0 for host in hosts: length = len(host.display_name()) if length > hostname_length: hostname_length = length Callbacks().set_hostname_length(hostname_length)
def info(self, host, msg, sep=':', foreground=None): from opsmop.callbacks.callbacks import Callbacks max_length = Callbacks().hostname_length() fmt = f"%-{max_length}s" msg = "%s %s %s" % (fmt % host.display_name(), sep, msg) if foreground: msg = foreground + msg + Style.RESET_ALL self.i3(msg)
def do_plan(self, resource): """ Ask a resource for the provider, and then see what the planned actions should be. The planned actions are kept on the provider object. We don't need to obtain the plan. Return the provider. """ # ask the resource for a provider instance provider = resource.provider() if provider.skip_plan_stage(): return provider # tell the context object we are about to run the plan stage. Callbacks.on_plan(provider) # compute the plan provider.plan() # copy the list of planned actions into the 'to do' list for the apply method # on the provider provider.commit_to_plan() return provider
def process_local_role(self, policy=None, role=None): host = self._local_host Context().set_host(host) policy.attach_child_scope_for(role) try: role.main() except Exception as e: tb = traceback.format_exc() # process *any* uncaught exceptions through the configured exception handlers # this includes any resources where failed_when / ignore_errors was not used # but also any random python exceptions Callbacks().on_fatal(e, tb)
def run(self): Callbacks().on_resource(self) provider = self.do_plan() if Context().is_apply(): result = self.do_apply(provider) else: result = self.do_simulate(provider) # copy over results self.changed = result.changed self.data = result.data self.rc = result.rc
def process_local_role(self, policy=None, role=None): host = self._local_host Context().set_host(host) role.pre() # set up the variable scope - this is done later by walk_handlers for lower-level objects in the tree policy.attach_child_scope_for(role) # tell the callbacks we are in validate mode - this may alter or quiet their output Callbacks().on_validate() # always validate the role in every mode (VALIDATE, CHECK ,or APPLY) self.validate_role(role) # skip the role if we need to if not role.conditions_true(): Callbacks().on_skipped(role) return # process the tree for real for non-validate modes if not Context().is_validate(): self.execute_role_resources(host, role) self.execute_role_handlers(host, role) # run any user hooks role.post()
def run_policy(self, policy=None): """ Runs one specific policy in CHECK, or APPLY mode """ # assign a new top scope to the policy object. policy.init_scope() roles = policy.get_roles() for role in roles.items: Context().set_role(role) if not self._push: self.process_local_role(policy, role) else: self.process_remote_role(policy, role) Callbacks().on_complete(policy)
def remote_fn(caller, params, sender): """ This is the remote function used for mitogen calls """ # FIXME: REFACTOR: we have a bit of a growing inconsistency between what is a constructor parameter and what is passed around in Context. # we should change this to have context objects that have more meat, but also get passed around versus acting globally, and smaller # function signatures across the board. import dill from opsmop.core.executor import Executor params = dill.loads(zlib.decompress(params)) host = params['host'] policy = params['policy'] role = params['role'] mode = params['mode'] tags = params['tags'] checksums = params['checksums'] relative_root = params['relative_root'] hostvars = params['hostvars'] extra_vars = params['extra_vars'] Context().set_mode(mode) Context().set_caller(caller) assert relative_root is not None Context().set_checksums(checksums) Context().update_globals(hostvars) policy.roles = Roles(role) Callbacks().set_callbacks([ EventStreamCallbacks(sender=sender), LocalCliCallbacks(), CommonCallbacks() ]) executor = Executor([policy], local_host=host, push=False, tags=params['tags'], extra_vars=extra_vars, relative_root=relative_root) # remove single_role # FIXME: care about mode executor.apply()
def remotify_role(self, host, policy, role, mode): if self.should_exclude_from_limits(host): return try: if not role.should_contact(host): Callbacks().on_skipped(role) return True else: role.before_contact(host) except Exception as e: print(str(e)) Context().record_host_failure(host, e) return False target_host = self.actual_host(role, host) target_host.reset_actions() import dill conn = self.connect(host, role) receiver = mitogen.core.Receiver(self.router) self.events_select.add(receiver) sender = self.status_recv.to_sender() params = dict(host=target_host, policy=policy, role=role, mode=mode, relative_root=Context().relative_root(), tags=self.tags, checksums=self.checksums, hostvars=host.all_variables(), extra_vars=Context().extra_vars()) params = zlib.compress(dill.dumps(params), level=9) call_recv = conn.call_async(remote_fn, self.myself, params, sender) self.calls_sel.add(call_recv) return True
def execute(self): """ Execute a command (a list or string) with input_text as input, appending the output of all commands to the build log. This code was derived from http://vespene.io/ though is slightly different because there are no database objects. """ Callbacks().on_execute_command(self.provider, self) command = self.cmd timeout_cmd = self.get_timeout() shell = True if type(command) == list: if self.timeout and timeout_cmd: command.insert(0, self.timeout) command.insert(0, timeout_cmd) shell = False else: if self.timeout and timeout_cmd: command = "%s %s %s" % (timeout_cmd, self.timeout, command) # keep SSH-agent working for executed commands sock = os.environ.get('SSH_AUTH_SOCK', None) if self.env and sock: self.env['SSH_AUTH_SOCK'] = sock process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell, env=self.env) if self.input_text is None: self.input_text = "" stdin = io.TextIOWrapper( process.stdin, encoding='utf-8', line_buffering=True, ) stdout = io.TextIOWrapper( process.stdout, encoding='utf-8', ) stdin.write(self.input_text) stdin.close() output = "" for line in stdout: if (self.echo or self.loud) and (not self.ignore_lines or not self.should_ignore(line)): Callbacks().on_command_echo(self.provider, line) output = output + line if output.strip() == "": Callbacks().on_command_echo(self.provider, "(no output)") process.wait() res = None rc = process.returncode if rc != 0: res = Result(self.provider, rc=rc, data=output, fatal=self.fatal, primary=self.primary) else: res = Result(self.provider, rc=rc, data=output, fatal=False, primary=self.primary) # this callback will, depending on implementation, usually note fatal result objects and raise an exception Callbacks().on_command_result(self.provider, res) return res
def echo(self, msg): Callbacks().on_echo(self, msg)
def do(self, action_name): """ marks off that an action has been completed. not marking off all planned actions (or any unplanned ones) will result in an error """ action = Action(action_name) self.actions_taken.append(action) Callbacks().on_do(self, action)
def needs(self, action_name): """ declares than an action 'should' take place during an apply step """ action = Action(action_name) self.actions_planned.append(action) Callbacks().on_needs(self, action)
def _on_walk(self): Callbacks.on_role(self)
def go(self): colorama_init() Callbacks().set_callbacks([LocalCliCallbacks(), CommonCallbacks()]) if len(self.args) < 3 or sys.argv[1] == "--help": print(USAGE) sys.exit(1) mode = self.args[1] path = sys.argv[2] callbacks = None extra_vars = dict() parser = argparse.ArgumentParser() parser.add_argument('--validate', action='store_true', help='policy file to validate') parser.add_argument('--apply', action='store_true', help="policy file to apply") parser.add_argument('--check', action='store_true', help="policy file to check") parser.add_argument('--push', action='store_true', help='run in push mode') parser.add_argument('--local', action='store_true', help='run in local mode') parser.add_argument('--verbose', action='store_true', help='(with --push) increase verbosity') parser.add_argument('--extra-vars', help="add extra variables from the command line") parser.add_argument( '--limit-groups', help= "(with --push) limit groups executed to this comma-separated list of patterns" ) parser.add_argument( '--limit-hosts', help= "(with --push) limit hosts executed to this comma-separated list of patterns" ) args = parser.parse_args(self.args[1:]) all_modes = [args.validate, args.apply, args.check] selected_modes = [x for x in all_modes if x is True] if len(selected_modes) != 1: print(selected_modes) print(USAGE) sys.exit(1) all_modes = [args.push, args.local] selected_modes = [x for x in all_modes if x is True] if len(selected_modes) != 1: print(USAGE) sys.exit(1) if args.extra_vars is not None: extra_vars = self.handle_extra_vars(args.extra_vars) Context().set_verbose(args.verbose) abspath = os.path.abspath(sys.modules[self.policy.__module__].__file__) relative_root = os.path.dirname(abspath) os.chdir(os.path.dirname(abspath)) api = Api(policies=[self.policy], push=args.push, extra_vars=extra_vars, limit_groups=args.limit_groups, limit_hosts=args.limit_hosts, relative_root=relative_root) try: if args.validate: # just check for missing files and invalid types api.validate() elif args.check: # operate in dry-run mode api.check() elif args.apply: # configure everything api.apply() else: print(USAGE) sys.exit(1) except OpsMopStop as oms: sys.exit(1) except OpsMopError as ome: print("") print(str(ome)) print("") sys.exit(1) print("") sys.exit(0)
def walk_children(self, items=None, which=None, fn=None, handlers=False, tags=None): """ A relatively complex iterator used by Executor() code. Walks the entire object tree calling fn() on each element. items - the kids to start the iteration with context - a Context() object for callback tracking which - 'resources' or 'handlers' fn - the function to call on each object """ self._on_walk() items_type = type(items) if items is None: return def maybe(v): # we'll visit every resource but only call the function on items if tags are *not* engaged if not tags or v.has_tag(tags): fn(v) if issubclass(items_type, Collection): self.attach_child_scope_for(items) proceed = items.conditions_true() if proceed: return items.walk_children(items=items.get_children(), which=which, fn=fn, tags=tags) else: Callbacks().on_skipped(items, is_handler=handlers) elif issubclass(items_type, Resource): self.attach_child_scope_for(items) if items.conditions_true(): return maybe(items) else: Callbacks().on_skipped(items, is_handler=handlers) elif items_type == list: for x in items: self.attach_child_scope_for(x) if x.conditions_true(): if issubclass(type(x), Collection): x.walk_children(items=x.get_children(), fn=fn, tags=tags) else: maybe(x) else: Callbacks().on_skipped(items, is_handler=handlers) elif items_type == dict: for (k, v) in items.items(): self.attach_child_scope_for(v) if v.conditions_true(): if issubclass(type(v), Collection): items.walk_children(items=v.get_children(), which=which, fn=fn, tags=tags) else: v.handles = k maybe(v) else: Callbacks().on_skipped(items, is_handler=handlers)