def process_role_internal(self, host, policy, role, local=True, sender=None):
if not local:
from opsmop.callbacks.event_stream import EventStreamStreamCallbacks
from opsmop.callbacks.common import CommonCallbacks
Context.set_callbacks([ EventStreamCallbacks(sender=sender), CommonCallbacks() ])
role.pre()
# set up the variable scope - this is done later by walk_handlers for lower-level objects in the tree
policy.attach_child_scope_for(role)
# tell the callbacks we are in validate mode - this may alter or quiet their output
Callbacks.on_validate()
# always validate the role in every mode (VALIDATE, CHECK ,or APPLY)
self.validate_role(role)
# skip the role if we need to
if not role.conditions_true():
Callbacks.on_skipped(role)
return
# process the tree for real for non-validate modes
if not Context.is_validate():
self.execute_role_resources(host, role)
self.execute_role_handlers(host, role)
# run any user hooks
role.post()
def go(self):
if len(self.args) < 3 or sys.argv[1] == "--help":
print(USAGE)
sys.exit(1)
mode = self.args[1]
path = sys.argv[2]
callbacks = None
parser = argparse.ArgumentParser()
parser.add_argument('--validate', help='policy file to validate')
parser.add_argument('--apply', help="policy file to apply")
parser.add_argument('--check', help="policy file to check")
parser.add_argument('--tags', help='optional comma seperated list of tags')
parser.add_argument('--event-stream', action='store_true', help=argparse.SUPPRESS)
args = parser.parse_args(self.args[1:])
all_modes = [ args.validate, args.apply, args.check ]
selected_modes = [ x for x in all_modes if x is not None ]
if len(selected_modes) != 1:
print(USAGE)
sys.exit(1)
path = args.validate or args.apply or args.check
if not args.event_stream:
Callbacks.set_callbacks([ LocalCliCallbacks(), CommonCallbacks() ])
else:
Callbacks.set_callbacks([ EventStreamCallbacks(), CommonCallbacks() ])
tags = None
if args.tags is not None:
tags = args.tags.strip().split(",")
api = Api.from_file(path=path, tags=tags)
try:
if args.validate:
# just check for missing files and invalid types
api.validate()
elif args.check:
# operate in dry-run mode
api.check()
elif args.apply:
# configure everything
api.apply()
else:
print(USAGE)
sys.exit(1)
except OpsMopError as ome:
print("")
print(str(ome))
print("")
sys.exit(1)
print("")
sys.exit(0)
def execute_resource(self, host, resource, handlers=False):
"""
This handles the plan/apply intercharge for a given resource in the resource tree.
It is called recursively via walk_children to run against all resources.
"""
assert host is not None
# we only care about processing leaf node objects
if issubclass(type(resource), Collection):
return
# if in handler mode we do not process the handler unless it was signaled
if handlers and not Context().has_seen_any_signal(
host, resource.all_handles()):
Callbacks().on_skipped(resource, is_handler=handlers)
return
# tell the callbacks we are about to process a resource
# they may use this to print information about the resource
Callbacks().on_resource(resource, handlers)
# plan always, apply() only if not in check mode, else assume
# the plan was executed.
provider = self.do_plan(resource)
assert provider is not None
if Context().is_apply():
self.do_apply(host, provider, handlers)
else: # is_check
self.do_simulate(host, provider)
# if anything has changed, let the callbacks know about it
self.signal_changes(host=host, provider=provider, resource=resource)
def do_apply(self, host, provider, handlers):
"""
Once a provider has a plan generated, see if we need to run the plan.
If so, also run any actions associated witht he apply step, which mostly means registering
variables from apply results.
"""
# some simple providers - like Echo, do not have a planning step
# we will always run the apply step for them. For others, we will only
# run the apply step if we have computed a plan
if (not provider.skip_plan_stage()) and (
not provider.has_planned_actions()):
return False
# indicate we are about take some actions
Callbacks().on_apply(provider)
# take them
result = provider.apply()
if not handlers:
# let the callbacks now we have taken some actions
Callbacks().on_taken_actions(provider, provider.actions_taken)
# all Provider apply() methods need to return Result objects or raise
# exceptions
assert issubclass(type(result), Result)
# the 'register' feature saves results into variable scope
resource = provider.resource
if resource.register is not None:
provider.handle_registration(result)
# determine if there was a failure
fatal = False
cond = resource.failed_when
if cond is not None:
if issubclass(type(cond), Lookup):
fatal = cond.evaluate(resource)
result.reason = cond
else:
fatal = cond
elif result.fatal:
fatal = True
result.fatal = fatal
result.changed = provider.has_changed()
# TODO: eliminate the actions class
if result.changed:
result.actions = [x.do for x in provider.actions_taken]
else:
result.actions = []
# tell the callbacks about the result
Callbacks().on_result(provider, result)
# if there was a failure, handle it
# (common callbacks should abort execution)
#if fatal:
# Callbacks().on_fatal(provider, result)
return True
def signal_changes(self, host=None, provider=None, resource=None):
"""
If any events were signaled, add them to the context here.
"""
assert host is not None
if not provider.has_changed():
return
if resource.signals:
# record the list of all events signaled while processing this role
Context.add_signal(host, resource.signals)
# tell the callbacks that a signal occurred
Callbacks.on_signaled(resource, resource.signals)
def execute_role_handlers(self, host, role):
"""
Processes handler resources for one role for CHECK or APPLY mode
"""
# see comments for prior method for details
Callbacks.on_begin_handlers()
def execute_handler(handler):
handler.pre()
result = self.execute_resource(host=host, resource=handler, handlers=True)
handler.post()
return result
role.walk_children(items=role.get_children('handlers'), which='handlers', fn=execute_handler, tags=self._tags)
def run_policy(self, policy=None):
"""
Runs one specific policy in VALIDATE, CHECK, or APPLY mode
"""
# assign a new top scope to the policy object.
policy.init_scope()
roles = policy.get_roles()
for role in roles.items:
if self._local:
self.process_role(policy, role)
else:
self.process_role_push(policy, role)
Callbacks.on_complete(policy)
def do_apply(self, provider):
"""
Once a provider has a plan generated, see if we need to run the plan.
"""
from opsmop.callbacks.callbacks import Callbacks
# some simple providers - like Echo, do not have a planning step
# we will always run the apply step for them. For others, we will only
# run the apply step if we have computed a plan
if not provider.has_planned_actions():
if not provider.skip_plan_stage():
return Result(provider=provider, changed=False, data=None)
# indicate we are about take some actions
Callbacks().on_apply(provider)
# take them
result = provider.apply()
Callbacks().on_taken_actions(provider, provider.actions_taken)
# all Provider apply() methods need to return Result objects or raise
# exceptions
assert issubclass(type(result), Result)
# the 'register' feature saves results into variable scope
resource = provider.resource
# determine if there was a failure
fatal = False
cond = resource.failed_when
if cond is not None:
if issubclass(type(cond), Lookup):
fatal = cond.evaluate(resource)
result.reason = cond
elif callable(cond):
fatal = cond(result)
else:
fatal = cond
elif result.fatal:
fatal = True
result.fatal = fatal
result.changed = provider.has_changed()
# TODO: eliminate the actions class
if result.changed:
result.actions = [x.do for x in provider.actions_taken]
else:
result.actions = []
Callbacks().on_result(provider, result)
return result
def execute_role_resources(self, host, role):
"""
Processes non-handler resources for one role for CHECK or APPLY mode
"""
# tell the context we are processing resources now, which may change their behavior
# of certain methods like on_resource()
Callbacks.on_begin_role(role)
def execute_resource(resource):
# execute each resource through plan() and if needed apply() stages, but before and after
# doing so, run any user pre() or post() hooks implemented on that object.
resource.pre()
result = self.execute_resource(host=host, resource=resource)
resource.post()
return result
role.walk_children(items=role.get_children('resources'), which='resources', fn=execute_resource, tags=self._tags)
def handle_registration(self, result):
assert result is not None
va = dict()
va[self.register] = result
Callbacks().on_update_variables(va)
self.resource.update_variables(va)
self.resource.update_parent_variables(va)
def compute_max_hostname_length(self, hosts):
hostname_length = 0
for host in hosts:
length = len(host.display_name())
if length > hostname_length:
hostname_length = length
Callbacks().set_hostname_length(hostname_length)
def info(self, host, msg, sep=':', foreground=None):
from opsmop.callbacks.callbacks import Callbacks
max_length = Callbacks().hostname_length()
fmt = f"%-{max_length}s"
msg = "%s %s %s" % (fmt % host.display_name(), sep, msg)
if foreground:
msg = foreground + msg + Style.RESET_ALL
self.i3(msg)
def do_plan(self, resource):
"""
Ask a resource for the provider, and then see what the planned actions should be.
The planned actions are kept on the provider object. We don't need to obtain the plan.
Return the provider.
"""
# ask the resource for a provider instance
provider = resource.provider()
if provider.skip_plan_stage():
return provider
# tell the context object we are about to run the plan stage.
Callbacks.on_plan(provider)
# compute the plan
provider.plan()
# copy the list of planned actions into the 'to do' list for the apply method
# on the provider
provider.commit_to_plan()
return provider
def process_local_role(self, policy=None, role=None):
host = self._local_host
Context().set_host(host)
policy.attach_child_scope_for(role)
try:
role.main()
except Exception as e:
tb = traceback.format_exc()
# process *any* uncaught exceptions through the configured exception handlers
# this includes any resources where failed_when / ignore_errors was not used
# but also any random python exceptions
Callbacks().on_fatal(e, tb)
def run(self):
Callbacks().on_resource(self)
provider = self.do_plan()
if Context().is_apply():
result = self.do_apply(provider)
else:
result = self.do_simulate(provider)
# copy over results
self.changed = result.changed
self.data = result.data
self.rc = result.rc
def process_local_role(self, policy=None, role=None):
host = self._local_host
Context().set_host(host)
role.pre()
# set up the variable scope - this is done later by walk_handlers for lower-level objects in the tree
policy.attach_child_scope_for(role)
# tell the callbacks we are in validate mode - this may alter or quiet their output
Callbacks().on_validate()
# always validate the role in every mode (VALIDATE, CHECK ,or APPLY)
self.validate_role(role)
# skip the role if we need to
if not role.conditions_true():
Callbacks().on_skipped(role)
return
# process the tree for real for non-validate modes
if not Context().is_validate():
self.execute_role_resources(host, role)
self.execute_role_handlers(host, role)
# run any user hooks
role.post()
def run_policy(self, policy=None):
"""
Runs one specific policy in CHECK, or APPLY mode
"""
# assign a new top scope to the policy object.
policy.init_scope()
roles = policy.get_roles()
for role in roles.items:
Context().set_role(role)
if not self._push:
self.process_local_role(policy, role)
else:
self.process_remote_role(policy, role)
Callbacks().on_complete(policy)
def remote_fn(caller, params, sender):
"""
This is the remote function used for mitogen calls
"""
# FIXME: REFACTOR: we have a bit of a growing inconsistency between what is a constructor parameter and what is passed around in Context.
# we should change this to have context objects that have more meat, but also get passed around versus acting globally, and smaller
# function signatures across the board.
import dill
from opsmop.core.executor import Executor
params = dill.loads(zlib.decompress(params))
host = params['host']
policy = params['policy']
role = params['role']
mode = params['mode']
tags = params['tags']
checksums = params['checksums']
relative_root = params['relative_root']
hostvars = params['hostvars']
extra_vars = params['extra_vars']
Context().set_mode(mode)
Context().set_caller(caller)
assert relative_root is not None
Context().set_checksums(checksums)
Context().update_globals(hostvars)
policy.roles = Roles(role)
Callbacks().set_callbacks([
EventStreamCallbacks(sender=sender),
LocalCliCallbacks(),
CommonCallbacks()
])
executor = Executor([policy],
local_host=host,
push=False,
tags=params['tags'],
extra_vars=extra_vars,
relative_root=relative_root) # remove single_role
# FIXME: care about mode
executor.apply()
def remotify_role(self, host, policy, role, mode):
if self.should_exclude_from_limits(host):
return
try:
if not role.should_contact(host):
Callbacks().on_skipped(role)
return True
else:
role.before_contact(host)
except Exception as e:
print(str(e))
Context().record_host_failure(host, e)
return False
target_host = self.actual_host(role, host)
target_host.reset_actions()
import dill
conn = self.connect(host, role)
receiver = mitogen.core.Receiver(self.router)
self.events_select.add(receiver)
sender = self.status_recv.to_sender()
params = dict(host=target_host,
policy=policy,
role=role,
mode=mode,
relative_root=Context().relative_root(),
tags=self.tags,
checksums=self.checksums,
hostvars=host.all_variables(),
extra_vars=Context().extra_vars())
params = zlib.compress(dill.dumps(params), level=9)
call_recv = conn.call_async(remote_fn, self.myself, params, sender)
self.calls_sel.add(call_recv)
return True
def execute(self):
"""
Execute a command (a list or string) with input_text as input, appending
the output of all commands to the build log.
This code was derived from http://vespene.io/ though is slightly different
because there are no database objects.
"""
Callbacks().on_execute_command(self.provider, self)
command = self.cmd
timeout_cmd = self.get_timeout()
shell = True
if type(command) == list:
if self.timeout and timeout_cmd:
command.insert(0, self.timeout)
command.insert(0, timeout_cmd)
shell = False
else:
if self.timeout and timeout_cmd:
command = "%s %s %s" % (timeout_cmd, self.timeout, command)
# keep SSH-agent working for executed commands
sock = os.environ.get('SSH_AUTH_SOCK', None)
if self.env and sock:
self.env['SSH_AUTH_SOCK'] = sock
process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell, env=self.env)
if self.input_text is None:
self.input_text = ""
stdin = io.TextIOWrapper(
process.stdin,
encoding='utf-8',
line_buffering=True,
)
stdout = io.TextIOWrapper(
process.stdout,
encoding='utf-8',
)
stdin.write(self.input_text)
stdin.close()
output = ""
for line in stdout:
if (self.echo or self.loud) and (not self.ignore_lines or not self.should_ignore(line)):
Callbacks().on_command_echo(self.provider, line)
output = output + line
if output.strip() == "":
Callbacks().on_command_echo(self.provider, "(no output)")
process.wait()
res = None
rc = process.returncode
if rc != 0:
res = Result(self.provider, rc=rc, data=output, fatal=self.fatal, primary=self.primary)
else:
res = Result(self.provider, rc=rc, data=output, fatal=False, primary=self.primary)
# this callback will, depending on implementation, usually note fatal result objects and raise an exception
Callbacks().on_command_result(self.provider, res)
return res
def echo(self, msg):
Callbacks().on_echo(self, msg)
def do(self, action_name):
""" marks off that an action has been completed. not marking off all planned actions (or any unplanned ones) will result in an error """
action = Action(action_name)
self.actions_taken.append(action)
Callbacks().on_do(self, action)
def needs(self, action_name):
""" declares than an action 'should' take place during an apply step """
action = Action(action_name)
self.actions_planned.append(action)
Callbacks().on_needs(self, action)
def _on_walk(self):
Callbacks.on_role(self)
def go(self):
colorama_init()
Callbacks().set_callbacks([LocalCliCallbacks(), CommonCallbacks()])
if len(self.args) < 3 or sys.argv[1] == "--help":
print(USAGE)
sys.exit(1)
mode = self.args[1]
path = sys.argv[2]
callbacks = None
extra_vars = dict()
parser = argparse.ArgumentParser()
parser.add_argument('--validate',
action='store_true',
help='policy file to validate')
parser.add_argument('--apply',
action='store_true',
help="policy file to apply")
parser.add_argument('--check',
action='store_true',
help="policy file to check")
parser.add_argument('--push',
action='store_true',
help='run in push mode')
parser.add_argument('--local',
action='store_true',
help='run in local mode')
parser.add_argument('--verbose',
action='store_true',
help='(with --push) increase verbosity')
parser.add_argument('--extra-vars',
help="add extra variables from the command line")
parser.add_argument(
'--limit-groups',
help=
"(with --push) limit groups executed to this comma-separated list of patterns"
)
parser.add_argument(
'--limit-hosts',
help=
"(with --push) limit hosts executed to this comma-separated list of patterns"
)
args = parser.parse_args(self.args[1:])
all_modes = [args.validate, args.apply, args.check]
selected_modes = [x for x in all_modes if x is True]
if len(selected_modes) != 1:
print(selected_modes)
print(USAGE)
sys.exit(1)
all_modes = [args.push, args.local]
selected_modes = [x for x in all_modes if x is True]
if len(selected_modes) != 1:
print(USAGE)
sys.exit(1)
if args.extra_vars is not None:
extra_vars = self.handle_extra_vars(args.extra_vars)
Context().set_verbose(args.verbose)
abspath = os.path.abspath(sys.modules[self.policy.__module__].__file__)
relative_root = os.path.dirname(abspath)
os.chdir(os.path.dirname(abspath))
api = Api(policies=[self.policy],
push=args.push,
extra_vars=extra_vars,
limit_groups=args.limit_groups,
limit_hosts=args.limit_hosts,
relative_root=relative_root)
try:
if args.validate:
# just check for missing files and invalid types
api.validate()
elif args.check:
# operate in dry-run mode
api.check()
elif args.apply:
# configure everything
api.apply()
else:
print(USAGE)
sys.exit(1)
except OpsMopStop as oms:
sys.exit(1)
except OpsMopError as ome:
print("")
print(str(ome))
print("")
sys.exit(1)
print("")
sys.exit(0)
def walk_children(self,
items=None,
which=None,
fn=None,
handlers=False,
tags=None):
"""
A relatively complex iterator used by Executor() code.
Walks the entire object tree calling fn() on each element.
items - the kids to start the iteration with
context - a Context() object for callback tracking
which - 'resources' or 'handlers'
fn - the function to call on each object
"""
self._on_walk()
items_type = type(items)
if items is None:
return
def maybe(v):
# we'll visit every resource but only call the function on items if tags are *not* engaged
if not tags or v.has_tag(tags):
fn(v)
if issubclass(items_type, Collection):
self.attach_child_scope_for(items)
proceed = items.conditions_true()
if proceed:
return items.walk_children(items=items.get_children(),
which=which,
fn=fn,
tags=tags)
else:
Callbacks().on_skipped(items, is_handler=handlers)
elif issubclass(items_type, Resource):
self.attach_child_scope_for(items)
if items.conditions_true():
return maybe(items)
else:
Callbacks().on_skipped(items, is_handler=handlers)
elif items_type == list:
for x in items:
self.attach_child_scope_for(x)
if x.conditions_true():
if issubclass(type(x), Collection):
x.walk_children(items=x.get_children(),
fn=fn,
tags=tags)
else:
maybe(x)
else:
Callbacks().on_skipped(items, is_handler=handlers)
elif items_type == dict:
for (k, v) in items.items():
self.attach_child_scope_for(v)
if v.conditions_true():
if issubclass(type(v), Collection):
items.walk_children(items=v.get_children(),
which=which,
fn=fn,
tags=tags)
else:
v.handles = k
maybe(v)
else:
Callbacks().on_skipped(items, is_handler=handlers)
