def _save_ssh_host_keys(self, filename):
'''
not using the paramiko save_ssh_host_keys function as we want to add new SSH keys at the bottom so folks
don't complain about it :)
'''
if not self._any_keys_added():
return False
path = os.path.expanduser("~/.ssh")
makedirs_safe(path)
with open(filename, 'w') as f:
for hostname, keys in iteritems(self.ssh._host_keys):
for keytype, key in iteritems(keys):
# was f.write
added_this_time = getattr(key,
'_added_by_assible_this_time',
False)
if not added_this_time:
f.write("%s %s %s\n" %
(hostname, keytype, key.get_base64()))
for hostname, keys in iteritems(self.ssh._host_keys):
for keytype, key in iteritems(keys):
added_this_time = getattr(key,
'_added_by_assible_this_time',
False)
if added_this_time:
f.write("%s %s %s\n" %
(hostname, keytype, key.get_base64()))
def __init__(
self, module, attrs=None, args=None, keys=None, from_argspec=False
):
args = [] if args is None else args
self._attributes = attrs or {}
self._module = module
for arg in args:
self._attributes[arg] = dict()
if from_argspec:
self._attributes[arg]["read_from"] = arg
if keys and arg in keys:
self._attributes[arg]["key"] = True
self.attr_names = frozenset(self._attributes.keys())
_has_key = False
for name, attr in iteritems(self._attributes):
if attr.get("read_from"):
if attr["read_from"] not in self._module.argument_spec:
module.fail_json(
msg="argument %s does not exist" % attr["read_from"]
)
spec = self._module.argument_spec.get(attr["read_from"])
for key, value in iteritems(spec):
if key not in attr:
attr[key] = value
if attr.get("key"):
if _has_key:
module.fail_json(msg="only one key value can be specified")
_has_key = True
attr["required"] = True
def _any_keys_added(self):
for hostname, keys in iteritems(self.ssh._host_keys):
for keytype, key in iteritems(keys):
added_this_time = getattr(key, '_added_by_assible_this_time',
False)
if added_this_time:
return True
return False
def remove_empties(cfg_dict):
"""
Generate final config dictionary
:param cfg_dict: A dictionary parsed in the facts system
:rtype: A dictionary
:returns: A dictionary by eliminating keys that have null values
"""
final_cfg = {}
if not cfg_dict:
return final_cfg
for key, val in iteritems(cfg_dict):
dct = None
if isinstance(val, dict):
child_val = remove_empties(val)
if child_val:
dct = {key: child_val}
elif (isinstance(val, list) and val
and all([isinstance(x, dict) for x in val])):
child_val = [remove_empties(x) for x in val]
if child_val:
dct = {key: child_val}
elif val not in [None, [], {}, (), '']:
dct = {key: val}
if dct:
final_cfg.update(dct)
return final_cfg
def run(self, tmp=None, task_vars=None):
if task_vars is None:
task_vars = dict()
result = super(ActionModule, self).run(tmp, task_vars)
del tmp # tmp no longer has any effect
facts = dict()
cacheable = boolean(self._task.args.pop('cacheable', False))
if self._task.args:
for (k, v) in iteritems(self._task.args):
k = self._templar.template(k)
if not isidentifier(k):
result['failed'] = True
result['msg'] = (
"The variable name '%s' is not valid. Variables must start with a letter or underscore character, and contain only "
"letters, numbers and underscores." % k)
return result
if not C.DEFAULT_JINJA2_NATIVE and isinstance(
v, string_types) and v.lower() in ('true', 'false',
'yes', 'no'):
v = boolean(v, strict=False)
facts[k] = v
result['changed'] = False
result['assible_facts'] = facts
result['_assible_facts_cacheable'] = cacheable
return result
def get_network_legacy_facts(
self, fact_legacy_obj_map, legacy_facts_type=None
):
if not legacy_facts_type:
legacy_facts_type = self._gather_subset
runable_subsets = self.gen_runable(
legacy_facts_type, frozenset(fact_legacy_obj_map.keys())
)
if runable_subsets:
facts = dict()
# default subset should always returned be with legacy facts subsets
if "default" not in runable_subsets:
runable_subsets.add("default")
self.assible_facts["assible_net_gather_subset"] = list(
runable_subsets
)
instances = list()
for key in runable_subsets:
instances.append(fact_legacy_obj_map[key](self._module))
for inst in instances:
inst.populate()
facts.update(inst.facts)
self._warnings.extend(inst.warnings)
for key, value in iteritems(facts):
key = "assible_net_%s" % key
self.assible_facts[key] = value
def preprocess_data(self, ds):
'''
Regorganizes the data for a PlaybookInclude datastructure to line
up with what we expect the proper attributes to be
'''
if not isinstance(ds, dict):
raise AssibleAssertionError(
'ds (%s) should be a dict but was a %s' % (ds, type(ds)))
# the new, cleaned datastructure, which will have legacy
# items reduced to a standard structure
new_ds = AssibleMapping()
if isinstance(ds, AssibleBaseYAMLObject):
new_ds.assible_pos = ds.assible_pos
for (k, v) in iteritems(ds):
if k in ('include', 'import_playbook'):
self._preprocess_import(ds, new_ds, k, v)
else:
# some basic error checking, to make sure vars are properly
# formatted and do not conflict with k=v parameters
if k == 'vars':
if 'vars' in new_ds:
raise AssibleParserError(
"import_playbook parameters cannot be mixed with 'vars' entries for import statements",
obj=ds)
elif not isinstance(v, dict):
raise AssibleParserError(
"vars for import_playbook statements must be specified as a dictionary",
obj=ds)
new_ds[k] = v
return super(PlaybookInclude, self).preprocess_data(new_ds)
def _parse_group(self, group, data):
group = self.inventory.add_group(group)
if not isinstance(data, dict):
data = {'hosts': data}
# is not those subkeys, then simplified syntax, host with vars
elif not any(k in data for k in ('hosts', 'vars', 'children')):
data = {'hosts': [group], 'vars': data}
if 'hosts' in data:
if not isinstance(data['hosts'], list):
raise AssibleError(
"You defined a group '%s' with bad data for the host list:\n %s"
% (group, data))
for hostname in data['hosts']:
self._hosts.add(hostname)
self.inventory.add_host(hostname, group)
if 'vars' in data:
if not isinstance(data['vars'], dict):
raise AssibleError(
"You defined a group '%s' with bad data for variables:\n %s"
% (group, data))
for k, v in iteritems(data['vars']):
self.inventory.set_variable(group, k, v)
if group != '_meta' and isinstance(data, dict) and 'children' in data:
for child_name in data['children']:
child_name = self.inventory.add_group(child_name)
self.inventory.add_child(group, child_name)
def validate(self, all_vars=None):
''' validation that is done at parse time, not load time '''
all_vars = {} if all_vars is None else all_vars
if not self._validated:
# walk all fields in the object
for (name, attribute) in iteritems(self._valid_attrs):
if name in self._alias_attrs:
name = self._alias_attrs[name]
# run validator only if present
method = getattr(self, '_validate_%s' % name, None)
if method:
method(attribute, name, getattr(self, name))
else:
# and make sure the attribute is of the type it should be
value = self._attributes[name]
if value is not None:
if attribute.isa == 'string' and isinstance(value, (list, dict)):
raise AssibleParserError(
"The field '%s' is supposed to be a string type,"
" however the incoming data structure is a %s" % (name, type(value)), obj=self.get_ds()
)
self._validated = True
def _split_role_params(self, ds):
'''
Splits any random role params off from the role spec and store
them in a dictionary of params for parsing later
'''
role_def = dict()
role_params = dict()
base_attribute_names = frozenset(self._valid_attrs.keys())
for (key, value) in iteritems(ds):
# use the list of FieldAttribute values to determine what is and is not
# an extra parameter for this role (or sub-class of this role)
# FIXME: hard-coded list of exception key names here corresponds to the
# connection fields in the Base class. There may need to be some
# other mechanism where we exclude certain kinds of field attributes,
# or make this list more automatic in some way so we don't have to
# remember to update it manually.
if key not in base_attribute_names:
# this key does not match a field attribute, so it must be a role param
role_params[key] = value
else:
# this is a field attribute, so copy it over directly
role_def[key] = value
return (role_def, role_params)
def main():
module = AssibleModule(argument_spec=dict(
jid=dict(type='str', required=True),
mode=dict(type='str', default='status', choices=['cleanup', 'status']),
# passed in from the async_status action plugin
_async_dir=dict(type='path', required=True),
))
mode = module.params['mode']
jid = module.params['jid']
async_dir = module.params['_async_dir']
# setup logging directory
logdir = os.path.expanduser(async_dir)
log_path = os.path.join(logdir, jid)
if not os.path.exists(log_path):
module.fail_json(msg="could not find job",
assible_job_id=jid,
started=1,
finished=1)
if mode == 'cleanup':
os.unlink(log_path)
module.exit_json(assible_job_id=jid, erased=log_path)
# NOT in cleanup mode, assume regular status mode
# no remote kill mode currently exists, but probably should
# consider log_path + ".pid" file and also unlink that above
data = None
try:
with open(log_path) as f:
data = json.loads(f.read())
except Exception:
if not data:
# file not written yet? That means it is running
module.exit_json(results_file=log_path,
assible_job_id=jid,
started=1,
finished=0)
else:
module.fail_json(assible_job_id=jid,
results_file=log_path,
msg="Could not parse job output: %s" % data,
started=1,
finished=1)
if 'started' not in data:
data['finished'] = 1
data['assible_job_id'] = jid
elif 'finished' not in data:
data['finished'] = 0
# Fix error: TypeError: exit_json() keywords must be strings
data = dict([(to_native(k), v) for k, v in iteritems(data)])
module.exit_json(**data)
def collect(self, module=None, collected_facts=None):
env_facts = {}
env_facts['env'] = {}
for k, v in iteritems(os.environ):
env_facts['env'][k] = v
return env_facts
def to_dict(self, value):
obj = {}
for name, attr in iteritems(self._attributes):
if attr.get('key'):
obj[name] = value
else:
obj[name] = attr.get('default')
return obj
def re_search(regex, value):
obj = {}
match = regex.search(value, re.M)
if match:
items = list(match.groups())
if regex.groupindex:
for name, index in iteritems(regex.groupindex):
obj[name] = items[index - 1]
return obj
def remove_aliases(self):
"""
The helper doesn't know what to do with aliased keys
"""
for k, v in iteritems(self.argspec):
if 'aliases' in v:
for alias in v['aliases']:
if alias in self.params:
self.params.pop(alias)
def __call__(self, value, strict=True):
if not isinstance(value, dict):
value = self.to_dict(value)
if strict:
unknown = set(value).difference(self.attr_names)
if unknown:
self._module.fail_json(
msg="invalid keys: %s" % ",".join(unknown)
)
for name, attr in iteritems(self._attributes):
if value.get(name) is None:
value[name] = attr.get("default")
if attr.get("fallback") and not value.get(name):
fallback = attr.get("fallback", (None,))
fallback_strategy = fallback[0]
fallback_args = []
fallback_kwargs = {}
if fallback_strategy is not None:
for item in fallback[1:]:
if isinstance(item, dict):
fallback_kwargs = item
else:
fallback_args = item
try:
value[name] = fallback_strategy(
*fallback_args, **fallback_kwargs
)
except basic.AssibleFallbackNotFound:
continue
if attr.get("required") and value.get(name) is None:
self._module.fail_json(
msg="missing required attribute %s" % name
)
if "choices" in attr:
if value[name] not in attr["choices"]:
self._module.fail_json(
msg="%s must be one of %s, got %s"
% (name, ", ".join(attr["choices"]), value[name])
)
if value[name] is not None:
value_type = attr.get("type", "str")
type_checker = self._module._CHECK_ARGUMENT_TYPES_DISPATCHER[
value_type
]
type_checker(value[name])
elif value.get(name):
value[name] = self._module.params[name]
return value
def load(role_include,
play,
parent_role=None,
from_files=None,
from_include=False):
if from_files is None:
from_files = {}
try:
# The ROLE_CACHE is a dictionary of role names, with each entry
# containing another dictionary corresponding to a set of parameters
# specified for a role as the key and the Role() object itself.
# We use frozenset to make the dictionary hashable.
params = role_include.get_role_params()
if role_include.when is not None:
params['when'] = role_include.when
if role_include.tags is not None:
params['tags'] = role_include.tags
if from_files is not None:
params['from_files'] = from_files
if role_include.vars:
params['vars'] = role_include.vars
params['from_include'] = from_include
hashed_params = hash_params(params)
if role_include.get_name() in play.ROLE_CACHE:
for (entry, role_obj) in iteritems(
play.ROLE_CACHE[role_include.get_name()]):
if hashed_params == entry:
if parent_role:
role_obj.add_parent(parent_role)
return role_obj
# TODO: need to fix cycle detection in role load (maybe use an empty dict
# for the in-flight in role cache as a sentinel that we're already trying to load
# that role?)
# see https://github.com/assible/assible/issues/61527
r = Role(play=play,
from_files=from_files,
from_include=from_include)
r._load_role_data(role_include, parent_role=parent_role)
if role_include.get_name() not in play.ROLE_CACHE:
play.ROLE_CACHE[role_include.get_name()] = dict()
# FIXME: how to handle cache keys for collection-based roles, since they're technically adjustable per task?
play.ROLE_CACHE[role_include.get_name()][hashed_params] = r
return r
except RuntimeError:
raise AssibleError(
"A recursion loop was detected with the roles specified. Make sure child roles do not have dependencies on parent roles",
obj=role_include._ds)
def get_groups_dict(self):
"""
We merge a 'magic' var 'groups' with group name keys and hostname list values into every host variable set. Cache for speed.
"""
if not self._groups_dict_cache:
for (group_name, group) in iteritems(self.groups):
self._groups_dict_cache[group_name] = [
h.name for h in group.get_hosts()
]
return self._groups_dict_cache
def get_all_device_owners(self):
try:
retval = collections.defaultdict(set)
for path in glob.glob('/sys/block/*/slaves/*'):
elements = path.split('/')
device = elements[3]
target = elements[5]
retval[target].add(device)
return dict((k, list(sorted(v))) for (k, v) in iteritems(retval))
except OSError:
return {}
def key_value_in_dict(have_key, have_value, want_dict):
"""
This function checks whether the key and values exist in dict
:param have_key:
:param have_value:
:param want_dict:
:return:
"""
for key, value in iteritems(want_dict):
if key == have_key and value == have_value:
return True
return False
def dump_attrs(self):
'''
Dumps all attributes to a dictionary
'''
attrs = {}
for (name, attribute) in iteritems(self._valid_attrs):
attr = getattr(self, name)
if attribute.isa == 'class' and hasattr(attr, 'serialize'):
attrs[name] = attr.serialize()
else:
attrs[name] = attr
return attrs
def re_matchall(regex, value):
objects = list()
for match in re.findall(regex.pattern, value, re.M):
obj = {}
if regex.groupindex:
for name, index in iteritems(regex.groupindex):
if len(regex.groupindex) == 1:
obj[name] = match
else:
obj[name] = match[index - 1]
objects.append(obj)
return objects
def from_attrs(self, attrs):
'''
Loads attributes from a dictionary
'''
for (attr, value) in iteritems(attrs):
if attr in self._valid_attrs:
attribute = self._valid_attrs[attr]
if attribute.isa == 'class' and isinstance(value, dict):
obj = attribute.class_type()
obj.deserialize(value)
setattr(self, attr, obj)
else:
setattr(self, attr, value)
def test_variable_manager_options_vars(self):
fake_loader = DictDataLoader({})
options_vars = dict(a=1, b=2, c=3)
mock_inventory = MagicMock()
v = VariableManager(loader=fake_loader, inventory=mock_inventory)
# override internal options_vars loading
v._extra_vars = options_vars
myvars = v.get_vars(use_cache=False)
for (key, val) in iteritems(options_vars):
self.assertEqual(myvars.get(key), val)
def check(self, phase):
"""Check the state in given phase and set it to `self.value`.
Args:
phase: The name of the phase to check.
Returns:
NO RETURN VALUE
"""
if phase == 'planned':
return
for key, value in iteritems(self.value):
value[phase] = self.get(key, phase)
def _merge_kv(self, ds):
if ds is None:
return ""
elif isinstance(ds, string_types):
return ds
elif isinstance(ds, dict):
buf = ""
for (k, v) in iteritems(ds):
if k.startswith('_'):
continue
buf = buf + "%s=%s " % (k, v)
buf = buf.strip()
return buf
def do_urlencode(value):
itemiter = None
if isinstance(value, dict):
itemiter = iteritems(value)
elif not isinstance(value, string_types):
try:
itemiter = iter(value)
except TypeError:
pass
if itemiter is None:
return unicode_urlencode(value)
return u'&'.join(
unicode_urlencode(k) + '=' + unicode_urlencode(v, for_qs=True)
for k, v in itemiter)
def run(self, tmp=None, task_vars=None):
if task_vars is None:
task_vars = dict()
result = super(ActionModule, self).run(tmp, task_vars)
del tmp # tmp no longer has any effect
stats = {'data': {}, 'per_host': False, 'aggregate': True}
if self._task.args:
data = self._task.args.get('data', {})
if not isinstance(data, dict):
data = self._templar.template(data,
convert_bare=False,
fail_on_undefined=True)
if not isinstance(data, dict):
result['failed'] = True
result[
'msg'] = "The 'data' option needs to be a dictionary/hash"
return result
# set boolean options, defaults are set above in stats init
for opt in ['per_host', 'aggregate']:
val = self._task.args.get(opt, None)
if val is not None:
if not isinstance(val, bool):
stats[opt] = boolean(self._templar.template(val),
strict=False)
else:
stats[opt] = val
for (k, v) in iteritems(data):
k = self._templar.template(k)
if not isidentifier(k):
result['failed'] = True
result['msg'] = (
"The variable name '%s' is not valid. Variables must start with a letter or underscore character, and contain only "
"letters, numbers and underscores." % k)
return result
stats['data'][k] = self._templar.template(v)
result['changed'] = False
result['assible_stats'] = stats
return result
def load_provider(spec, args):
provider = args.get('provider') or {}
for key, value in iteritems(spec):
if key not in provider:
if 'fallback' in value:
provider[key] = _fallback(value['fallback'])
elif 'default' in value:
provider[key] = value['default']
else:
provider[key] = None
if 'authorize' in provider:
# Coerce authorize to provider if a string has somehow snuck in.
provider['authorize'] = boolean(provider['authorize'] or False)
args['provider'] = provider
return provider
def get_device_links(self, link_dir):
if not os.path.exists(link_dir):
return {}
try:
retval = collections.defaultdict(set)
for entry in os.listdir(link_dir):
try:
target = os.path.basename(
os.readlink(os.path.join(link_dir, entry)))
retval[target].add(entry)
except OSError:
continue
return dict((k, list(sorted(v))) for (k, v) in iteritems(retval))
except OSError:
return {}