def __init__(self, paths=None, scan_sys_paths=True): # TODO: accept metadata loader override self._assible_pkg_path = to_native(os.path.dirname(to_bytes(sys.modules['assible'].__file__))) if isinstance(paths, string_types): paths = [paths] elif paths is None: paths = [] # expand any placeholders in configured paths paths = [os.path.expanduser(to_native(p, errors='surrogate_or_strict')) for p in paths] if scan_sys_paths: # append all sys.path entries with an assible_collections package for path in sys.path: if ( path not in paths and os.path.isdir(to_bytes( os.path.join(path, 'assible_collections'), errors='surrogate_or_strict', )) ): paths.append(path) self._n_configured_paths = paths self._n_cached_collection_paths = None self._n_cached_collection_qualified_paths = None self._n_playbook_paths = []
def _iter_modules_impl(paths, prefix=''): # NB: this currently only iterates what's on disk- redirected modules are not considered if not prefix: prefix = '' else: prefix = to_native(prefix) # yield (module_loader, name, ispkg) for each module/pkg under path # TODO: implement ignore/silent catch for unreadable? for b_path in map(to_bytes, paths): if not os.path.isdir(b_path): continue for b_basename in sorted(os.listdir(b_path)): b_candidate_module_path = os.path.join(b_path, b_basename) if os.path.isdir(b_candidate_module_path): # exclude things that obviously aren't Python package dirs # FIXME: this dir is adjustable in py3.8+, check for it if b'.' in b_basename or b_basename == b'__pycache__': continue # TODO: proper string handling? yield prefix + to_native(b_basename), True else: # FIXME: match builtin ordering for package/dir/file, support compiled? if b_basename.endswith(b'.py') and b_basename != b'__init__.py': yield prefix + to_native(os.path.splitext(b_basename)[0]), False
def yaml_to_dict(yaml, content_id): """ Return a Python dict version of the provided YAML. Conversion is done in a subprocess since the current Python interpreter does not have access to PyYAML. """ if content_id in yaml_to_dict_cache: return yaml_to_dict_cache[content_id] try: cmd = [external_python, yaml_to_json_path] proc = subprocess.Popen([to_bytes(c) for c in cmd], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_bytes, stderr_bytes = proc.communicate(to_bytes(yaml)) if proc.returncode != 0: raise Exception( 'command %s failed with return code %d: %s' % ([to_native(c) for c in cmd ], proc.returncode, to_native(stderr_bytes))) data = yaml_to_dict_cache[content_id] = json.loads( to_text(stdout_bytes), object_hook=object_hook) return data except Exception as ex: raise Exception( 'internal importer error - failed to parse yaml: %s' % to_native(ex))
def __init__(self, collection_name, subdirs, resource, ref_type): """ Create an AssibleCollectionRef from components :param collection_name: a collection name of the form 'namespace.collectionname' :param subdirs: optional subdir segments to be appended below the plugin type (eg, 'subdir1.subdir2') :param resource: the name of the resource being references (eg, 'mymodule', 'someaction', 'a_role') :param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment' """ collection_name = to_text(collection_name, errors='strict') if subdirs is not None: subdirs = to_text(subdirs, errors='strict') resource = to_text(resource, errors='strict') ref_type = to_text(ref_type, errors='strict') if not self.is_valid_collection_name(collection_name): raise ValueError('invalid collection name (must be of the form namespace.collection): {0}'.format(to_native(collection_name))) if ref_type not in self.VALID_REF_TYPES: raise ValueError('invalid collection ref_type: {0}'.format(ref_type)) self.collection = collection_name if subdirs: if not re.match(self.VALID_SUBDIRS_RE, subdirs): raise ValueError('invalid subdirs entry: {0} (must be empty/None or of the form subdir1.subdir2)'.format(to_native(subdirs))) self.subdirs = subdirs else: self.subdirs = u'' self.resource = resource self.ref_type = ref_type package_components = [u'assible_collections', self.collection] fqcr_components = [self.collection] self.n_python_collection_package_name = to_native('.'.join(package_components)) if self.ref_type == u'role': package_components.append(u'roles') else: # we assume it's a plugin package_components += [u'plugins', self.ref_type] if self.subdirs: package_components.append(self.subdirs) fqcr_components.append(self.subdirs) if self.ref_type == u'role': # roles are their own resource package_components.append(self.resource) fqcr_components.append(self.resource) self.n_python_package_name = to_native('.'.join(package_components)) self._fqcr = u'.'.join(fqcr_components)
def load_module(self, fullname): if not _meta_yml_to_dict: raise ValueError('assible.utils.collection_loader._meta_yml_to_dict is not set') module = super(_AssibleCollectionPkgLoader, self).load_module(fullname) module._collection_meta = {} # TODO: load collection metadata, cache in __loader__ state collection_name = '.'.join(self._split_name[1:3]) if collection_name == 'assible.builtin': # assible.builtin is a synthetic collection, get its routing config from the Assible distro assible_pkg_path = os.path.dirname(import_module('assible').__file__) metadata_path = os.path.join(assible_pkg_path, 'config/assible_builtin_runtime.yml') with open(to_bytes(metadata_path), 'rb') as fd: raw_routing = fd.read() else: b_routing_meta_path = to_bytes(os.path.join(module.__path__[0], 'meta/runtime.yml')) if os.path.isfile(b_routing_meta_path): with open(b_routing_meta_path, 'rb') as fd: raw_routing = fd.read() else: raw_routing = '' try: if raw_routing: routing_dict = _meta_yml_to_dict(raw_routing, (collection_name, 'runtime.yml')) module._collection_meta = self._canonicalize_meta(routing_dict) except Exception as ex: raise ValueError('error parsing collection metadata: {0}'.format(to_native(ex))) AssibleCollectionConfig.on_collection_load.fire(collection_name=collection_name, collection_path=os.path.dirname(module.__file__)) return module
def from_fqcr(ref, ref_type): """ Parse a string as a fully-qualified collection reference, raises ValueError if invalid :param ref: collection reference to parse (a valid ref is of the form 'ns.coll.resource' or 'ns.coll.subdir1.subdir2.resource') :param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment' :return: a populated AssibleCollectionRef object """ # assuming the fq_name is of the form (ns).(coll).(optional_subdir_N).(resource_name), # we split the resource name off the right, split ns and coll off the left, and we're left with any optional # subdirs that need to be added back below the plugin-specific subdir we'll add. So: # ns.coll.resource -> assible_collections.ns.coll.plugins.(plugintype).resource # ns.coll.subdir1.resource -> assible_collections.ns.coll.plugins.subdir1.(plugintype).resource # ns.coll.rolename -> assible_collections.ns.coll.roles.rolename if not AssibleCollectionRef.is_valid_fqcr(ref): raise ValueError('{0} is not a valid collection reference'.format(to_native(ref))) ref = to_text(ref, errors='strict') ref_type = to_text(ref_type, errors='strict') resource_splitname = ref.rsplit(u'.', 1) package_remnant = resource_splitname[0] resource = resource_splitname[1] # split the left two components of the collection package name off, anything remaining is plugin-type # specific subdirs to be added back on below the plugin type package_splitname = package_remnant.split(u'.', 2) if len(package_splitname) == 3: subdirs = package_splitname[2] else: subdirs = u'' collection_name = u'.'.join(package_splitname[0:2]) return AssibleCollectionRef(collection_name, subdirs, resource, ref_type)
def __init__(self, collection_finder, pathctx): # when called from a path_hook, find_module doesn't usually get the path arg, so this provides our context self._pathctx = to_native(pathctx) self._collection_finder = collection_finder if PY3: # cache the native FileFinder (take advantage of its filesystem cache for future find/load requests) self._file_finder = None
def _module_file_from_path(leaf_name, path): has_code = True package_path = os.path.join(to_native(path), to_native(leaf_name)) module_path = None # if the submodule is a package, assemble valid submodule paths, but stop looking for a module if os.path.isdir(to_bytes(package_path)): # is there a package init? module_path = os.path.join(package_path, '__init__.py') if not os.path.isfile(to_bytes(module_path)): module_path = os.path.join(package_path, '__synthetic__') has_code = False else: module_path = package_path + '.py' package_path = None if not os.path.isfile(to_bytes(module_path)): raise ImportError('{0} not found at {1}'.format(leaf_name, path)) return module_path, has_code, package_path
def run_scm_cmd(cmd, tempdir): try: stdout = '' stderr = '' popen = Popen(cmd, cwd=tempdir, stdout=PIPE, stderr=PIPE) stdout, stderr = popen.communicate() except Exception as e: ran = " ".join(cmd) display.debug("ran %s:" % ran) raise AssibleError("when executing %s: %s" % (ran, to_native(e))) if popen.returncode != 0: raise AssibleError("- command %s failed in directory %s (rc=%s) - %s" % (' '.join(cmd), tempdir, popen.returncode, to_native(stderr)))
def _assible_collection_path_hook(self, path): path = to_native(path) interesting_paths = self._n_cached_collection_qualified_paths if not interesting_paths: interesting_paths = [os.path.join(p, 'assible_collections') for p in self._n_collection_paths] interesting_paths.insert(0, self._assible_pkg_path) self._n_cached_collection_qualified_paths = interesting_paths if any(path.startswith(p) for p in interesting_paths): return _AssiblePathHookFinder(self, path) raise ImportError('not interested')
def _get_collection_name_from_path(path): """ Return the containing collection name for a given path, or None if the path is not below a configured collection, or the collection cannot be loaded (eg, the collection is masked by another of the same name higher in the configured collection roots). :param path: path to evaluate for collection containment :return: collection name or None """ # FIXME: mess with realpath canonicalization or not? path = to_native(path) path_parts = path.split('/') if path_parts.count('assible_collections') != 1: return None ac_pos = path_parts.index('assible_collections') # make sure it's followed by at least a namespace and collection name if len(path_parts) < ac_pos + 3: return None candidate_collection_name = '.'.join(path_parts[ac_pos + 1:ac_pos + 3]) try: # we've got a name for it, now see if the path prefix matches what the loader sees imported_pkg_path = to_native(os.path.dirname(to_bytes(import_module('assible_collections.' + candidate_collection_name).__file__))) except ImportError: return None # reassemble the original path prefix up the collection name, and it should match what we just imported. If not # this is probably a collection root that's not configured. original_path_prefix = os.path.join('/', *path_parts[0:ac_pos + 3]) if original_path_prefix != imported_pkg_path: return None return candidate_collection_name
def set_playbook_paths(self, playbook_paths): if isinstance(playbook_paths, string_types): playbook_paths = [playbook_paths] # track visited paths; we have to preserve the dir order as-passed in case there are duplicate collections (first one wins) added_paths = set() # de-dupe self._n_playbook_paths = [os.path.join(to_native(p), 'collections') for p in playbook_paths if not (p in added_paths or added_paths.add(p))] self._n_cached_collection_paths = None # HACK: playbook CLI sets this relatively late, so we've already loaded some packages whose paths might depend on this. Fix those up. # NB: this should NOT be used for late additions; ideally we'd fix the playbook dir setup earlier in Assible init # to prevent this from occurring for pkg in ['assible_collections', 'assible_collections.assible']: self._reload_hack(pkg)
def _get_collection_metadata(collection_name): collection_name = to_native(collection_name) if not collection_name or not isinstance(collection_name, string_types) or len(collection_name.split('.')) != 2: raise ValueError('collection_name must be a non-empty string of the form namespace.collection') try: collection_pkg = import_module('assible_collections.' + collection_name) except ImportError: raise ValueError('unable to locate collection {0}'.format(collection_name)) _collection_meta = getattr(collection_pkg, '_collection_meta', None) if _collection_meta is None: raise ValueError('collection metadata was not loaded for collection {0}'.format(collection_name)) return _collection_meta
def __init__(self, fullname, path_list=None): self._fullname = fullname self._redirect_module = None self._split_name = fullname.split('.') self._rpart_name = fullname.rpartition('.') self._parent_package_name = self._rpart_name[0] # eg assible_collections for assible_collections.somens, '' for toplevel self._package_to_load = self._rpart_name[2] # eg somens for assible_collections.somens self._source_code_path = None self._decoded_source = None self._compiled_code = None self._validate_args() self._candidate_paths = self._get_candidate_paths([to_native(p) for p in path_list]) self._subpackage_search_paths = self._get_subpackage_search_paths(self._candidate_paths) self._validate_final()
def read(self): # Read in the crontab from the system self.lines = [] if self.cron_file: # read the cronfile try: f = open(self.b_cron_file, 'rb') self.n_existing = to_native(f.read(), errors='surrogate_or_strict') self.lines = self.n_existing.splitlines() f.close() except IOError: # cron file does not exist return except Exception: raise CronTabError("Unexpected error:", sys.exc_info()[0]) else: # using safely quoted shell for now, but this really should be two non-shell calls instead. FIXME (rc, out, err) = self.module.run_command(self._read_user_execute(), use_unsafe_shell=True) if rc != 0 and rc != 1: # 1 can mean that there are no jobs. raise CronTabError("Unable to read crontab") self.n_existing = out lines = out.splitlines() count = 0 for l in lines: if count > 2 or (not re.match( r'# DO NOT EDIT THIS FILE - edit the master and reinstall.', l) and not re.match(r'# \(/tmp/.*installed on.*\)', l) and not re.match(r'# \(.*version.*\)', l)): self.lines.append(l) else: pattern = re.escape(l) + '[\r\n]?' self.n_existing = re.sub(pattern, '', self.n_existing, 1) count += 1
def test_to_native(in_string, encoding, expected): """test happy path of encoding to native strings""" assert to_native(in_string, encoding) == expected
def safe_eval(expr, locals=None, include_exceptions=False): ''' This is intended for allowing things like: with_items: a_list_variable Where Jinja2 would return a string but we do not want to allow it to call functions (outside of Jinja2, where the env is constrained). Based on: http://stackoverflow.com/questions/12523516/using-ast-and-whitelists-to-make-pythons-eval-safe ''' locals = {} if locals is None else locals # define certain JSON types # eg. JSON booleans are unknown to python eval() OUR_GLOBALS = { '__builtins__': {}, # avoid global builtins as per eval docs 'false': False, 'null': None, 'true': True, # also add back some builtins we do need 'True': True, 'False': False, 'None': None } # this is the whitelist of AST nodes we are going to # allow in the evaluation. Any node type other than # those listed here will raise an exception in our custom # visitor class defined below. SAFE_NODES = set(( ast.Add, ast.BinOp, # ast.Call, ast.Compare, ast.Dict, ast.Div, ast.Expression, ast.List, ast.Load, ast.Mult, ast.Num, ast.Name, ast.Str, ast.Sub, ast.USub, ast.Tuple, ast.UnaryOp, )) # AST node types were expanded after 2.6 if sys.version_info[:2] >= (2, 7): SAFE_NODES.update(set((ast.Set, ))) # And in Python 3.4 too if sys.version_info[:2] >= (3, 4): SAFE_NODES.update(set((ast.NameConstant, ))) # And in Python 3.6 too, although not encountered until Python 3.8, see https://bugs.python.org/issue32892 if sys.version_info[:2] >= (3, 6): SAFE_NODES.update(set((ast.Constant, ))) filter_list = [] for filter_ in filter_loader.all(): filter_list.extend(filter_.filters().keys()) test_list = [] for test in test_loader.all(): test_list.extend(test.tests().keys()) CALL_WHITELIST = C.DEFAULT_CALLABLE_WHITELIST + filter_list + test_list class CleansingNodeVisitor(ast.NodeVisitor): def generic_visit(self, node, inside_call=False): if type(node) not in SAFE_NODES: raise Exception("invalid expression (%s)" % expr) elif isinstance(node, ast.Call): inside_call = True elif isinstance(node, ast.Name) and inside_call: # Disallow calls to builtin functions that we have not vetted # as safe. Other functions are excluded by setting locals in # the call to eval() later on if hasattr(builtins, node.id) and node.id not in CALL_WHITELIST: raise Exception("invalid function: %s" % node.id) # iterate over all child nodes for child_node in ast.iter_child_nodes(node): self.generic_visit(child_node, inside_call) if not isinstance(expr, string_types): # already templated to a datastructure, perhaps? if include_exceptions: return (expr, None) return expr cnv = CleansingNodeVisitor() try: parsed_tree = ast.parse(expr, mode='eval') cnv.visit(parsed_tree) compiled = compile(parsed_tree, to_native(expr), 'eval') # Note: passing our own globals and locals here constrains what # callables (and other identifiers) are recognized. this is in # addition to the filtering of builtins done in CleansingNodeVisitor result = eval(compiled, OUR_GLOBALS, dict(locals)) if PY2: # On Python 2 u"{'key': 'value'}" is evaluated to {'key': 'value'}, # ensure it is converted to {u'key': u'value'}. result = container_to_text(result) if include_exceptions: return (result, None) else: return result except SyntaxError as e: # special handling for syntax errors, we just return # the expression string back as-is to support late evaluation if include_exceptions: return (expr, None) return expr except Exception as e: if include_exceptions: return (expr, e) return expr
def legacy_plugin_dir_to_plugin_type(legacy_plugin_dir_name): """ Utility method to convert from a PluginLoader dir name to a plugin ref_type :param legacy_plugin_dir_name: PluginLoader dir name (eg, 'action_plugins', 'library') :return: the corresponding plugin ref_type (eg, 'action', 'role') """ legacy_plugin_dir_name = to_text(legacy_plugin_dir_name) plugin_type = legacy_plugin_dir_name.replace(u'_plugins', u'') if plugin_type == u'library': plugin_type = u'modules' if plugin_type not in AssibleCollectionRef.VALID_REF_TYPES: raise ValueError('{0} cannot be mapped to a valid collection ref type'.format(to_native(legacy_plugin_dir_name))) return plugin_type