def spec_content_validator(spec_content):
"""validates that spec (YAML) file has all required fields
:param spec_content: content of spec file
:raise IRFailedToAddPlugin: when mandatory data is missing in spec file
:return: Dictionary with data loaded from a spec (YAML) file
"""
spec_dict = yaml.load(spec_content)
if not isinstance(spec_dict, dict):
raise IRFailedToAddPlugin(
"Spec file is empty or corrupted: '{}'".format(spec_content))
plugin_type_key = 'plugin_type'
if plugin_type_key not in spec_dict:
raise IRFailedToAddPlugin(
"Required key '{}' is missing in plugin spec "
"file: {}".format(plugin_type_key, spec_content))
if not isinstance(spec_dict[plugin_type_key], str):
raise IRFailedToAddPlugin(
"Value of 'str' is expected for key '{}' in spec "
"file '{}'".format(plugin_type_key, spec_content))
if not len(spec_dict[plugin_type_key]):
raise IRFailedToAddPlugin(
"String value of key '{}' in spec file '{}' can't "
"be empty.".format(plugin_type_key, spec_content))
subparsers_key = 'subparsers'
if subparsers_key not in spec_dict:
raise IRFailedToAddPlugin(
"'{}' key is missing in spec file: '{}'".format(
subparsers_key, spec_content))
if not isinstance(spec_dict[subparsers_key], dict):
raise IRFailedToAddPlugin(
"Value of '{}' in spec file '{}' should be "
"'dict' type".format(subparsers_key, spec_content))
if 'description' not in spec_dict \
and 'description' not in spec_dict[subparsers_key].values()[0]:
raise IRFailedToAddPlugin(
"Required key 'description' is missing for supbarser '{}' in "
"spec file '{}'".format(spec_dict[subparsers_key].keys()[0],
spec_content))
if len(spec_dict[subparsers_key]) != 1:
raise IRFailedToAddPlugin(
"One subparser should be defined under '{}' in "
"spec file '{}'".format(subparsers_key, spec_content))
if not isinstance(spec_dict[subparsers_key].values()[0], dict):
raise IRFailedToAddPlugin(
"Subparser '{}' should be 'dict' type and not '{}' type in "
"spec file '{}'".format(
spec_dict[subparsers_key].keys()[0],
type(spec_dict[subparsers_key].values()[0]), spec_content))
if spec_dict[subparsers_key].keys()[0].lower() == 'all':
raise IRFailedToAddPlugin(
"Adding a plugin named 'all' isn't allowed")
return spec_dict
def spec_validator(spec_file):
"""Loads & validates that spec (YAML) file has all required fields
:param spec_file: Path to plugin's spec file
:raise IRFailedToAddPlugin: when mandatory data is missing in spec file
:return: Dictionary with data loaded from a spec (YAML) file
"""
if not os.path.isfile(spec_file):
raise IRFailedToAddPlugin(
"Plugin spec doesn't exist: {}".format(spec_file))
import yaml
with open(spec_file) as fp:
spec_dict = yaml.load(fp)
if not isinstance(spec_dict, dict):
raise IRFailedToAddPlugin(
"Spec file is empty or corrupted: '{}'".format(spec_file))
for required_key in ('plugin_type', 'description'):
if required_key not in spec_dict:
raise IRFailedToAddPlugin(
"Required key '{}' is missing in plugin spec "
"file: {}".format(required_key, spec_file))
if not isinstance(spec_dict[required_key], str):
raise IRFailedToAddPlugin(
"Value of 'str' is expected for key '{}' in spec "
"file '{}'".format(required_key, spec_file))
if not len(spec_dict[required_key]):
raise IRFailedToAddPlugin(
"String value of key '{}' in spec file '{}' can't "
"be empty.".format(required_key, spec_file))
key = 'subparsers'
if key not in spec_dict:
raise IRFailedToAddPlugin(
"'{}' key is missing in spec file: '{}'".format(
key, spec_file))
if not isinstance(spec_dict[key], dict):
raise IRFailedToAddPlugin(
"Value of '{}' in spec file '{}' should be "
"'dict' type".format(key, spec_file))
if len(spec_dict[key]) != 1:
raise IRFailedToAddPlugin(
"One subparser should be defined under '{}' in "
"spec file '{}'".format(key, spec_file))
if not isinstance(spec_dict[key].values()[0], dict):
raise IRFailedToAddPlugin(
"Subparser '{}' should be 'dict' type and not '{}' type in "
"spec file '{}'".format(spec_dict[key].keys()[0],
type(spec_dict[key].values()[0]),
spec_file))
if spec_dict[key].keys()[0].lower() == 'all':
raise IRFailedToAddPlugin(
"Adding a plugin named 'all' isn't allowed")
return spec_dict
def add_plugin(self, plugin_source, rev=None, dest=None):
"""Adds (install) a plugin
:param plugin_source: Plugin source.
Can be:
1. Plugin name (from available in registry)
2. Path to a local directory
3. Git URL
:param dest: destination where to clone a plugin into (if 'source' is
a Git URL)
:param rev: git branch/tag/revision
"""
plugin_data = {}
# Check if a plugin is in the registry
if plugin_source in PLUGINS_REGISTRY:
plugin_data = PLUGINS_REGISTRY[plugin_source]
plugin_source = PLUGINS_REGISTRY[plugin_source]['src']
# Local dir plugin
if os.path.exists(plugin_source):
pass
# Git Plugin
else:
if rev is None:
rev = plugin_data.get('rev')
plugin_src_path = plugin_data.get('src_path', '')
plugin_source = self._clone_git_plugin(
plugin_source, plugin_src_path, rev,
dest)
plugin = InfraredPlugin(plugin_source)
plugin_type = plugin.config['plugin_type']
# FIXME(yfried) validate spec and throw exception on missing input
if plugin_type not in self.config.options(
self.SUPPORTED_TYPES_SECTION):
raise IRFailedToAddPlugin(
"Unsupported plugin type: '{}'".format(plugin_type))
if not self.config.has_section(plugin_type):
self.config.add_section(plugin_type)
elif self.config.has_option(plugin_type, plugin.name):
raise IRFailedToAddPlugin(
"Plugin with the same name & type already exists")
self.config.set(plugin_type, plugin.name, plugin.path)
with open(self.config_file, 'w') as fp:
self.config.write(fp)
self._install_requirements(plugin_source)
self._load_plugins()
def _clone_git_plugin(git_url, dest_dir=None):
"""Clone a plugin into a given destination directory
:param git_url: Plugin's Git URL
:param dest_dir: destination where to clone a plugin into (if 'source'
is a Git URL)
:return: Path to plugin cloned directory (str)
"""
dest_dir = os.path.abspath(dest_dir or "plugins")
tmpdir = tempfile.mkdtemp(prefix="ir-")
cwd = os.getcwdu()
os.chdir(tmpdir)
try:
subprocess.check_output(["git", "clone", git_url])
except subprocess.CalledProcessError:
shutil.rmtree(tmpdir)
raise IRFailedToAddPlugin(
"Cloning git repo {} is failed".format(git_url))
cloned = os.listdir(tmpdir)
plugin_dir_name = cloned[0]
plugin_source = os.path.join(dest_dir, plugin_dir_name)
shutil.copytree(os.path.join(tmpdir, plugin_dir_name), plugin_source)
os.chdir(cwd)
shutil.rmtree(tmpdir)
return plugin_source
def _clone_git_plugin(git_url, rev=None):
"""Clone a plugin into a given destination directory
:param git_url: Plugin's Git URL
:param rev: git branch/tag/revision
:return: Path to plugin cloned directory (str)
"""
plugin_git_name = os.path.split(git_url)[-1].split('.')[0]
tmpdir = tempfile.mkdtemp(prefix="ir-")
cwd = getcwd()
os.chdir(tmpdir)
try:
repo = git.Repo.clone_from(url=git_url,
to_path=os.path.join(
tmpdir, plugin_git_name))
if rev is not None:
repo.git.checkout(rev)
except (git.exc.GitCommandError) as e:
shutil.rmtree(tmpdir)
raise IRFailedToAddPlugin(
"Cloning git repo {} is failed: {}".format(git_url, e))
plugin_tmp_source = os.path.join(tmpdir, plugin_git_name)
os.chdir(cwd)
return plugin_tmp_source
def _clone_git_plugin(git_url,
repo_plugin_path=None,
rev=None,
dest_dir=None):
"""Clone a plugin into a given destination directory
:param git_url: Plugin's Git URL
:param dest_dir: destination where to clone a plugin into (if 'source'
is a Git URL)
:param repo_plugin_path: path in the Git repo where the infrared plugin
is defined
:param rev: git branch/tag/revision
:return: Path to plugin cloned directory (str)
"""
dest_dir = os.path.abspath(dest_dir or "plugins")
plugin_git_name = os.path.split(git_url)[-1].split('.')[0]
tmpdir = tempfile.mkdtemp(prefix="ir-")
cwd = os.getcwdu()
os.chdir(tmpdir)
try:
repo = git.Repo.clone_from(url=git_url,
to_path=os.path.join(
tmpdir, plugin_git_name))
if rev is not None:
repo.git.checkout(rev)
except (git.exc.GitCommandError) as e:
shutil.rmtree(tmpdir)
raise IRFailedToAddPlugin(
"Cloning git repo {} is failed: {}".format(git_url, e))
plugin_tmp_source = os.path.join(tmpdir, plugin_git_name)
if repo_plugin_path:
plugin_tmp_source = os.path.join(plugin_tmp_source,
repo_plugin_path)
# validate & load spec data in order to pull the name of the plugin
spec_data = InfraredPlugin.spec_validator(
os.path.join(plugin_tmp_source, InfraredPlugin.PLUGIN_SPEC_FILE))
# get the real plugin name from spec
plugin_dir_name = spec_data["subparsers"].keys()[0]
plugin_source = os.path.join(dest_dir, plugin_dir_name)
if os.path.islink(plugin_source):
LOG.debug(
"%s found as symlink pointing to %s, unlinking it, not touching the target.",
plugin_source, os.path.realpath(plugin_source))
os.unlink(plugin_source)
elif os.path.exists(plugin_source):
shutil.rmtree(plugin_source)
shutil.copytree(os.path.join(tmpdir, plugin_git_name), plugin_source)
if repo_plugin_path:
plugin_source = plugin_source + '/' + repo_plugin_path
os.chdir(cwd)
shutil.rmtree(tmpdir)
return plugin_source
def spec_validator(spec_file):
"""Loads & validates that spec (YAML) file has all required fields
:param spec_file: Path to plugin's spec file
:raise IRFailedToAddPlugin: when mandatory data is missing in spec file
:return: Dictionary with data loaded from a spec (YAML) file
"""
if not os.path.isfile(spec_file):
raise IRFailedToAddPlugin(
"Plugin spec doesn't exist: {}".format(spec_file))
with open(spec_file) as fp:
spec_dict = InfraredPlugin.spec_content_validator(fp)
return spec_dict
def _install_roles_from_file(plugin_path):
# Ansible Galaxy - install roles from file
for req_file in ['requirements.yml', 'requirements.yaml']:
galaxy_reqs_file = os.path.join(plugin_path, req_file)
if not os.path.isfile(galaxy_reqs_file):
continue
LOG.debug("Installing Galaxy "
"requirements... ({})".format(galaxy_reqs_file))
from ansible.cli.galaxy import GalaxyCLI
from ansible.errors import AnsibleError
glxy_cli = GalaxyCLI(['install'])
glxy_cli.parse()
glxy_cli.options.role_file = galaxy_reqs_file
try:
glxy_cli.execute_install()
except AnsibleError as e:
raise IRFailedToAddPlugin(e.message)
else:
LOG.debug("Galaxy requirements files weren't found.")
def _clone_git_plugin(git_url, repo_plugin_path=None, dest_dir=None):
"""Clone a plugin into a given destination directory
:param git_url: Plugin's Git URL
:param dest_dir: destination where to clone a plugin into (if 'source'
is a Git URL)
:param repo_plugin_path: path in the Git repo where the infrared plugin
is defined
:return: Path to plugin cloned directory (str)
"""
dest_dir = os.path.abspath(dest_dir or "plugins")
plugin_dir_name = os.path.split(git_url)[-1].split('.')[0]
tmpdir = tempfile.mkdtemp(prefix="ir-")
cwd = os.getcwdu()
os.chdir(tmpdir)
try:
git.Repo.clone_from(url=git_url,
to_path=os.path.join(tmpdir, plugin_dir_name))
except (git.exc.GitCommandError):
shutil.rmtree(tmpdir)
raise IRFailedToAddPlugin(
"Cloning git repo {} is failed".format(git_url))
plugin_source = os.path.join(dest_dir, plugin_dir_name)
if os.path.exists(plugin_source):
shutil.rmtree(plugin_source)
shutil.copytree(os.path.join(tmpdir, plugin_dir_name),
plugin_source)
if repo_plugin_path:
plugin_source = plugin_source + '/' + repo_plugin_path
os.chdir(cwd)
shutil.rmtree(tmpdir)
return plugin_source
def path(self, plugin_dir):
full_path = os.path.abspath(os.path.expanduser(plugin_dir))
if not os.path.isdir(full_path):
raise IRFailedToAddPlugin(
"Path to plugin dir '{}' doesn't exist".format(plugin_dir))
self._path = full_path
def add_plugin(self,
plugin_source,
rev=None,
plugins_registry=None,
plugin_src_path=None,
skip_roles=False,
link_roles=False):
"""Adds (install) a plugin
:param plugin_source: Plugin source.
Can be:
1. Plugin name (from available in registry)
2. Path to a local directory
3. Git URL
:param rev: git branch/tag/revision
:param plugins_registry: content of plugin registry dictionary
:param plugin_src_path: relative path to the plugin location inside the
source
:param skip_roles: Skip the from file roles installation
:param link_roles: Auto create symlink from {plugin_src_path}/roles to
the 'roles' directory inside the plugin root dir or to the
plugin root dir itself
"""
plugins_registry = plugins_registry or PLUGINS_REGISTRY
plugin_data = {}
# Check if a plugin is in the registry
if plugin_source in plugins_registry:
plugin_data = plugins_registry[plugin_source]
plugin_source = plugins_registry[plugin_source]['src']
if plugin_src_path is None:
plugin_src_path = plugin_data.get('src_path', '')
_link_roles = link_roles or bool(
strtobool(plugin_data.get('link_roles', 'false')))
# Local dir plugin
if os.path.exists(plugin_source):
rm_source = False
# Git Plugin
else:
if rev is None:
rev = plugin_data.get('rev')
plugin_source = \
self._clone_git_plugin(plugin_source, rev)
rm_source = True
plugin = InfraredPlugin(os.path.join(plugin_source, plugin_src_path))
plugin_type = plugin.type
if plugin_type not in self.supported_plugin_types:
raise IRUnsupportedPluginType(plugin_type)
if not self.config.has_section(plugin_type):
self.config.add_section(plugin_type)
elif self.config.has_option(plugin_type, plugin.name):
raise IRPluginExistsException(
"Plugin with the same name & type already exists")
dest = os.path.join(self.plugins_dir, plugin.name)
if os.path.abspath(plugin_source) != os.path.abspath(dest):
# copy only if plugin was added from a location which is
# different from the location of the plugins dir
if os.path.islink(dest):
LOG.debug(
"%s found as symlink pointing to %s, "
"unlinking it, not touching the target.", dest,
os.path.realpath(dest))
os.unlink(dest)
elif os.path.exists(dest):
shutil.rmtree(dest)
shutil.copytree(plugin_source, dest)
if rm_source:
shutil.rmtree(plugin_source)
if _link_roles:
if not plugin_src_path:
raise IRFailedToAddPlugin(
"'--src-path' is required with '--link-roles'")
roles_dir = os.path.join(dest, plugin_src_path, 'roles')
if os.path.exists(roles_dir):
raise IRFailedToAddPlugin(
"Can't create a symbolic link to a 'roles' directory in "
"'{roles_dir}', because one is already exists".format(
roles_dir=roles_dir))
src_dir = dest
if os.path.exists(dest + '/roles'):
if not os.path.isdir(dest + '/roles'):
raise IRFailedToAddPlugin(
"The plugin directory ('{dest}') contains"
"a 'roles' file that can't be sym-linked".format(
dest=dest))
src_dir += '/roles'
os.mkdir(roles_dir)
dest_dir = roles_dir + '/' + dest.split('/')[-1]
os.symlink(src_dir, dest_dir)
self.config.set(plugin_type, plugin.name,
os.path.join(dest, plugin_src_path))
with open(self.config_file, 'w') as fp:
self.config.write(fp)
self._install_requirements(dest)
if not skip_roles:
# roles are skipped only in infrared's internal tests
# param for this is not exposed in cli/spec files
self._install_roles_from_file(dest)
self._load_plugins()