def _pip_install(self, source, args, constraint):
plugin_dir = None
try:
if os.path.isabs(source):
plugin_dir = source
else:
self.logger.debug('Extracting archive: {0}'.format(source))
plugin_dir = extract_package_to_dir(source)
package_name = extract_package_name(plugin_dir)
if self._package_installed_in_agent_env(constraint, package_name):
self.logger.warn('Skipping source plugin {0} installation, '
'as the plugin is already installed in the '
'agent virtualenv.'.format(package_name))
return
self.logger.debug('Installing from directory: {0} '
'[args={1}, package_name={2}]'.format(
plugin_dir, args, package_name))
command = '{0} install {1} {2}'.format(get_pip_path(), plugin_dir,
args)
self.runner.run(command, cwd=plugin_dir)
self.logger.debug(
'Retrieved package name: {0}'.format(package_name))
except CommandExecutionException as e:
self.logger.debug(
'Failed running pip install. Output:\n{0}'.format(e.output))
raise exceptions.PluginInstallationError(
'Failed running pip install. ({0})'.format(e.error))
finally:
if plugin_dir and not os.path.isabs(source):
self.logger.debug('Removing directory: {0}'.format(plugin_dir))
self._rmtree(plugin_dir)
def _remove_tempdir_and_raise_proper_exception(package_url, tempdir):
if tempdir and os.path.exists(tempdir):
shutil.rmtree(tempdir)
raise exceptions.PluginInstallationError(
'Failed to download package from {0}.'
'You may consider uploading the plugin\'s Wagon archive '
'to the manager, For more information please refer to '
'the documentation.'.format(package_url))
def parse_pip_version(pip_version=''):
"""
Parses a pip version string to identify major, minor, micro versions.
:param pip_version: the version of pip
:return: major, minor, micro version of pip
:rtype: tuple
"""
if not pip_version:
try:
pip_version = pip.__version__
except AttributeError as e:
raise exceptions.PluginInstallationError(
'Failed to get pip version: ', str(e))
if not isinstance(pip_version, basestring):
raise exceptions.PluginInstallationError(
'Invalid pip version: {0} is not a string'.format(pip_version))
if not pip_version.__contains__("."):
raise exceptions.PluginInstallationError(
'Unknown formatting of pip version: "{0}", expected '
'dot-delimited numbers (e.g. "1.5.4", "6.0")'.format(pip_version))
version_parts = pip_version.split('.')
major = version_parts[0]
minor = version_parts[1]
micro = ''
if len(version_parts) > 2:
micro = version_parts[2]
if not str(major).isdigit():
raise exceptions.PluginInstallationError(
'Invalid pip version: "{0}", major version is "{1}" '
'while expected to be a number'.format(pip_version, major))
if not str(minor).isdigit():
raise exceptions.PluginInstallationError(
'Invalid pip version: "{0}", minor version is "{1}" while '
'expected to be a number'.format(pip_version, minor))
return major, minor, micro
def _install_source_plugin(self, deployment_id, plugin, source, args,
tmp_plugin_dir, constraint):
dst_dir = '{0}-{1}'.format(deployment_id, plugin['name'])
dst_dir = self._full_dst_dir(dst_dir)
if os.path.exists(dst_dir):
raise exceptions.PluginInstallationError(
'Source plugin {0} already exists for deployment {1}. '
'This probably means a previous deployment with the '
'same name was not cleaned properly.'.format(
plugin['name'], deployment_id))
self.logger.info('Installing plugin from source')
self._pip_install(source=source, args=args, constraint=constraint)
shutil.move(tmp_plugin_dir, dst_dir)
def extract_package_to_dir(package_url):
"""
Extracts a pip package to a temporary directory.
:param package_url: the URL to the package source.
:return: the directory the package was extracted to.
"""
plugin_dir = None
try:
plugin_dir = tempfile.mkdtemp()
# check pip version and unpack plugin_url accordingly
if is_pip6_or_higher():
pip.download.unpack_url(link=pip.index.Link(package_url),
location=plugin_dir,
download_dir=None,
only_download=False)
else:
req_set = pip.req.RequirementSet(build_dir=None,
src_dir=None,
download_dir=None)
req_set.unpack_url(link=pip.index.Link(package_url),
location=plugin_dir,
download_dir=None,
only_download=False)
except Exception as e:
if plugin_dir and os.path.exists(plugin_dir):
shutil.rmtree(plugin_dir)
raise exceptions.PluginInstallationError(
'Failed to download and unpack package from {0}: {1}'.format(
package_url, str(e)))
return plugin_dir
def extract_package_to_dir(package_url):
"""
Extracts a pip package to a temporary directory.
:param package_url: the URL to the package source.
:return: the directory the package was extracted to.
"""
# 1) Plugin installation during deployment creation occurs not in the main
# thread, but rather in the local task thread pool.
# When installing source based plugins, pip will install an
# interrupt handler using signal.signal, this will fail saying something
# like "signals are only allowed in the main thread"
# from examining the code, I found patching signal in pip.util.ui
# is the cleanest form. No "official" way of disabling this was found.
# 2) pip.utils logger may be used internally by pip during some
# ImportError. This interferes with our ZMQLoggingHandler which during
# the first time it is invoked tries importing some stuff. This causes
# a deadlock between the handler lock and the global import lock. One side
# holds the import lock and tried writing to the logger and is blocked on
# the handler lock, while the other side holds the handler lock and is
# blocked on the import lock. This is why we patch the logging level
# of this logger - by name, before importing pip. (see CFY-4866)
_previous_signal = []
_previous_level = []
def _patch_pip_download():
pip_utils_logger = logging.getLogger('pip.utils')
_previous_level.append(pip_utils_logger.level)
pip_utils_logger.setLevel(logging.CRITICAL)
try:
import pip.utils.ui
def _stub_signal(sig, action):
return None
if hasattr(pip.utils.ui, 'signal'):
_previous_signal.append(pip.utils.ui.signal)
pip.utils.ui.signal = _stub_signal
except ImportError:
pass
def _restore_pip_download():
try:
import pip.utils.ui
if hasattr(pip.utils.ui, 'signal') and _previous_signal:
pip.utils.ui.signal = _previous_signal[0]
except ImportError:
pass
pip_utils_logger = logging.getLogger('pip.utils')
pip_utils_logger.setLevel(_previous_level[0])
plugin_dir = None
try:
plugin_dir = tempfile.mkdtemp()
_patch_pip_download()
# Import here, after patch
import pip
pip.download.unpack_url(link=pip.index.Link(package_url),
location=plugin_dir,
download_dir=None,
only_download=False)
except Exception as e:
if plugin_dir and os.path.exists(plugin_dir):
shutil.rmtree(plugin_dir)
raise exceptions.PluginInstallationError(
'Failed to download and unpack package from {0}: {1}.'
'You may consider uploading the plugin\'s Wagon archive '
'to the manager, For more information please refer to '
'the documentation.'.format(package_url, str(e)))
finally:
_restore_pip_download()
return plugin_dir
def _pip_freeze(self):
try:
return self.runner.run('{0} freeze'.format(get_pip_path())).std_out
except CommandExecutionException as e:
raise exceptions.PluginInstallationError(
'Failed running pip freeze. ({0})'.format(e))
def _install_managed_plugin(self,
deployment_id,
managed_plugin,
plugin,
args,
tmp_plugin_dir):
matching_existing_installation = False
dst_dir = '{0}-{1}'.format(managed_plugin.package_name,
managed_plugin.package_version)
dst_dir = self._full_dst_dir(dst_dir)
lock = self._lock(dst_dir)
lock.acquire()
try:
if os.path.exists(dst_dir):
plugin_id_path = os.path.join(dst_dir, 'plugin.id')
if os.path.exists(plugin_id_path):
with open(plugin_id_path) as f:
existing_plugin_id = f.read().strip()
matching_existing_installation = (
existing_plugin_id == managed_plugin.id)
if not matching_existing_installation:
raise exceptions.PluginInstallationError(
'Managed plugin installation found but its ID '
'does not match the ID of the plugin currently '
'on the manager. [existing: {0}, new: {1}]'
.format(existing_plugin_id,
managed_plugin.id))
else:
raise exceptions.PluginInstallationError(
'Managed plugin installation found but it is '
'in a corrupted state. [{0}]'.format(managed_plugin))
fields = ['package_name',
'package_version',
'supported_platform',
'distribution',
'distribution_release']
description = ', '.join('{0}: {1}'.format(
field, managed_plugin.get(field))
for field in fields if managed_plugin.get(field))
if matching_existing_installation:
self.logger.info(
'Using existing installation of managed plugin: {0} [{1}]'
.format(managed_plugin.id, description))
elif (deployment_id != SYSTEM_DEPLOYMENT and
plugin['executor'] == 'central_deployment_agent'):
raise exceptions.PluginInstallationError(
'Central deployment agent managed plugins can only be '
'installed using the REST plugins API. [{0}]'
.format(managed_plugin))
else:
self.logger.info('Installing managed plugin: {0} [{1}]'
.format(managed_plugin.id, description))
try:
self._wagon_install(plugin=managed_plugin, args=args)
shutil.move(tmp_plugin_dir, dst_dir)
with open(os.path.join(dst_dir, 'plugin.id'), 'w') as f:
f.write(managed_plugin.id)
except Exception as e:
tpe, value, tb = sys.exc_info()
raise NonRecoverableError('Failed installing managed '
'plugin: {0} [{1}][{2}]'
.format(managed_plugin.id,
plugin, e)), None, tb
finally:
if lock:
lock.release()
def _install_managed_plugin(self,
deployment_id,
managed_plugin,
plugin,
args,
tmp_plugin_dir):
matching_existing_installation = False
dst_dir = '{0}-{1}'.format(managed_plugin.package_name,
managed_plugin.package_version)
dst_dir = self._full_dst_dir(dst_dir, managed_plugin)
self.logger.debug('Checking if managed plugin installation exists '
'in {0}'.format(dst_dir))
# create_deployment_env should wait for the install_plugin wf to finish
deadline = time.time() + INSTALLATION_TIMEOUT
while (self._is_plugin_installing(managed_plugin.id) and
deployment_id != SYSTEM_DEPLOYMENT):
if time.time() < deadline:
time.sleep(PLUGIN_QUERY_INTERVAL)
else:
raise exceptions.PluginInstallationError(
'Timeout waiting for plugin to be installed. '
'Plugin info: [{0}] '.format(managed_plugin))
if os.path.exists(dst_dir):
self.logger.debug('Plugin path exists {0}'.format(dst_dir))
plugin_id_path = os.path.join(dst_dir, 'plugin.id')
if os.path.exists(plugin_id_path):
self.logger.debug('Plugin id path exists {0}'.
format(plugin_id_path))
with open(plugin_id_path) as f:
existing_plugin_id = f.read().strip()
matching_existing_installation = (
existing_plugin_id == managed_plugin.id)
if not matching_existing_installation:
raise exceptions.PluginInstallationError(
'Managed plugin installation found but its ID '
'does not match the ID of the plugin currently '
'on the manager. [existing: {0}, new: {1}]'
.format(existing_plugin_id,
managed_plugin.id))
else:
raise exceptions.PluginInstallationError(
'Managed plugin installation found but it is '
'in a corrupted state. [{0}]'.format(managed_plugin))
fields = ['package_name',
'package_version',
'supported_platform',
'distribution',
'distribution_release']
description = ', '.join('{0}: {1}'.format(
field, managed_plugin.get(field))
for field in fields if managed_plugin.get(field))
if matching_existing_installation:
self.logger.info(
'Using existing installation of managed plugin: {0} [{1}]'
.format(managed_plugin.id, description))
elif (deployment_id != SYSTEM_DEPLOYMENT and
plugin['executor'] == 'central_deployment_agent'):
raise exceptions.PluginInstallationError(
'Central deployment agent managed plugins can only be '
'installed using the REST plugins API. [{0}]'
.format(managed_plugin))
else:
try:
self.logger.info('Installing managed plugin: {0} [{1}]'
.format(managed_plugin.id, description))
wait_for_wagon_in_directory(managed_plugin.id)
# Wait for Syncthing to sync resources for the wagons
if syncthing_utils:
syncthing_utils.wait_for_plugins_sync(
sync_dir=syncthing_utils.RESOURCES_DIR)
self._wagon_install(plugin=managed_plugin, args=args)
self.logger.debug('Moving plugin from tmp dir `{0}` to `{1}`'.
format(tmp_plugin_dir, dst_dir))
shutil.move(tmp_plugin_dir, dst_dir)
with open(os.path.join(dst_dir, 'plugin.id'), 'w') as f:
f.write(managed_plugin.id)
if self._is_plugin_installing(managed_plugin.id):
# Wait for Syncthing to sync plugin files on all managers
if syncthing_utils:
syncthing_utils.wait_for_plugins_sync()
self._update_plugin_status(managed_plugin.id)
except Exception as e:
tpe, value, tb = sys.exc_info()
exc = NonRecoverableError('Failed installing managed '
'plugin: {0} [{1}][{2}]'
.format(managed_plugin.id,
plugin, e))
reraise(NonRecoverableError, exc, tb)
