def _pip_install(self, source, args, constraint): plugin_dir = None try: if os.path.isabs(source): plugin_dir = source else: self.logger.debug('Extracting archive: {0}'.format(source)) plugin_dir = extract_package_to_dir(source) package_name = extract_package_name(plugin_dir) if self._package_installed_in_agent_env(constraint, package_name): self.logger.warn('Skipping source plugin {0} installation, ' 'as the plugin is already installed in the ' 'agent virtualenv.'.format(package_name)) return self.logger.debug('Installing from directory: {0} ' '[args={1}, package_name={2}]'.format( plugin_dir, args, package_name)) command = '{0} install {1} {2}'.format(get_pip_path(), plugin_dir, args) self.runner.run(command, cwd=plugin_dir) self.logger.debug( 'Retrieved package name: {0}'.format(package_name)) except CommandExecutionException as e: self.logger.debug( 'Failed running pip install. Output:\n{0}'.format(e.output)) raise exceptions.PluginInstallationError( 'Failed running pip install. ({0})'.format(e.error)) finally: if plugin_dir and not os.path.isabs(source): self.logger.debug('Removing directory: {0}'.format(plugin_dir)) self._rmtree(plugin_dir)
def _remove_tempdir_and_raise_proper_exception(package_url, tempdir): if tempdir and os.path.exists(tempdir): shutil.rmtree(tempdir) raise exceptions.PluginInstallationError( 'Failed to download package from {0}.' 'You may consider uploading the plugin\'s Wagon archive ' 'to the manager, For more information please refer to ' 'the documentation.'.format(package_url))
def parse_pip_version(pip_version=''): """ Parses a pip version string to identify major, minor, micro versions. :param pip_version: the version of pip :return: major, minor, micro version of pip :rtype: tuple """ if not pip_version: try: pip_version = pip.__version__ except AttributeError as e: raise exceptions.PluginInstallationError( 'Failed to get pip version: ', str(e)) if not isinstance(pip_version, basestring): raise exceptions.PluginInstallationError( 'Invalid pip version: {0} is not a string'.format(pip_version)) if not pip_version.__contains__("."): raise exceptions.PluginInstallationError( 'Unknown formatting of pip version: "{0}", expected ' 'dot-delimited numbers (e.g. "1.5.4", "6.0")'.format(pip_version)) version_parts = pip_version.split('.') major = version_parts[0] minor = version_parts[1] micro = '' if len(version_parts) > 2: micro = version_parts[2] if not str(major).isdigit(): raise exceptions.PluginInstallationError( 'Invalid pip version: "{0}", major version is "{1}" ' 'while expected to be a number'.format(pip_version, major)) if not str(minor).isdigit(): raise exceptions.PluginInstallationError( 'Invalid pip version: "{0}", minor version is "{1}" while ' 'expected to be a number'.format(pip_version, minor)) return major, minor, micro
def _install_source_plugin(self, deployment_id, plugin, source, args, tmp_plugin_dir, constraint): dst_dir = '{0}-{1}'.format(deployment_id, plugin['name']) dst_dir = self._full_dst_dir(dst_dir) if os.path.exists(dst_dir): raise exceptions.PluginInstallationError( 'Source plugin {0} already exists for deployment {1}. ' 'This probably means a previous deployment with the ' 'same name was not cleaned properly.'.format( plugin['name'], deployment_id)) self.logger.info('Installing plugin from source') self._pip_install(source=source, args=args, constraint=constraint) shutil.move(tmp_plugin_dir, dst_dir)
def extract_package_to_dir(package_url): """ Extracts a pip package to a temporary directory. :param package_url: the URL to the package source. :return: the directory the package was extracted to. """ plugin_dir = None try: plugin_dir = tempfile.mkdtemp() # check pip version and unpack plugin_url accordingly if is_pip6_or_higher(): pip.download.unpack_url(link=pip.index.Link(package_url), location=plugin_dir, download_dir=None, only_download=False) else: req_set = pip.req.RequirementSet(build_dir=None, src_dir=None, download_dir=None) req_set.unpack_url(link=pip.index.Link(package_url), location=plugin_dir, download_dir=None, only_download=False) except Exception as e: if plugin_dir and os.path.exists(plugin_dir): shutil.rmtree(plugin_dir) raise exceptions.PluginInstallationError( 'Failed to download and unpack package from {0}: {1}'.format( package_url, str(e))) return plugin_dir
def extract_package_to_dir(package_url): """ Extracts a pip package to a temporary directory. :param package_url: the URL to the package source. :return: the directory the package was extracted to. """ # 1) Plugin installation during deployment creation occurs not in the main # thread, but rather in the local task thread pool. # When installing source based plugins, pip will install an # interrupt handler using signal.signal, this will fail saying something # like "signals are only allowed in the main thread" # from examining the code, I found patching signal in pip.util.ui # is the cleanest form. No "official" way of disabling this was found. # 2) pip.utils logger may be used internally by pip during some # ImportError. This interferes with our ZMQLoggingHandler which during # the first time it is invoked tries importing some stuff. This causes # a deadlock between the handler lock and the global import lock. One side # holds the import lock and tried writing to the logger and is blocked on # the handler lock, while the other side holds the handler lock and is # blocked on the import lock. This is why we patch the logging level # of this logger - by name, before importing pip. (see CFY-4866) _previous_signal = [] _previous_level = [] def _patch_pip_download(): pip_utils_logger = logging.getLogger('pip.utils') _previous_level.append(pip_utils_logger.level) pip_utils_logger.setLevel(logging.CRITICAL) try: import pip.utils.ui def _stub_signal(sig, action): return None if hasattr(pip.utils.ui, 'signal'): _previous_signal.append(pip.utils.ui.signal) pip.utils.ui.signal = _stub_signal except ImportError: pass def _restore_pip_download(): try: import pip.utils.ui if hasattr(pip.utils.ui, 'signal') and _previous_signal: pip.utils.ui.signal = _previous_signal[0] except ImportError: pass pip_utils_logger = logging.getLogger('pip.utils') pip_utils_logger.setLevel(_previous_level[0]) plugin_dir = None try: plugin_dir = tempfile.mkdtemp() _patch_pip_download() # Import here, after patch import pip pip.download.unpack_url(link=pip.index.Link(package_url), location=plugin_dir, download_dir=None, only_download=False) except Exception as e: if plugin_dir and os.path.exists(plugin_dir): shutil.rmtree(plugin_dir) raise exceptions.PluginInstallationError( 'Failed to download and unpack package from {0}: {1}.' 'You may consider uploading the plugin\'s Wagon archive ' 'to the manager, For more information please refer to ' 'the documentation.'.format(package_url, str(e))) finally: _restore_pip_download() return plugin_dir
def _pip_freeze(self): try: return self.runner.run('{0} freeze'.format(get_pip_path())).std_out except CommandExecutionException as e: raise exceptions.PluginInstallationError( 'Failed running pip freeze. ({0})'.format(e))
def _install_managed_plugin(self, deployment_id, managed_plugin, plugin, args, tmp_plugin_dir): matching_existing_installation = False dst_dir = '{0}-{1}'.format(managed_plugin.package_name, managed_plugin.package_version) dst_dir = self._full_dst_dir(dst_dir) lock = self._lock(dst_dir) lock.acquire() try: if os.path.exists(dst_dir): plugin_id_path = os.path.join(dst_dir, 'plugin.id') if os.path.exists(plugin_id_path): with open(plugin_id_path) as f: existing_plugin_id = f.read().strip() matching_existing_installation = ( existing_plugin_id == managed_plugin.id) if not matching_existing_installation: raise exceptions.PluginInstallationError( 'Managed plugin installation found but its ID ' 'does not match the ID of the plugin currently ' 'on the manager. [existing: {0}, new: {1}]' .format(existing_plugin_id, managed_plugin.id)) else: raise exceptions.PluginInstallationError( 'Managed plugin installation found but it is ' 'in a corrupted state. [{0}]'.format(managed_plugin)) fields = ['package_name', 'package_version', 'supported_platform', 'distribution', 'distribution_release'] description = ', '.join('{0}: {1}'.format( field, managed_plugin.get(field)) for field in fields if managed_plugin.get(field)) if matching_existing_installation: self.logger.info( 'Using existing installation of managed plugin: {0} [{1}]' .format(managed_plugin.id, description)) elif (deployment_id != SYSTEM_DEPLOYMENT and plugin['executor'] == 'central_deployment_agent'): raise exceptions.PluginInstallationError( 'Central deployment agent managed plugins can only be ' 'installed using the REST plugins API. [{0}]' .format(managed_plugin)) else: self.logger.info('Installing managed plugin: {0} [{1}]' .format(managed_plugin.id, description)) try: self._wagon_install(plugin=managed_plugin, args=args) shutil.move(tmp_plugin_dir, dst_dir) with open(os.path.join(dst_dir, 'plugin.id'), 'w') as f: f.write(managed_plugin.id) except Exception as e: tpe, value, tb = sys.exc_info() raise NonRecoverableError('Failed installing managed ' 'plugin: {0} [{1}][{2}]' .format(managed_plugin.id, plugin, e)), None, tb finally: if lock: lock.release()
def _install_managed_plugin(self, deployment_id, managed_plugin, plugin, args, tmp_plugin_dir): matching_existing_installation = False dst_dir = '{0}-{1}'.format(managed_plugin.package_name, managed_plugin.package_version) dst_dir = self._full_dst_dir(dst_dir, managed_plugin) self.logger.debug('Checking if managed plugin installation exists ' 'in {0}'.format(dst_dir)) # create_deployment_env should wait for the install_plugin wf to finish deadline = time.time() + INSTALLATION_TIMEOUT while (self._is_plugin_installing(managed_plugin.id) and deployment_id != SYSTEM_DEPLOYMENT): if time.time() < deadline: time.sleep(PLUGIN_QUERY_INTERVAL) else: raise exceptions.PluginInstallationError( 'Timeout waiting for plugin to be installed. ' 'Plugin info: [{0}] '.format(managed_plugin)) if os.path.exists(dst_dir): self.logger.debug('Plugin path exists {0}'.format(dst_dir)) plugin_id_path = os.path.join(dst_dir, 'plugin.id') if os.path.exists(plugin_id_path): self.logger.debug('Plugin id path exists {0}'. format(plugin_id_path)) with open(plugin_id_path) as f: existing_plugin_id = f.read().strip() matching_existing_installation = ( existing_plugin_id == managed_plugin.id) if not matching_existing_installation: raise exceptions.PluginInstallationError( 'Managed plugin installation found but its ID ' 'does not match the ID of the plugin currently ' 'on the manager. [existing: {0}, new: {1}]' .format(existing_plugin_id, managed_plugin.id)) else: raise exceptions.PluginInstallationError( 'Managed plugin installation found but it is ' 'in a corrupted state. [{0}]'.format(managed_plugin)) fields = ['package_name', 'package_version', 'supported_platform', 'distribution', 'distribution_release'] description = ', '.join('{0}: {1}'.format( field, managed_plugin.get(field)) for field in fields if managed_plugin.get(field)) if matching_existing_installation: self.logger.info( 'Using existing installation of managed plugin: {0} [{1}]' .format(managed_plugin.id, description)) elif (deployment_id != SYSTEM_DEPLOYMENT and plugin['executor'] == 'central_deployment_agent'): raise exceptions.PluginInstallationError( 'Central deployment agent managed plugins can only be ' 'installed using the REST plugins API. [{0}]' .format(managed_plugin)) else: try: self.logger.info('Installing managed plugin: {0} [{1}]' .format(managed_plugin.id, description)) wait_for_wagon_in_directory(managed_plugin.id) # Wait for Syncthing to sync resources for the wagons if syncthing_utils: syncthing_utils.wait_for_plugins_sync( sync_dir=syncthing_utils.RESOURCES_DIR) self._wagon_install(plugin=managed_plugin, args=args) self.logger.debug('Moving plugin from tmp dir `{0}` to `{1}`'. format(tmp_plugin_dir, dst_dir)) shutil.move(tmp_plugin_dir, dst_dir) with open(os.path.join(dst_dir, 'plugin.id'), 'w') as f: f.write(managed_plugin.id) if self._is_plugin_installing(managed_plugin.id): # Wait for Syncthing to sync plugin files on all managers if syncthing_utils: syncthing_utils.wait_for_plugins_sync() self._update_plugin_status(managed_plugin.id) except Exception as e: tpe, value, tb = sys.exc_info() exc = NonRecoverableError('Failed installing managed ' 'plugin: {0} [{1}][{2}]' .format(managed_plugin.id, plugin, e)) reraise(NonRecoverableError, exc, tb)