def _setup_pack_virtualenv(self, pack_name, update=False): """ Setup virtual environment for the provided pack. :param pack_name: Pack name. :type pack_name: ``str`` """ # Prevent directory traversal by whitelisting allowed characters in the # pack name if not re.match(PACK_NAME_WHITELIST, pack_name): raise ValueError('Invalid pack name "%s"' % (pack_name)) self.logger.debug('Setting up virtualenv for pack "%s"' % (pack_name)) virtualenv_path = os.path.join(self._base_virtualenvs_path, quote_unix(pack_name)) # Ensure pack directory exists in one of the search paths pack_path = get_pack_directory(pack_name=pack_name) if not pack_path: packs_base_paths = get_packs_base_paths() search_paths = ', '.join(packs_base_paths) msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths) raise Exception(msg) if not os.path.exists(self._base_virtualenvs_path): os.makedirs(self._base_virtualenvs_path) # If we don't want to update, or if the virtualenv doesn't exist, let's create it. if not update or not os.path.exists(virtualenv_path): # 0. Delete virtual environment if it exists self._remove_virtualenv(virtualenv_path=virtualenv_path) # 1. Create virtual environment self.logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path)) self._create_virtualenv(virtualenv_path=virtualenv_path) # 2. Install base requirements which are common to all the packs self.logger.debug('Installing base requirements') for requirement in BASE_PACK_REQUIREMENTS: self._install_requirement(virtualenv_path=virtualenv_path, requirement=requirement) # 3. Install pack-specific requirements requirements_file_path = os.path.join(pack_path, 'requirements.txt') has_requirements = os.path.isfile(requirements_file_path) if has_requirements: self.logger.debug('Installing pack specific requirements from "%s"' % (requirements_file_path)) self._install_requirements(virtualenv_path, requirements_file_path) else: self.logger.debug('No pack specific requirements found') self.logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' % (pack_name, 'updated' if update else 'created', virtualenv_path))
def _setup_pack_virtualenv(self, pack_name): """ Setup virtual environment for the provided pack. :param pack_name: Pack name. :type pack_name: ``str`` """ # Prevent directory traversal by whitelisting allowed characters in the # pack name if not re.match(PACK_NAME_WHITELIST, pack_name): raise ValueError('Invalid pack name "%s"' % (pack_name)) self.logger.debug('Setting up virtualenv for pack "%s"' % (pack_name)) virtualenv_path = os.path.join(self._base_virtualenvs_path, quote_unix(pack_name)) # Ensure pack directory exists in one of the search paths pack_path = get_pack_directory(pack_name=pack_name) if not pack_path: packs_base_paths = get_packs_base_paths() search_paths = ', '.join(packs_base_paths) msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths) raise Exception(msg) if not os.path.exists(self._base_virtualenvs_path): os.makedirs(self._base_virtualenvs_path) # 0. Delete virtual environment if it exists self._remove_virtualenv(virtualenv_path=virtualenv_path) # 1. Create virtual environment self.logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path)) self._create_virtualenv(virtualenv_path=virtualenv_path) # 2. Install base requirements which are common to all the packs self.logger.debug('Installing base requirements') for requirement in BASE_PACK_REQUIREMENTS: self._install_requirement(virtualenv_path=virtualenv_path, requirement=requirement) # 3. Install pack-specific requirements requirements_file_path = os.path.join(pack_path, 'requirements.txt') has_requirements = os.path.isfile(requirements_file_path) if has_requirements: self.logger.debug( 'Installing pack specific requirements from "%s"' % (requirements_file_path)) self._install_requirements(virtualenv_path, requirements_file_path) else: self.logger.debug('No pack specific requirements found') self.logger.debug( 'Virtualenv for pack "%s" successfully created in "%s"' % (pack_name, virtualenv_path))
def create_git_worktree(self, content_version): """ Create a git worktree for the provided git content version. :return: Path to the created git worktree directory. :rtype: ``str`` """ pack_name = self.get_pack_name() pack_directory = get_pack_directory(pack_name=pack_name) worktree_path = tempfile.mkdtemp(prefix=self.WORKTREE_DIRECTORY_PREFIX) # Set class variables self.git_worktree_revision = content_version self.git_worktree_path = worktree_path extra = { 'pack_name': pack_name, 'pack_directory': pack_directory, 'content_version': content_version, 'worktree_path': worktree_path } if not os.path.isdir(pack_directory): msg = ( 'Failed to create git worktree for pack "%s". Pack directory "%s" doesn\'t ' 'exist.' % (pack_name, pack_directory)) raise ValueError(msg) args = [ 'git', '-C', pack_directory, 'worktree', 'add', worktree_path, content_version ] cmd = list2cmdline(args) LOG.debug( 'Creating git worktree for pack "%s", content version "%s" and execution ' 'id "%s" in "%s"' % (pack_name, content_version, self.execution_id, worktree_path), extra=extra) LOG.debug('Command: %s' % (cmd)) exit_code, stdout, stderr, timed_out = run_command( cmd=cmd, cwd=pack_directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) if exit_code != 0: self._handle_git_worktree_error(pack_name=pack_name, pack_directory=pack_directory, content_version=content_version, exit_code=exit_code, stdout=stdout, stderr=stderr) else: LOG.debug('Git worktree created in "%s"' % (worktree_path), extra=extra) # Make sure system / action runner user can access that directory args = ['chmod', '777', worktree_path] cmd = list2cmdline(args) run_command(cmd=cmd, shell=True) return worktree_path
def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True, include_setuptools=True, include_wheel=True, proxy_config=None, no_download=True, force_owner_group=True): """ Setup virtual environment for the provided pack. :param pack_name: Name of the pack to setup the virtualenv for. :type pack_name: ``str`` :param update: True to update dependencies inside the virtual environment. :type update: ``bool`` :param logger: Optional logger instance to use. If not provided it defaults to the module level logger. :param no_download: Do not download and install latest version of pre-installed packages such as pip and distutils. :type no_download: ``bool`` """ logger = logger or LOG if not re.match(PACK_REF_WHITELIST_REGEX, pack_name): raise ValueError('Invalid pack name "%s"' % (pack_name)) base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/') virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name)) # Ensure pack directory exists in one of the search paths pack_path = get_pack_directory(pack_name=pack_name) logger.debug('Setting up virtualenv for pack "%s" (%s)' % (pack_name, pack_path)) if not pack_path: packs_base_paths = get_packs_base_paths() search_paths = ', '.join(packs_base_paths) msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths) raise Exception(msg) # 1. Create virtualenv if it doesn't exist if not update or not os.path.exists(virtualenv_path): # 0. Delete virtual environment if it exists remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger) # 1. Create virtual environment logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path)) create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip, include_setuptools=include_setuptools, include_wheel=include_wheel, no_download=no_download) # 2. Install base requirements which are common to all the packs logger.debug('Installing base requirements') for requirement in BASE_PACK_REQUIREMENTS: install_requirement(virtualenv_path=virtualenv_path, requirement=requirement, proxy_config=proxy_config, logger=logger) # 3. Install pack-specific requirements requirements_file_path = os.path.join(pack_path, 'requirements.txt') has_requirements = os.path.isfile(requirements_file_path) if has_requirements: logger.debug('Installing pack specific requirements from "%s"' % (requirements_file_path)) install_requirements(virtualenv_path=virtualenv_path, requirements_file_path=requirements_file_path, proxy_config=proxy_config, logger=logger) else: logger.debug('No pack specific requirements found') # 4. Set the owner group if force_owner_group: apply_pack_owner_group(pack_path=virtualenv_path) action = 'updated' if update else 'created' logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' % (pack_name, action, virtualenv_path))
def setup_pack_virtualenv(pack_name, update=False, logger=None): """ Setup virtual environment for the provided pack. :param pack_name: Name of the pack to setup the virtualenv for. :type pack_name: ``str`` :param update: True to update dependencies inside the virtual environment. :type update: ``bool`` :param logger: Optional logger instance to use. If not provided it defaults to the module level logger. """ logger = logger or LOG if not re.match(PACK_NAME_WHITELIST, pack_name): raise ValueError('Invalid pack name "%s"' % (pack_name)) base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/') logger.debug('Setting up virtualenv for pack "%s"' % (pack_name)) virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name)) # Ensure pack directory exists in one of the search paths pack_path = get_pack_directory(pack_name=pack_name) if not pack_path: packs_base_paths = get_packs_base_paths() search_paths = ', '.join(packs_base_paths) msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths) raise Exception(msg) # 1. Create virtualenv if it doesn't exist if not update or not os.path.exists(virtualenv_path): # 0. Delete virtual environment if it exists remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger) # 1. Create virtual environment logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path)) create_virtualenv(virtualenv_path=virtualenv_path, logger=logger) # 2. Install base requirements which are common to all the packs logger.debug('Installing base requirements') for requirement in BASE_PACK_REQUIREMENTS: install_requirement(virtualenv_path=virtualenv_path, requirement=requirement) # 3. Install pack-specific requirements requirements_file_path = os.path.join(pack_path, 'requirements.txt') has_requirements = os.path.isfile(requirements_file_path) if has_requirements: logger.debug('Installing pack specific requirements from "%s"' % (requirements_file_path)) install_requirements(virtualenv_path=virtualenv_path, requirements_file_path=requirements_file_path) else: logger.debug('No pack specific requirements found') action = 'updated' if update else 'created' logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' % (pack_name, action, virtualenv_path))
def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True, include_setuptools=True, include_wheel=True, proxy_config=None, use_python3=False, no_download=True, force_owner_group=True): """ Setup virtual environment for the provided pack. :param pack_name: Name of the pack to setup the virtualenv for. :type pack_name: ``str`` :param update: True to update dependencies inside the virtual environment. :type update: ``bool`` :param logger: Optional logger instance to use. If not provided it defaults to the module level logger. :param use_python3: Use Python3 binary when creating virtualenv for this pack. :type use_python3: ``bool`` :param no_download: Do not download and install latest version of pre-installed packages such as pip and distutils. :type no_download: ``bool`` """ logger = logger or LOG if not re.match(PACK_REF_WHITELIST_REGEX, pack_name): raise ValueError('Invalid pack name "%s"' % (pack_name)) base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/') virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name)) # Ensure pack directory exists in one of the search paths pack_path = get_pack_directory(pack_name=pack_name) logger.debug('Setting up virtualenv for pack "%s" (%s)' % (pack_name, pack_path)) if not pack_path: packs_base_paths = get_packs_base_paths() search_paths = ', '.join(packs_base_paths) msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths) raise Exception(msg) # 1. Create virtualenv if it doesn't exist if not update or not os.path.exists(virtualenv_path): # 0. Delete virtual environment if it exists remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger) # 1. Create virtual environment logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path)) create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip, include_setuptools=include_setuptools, include_wheel=include_wheel, use_python3=use_python3, no_download=no_download) # 2. Install base requirements which are common to all the packs logger.debug('Installing base requirements') for requirement in BASE_PACK_REQUIREMENTS: install_requirement(virtualenv_path=virtualenv_path, requirement=requirement, proxy_config=proxy_config, logger=logger) # 3. Install base Python 3 requirements which are common to all the packs if use_python3: logger.debug('Installing base Python 3 requirements') for requirement in BASE_PACK_PYTHON3_REQUIREMENTS: install_requirement(virtualenv_path=virtualenv_path, requirement=requirement, proxy_config=proxy_config, logger=logger) # 4. Install pack-specific requirements requirements_file_path = os.path.join(pack_path, 'requirements.txt') has_requirements = os.path.isfile(requirements_file_path) if has_requirements: logger.debug('Installing pack specific requirements from "%s"' % (requirements_file_path)) install_requirements(virtualenv_path=virtualenv_path, requirements_file_path=requirements_file_path, proxy_config=proxy_config, logger=logger) else: logger.debug('No pack specific requirements found') # 5. Set the owner group if force_owner_group: apply_pack_owner_group(pack_path=virtualenv_path) action = 'updated' if update else 'created' logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' % (pack_name, action, virtualenv_path))
def create_git_worktree(self, content_version): """ Create a git worktree for the provided git content version. :return: Path to the created git worktree directory. :rtype: ``str`` """ pack_name = self.get_pack_name() pack_directory = get_pack_directory(pack_name=pack_name) worktree_path = tempfile.mkdtemp(prefix=self.WORKTREE_DIRECTORY_PREFIX) # Set class variables self.git_worktree_revision = content_version self.git_worktree_path = worktree_path extra = { 'pack_name': pack_name, 'pack_directory': pack_directory, 'content_version': content_version, 'worktree_path': worktree_path } if not os.path.isdir(pack_directory): msg = ('Failed to create git worktree for pack "%s". Pack directory "%s" doesn\'t ' 'exist.' % (pack_name, pack_directory)) raise ValueError(msg) args = [ 'git', '-C', pack_directory, 'worktree', 'add', worktree_path, content_version ] cmd = list2cmdline(args) LOG.debug('Creating git worktree for pack "%s", content version "%s" and execution ' 'id "%s" in "%s"' % (pack_name, content_version, self.execution_id, worktree_path), extra=extra) LOG.debug('Command: %s' % (cmd)) exit_code, stdout, stderr, timed_out = run_command(cmd=cmd, cwd=pack_directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) if exit_code != 0: self._handle_git_worktree_error(pack_name=pack_name, pack_directory=pack_directory, content_version=content_version, exit_code=exit_code, stdout=stdout, stderr=stderr) else: LOG.debug('Git worktree created in "%s"' % (worktree_path), extra=extra) # Make sure system / action runner user can access that directory args = [ 'chmod', '777', worktree_path ] cmd = list2cmdline(args) run_command(cmd=cmd, shell=True) return worktree_path