def _setup_pack_virtualenv(self, pack_name, update=False):
"""
Setup virtual environment for the provided pack.
:param pack_name: Pack name.
:type pack_name: ``str``
"""
# Prevent directory traversal by whitelisting allowed characters in the
# pack name
if not re.match(PACK_NAME_WHITELIST, pack_name):
raise ValueError('Invalid pack name "%s"' % (pack_name))
self.logger.debug('Setting up virtualenv for pack "%s"' % (pack_name))
virtualenv_path = os.path.join(self._base_virtualenvs_path, quote_unix(pack_name))
# Ensure pack directory exists in one of the search paths
pack_path = get_pack_directory(pack_name=pack_name)
if not pack_path:
packs_base_paths = get_packs_base_paths()
search_paths = ', '.join(packs_base_paths)
msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
raise Exception(msg)
if not os.path.exists(self._base_virtualenvs_path):
os.makedirs(self._base_virtualenvs_path)
# If we don't want to update, or if the virtualenv doesn't exist, let's create it.
if not update or not os.path.exists(virtualenv_path):
# 0. Delete virtual environment if it exists
self._remove_virtualenv(virtualenv_path=virtualenv_path)
# 1. Create virtual environment
self.logger.debug('Creating virtualenv for pack "%s" in "%s"' %
(pack_name, virtualenv_path))
self._create_virtualenv(virtualenv_path=virtualenv_path)
# 2. Install base requirements which are common to all the packs
self.logger.debug('Installing base requirements')
for requirement in BASE_PACK_REQUIREMENTS:
self._install_requirement(virtualenv_path=virtualenv_path,
requirement=requirement)
# 3. Install pack-specific requirements
requirements_file_path = os.path.join(pack_path, 'requirements.txt')
has_requirements = os.path.isfile(requirements_file_path)
if has_requirements:
self.logger.debug('Installing pack specific requirements from "%s"' %
(requirements_file_path))
self._install_requirements(virtualenv_path, requirements_file_path)
else:
self.logger.debug('No pack specific requirements found')
self.logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
(pack_name,
'updated' if update else 'created',
virtualenv_path))
def _setup_pack_virtualenv(self, pack_name):
"""
Setup virtual environment for the provided pack.
:param pack_name: Pack name.
:type pack_name: ``str``
"""
# Prevent directory traversal by whitelisting allowed characters in the
# pack name
if not re.match(PACK_NAME_WHITELIST, pack_name):
raise ValueError('Invalid pack name "%s"' % (pack_name))
self.logger.debug('Setting up virtualenv for pack "%s"' % (pack_name))
virtualenv_path = os.path.join(self._base_virtualenvs_path,
quote_unix(pack_name))
# Ensure pack directory exists in one of the search paths
pack_path = get_pack_directory(pack_name=pack_name)
if not pack_path:
packs_base_paths = get_packs_base_paths()
search_paths = ', '.join(packs_base_paths)
msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name,
search_paths)
raise Exception(msg)
if not os.path.exists(self._base_virtualenvs_path):
os.makedirs(self._base_virtualenvs_path)
# 0. Delete virtual environment if it exists
self._remove_virtualenv(virtualenv_path=virtualenv_path)
# 1. Create virtual environment
self.logger.debug('Creating virtualenv for pack "%s" in "%s"' %
(pack_name, virtualenv_path))
self._create_virtualenv(virtualenv_path=virtualenv_path)
# 2. Install base requirements which are common to all the packs
self.logger.debug('Installing base requirements')
for requirement in BASE_PACK_REQUIREMENTS:
self._install_requirement(virtualenv_path=virtualenv_path,
requirement=requirement)
# 3. Install pack-specific requirements
requirements_file_path = os.path.join(pack_path, 'requirements.txt')
has_requirements = os.path.isfile(requirements_file_path)
if has_requirements:
self.logger.debug(
'Installing pack specific requirements from "%s"' %
(requirements_file_path))
self._install_requirements(virtualenv_path, requirements_file_path)
else:
self.logger.debug('No pack specific requirements found')
self.logger.debug(
'Virtualenv for pack "%s" successfully created in "%s"' %
(pack_name, virtualenv_path))
def create_git_worktree(self, content_version):
"""
Create a git worktree for the provided git content version.
:return: Path to the created git worktree directory.
:rtype: ``str``
"""
pack_name = self.get_pack_name()
pack_directory = get_pack_directory(pack_name=pack_name)
worktree_path = tempfile.mkdtemp(prefix=self.WORKTREE_DIRECTORY_PREFIX)
# Set class variables
self.git_worktree_revision = content_version
self.git_worktree_path = worktree_path
extra = {
'pack_name': pack_name,
'pack_directory': pack_directory,
'content_version': content_version,
'worktree_path': worktree_path
}
if not os.path.isdir(pack_directory):
msg = (
'Failed to create git worktree for pack "%s". Pack directory "%s" doesn\'t '
'exist.' % (pack_name, pack_directory))
raise ValueError(msg)
args = [
'git', '-C', pack_directory, 'worktree', 'add', worktree_path,
content_version
]
cmd = list2cmdline(args)
LOG.debug(
'Creating git worktree for pack "%s", content version "%s" and execution '
'id "%s" in "%s"' %
(pack_name, content_version, self.execution_id, worktree_path),
extra=extra)
LOG.debug('Command: %s' % (cmd))
exit_code, stdout, stderr, timed_out = run_command(
cmd=cmd,
cwd=pack_directory,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
if exit_code != 0:
self._handle_git_worktree_error(pack_name=pack_name,
pack_directory=pack_directory,
content_version=content_version,
exit_code=exit_code,
stdout=stdout,
stderr=stderr)
else:
LOG.debug('Git worktree created in "%s"' % (worktree_path),
extra=extra)
# Make sure system / action runner user can access that directory
args = ['chmod', '777', worktree_path]
cmd = list2cmdline(args)
run_command(cmd=cmd, shell=True)
return worktree_path
def setup_pack_virtualenv(pack_name,
update=False,
logger=None,
include_pip=True,
include_setuptools=True,
include_wheel=True,
proxy_config=None,
no_download=True,
force_owner_group=True):
"""
Setup virtual environment for the provided pack.
:param pack_name: Name of the pack to setup the virtualenv for.
:type pack_name: ``str``
:param update: True to update dependencies inside the virtual environment.
:type update: ``bool``
:param logger: Optional logger instance to use. If not provided it defaults to the module
level logger.
:param no_download: Do not download and install latest version of pre-installed packages such
as pip and distutils.
:type no_download: ``bool``
"""
logger = logger or LOG
if not re.match(PACK_REF_WHITELIST_REGEX, pack_name):
raise ValueError('Invalid pack name "%s"' % (pack_name))
base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path,
'virtualenvs/')
virtualenv_path = os.path.join(base_virtualenvs_path,
quote_unix(pack_name))
# Ensure pack directory exists in one of the search paths
pack_path = get_pack_directory(pack_name=pack_name)
logger.debug('Setting up virtualenv for pack "%s" (%s)' %
(pack_name, pack_path))
if not pack_path:
packs_base_paths = get_packs_base_paths()
search_paths = ', '.join(packs_base_paths)
msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name,
search_paths)
raise Exception(msg)
# 1. Create virtualenv if it doesn't exist
if not update or not os.path.exists(virtualenv_path):
# 0. Delete virtual environment if it exists
remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)
# 1. Create virtual environment
logger.debug('Creating virtualenv for pack "%s" in "%s"' %
(pack_name, virtualenv_path))
create_virtualenv(virtualenv_path=virtualenv_path,
logger=logger,
include_pip=include_pip,
include_setuptools=include_setuptools,
include_wheel=include_wheel,
no_download=no_download)
# 2. Install base requirements which are common to all the packs
logger.debug('Installing base requirements')
for requirement in BASE_PACK_REQUIREMENTS:
install_requirement(virtualenv_path=virtualenv_path,
requirement=requirement,
proxy_config=proxy_config,
logger=logger)
# 3. Install pack-specific requirements
requirements_file_path = os.path.join(pack_path, 'requirements.txt')
has_requirements = os.path.isfile(requirements_file_path)
if has_requirements:
logger.debug('Installing pack specific requirements from "%s"' %
(requirements_file_path))
install_requirements(virtualenv_path=virtualenv_path,
requirements_file_path=requirements_file_path,
proxy_config=proxy_config,
logger=logger)
else:
logger.debug('No pack specific requirements found')
# 4. Set the owner group
if force_owner_group:
apply_pack_owner_group(pack_path=virtualenv_path)
action = 'updated' if update else 'created'
logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
(pack_name, action, virtualenv_path))
def setup_pack_virtualenv(pack_name, update=False, logger=None):
"""
Setup virtual environment for the provided pack.
:param pack_name: Name of the pack to setup the virtualenv for.
:type pack_name: ``str``
:param update: True to update dependencies inside the virtual environment.
:type update: ``bool``
:param logger: Optional logger instance to use. If not provided it defaults to the module
level logger.
"""
logger = logger or LOG
if not re.match(PACK_NAME_WHITELIST, pack_name):
raise ValueError('Invalid pack name "%s"' % (pack_name))
base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/')
logger.debug('Setting up virtualenv for pack "%s"' % (pack_name))
virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name))
# Ensure pack directory exists in one of the search paths
pack_path = get_pack_directory(pack_name=pack_name)
if not pack_path:
packs_base_paths = get_packs_base_paths()
search_paths = ', '.join(packs_base_paths)
msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
raise Exception(msg)
# 1. Create virtualenv if it doesn't exist
if not update or not os.path.exists(virtualenv_path):
# 0. Delete virtual environment if it exists
remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)
# 1. Create virtual environment
logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path))
create_virtualenv(virtualenv_path=virtualenv_path, logger=logger)
# 2. Install base requirements which are common to all the packs
logger.debug('Installing base requirements')
for requirement in BASE_PACK_REQUIREMENTS:
install_requirement(virtualenv_path=virtualenv_path, requirement=requirement)
# 3. Install pack-specific requirements
requirements_file_path = os.path.join(pack_path, 'requirements.txt')
has_requirements = os.path.isfile(requirements_file_path)
if has_requirements:
logger.debug('Installing pack specific requirements from "%s"' %
(requirements_file_path))
install_requirements(virtualenv_path=virtualenv_path,
requirements_file_path=requirements_file_path)
else:
logger.debug('No pack specific requirements found')
action = 'updated' if update else 'created'
logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
(pack_name, action, virtualenv_path))
def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True,
include_setuptools=True, include_wheel=True, proxy_config=None,
use_python3=False, no_download=True, force_owner_group=True):
"""
Setup virtual environment for the provided pack.
:param pack_name: Name of the pack to setup the virtualenv for.
:type pack_name: ``str``
:param update: True to update dependencies inside the virtual environment.
:type update: ``bool``
:param logger: Optional logger instance to use. If not provided it defaults to the module
level logger.
:param use_python3: Use Python3 binary when creating virtualenv for this pack.
:type use_python3: ``bool``
:param no_download: Do not download and install latest version of pre-installed packages such
as pip and distutils.
:type no_download: ``bool``
"""
logger = logger or LOG
if not re.match(PACK_REF_WHITELIST_REGEX, pack_name):
raise ValueError('Invalid pack name "%s"' % (pack_name))
base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/')
virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name))
# Ensure pack directory exists in one of the search paths
pack_path = get_pack_directory(pack_name=pack_name)
logger.debug('Setting up virtualenv for pack "%s" (%s)' % (pack_name, pack_path))
if not pack_path:
packs_base_paths = get_packs_base_paths()
search_paths = ', '.join(packs_base_paths)
msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
raise Exception(msg)
# 1. Create virtualenv if it doesn't exist
if not update or not os.path.exists(virtualenv_path):
# 0. Delete virtual environment if it exists
remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)
# 1. Create virtual environment
logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path))
create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip,
include_setuptools=include_setuptools, include_wheel=include_wheel,
use_python3=use_python3, no_download=no_download)
# 2. Install base requirements which are common to all the packs
logger.debug('Installing base requirements')
for requirement in BASE_PACK_REQUIREMENTS:
install_requirement(virtualenv_path=virtualenv_path, requirement=requirement,
proxy_config=proxy_config, logger=logger)
# 3. Install base Python 3 requirements which are common to all the packs
if use_python3:
logger.debug('Installing base Python 3 requirements')
for requirement in BASE_PACK_PYTHON3_REQUIREMENTS:
install_requirement(virtualenv_path=virtualenv_path, requirement=requirement,
proxy_config=proxy_config, logger=logger)
# 4. Install pack-specific requirements
requirements_file_path = os.path.join(pack_path, 'requirements.txt')
has_requirements = os.path.isfile(requirements_file_path)
if has_requirements:
logger.debug('Installing pack specific requirements from "%s"' %
(requirements_file_path))
install_requirements(virtualenv_path=virtualenv_path,
requirements_file_path=requirements_file_path,
proxy_config=proxy_config,
logger=logger)
else:
logger.debug('No pack specific requirements found')
# 5. Set the owner group
if force_owner_group:
apply_pack_owner_group(pack_path=virtualenv_path)
action = 'updated' if update else 'created'
logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
(pack_name, action, virtualenv_path))
def create_git_worktree(self, content_version):
"""
Create a git worktree for the provided git content version.
:return: Path to the created git worktree directory.
:rtype: ``str``
"""
pack_name = self.get_pack_name()
pack_directory = get_pack_directory(pack_name=pack_name)
worktree_path = tempfile.mkdtemp(prefix=self.WORKTREE_DIRECTORY_PREFIX)
# Set class variables
self.git_worktree_revision = content_version
self.git_worktree_path = worktree_path
extra = {
'pack_name': pack_name,
'pack_directory': pack_directory,
'content_version': content_version,
'worktree_path': worktree_path
}
if not os.path.isdir(pack_directory):
msg = ('Failed to create git worktree for pack "%s". Pack directory "%s" doesn\'t '
'exist.' % (pack_name, pack_directory))
raise ValueError(msg)
args = [
'git',
'-C',
pack_directory,
'worktree',
'add',
worktree_path,
content_version
]
cmd = list2cmdline(args)
LOG.debug('Creating git worktree for pack "%s", content version "%s" and execution '
'id "%s" in "%s"' % (pack_name, content_version, self.execution_id,
worktree_path), extra=extra)
LOG.debug('Command: %s' % (cmd))
exit_code, stdout, stderr, timed_out = run_command(cmd=cmd,
cwd=pack_directory,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
if exit_code != 0:
self._handle_git_worktree_error(pack_name=pack_name, pack_directory=pack_directory,
content_version=content_version,
exit_code=exit_code, stdout=stdout, stderr=stderr)
else:
LOG.debug('Git worktree created in "%s"' % (worktree_path), extra=extra)
# Make sure system / action runner user can access that directory
args = [
'chmod',
'777',
worktree_path
]
cmd = list2cmdline(args)
run_command(cmd=cmd, shell=True)
return worktree_path