def update():
join = functools.partial(posixpath.join, env.code_root)
exists = functools.partial(files.exists, use_sudo=True)
# Optimization if we have current setup (i.e. not the first deploy)
if exists(env.py3_virtualenv_current) and not exists(env.py3_virtualenv_root):
_clone_virtual_env(env.py3_virtualenv_current, env.py3_virtualenv_root)
elif not exists(env.py3_virtualenv_current):
raise EnvironmentException(
"Virtual environment not found: {}".format(env.py3_virtualenv_current)
)
requirements_files = [join("requirements", "prod-requirements.txt")]
requirements_files.extend(
join(repo.relative_dest, repo.requirements_path)
for repo in env.ccc_environment.meta_config.git_repositories
if repo.requirements_path
)
with cd(env.code_root):
cmd_prefix = 'export HOME=/home/{} && source {}/bin/activate && '.format(
env.sudo_user, env.py3_virtualenv_root)
proxy = " --proxy={}".format(env.http_proxy) if env.http_proxy else ""
sudo("{} pip install --quiet --upgrade --timeout=60{} pip-tools".format(
cmd_prefix, proxy))
sudo("{} pip-sync --quiet --pip-args='--timeout=60{}' {}".format(
cmd_prefix, proxy, " ".join(requirements_files)))
def check_known_hosts(self):
if not os.path.exists(self.paths.known_hosts):
return
blacklist = set(self.groups.get('ansible_skip', []))
hostname_to_sshable = {
inventory_hostname: sshable.split(':')[0]
for sshable, inventory_hostname in self.inventory_hostname_map.items()
}
inventory_hostnames = {
host for hosts in self.groups.values()
for host in hosts if host not in blacklist
}
expected_hosts = {
(hostname_to_sshable[host], self.hostname_map.get(host))
for host in inventory_hostnames
}
with open(self.paths.known_hosts) as f:
known_hosts_contents = f.read()
missing_hosts = {
(sshable, hostname) for sshable, hostname in expected_hosts
if not re.search(r'\b{}\b'.format(sshable), known_hosts_contents)
}
if missing_hosts:
raise EnvironmentException(
'The following hosts are missing from known_hosts:\n{}'.format(
'\n'.join(
"{} ({})".format(sshable, hostname) for sshable, hostname in missing_hosts
)
)
)
def repo(self):
match = re.match(r"[email protected]:(.*?).git", self.url)
if not match:
raise EnvironmentException(
"Unable to parse repository URL: {}".format(self.url))
repo = match.group(1)
return get_github(self).get_repo(repo)
def get_github(repo):
login_or_token, password = None, None
if repo.is_private:
message = "This environment references a private repository: {}".format(
repo.url)
login_or_token, password = get_github_credentials(message, force=True)
if not login_or_token and not password:
raise EnvironmentException("Github credentials required")
return Github(login_or_token=login_or_token, password=password)
def get_github(repo):
token = None
if repo.is_private:
message = "This environment references a private repository: {}".format(
repo.url)
token = get_github_token(message, required=True)
if not token:
raise EnvironmentException("Github credentials required")
return Github(login_or_token=token)
def check_user_group_absent_present_overlaps(self, absent_users,
present_users):
if not set(present_users).isdisjoint(absent_users):
repeated_users = list(
(Counter(present_users) & Counter(absent_users)).elements())
raise EnvironmentException(
'The user(s) {} appear(s) in both the absent and present users list for '
'the environment {}. Please fix this and try again.'.format(
(', '.join(map(str, repeated_users))),
self.meta_config.deploy_env))
def inventory_source(self):
sources = []
for path in [self.inventory_ini, self.inventory_csv]:
if os.path.exists(path) and os.access(path, os.R_OK):
sources.append(path)
if os.path.exists(self.inventory_directory) and os.path.isdir(
self.inventory_directory):
sources.append(self.inventory_directory)
if not sources or len(sources) > 1:
raise EnvironmentException(
"Exactly one inventory source must be provided. Either a single 'inventory.ini' file "
"or a single 'inventory.csv' file or an 'inventory' folder containing inventory files."
)
return sources[0]
def _clone_virtualenv(env):
print('Cloning virtual env')
python_version = env.ccc_environment.python_version
virtualenv_name = f"python_env-{python_version}"
virtualenv_current = posixpath.join(env.code_current, virtualenv_name)
# There's a bug in virtualenv-clone that doesn't allow us to clone envs from symlinks
old_virtualenv_path = sudo(f'readlink -f {virtualenv_current}')
if not files.exists(old_virtualenv_path, use_sudo=True):
raise EnvironmentException(
f"virtualenv not found: {old_virtualenv_path}")
join = functools.partial(posixpath.join, env.code_root)
new_virtualenv_path = join(virtualenv_name)
python_env = join(
"python_env") if python_version == '3.6' else env.virtualenv_root
assert os.path.basename(python_env) != virtualenv_name, python_env
sudo(f"virtualenv-clone {old_virtualenv_path} {new_virtualenv_path}")
sudo(f"ln -nfs {virtualenv_name} {python_env}")