def run_action_with_check_mode(run_check, run_apply, skip_check, quiet=False, always_skip_check=False):
if always_skip_check:
user_wants_to_apply = ask(
'This command will apply without running the check first. Continue?',
quiet=quiet)
elif skip_check:
user_wants_to_apply = ask('Do you want to apply without running the check first?',
quiet=quiet)
else:
exit_code = run_check()
if exit_code == 1:
# this means there was an error before ansible was able to start running
return exit_code
elif exit_code == 0:
puts(color_success(u"✓ Check completed with status code {}".format(exit_code)))
user_wants_to_apply = ask('Do you want to apply these changes?',
quiet=quiet)
else:
puts(color_error(u"✗ Check failed with status code {}".format(exit_code)))
user_wants_to_apply = ask('Do you want to try to apply these changes anyway?',
quiet=quiet)
exit_code = 0
if user_wants_to_apply:
exit_code = run_apply()
if exit_code == 0:
puts(color_success(u"✓ Apply completed with status code {}".format(exit_code)))
else:
puts(color_error(u"✗ Apply failed with status code {}".format(exit_code)))
return exit_code
def run(self, args, unknown_args):
check_branch(args)
environment = get_environment(args.env_name)
if args.resume:
try:
# use cached env to ensure consistency with last deploy
cached_fab_env = retrieve_cached_deploy_env(environment.deploy_env)
except Exception:
print(color_error('Unable to resume deploy, please start anew'))
else:
environment = cached_fab_env.ccc_environment
deploy_component = args.component
if not deploy_component:
deploy_component = ['commcare']
if environment.meta_config.always_deploy_formplayer:
deploy_component.append('formplayer')
rc = 0
if 'commcare' in deploy_component:
if 'formplayer' not in deploy_component:
_warn_no_formplayer()
rc = deploy_commcare(environment, args, unknown_args)
if 'formplayer' in deploy_component:
if 'commcare' not in deploy_component:
if args.commcare_rev:
print(color_warning('--commcare-rev does not apply to a formplayer deploy and will be ignored'))
if args.fab_settings:
print(color_warning('--set does not apply to a formplayer deploy and will be ignored'))
if rc:
print(color_error("Skipping formplayer because commcare failed"))
else:
rc = deploy_formplayer(environment, args)
return rc
def commit(migration, ansible_context):
print_allocation(migration)
alloc_docs_by_db = {plan.db_name: plan for plan in migration.shard_plan}
puts(color_summary("Checking shards on disk vs plan. Please wait."))
if not assert_files(migration, alloc_docs_by_db, ansible_context):
puts(color_error("Some shard files are not where we expect. Have you run 'migrate'?"))
puts(color_error("Aborting"))
return 1
else:
puts(color_success("All shards appear to be where we expect according to the plan."))
if ask("Are you sure you want to update the Couch Database config?"):
commit_migration(migration)
diff_with_db = diff_plan(migration)
if diff_with_db:
puts(color_error('DB allocation differs from expected:\n'))
puts("{}\n\n".format(diff_with_db))
puts("Check the DB state and logs and maybe try running 'commit' again?")
return 1
puts(color_highlight("New shard allocation:\n"))
print_shard_table([
get_shard_allocation(migration.target_couch_config, db_name)
for db_name in sorted(get_db_list(migration.target_couch_config.get_control_node()))
])
return 0
def clean(migration, ansible_context, skip_check, limit):
diff_with_db = diff_plan(migration)
if diff_with_db:
puts(color_warning("Current plan differs with database:\n"))
puts("{}\n\n".format(diff_with_db))
puts(
color_notice(
"This could mean that the plan hasn't been committed yet\n"
"or that the plan was re-generated.\n"
"Performing the 'clean' operation is still safe but may\n"
"not have the outcome you are expecting.\n"))
if not ask("Do you wish to continue?"):
puts(color_error('Abort.'))
return 0
alloc_docs_by_db = get_db_allocations(migration.target_couch_config)
puts(color_summary("Checking shards on disk vs DB. Please wait."))
if not assert_files(migration, alloc_docs_by_db, ansible_context):
puts(color_error("Not all couch files are accounted for. Aborting."))
return 1
nodes = generate_shard_prune_playbook(migration)
if nodes:
return run_ansible_playbook(migration.target_environment,
migration.prune_playbook_path,
ansible_context,
skip_check=skip_check,
limit=limit)
def run(self, args, unknown_args):
if 'destroy' in unknown_args:
puts(color_error("Refusing to run a terraform command containing the argument 'destroy'."))
puts(color_error("It's simply not worth the risk."))
exit(-1)
environment = get_environment(args.env_name)
run_dir = environment.paths.get_env_file_path('.generated-terraform')
modules_dir = os.path.join(TERRAFORM_DIR, 'modules')
modules_dest = os.path.join(run_dir, 'modules')
if not os.path.isdir(run_dir):
os.mkdir(run_dir)
if not os.path.isdir(run_dir):
os.mkdir(run_dir)
if not (os.path.exists(modules_dest) and os.readlink(modules_dest) == modules_dir):
os.symlink(modules_dir, modules_dest)
if args.username != get_default_username():
print_help_message_about_the_commcare_cloud_default_username_env_var(args.username)
key_name = args.username
try:
generate_terraform_entrypoint(environment, key_name, run_dir,
apply_immediately=args.apply_immediately)
except UnauthorizedUser as e:
allowed_users = environment.users_config.dev_users.present
puts(color_error(
"Unauthorized user {}.\n\n"
"Use COMMCARE_CLOUD_DEFAULT_USERNAME or --username to pass in one of the allowed ssh users:{}"
.format(e.username, '\n - '.join([''] + allowed_users))))
return -1
if not args.skip_secrets and unknown_args and unknown_args[0] in ('plan', 'apply'):
rds_password = (
environment.get_secret('POSTGRES_USERS.root.password')
if environment.terraform_config.rds_instances
else ''
)
with open(os.path.join(run_dir, 'secrets.auto.tfvars'), 'w', encoding='utf-8') as f:
print('rds_password = {}'.format(json.dumps(rds_password)), file=f)
env_vars = {'AWS_PROFILE': aws_sign_in(environment)}
all_env_vars = os.environ.copy()
all_env_vars.update(env_vars)
cmd_parts = ['terraform'] + unknown_args
cmd = ' '.join(shlex_quote(arg) for arg in cmd_parts)
print_command('cd {}; {} {}; cd -'.format(
run_dir,
' '.join('{}={}'.format(key, value) for key, value in env_vars.items()),
cmd,
))
return subprocess.call(cmd, shell=True, env=all_env_vars, cwd=run_dir)
def check_branch(args):
branch = git_branch()
if branch is None:
# not in a git repo
if args.branch != 'master':
puts(color_error("You are not in a git repo. To deploy, remove --branch={}".format(branch)))
exit(-1)
elif args.branch != branch:
puts(color_error("You are not currently on the branch specified with the --branch tag. To deploy on "
"this branch, use --branch={}, otherwise, change branches".format(branch)))
exit(-1)
def _get_lines(proc):
"""Read lines from process stdout
:returns tuple(error_message, lines)
"""
out = proc.stdout.read()
if proc.returncode != 0:
err = proc.stderr.read()
return str(color_error('error: {}\n'.format(err))), []
elif not out:
return str(color_error('timeout\n')), []
else:
return None, out.splitlines()
def _get_lines(proc):
"""Read lines from process stdout
:returns tuple(error_message: AnyStr, lines: [AnyStr])
To return a str, specify an encoding when creating the subprocess, otherwise bytes-like objects are returned
"""
out = proc.stdout.read()
if proc.returncode != 0:
err = proc.stderr.read()
return str(color_error('error: {}\n'.format(err))), []
elif not out:
return str(color_error('timeout\n')), []
else:
return None, out.splitlines()
def ansible_playbook(environment, playbook, *cmd_args):
if os.path.isabs(playbook):
playbook_path = playbook
else:
playbook_path = os.path.join(
ANSIBLE_DIR, '{playbook}'.format(playbook=playbook))
cmd_parts = (
'ansible-playbook',
playbook_path,
'-i',
environment.paths.inventory_source,
'-e',
'@{}'.format(environment.paths.vault_yml),
'-e',
'@{}'.format(environment.paths.public_yml),
'-e',
'@{}'.format(environment.paths.generated_yml),
'--diff',
) + get_limit() + cmd_args
public_vars = environment.public_vars
cmd_parts += get_user_arg(public_vars, unknown_args, use_factory_auth)
if has_arg(unknown_args, '-D', '--diff') or has_arg(
unknown_args, '-C', '--check'):
puts(
color_error("Options --diff and --check not allowed. "
"Please remove -D, --diff, -C, --check."))
puts(
color_error(
"These ansible-playbook options are managed automatically "
"by commcare-cloud and cannot be set manually."))
return 2 # exit code
ask_vault_pass = public_vars.get('commcare_cloud_use_vault', True)
if ask_vault_pass:
cmd_parts += ('--vault-password-file={}/echo_vault_password.sh'.
format(ANSIBLE_DIR), )
cmd_parts_with_common_ssh_args = get_common_ssh_args(
environment, use_factory_auth=use_factory_auth)
cmd_parts += cmd_parts_with_common_ssh_args
cmd = ' '.join(shlex_quote(arg) for arg in cmd_parts)
print_command(cmd)
env_vars = ansible_context.env_vars
if ask_vault_pass:
env_vars[
'ANSIBLE_VAULT_PASSWORD'] = environment.get_ansible_vault_password(
)
return subprocess.call(cmd_parts, env=env_vars)
def get_dev_username(env_name):
from .commands.terraform.aws import (
get_default_username,
print_help_message_about_the_commcare_cloud_default_username_env_var,
)
environment = get_environment(env_name)
username = default_username = get_default_username()
while True:
if not username or default_username.is_guess:
username = input(f"Enter your SSH username ({default_username}): ")
if not username:
username = default_username
if username in environment.users_config.dev_users.present:
break
allowed_users = environment.users_config.dev_users.present
env_users = '\n - '.join([''] + allowed_users)
puts(
color_error(
f"Unauthorized user {username}.\n\n"
f"Please pass in one of the allowed ssh users:{env_users}"))
username = ""
if default_username.is_guess:
print_help_message_about_the_commcare_cloud_default_username_env_var(
username)
return username
def update_sentry_post_deploy(environment, sentry_project, github_repo, diff,
deploy_start, deploy_end):
localsettings = environment.public_vars["localsettings"]
try:
sentry_api_key = environment.get_secret('SENTRY_API_KEY')
except KeyError:
return
client = SentryClient(
sentry_api_key, localsettings.get('SENTRY_ORGANIZATION_SLUG',
'dimagi'), sentry_project)
if client.is_valid():
# this must match the release name used in commcare-hq when configuring the Sentry API
release_name = f"{environment.new_release_name()}-{environment.meta_config.env_monitoring_id}"
if environment.fab_settings_config.generate_deploy_diffs:
try:
commits = get_release_commits(diff)
except GithubException as e:
commits = None
print(color_error(f"Error getting release commits: {e}"))
else:
commits = None
client.create_release(release_name, commits)
client.create_deploy(release_name,
environment.meta_config.env_monitoring_id,
deploy_start, deploy_end)
def run(self, args, unknown_args):
environment = get_environment(args.env_name)
from_backend = all_secrets_backends_by_name[
args.from_backend].from_environment(environment)
if args.to_backend:
to_backend = all_secrets_backends_by_name[
args.to_backend].from_environment(environment)
else:
to_backend = environment.secrets_backend
if from_backend.name == to_backend.name:
puts(
color_error(
'Refusing to copy from {from_backend.name} to {to_backend.name}: backends must differ'
.format(from_backend=from_backend, to_backend=to_backend)))
exit(-1)
print("Copying data from {from_backend.name} to {to_backend.name}:".
format(from_backend=from_backend, to_backend=to_backend))
for secret_spec in get_known_secret_specs():
try:
secret_value = from_backend.get_secret(secret_spec.name)
except KeyError:
print("No value for {secret_spec.name}... Skipping".format(
secret_spec=secret_spec))
continue
to_backend.set_secret(secret_spec.name, secret_value)
print("Copied value for {secret_spec.name}".format(
secret_spec=secret_spec))
def call_commcare_cloud(input_argv=sys.argv):
# throw error if user is attempting to use python 2
if not os.environ.get("TRAVIS_TEST") and sys.version_info[0] == 2:
exit(
dedent("""
Error: you must upgrade to Python 3. Python 2 is no longer supported.
To setup Python 3.6, see
https://dimagi.github.io/commcare-cloud/setup/installation.html
"""))
put_virtualenv_bin_on_the_path()
parser, subparsers, commands = make_command_parser(
available_envs=get_available_envs())
raw_args = input_argv[1:]
if not raw_args:
parser.print_help()
return
args, unknown_args = parser.parse_known_args(raw_args)
if args.control:
run_on_control_instead(args, input_argv)
try:
exit_code = commands[args.command].run(args, unknown_args)
except CommandError as e:
puts(color_error(str(e), bold=True))
return 1
return exit_code
def run(self, args, unknown_args):
environment = get_environment(args.env_name)
try:
environment.check()
except Exception:
puts(color_error(u"✗ The environment has the following error:"))
raise
else:
puts(color_success(u"✓ The environment configuration is valid."))
def get_diff_context(self):
context = {
"new_version_details": self.new_version_details,
"user": get_default_username(),
"LABELS_TO_EXPAND": LABELS_TO_EXPAND,
"errors": [],
"warnings": []
}
if self.deployed_commit_matches_latest_commit:
context["errors"].append(
"Versions are identical. No changes since last deploy.")
return context
if not (self.current_commit and self.deploy_commit):
context["warnings"].append("Insufficient info to get deploy diff.")
return context
context["compare_url"] = self.url
if not self.generate_diff:
disabled_msg = "Deploy diffs disabled for this environment."
print(color_warning(disabled_msg))
context["warnings"].append(disabled_msg)
return context
if not self.repo.permissions:
# using unauthenticated API calls, skip diff creation to avoid hitting rate limits
print(
color_warning(
"Diff generation skipped. Supply a Github token to see deploy diffs."
))
context["warnings"].append("Diff omitted.")
return context
try:
pr_numbers = self._get_pr_numbers()
except GithubException as e:
print(color_error(f"Error getting diff commits: {e}"))
context["warnings"].append(
"There was an error fetching the PRs since the last deploy.")
return context
if len(pr_numbers) > 500:
context["warnings"].append("There are too many PRs to display.")
return context
elif not pr_numbers:
context["warnings"].append("No PRs merged since last release.")
return context
pool = Pool(5)
pr_infos = [_f for _f in pool.map(self._get_pr_info, pr_numbers) if _f]
context["pr_infos"] = pr_infos
prs_by_label = self._get_prs_by_label(pr_infos)
context["prs_by_label"] = prs_by_label
return context
def ansible_playbook(environment, playbook, *cmd_args):
if os.path.isabs(playbook):
playbook_path = playbook
else:
playbook_path = os.path.join(
ANSIBLE_DIR, '{playbook}'.format(playbook=playbook))
cmd_parts = (
'ansible-playbook',
playbook_path,
'-i',
environment.paths.inventory_source,
'-e',
'@{}'.format(environment.paths.public_yml),
'-e',
'@{}'.format(environment.paths.generated_yml),
'--diff',
) + get_limit() + cmd_args
public_vars = environment.public_vars
env_vars = ansible_context.env_vars
cmd_parts += get_user_arg(public_vars, unknown_args, use_factory_auth)
if has_arg(unknown_args, '-D', '--diff') or has_arg(
unknown_args, '-C', '--check'):
puts(
color_error("Options --diff and --check not allowed. "
"Please remove -D, --diff, -C, --check."))
puts(
color_error(
"These ansible-playbook options are managed automatically "
"by commcare-cloud and cannot be set manually."))
return 2 # exit code
cmd_parts += environment.secrets_backend.get_extra_ansible_args()
cmd_parts_with_common_ssh_args = get_common_ssh_args(
environment, use_factory_auth=use_factory_auth)
cmd_parts += cmd_parts_with_common_ssh_args
cmd = ' '.join(shlex_quote(arg) for arg in cmd_parts)
print_command(cmd)
env_vars.update(
environment.secrets_backend.get_extra_ansible_env_vars())
return subprocess.call(cmd_parts, env=env_vars)
def create_release_tag(environment, repo, diff):
if environment.fab_settings_config.tag_deploy_commits:
try:
repo.create_git_ref(
ref='refs/tags/{}-{}-deploy'.format(
environment.new_release_name(), environment.name),
sha=diff.deploy_commit,
)
except GithubException as e:
print(color_error(f"Error creating release tag: {e}"))
def call_commcare_cloud(input_argv=sys.argv):
put_virtualenv_bin_on_the_path()
parser, subparsers, commands = make_command_parser(available_envs=get_available_envs())
args, unknown_args = parser.parse_known_args(input_argv[1:])
if args.control:
run_on_control_instead(args, input_argv)
try:
exit_code = commands[args.command].run(args, unknown_args)
except CommandError as e:
puts(color_error(str(e), bold=True))
return 1
return exit_code
def check_branch(args):
branch = git_branch()
if branch is None:
# not in a git repo
if args.branch != 'master':
puts(
color_error(
"You are not in a git repo. To deploy, remove --branch={}".
format(branch)))
exit(-1)
elif args.branch != branch:
puts(
color_error(
"You are not currently on the branch specified with the --branch tag. To deploy on "
"this branch, use --branch={}, otherwise, change branches".
format(branch)))
exit(-1)
elif branch == 'master' and not any(
a.startswith("--branch") for a in sys.argv):
require_clean_working_tree()
local_rev = git("rev-parse", branch).strip()
remote = "https://github.com/dimagi/commcare-cloud.git"
remote_rev = git("ls-remote", remote,
"refs/heads/master").split(maxsplit=1)[0]
if local_rev != remote_rev:
print("Your local 'master' branch has diverged from upstream")
print(f"local: {local_rev}")
print(f"remote: {remote_rev}")
print("")
print("Run 'git pull' to get the latest code, then try again.")
print(
"Create a pull request if you have new commits to contribute.")
print(
"Or add --branch=master to use your local branch (not recommended)."
)
exit(-1)
def run(self, action, host_pattern=None, process_pattern=None):
if action == 'help':
self.print_help()
return 0
elif action == 'logs':
print("Logs can be found at:\n{}".format(self.log_location.format(env=self.environment.name)))
return 0
try:
return self.execute_action(action, host_pattern, process_pattern)
except NoHostsMatch:
only = limit = ''
if process_pattern:
only = " '--only={}'".format(process_pattern)
if host_pattern:
limit = " '--limit={}'".format(host_pattern)
puts(color_error("No '{}' hosts match{}{}".format(self.name, limit, only)))
return 1
def _check_username(env_name, username, message):
default_username = username
allowed_users = ["ansible"] + get_environment(env_name).users_config.dev_users.present
while True:
if not username or default_username.is_guess:
username = input(f"Enter your SSH username ({default_username}): ")
if not username:
username = default_username
if username in allowed_users:
break
env_users = '\n - '.join([''] + allowed_users)
puts(color_error(
f"Unauthorized user {username}.\n\n"
f"Please pass in one of the allowed ssh users:{env_users}"
))
username = ""
if default_username.is_guess:
print(color_notice(message.format(username=username)))
return username
def _get_pr_info(self, pr_number):
try:
pr_response = self.repo.get_pull(pr_number)
except GithubException as e:
print(
color_error(f"Error getting PR details for {pr_number}: {e}"))
return None
if not pr_response.number:
# Likely rate limited by Github API
return None
assert pr_number == pr_response.number, (pr_number, pr_response.number)
return {
'number': pr_response.number,
'title': pr_response.title,
'url': pr_response.html_url,
'labels': pr_response.labels,
'additions': pr_response.additions,
'deletions': pr_response.deletions,
'opened_by': pr_response.user.login,
'body': pr_response.body,
}
def run(self, args, unknown_args):
j2 = jinja2.Environment(loader=jinja2.FileSystemLoader(
os.path.dirname(__file__)),
keep_trailing_newline=True)
changelog_dir = 'changelog'
for filename in _sort_files(changelog_dir):
if filename.endswith(".yml"):
last_log = filename
break
else:
puts(
color_error(
"Unable to find last changelog file. Please create a changelog manually."
))
return 1
last_index = int(re.search(r"^(\d+)", last_log).group())
name = args.name
date = datetime.utcnow()
if not name:
name = "auto {}".format(date.strftime("%Y%m%d_%H%M"))
name = re.sub("[\n\r\t]", " ", name)
key = name.replace(" ", "_")
file_name = "{:04d}-{}.yml".format(last_index + 1, key)
template = j2.get_template('changelog-template.yml.j2')
path = os.path.join(changelog_dir, file_name)
with open(path, 'w') as f:
f.write(
template.render(name=name,
key=key,
date=date.strftime("%Y-%m-%d")).rstrip())
print("Changelog created at {}".format(path))
def run(self, args, unknown_args):
limit = args.limit
environment = get_environment(args.env_name)
if limit:
environment.inventory_manager.subset(limit)
with open(environment.paths.known_hosts, 'r', encoding='utf-8') as known_hosts:
original_keys_by_host = _get_host_key_map(
[line.strip() for line in known_hosts.readlines()]
)
procs = {}
for hostname in environment.inventory_hostname_map:
port = '22'
if ':' in hostname:
hostname, port = hostname.split(':')
cmd = 'ssh-keyscan -T 10 -p {port} {hostname},$(dig +short {hostname})'.format(
hostname=hostname,
port=port
)
procs[hostname] = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True)
lines = []
error_hosts = set()
for hostname, proc in procs.items():
sys.stdout.write('[{}]: '.format(hostname))
proc.wait()
error, host_lines = _get_lines(proc)
if error:
sys.stdout.write(error)
else:
sys.stdout.write(str(color_success('fetched key\n')))
lines.extend(host_lines)
updated_keys_by_host = _get_host_key_map(lines)
all_keys = set(original_keys_by_host) | set(updated_keys_by_host)
lines = []
for host_key_type in sorted(all_keys):
host, key_type = host_key_type
original = original_keys_by_host.pop(host_key_type, None)
updated = updated_keys_by_host.get(host_key_type, None)
if updated and original:
if updated != original:
print(color_changed('Updating key: {} {}'.format(*host_key_type)))
elif updated:
print(color_added('Adding key: {} {}'.format(*host_key_type)))
elif original:
if limit or host in error_hosts:
# if we're limiting or there was an error keep original key
updated = original
else:
print(color_removed('Removing key: {} {}'.format(*host_key_type)))
if updated:
lines.append('{} {} {}'.format(host, key_type, updated))
with open(environment.paths.known_hosts, 'w', encoding='utf-8') as known_hosts:
known_hosts.write('\n'.join(sorted(lines)))
try:
environment.check_known_hosts()
except EnvironmentException as e:
print(color_error(str(e)))
return 1
return 0
