def run_action_with_check_mode(run_check, run_apply, skip_check, quiet=False, always_skip_check=False): if always_skip_check: user_wants_to_apply = ask( 'This command will apply without running the check first. Continue?', quiet=quiet) elif skip_check: user_wants_to_apply = ask('Do you want to apply without running the check first?', quiet=quiet) else: exit_code = run_check() if exit_code == 1: # this means there was an error before ansible was able to start running return exit_code elif exit_code == 0: puts(color_success(u"✓ Check completed with status code {}".format(exit_code))) user_wants_to_apply = ask('Do you want to apply these changes?', quiet=quiet) else: puts(color_error(u"✗ Check failed with status code {}".format(exit_code))) user_wants_to_apply = ask('Do you want to try to apply these changes anyway?', quiet=quiet) exit_code = 0 if user_wants_to_apply: exit_code = run_apply() if exit_code == 0: puts(color_success(u"✓ Apply completed with status code {}".format(exit_code))) else: puts(color_error(u"✗ Apply failed with status code {}".format(exit_code))) return exit_code
def run(self, args, unknown_args): check_branch(args) environment = get_environment(args.env_name) if args.resume: try: # use cached env to ensure consistency with last deploy cached_fab_env = retrieve_cached_deploy_env(environment.deploy_env) except Exception: print(color_error('Unable to resume deploy, please start anew')) else: environment = cached_fab_env.ccc_environment deploy_component = args.component if not deploy_component: deploy_component = ['commcare'] if environment.meta_config.always_deploy_formplayer: deploy_component.append('formplayer') rc = 0 if 'commcare' in deploy_component: if 'formplayer' not in deploy_component: _warn_no_formplayer() rc = deploy_commcare(environment, args, unknown_args) if 'formplayer' in deploy_component: if 'commcare' not in deploy_component: if args.commcare_rev: print(color_warning('--commcare-rev does not apply to a formplayer deploy and will be ignored')) if args.fab_settings: print(color_warning('--set does not apply to a formplayer deploy and will be ignored')) if rc: print(color_error("Skipping formplayer because commcare failed")) else: rc = deploy_formplayer(environment, args) return rc
def commit(migration, ansible_context): print_allocation(migration) alloc_docs_by_db = {plan.db_name: plan for plan in migration.shard_plan} puts(color_summary("Checking shards on disk vs plan. Please wait.")) if not assert_files(migration, alloc_docs_by_db, ansible_context): puts(color_error("Some shard files are not where we expect. Have you run 'migrate'?")) puts(color_error("Aborting")) return 1 else: puts(color_success("All shards appear to be where we expect according to the plan.")) if ask("Are you sure you want to update the Couch Database config?"): commit_migration(migration) diff_with_db = diff_plan(migration) if diff_with_db: puts(color_error('DB allocation differs from expected:\n')) puts("{}\n\n".format(diff_with_db)) puts("Check the DB state and logs and maybe try running 'commit' again?") return 1 puts(color_highlight("New shard allocation:\n")) print_shard_table([ get_shard_allocation(migration.target_couch_config, db_name) for db_name in sorted(get_db_list(migration.target_couch_config.get_control_node())) ]) return 0
def clean(migration, ansible_context, skip_check, limit): diff_with_db = diff_plan(migration) if diff_with_db: puts(color_warning("Current plan differs with database:\n")) puts("{}\n\n".format(diff_with_db)) puts( color_notice( "This could mean that the plan hasn't been committed yet\n" "or that the plan was re-generated.\n" "Performing the 'clean' operation is still safe but may\n" "not have the outcome you are expecting.\n")) if not ask("Do you wish to continue?"): puts(color_error('Abort.')) return 0 alloc_docs_by_db = get_db_allocations(migration.target_couch_config) puts(color_summary("Checking shards on disk vs DB. Please wait.")) if not assert_files(migration, alloc_docs_by_db, ansible_context): puts(color_error("Not all couch files are accounted for. Aborting.")) return 1 nodes = generate_shard_prune_playbook(migration) if nodes: return run_ansible_playbook(migration.target_environment, migration.prune_playbook_path, ansible_context, skip_check=skip_check, limit=limit)
def run(self, args, unknown_args): if 'destroy' in unknown_args: puts(color_error("Refusing to run a terraform command containing the argument 'destroy'.")) puts(color_error("It's simply not worth the risk.")) exit(-1) environment = get_environment(args.env_name) run_dir = environment.paths.get_env_file_path('.generated-terraform') modules_dir = os.path.join(TERRAFORM_DIR, 'modules') modules_dest = os.path.join(run_dir, 'modules') if not os.path.isdir(run_dir): os.mkdir(run_dir) if not os.path.isdir(run_dir): os.mkdir(run_dir) if not (os.path.exists(modules_dest) and os.readlink(modules_dest) == modules_dir): os.symlink(modules_dir, modules_dest) if args.username != get_default_username(): print_help_message_about_the_commcare_cloud_default_username_env_var(args.username) key_name = args.username try: generate_terraform_entrypoint(environment, key_name, run_dir, apply_immediately=args.apply_immediately) except UnauthorizedUser as e: allowed_users = environment.users_config.dev_users.present puts(color_error( "Unauthorized user {}.\n\n" "Use COMMCARE_CLOUD_DEFAULT_USERNAME or --username to pass in one of the allowed ssh users:{}" .format(e.username, '\n - '.join([''] + allowed_users)))) return -1 if not args.skip_secrets and unknown_args and unknown_args[0] in ('plan', 'apply'): rds_password = ( environment.get_secret('POSTGRES_USERS.root.password') if environment.terraform_config.rds_instances else '' ) with open(os.path.join(run_dir, 'secrets.auto.tfvars'), 'w', encoding='utf-8') as f: print('rds_password = {}'.format(json.dumps(rds_password)), file=f) env_vars = {'AWS_PROFILE': aws_sign_in(environment)} all_env_vars = os.environ.copy() all_env_vars.update(env_vars) cmd_parts = ['terraform'] + unknown_args cmd = ' '.join(shlex_quote(arg) for arg in cmd_parts) print_command('cd {}; {} {}; cd -'.format( run_dir, ' '.join('{}={}'.format(key, value) for key, value in env_vars.items()), cmd, )) return subprocess.call(cmd, shell=True, env=all_env_vars, cwd=run_dir)
def check_branch(args): branch = git_branch() if branch is None: # not in a git repo if args.branch != 'master': puts(color_error("You are not in a git repo. To deploy, remove --branch={}".format(branch))) exit(-1) elif args.branch != branch: puts(color_error("You are not currently on the branch specified with the --branch tag. To deploy on " "this branch, use --branch={}, otherwise, change branches".format(branch))) exit(-1)
def _get_lines(proc): """Read lines from process stdout :returns tuple(error_message, lines) """ out = proc.stdout.read() if proc.returncode != 0: err = proc.stderr.read() return str(color_error('error: {}\n'.format(err))), [] elif not out: return str(color_error('timeout\n')), [] else: return None, out.splitlines()
def _get_lines(proc): """Read lines from process stdout :returns tuple(error_message: AnyStr, lines: [AnyStr]) To return a str, specify an encoding when creating the subprocess, otherwise bytes-like objects are returned """ out = proc.stdout.read() if proc.returncode != 0: err = proc.stderr.read() return str(color_error('error: {}\n'.format(err))), [] elif not out: return str(color_error('timeout\n')), [] else: return None, out.splitlines()
def ansible_playbook(environment, playbook, *cmd_args): if os.path.isabs(playbook): playbook_path = playbook else: playbook_path = os.path.join( ANSIBLE_DIR, '{playbook}'.format(playbook=playbook)) cmd_parts = ( 'ansible-playbook', playbook_path, '-i', environment.paths.inventory_source, '-e', '@{}'.format(environment.paths.vault_yml), '-e', '@{}'.format(environment.paths.public_yml), '-e', '@{}'.format(environment.paths.generated_yml), '--diff', ) + get_limit() + cmd_args public_vars = environment.public_vars cmd_parts += get_user_arg(public_vars, unknown_args, use_factory_auth) if has_arg(unknown_args, '-D', '--diff') or has_arg( unknown_args, '-C', '--check'): puts( color_error("Options --diff and --check not allowed. " "Please remove -D, --diff, -C, --check.")) puts( color_error( "These ansible-playbook options are managed automatically " "by commcare-cloud and cannot be set manually.")) return 2 # exit code ask_vault_pass = public_vars.get('commcare_cloud_use_vault', True) if ask_vault_pass: cmd_parts += ('--vault-password-file={}/echo_vault_password.sh'. format(ANSIBLE_DIR), ) cmd_parts_with_common_ssh_args = get_common_ssh_args( environment, use_factory_auth=use_factory_auth) cmd_parts += cmd_parts_with_common_ssh_args cmd = ' '.join(shlex_quote(arg) for arg in cmd_parts) print_command(cmd) env_vars = ansible_context.env_vars if ask_vault_pass: env_vars[ 'ANSIBLE_VAULT_PASSWORD'] = environment.get_ansible_vault_password( ) return subprocess.call(cmd_parts, env=env_vars)
def get_dev_username(env_name): from .commands.terraform.aws import ( get_default_username, print_help_message_about_the_commcare_cloud_default_username_env_var, ) environment = get_environment(env_name) username = default_username = get_default_username() while True: if not username or default_username.is_guess: username = input(f"Enter your SSH username ({default_username}): ") if not username: username = default_username if username in environment.users_config.dev_users.present: break allowed_users = environment.users_config.dev_users.present env_users = '\n - '.join([''] + allowed_users) puts( color_error( f"Unauthorized user {username}.\n\n" f"Please pass in one of the allowed ssh users:{env_users}")) username = "" if default_username.is_guess: print_help_message_about_the_commcare_cloud_default_username_env_var( username) return username
def update_sentry_post_deploy(environment, sentry_project, github_repo, diff, deploy_start, deploy_end): localsettings = environment.public_vars["localsettings"] try: sentry_api_key = environment.get_secret('SENTRY_API_KEY') except KeyError: return client = SentryClient( sentry_api_key, localsettings.get('SENTRY_ORGANIZATION_SLUG', 'dimagi'), sentry_project) if client.is_valid(): # this must match the release name used in commcare-hq when configuring the Sentry API release_name = f"{environment.new_release_name()}-{environment.meta_config.env_monitoring_id}" if environment.fab_settings_config.generate_deploy_diffs: try: commits = get_release_commits(diff) except GithubException as e: commits = None print(color_error(f"Error getting release commits: {e}")) else: commits = None client.create_release(release_name, commits) client.create_deploy(release_name, environment.meta_config.env_monitoring_id, deploy_start, deploy_end)
def run(self, args, unknown_args): environment = get_environment(args.env_name) from_backend = all_secrets_backends_by_name[ args.from_backend].from_environment(environment) if args.to_backend: to_backend = all_secrets_backends_by_name[ args.to_backend].from_environment(environment) else: to_backend = environment.secrets_backend if from_backend.name == to_backend.name: puts( color_error( 'Refusing to copy from {from_backend.name} to {to_backend.name}: backends must differ' .format(from_backend=from_backend, to_backend=to_backend))) exit(-1) print("Copying data from {from_backend.name} to {to_backend.name}:". format(from_backend=from_backend, to_backend=to_backend)) for secret_spec in get_known_secret_specs(): try: secret_value = from_backend.get_secret(secret_spec.name) except KeyError: print("No value for {secret_spec.name}... Skipping".format( secret_spec=secret_spec)) continue to_backend.set_secret(secret_spec.name, secret_value) print("Copied value for {secret_spec.name}".format( secret_spec=secret_spec))
def call_commcare_cloud(input_argv=sys.argv): # throw error if user is attempting to use python 2 if not os.environ.get("TRAVIS_TEST") and sys.version_info[0] == 2: exit( dedent(""" Error: you must upgrade to Python 3. Python 2 is no longer supported. To setup Python 3.6, see https://dimagi.github.io/commcare-cloud/setup/installation.html """)) put_virtualenv_bin_on_the_path() parser, subparsers, commands = make_command_parser( available_envs=get_available_envs()) raw_args = input_argv[1:] if not raw_args: parser.print_help() return args, unknown_args = parser.parse_known_args(raw_args) if args.control: run_on_control_instead(args, input_argv) try: exit_code = commands[args.command].run(args, unknown_args) except CommandError as e: puts(color_error(str(e), bold=True)) return 1 return exit_code
def run(self, args, unknown_args): environment = get_environment(args.env_name) try: environment.check() except Exception: puts(color_error(u"✗ The environment has the following error:")) raise else: puts(color_success(u"✓ The environment configuration is valid."))
def get_diff_context(self): context = { "new_version_details": self.new_version_details, "user": get_default_username(), "LABELS_TO_EXPAND": LABELS_TO_EXPAND, "errors": [], "warnings": [] } if self.deployed_commit_matches_latest_commit: context["errors"].append( "Versions are identical. No changes since last deploy.") return context if not (self.current_commit and self.deploy_commit): context["warnings"].append("Insufficient info to get deploy diff.") return context context["compare_url"] = self.url if not self.generate_diff: disabled_msg = "Deploy diffs disabled for this environment." print(color_warning(disabled_msg)) context["warnings"].append(disabled_msg) return context if not self.repo.permissions: # using unauthenticated API calls, skip diff creation to avoid hitting rate limits print( color_warning( "Diff generation skipped. Supply a Github token to see deploy diffs." )) context["warnings"].append("Diff omitted.") return context try: pr_numbers = self._get_pr_numbers() except GithubException as e: print(color_error(f"Error getting diff commits: {e}")) context["warnings"].append( "There was an error fetching the PRs since the last deploy.") return context if len(pr_numbers) > 500: context["warnings"].append("There are too many PRs to display.") return context elif not pr_numbers: context["warnings"].append("No PRs merged since last release.") return context pool = Pool(5) pr_infos = [_f for _f in pool.map(self._get_pr_info, pr_numbers) if _f] context["pr_infos"] = pr_infos prs_by_label = self._get_prs_by_label(pr_infos) context["prs_by_label"] = prs_by_label return context
def ansible_playbook(environment, playbook, *cmd_args): if os.path.isabs(playbook): playbook_path = playbook else: playbook_path = os.path.join( ANSIBLE_DIR, '{playbook}'.format(playbook=playbook)) cmd_parts = ( 'ansible-playbook', playbook_path, '-i', environment.paths.inventory_source, '-e', '@{}'.format(environment.paths.public_yml), '-e', '@{}'.format(environment.paths.generated_yml), '--diff', ) + get_limit() + cmd_args public_vars = environment.public_vars env_vars = ansible_context.env_vars cmd_parts += get_user_arg(public_vars, unknown_args, use_factory_auth) if has_arg(unknown_args, '-D', '--diff') or has_arg( unknown_args, '-C', '--check'): puts( color_error("Options --diff and --check not allowed. " "Please remove -D, --diff, -C, --check.")) puts( color_error( "These ansible-playbook options are managed automatically " "by commcare-cloud and cannot be set manually.")) return 2 # exit code cmd_parts += environment.secrets_backend.get_extra_ansible_args() cmd_parts_with_common_ssh_args = get_common_ssh_args( environment, use_factory_auth=use_factory_auth) cmd_parts += cmd_parts_with_common_ssh_args cmd = ' '.join(shlex_quote(arg) for arg in cmd_parts) print_command(cmd) env_vars.update( environment.secrets_backend.get_extra_ansible_env_vars()) return subprocess.call(cmd_parts, env=env_vars)
def create_release_tag(environment, repo, diff): if environment.fab_settings_config.tag_deploy_commits: try: repo.create_git_ref( ref='refs/tags/{}-{}-deploy'.format( environment.new_release_name(), environment.name), sha=diff.deploy_commit, ) except GithubException as e: print(color_error(f"Error creating release tag: {e}"))
def call_commcare_cloud(input_argv=sys.argv): put_virtualenv_bin_on_the_path() parser, subparsers, commands = make_command_parser(available_envs=get_available_envs()) args, unknown_args = parser.parse_known_args(input_argv[1:]) if args.control: run_on_control_instead(args, input_argv) try: exit_code = commands[args.command].run(args, unknown_args) except CommandError as e: puts(color_error(str(e), bold=True)) return 1 return exit_code
def check_branch(args): branch = git_branch() if branch is None: # not in a git repo if args.branch != 'master': puts( color_error( "You are not in a git repo. To deploy, remove --branch={}". format(branch))) exit(-1) elif args.branch != branch: puts( color_error( "You are not currently on the branch specified with the --branch tag. To deploy on " "this branch, use --branch={}, otherwise, change branches". format(branch))) exit(-1) elif branch == 'master' and not any( a.startswith("--branch") for a in sys.argv): require_clean_working_tree() local_rev = git("rev-parse", branch).strip() remote = "https://github.com/dimagi/commcare-cloud.git" remote_rev = git("ls-remote", remote, "refs/heads/master").split(maxsplit=1)[0] if local_rev != remote_rev: print("Your local 'master' branch has diverged from upstream") print(f"local: {local_rev}") print(f"remote: {remote_rev}") print("") print("Run 'git pull' to get the latest code, then try again.") print( "Create a pull request if you have new commits to contribute.") print( "Or add --branch=master to use your local branch (not recommended)." ) exit(-1)
def run(self, action, host_pattern=None, process_pattern=None): if action == 'help': self.print_help() return 0 elif action == 'logs': print("Logs can be found at:\n{}".format(self.log_location.format(env=self.environment.name))) return 0 try: return self.execute_action(action, host_pattern, process_pattern) except NoHostsMatch: only = limit = '' if process_pattern: only = " '--only={}'".format(process_pattern) if host_pattern: limit = " '--limit={}'".format(host_pattern) puts(color_error("No '{}' hosts match{}{}".format(self.name, limit, only))) return 1
def _check_username(env_name, username, message): default_username = username allowed_users = ["ansible"] + get_environment(env_name).users_config.dev_users.present while True: if not username or default_username.is_guess: username = input(f"Enter your SSH username ({default_username}): ") if not username: username = default_username if username in allowed_users: break env_users = '\n - '.join([''] + allowed_users) puts(color_error( f"Unauthorized user {username}.\n\n" f"Please pass in one of the allowed ssh users:{env_users}" )) username = "" if default_username.is_guess: print(color_notice(message.format(username=username))) return username
def _get_pr_info(self, pr_number): try: pr_response = self.repo.get_pull(pr_number) except GithubException as e: print( color_error(f"Error getting PR details for {pr_number}: {e}")) return None if not pr_response.number: # Likely rate limited by Github API return None assert pr_number == pr_response.number, (pr_number, pr_response.number) return { 'number': pr_response.number, 'title': pr_response.title, 'url': pr_response.html_url, 'labels': pr_response.labels, 'additions': pr_response.additions, 'deletions': pr_response.deletions, 'opened_by': pr_response.user.login, 'body': pr_response.body, }
def run(self, args, unknown_args): j2 = jinja2.Environment(loader=jinja2.FileSystemLoader( os.path.dirname(__file__)), keep_trailing_newline=True) changelog_dir = 'changelog' for filename in _sort_files(changelog_dir): if filename.endswith(".yml"): last_log = filename break else: puts( color_error( "Unable to find last changelog file. Please create a changelog manually." )) return 1 last_index = int(re.search(r"^(\d+)", last_log).group()) name = args.name date = datetime.utcnow() if not name: name = "auto {}".format(date.strftime("%Y%m%d_%H%M")) name = re.sub("[\n\r\t]", " ", name) key = name.replace(" ", "_") file_name = "{:04d}-{}.yml".format(last_index + 1, key) template = j2.get_template('changelog-template.yml.j2') path = os.path.join(changelog_dir, file_name) with open(path, 'w') as f: f.write( template.render(name=name, key=key, date=date.strftime("%Y-%m-%d")).rstrip()) print("Changelog created at {}".format(path))
def run(self, args, unknown_args): limit = args.limit environment = get_environment(args.env_name) if limit: environment.inventory_manager.subset(limit) with open(environment.paths.known_hosts, 'r', encoding='utf-8') as known_hosts: original_keys_by_host = _get_host_key_map( [line.strip() for line in known_hosts.readlines()] ) procs = {} for hostname in environment.inventory_hostname_map: port = '22' if ':' in hostname: hostname, port = hostname.split(':') cmd = 'ssh-keyscan -T 10 -p {port} {hostname},$(dig +short {hostname})'.format( hostname=hostname, port=port ) procs[hostname] = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) lines = [] error_hosts = set() for hostname, proc in procs.items(): sys.stdout.write('[{}]: '.format(hostname)) proc.wait() error, host_lines = _get_lines(proc) if error: sys.stdout.write(error) else: sys.stdout.write(str(color_success('fetched key\n'))) lines.extend(host_lines) updated_keys_by_host = _get_host_key_map(lines) all_keys = set(original_keys_by_host) | set(updated_keys_by_host) lines = [] for host_key_type in sorted(all_keys): host, key_type = host_key_type original = original_keys_by_host.pop(host_key_type, None) updated = updated_keys_by_host.get(host_key_type, None) if updated and original: if updated != original: print(color_changed('Updating key: {} {}'.format(*host_key_type))) elif updated: print(color_added('Adding key: {} {}'.format(*host_key_type))) elif original: if limit or host in error_hosts: # if we're limiting or there was an error keep original key updated = original else: print(color_removed('Removing key: {} {}'.format(*host_key_type))) if updated: lines.append('{} {} {}'.format(host, key_type, updated)) with open(environment.paths.known_hosts, 'w', encoding='utf-8') as known_hosts: known_hosts.write('\n'.join(sorted(lines))) try: environment.check_known_hosts() except EnvironmentException as e: print(color_error(str(e))) return 1 return 0