def run(self, args, unknown_args): check_branch(args) environment = get_environment(args.env_name) deploy_revs = self._confirm_deploy_revs(environment, args, quiet=args.quiet) deploy_component = args.component if deploy_component == None: deploy_component = 'both' if environment.meta_config.always_deploy_formplayer else 'commcare' if deploy_component in ['commcare', 'both']: print(color_summary("You are about to deploy the following code:")) with indent(): for name, rev in deploy_revs: print(color_summary("{}: {}".format(name, rev))) if ask('Continue with deploy?', quiet=args.quiet): if deploy_component != 'both': _warn_no_formplayer() self.deploy_commcare(environment, deploy_revs, args, unknown_args) if deploy_component in ['formplayer', 'both']: if deploy_component != 'both': if args.commcare_rev: print(color_warning('--commcare-rev does not apply to a formplayer deploy and will be ignored')) if args.fab_settings: print(color_warning('--set does not apply to a formplayer deploy and will be ignored')) self._announce_formplayer_deploy_start(environment) self.deploy_formplayer(environment, args, unknown_args)
def run(self, args, unknown_args): check_branch(args) environment = get_environment(args.env_name) deploy_component = args.component if deploy_component is None: deploy_component = 'both' if environment.meta_config.always_deploy_formplayer else 'commcare' if deploy_component in ['commcare', 'both']: if deploy_component != 'both': _warn_no_formplayer() return deploy_commcare(environment, args, unknown_args) if deploy_component in ['formplayer', 'both']: if deploy_component != 'both': if args.commcare_rev: print( color_warning( '--commcare-rev does not apply to a formplayer deploy and will be ignored' )) if args.fab_settings: print( color_warning( '--set does not apply to a formplayer deploy and will be ignored' )) return deploy_formplayer(environment, args)
def run(self, args, unknown_args): check_branch(args) environment = get_environment(args.env_name) if args.resume: try: # use cached env to ensure consistency with last deploy cached_fab_env = retrieve_cached_deploy_env(environment.deploy_env) except Exception: print(color_error('Unable to resume deploy, please start anew')) else: environment = cached_fab_env.ccc_environment deploy_component = args.component if not deploy_component: deploy_component = ['commcare'] if environment.meta_config.always_deploy_formplayer: deploy_component.append('formplayer') rc = 0 if 'commcare' in deploy_component: if 'formplayer' not in deploy_component: _warn_no_formplayer() rc = deploy_commcare(environment, args, unknown_args) if 'formplayer' in deploy_component: if 'commcare' not in deploy_component: if args.commcare_rev: print(color_warning('--commcare-rev does not apply to a formplayer deploy and will be ignored')) if args.fab_settings: print(color_warning('--set does not apply to a formplayer deploy and will be ignored')) if rc: print(color_error("Skipping formplayer because commcare failed")) else: rc = deploy_formplayer(environment, args) return rc
def get_diff_context(self): context = { "new_version_details": self.new_version_details, "user": get_default_username(), "LABELS_TO_EXPAND": LABELS_TO_EXPAND, "errors": [], "warnings": [] } if self.deployed_commit_matches_latest_commit: context["errors"].append( "Versions are identical. No changes since last deploy.") return context if not (self.current_commit and self.deploy_commit): context["warnings"].append("Insufficient info to get deploy diff.") return context context["compare_url"] = self.url if not self.generate_diff: disabled_msg = "Deploy diffs disabled for this environment." print(color_warning(disabled_msg)) context["warnings"].append(disabled_msg) return context if not self.repo.permissions: # using unauthenticated API calls, skip diff creation to avoid hitting rate limits print( color_warning( "Diff generation skipped. Supply a Github token to see deploy diffs." )) context["warnings"].append("Diff omitted.") return context try: pr_numbers = self._get_pr_numbers() except GithubException as e: print(color_error(f"Error getting diff commits: {e}")) context["warnings"].append( "There was an error fetching the PRs since the last deploy.") return context if len(pr_numbers) > 500: context["warnings"].append("There are too many PRs to display.") return context elif not pr_numbers: context["warnings"].append("No PRs merged since last release.") return context pool = Pool(5) pr_infos = [_f for _f in pool.map(self._get_pr_info, pr_numbers) if _f] context["pr_infos"] = pr_infos prs_by_label = self._get_prs_by_label(pr_infos) context["prs_by_label"] = prs_by_label return context
def clean(migration, ansible_context, skip_check, limit): diff_with_db = diff_plan(migration) if diff_with_db: puts(color_warning("Current plan differs with database:\n")) puts("{}\n\n".format(diff_with_db)) puts( color_notice( "This could mean that the plan hasn't been committed yet\n" "or that the plan was re-generated.\n" "Performing the 'clean' operation is still safe but may\n" "not have the outcome you are expecting.\n")) if not ask("Do you wish to continue?"): puts(color_error('Abort.')) return 0 alloc_docs_by_db = get_db_allocations(migration.target_couch_config) puts(color_summary("Checking shards on disk vs DB. Please wait.")) if not assert_files(migration, alloc_docs_by_db, ansible_context): puts(color_error("Not all couch files are accounted for. Aborting.")) return 1 nodes = generate_shard_prune_playbook(migration) if nodes: return run_ansible_playbook(migration.target_environment, migration.prune_playbook_path, ansible_context, skip_check=skip_check, limit=limit)
def wrap(cls, data): if 'encrypted' in data: puts( color_warning( 'Warning! The "encrypted" option on block_device is experimental ' 'and not well-integrated into provisioning scripts.')) return super(BlockDevice, cls).wrap(data)
def get_latest_formplayer_version(env_name): """Get version info of latest available version. This fetches meta files from S3 and parses them to get the data. """ def get_url_content(url): res = requests.get(url) res.raise_for_status() return res.text def extract_vals_from_property_data(data, mapping): out = {} for line in data.splitlines(keepends=False): if not line.strip(): continue key, value = line.strip().split("=") if key in mapping: out[mapping[key]] = strip_escapes(value) return out git_info_url, build_info_url = get_info_urls(env_name) try: git_info = get_url_content(git_info_url) build_info = get_url_content(build_info_url) except RequestException as e: print(color_warning(f"Error getting latest formplayer version: {e}")) return git_data = extract_vals_from_property_data(git_info, { "git.commit.id": "commit", "git.commit.message.short": "message", "git.commit.time": "time" }) build_data = extract_vals_from_property_data(build_info, {"build.time": "build_time"}) return VersionInfo(**git_data, **build_data)
def notify_slack_deploy_end(environment, context, is_success): try: client = SlackClient(environment) except SlackException: return try: client.send_deploy_end_message(context, is_success) except SlackException as e: puts(color_warning(f"Error sending Slack notification: {e}"))
def patch_environ(): if 'ANSIBLE_CONFIG' not in os.environ: from commcare_cloud.environment.paths import ANSIBLE_DIR constants_module = 'ansible.constants' if constants_module in sys.modules: puts(color_warning( "\nSettings in 'ansible.cfg' have not been applied. " "'ANSIBLE_CONFIG' environment variable must be set before the '{}' module is imported.\n" ).format(constants_module)) os.environ['ANSIBLE_CONFIG'] = os.path.join(ANSIBLE_DIR, 'ansible.cfg')
def get_current_formplayer_version(environment): """Get version of currently deployed Formplayer by querying the Formplayer management endpoint to get the build info. """ formplayer0 = environment.groups["formplayer"][0] try: res = requests.get(f"http://{formplayer0}:8081/info", timeout=5) res.raise_for_status() except RequestException as e: print(color_warning(f"Error getting current formplayer version: {e}")) return info = res.json() return info.get("git", {}).get("commit", {}).get("id", None)
def run(self, args, unknown_args): check_branch(args) environment = get_environment(args.env_name) commcare_rev = self._confirm_commcare_rev(environment, args.commcare_rev, quiet=args.quiet) deploy_component = args.component if deploy_component == None: deploy_component = 'both' if environment.meta_config.always_deploy_formplayer else 'commcare' if deploy_component in ['commcare', 'both']: print(color_summary("You are about to deploy commcare from {}".format(commcare_rev))) if ask('Deploy commcare?', quiet=args.quiet): if deploy_component != 'both': _warn_no_formplayer() self.deploy_commcare(environment, commcare_rev, args, unknown_args) if deploy_component in ['formplayer', 'both']: if deploy_component != 'both': if args.commcare_rev: print(color_warning('--commcare-rev does not apply to a formplayer deploy and will be ignored')) if args.fab_settings: print(color_warning('--set does not apply to a formplayer deploy and will be ignored')) self._announce_formplayer_deploy_start(environment) self.deploy_formplayer(environment, args, unknown_args)
def filter_out_deprecated_pillows(environment, pillows): deprecated_pillows = ['GeographyFluffPillow', 'FarmerRecordFluffPillow'] good_pillows = {} bad_pillows = set() for host, pillow_configs in pillows.items(): good_pillows[host] = {} for pillow_name, pillow_config in pillow_configs.items(): if pillow_name not in deprecated_pillows: good_pillows[host][pillow_name] = pillow_config else: bad_pillows.add(pillow_name) if bad_pillows: puts( color_warning( 'This environment references deprecated pillow(s):\n')) with indent(): for pillow_name in sorted(bad_pillows): puts(color_warning('- {}'.format(pillow_name))) puts( color_warning( '\nThis pillows are unused and no longer needed.\n' 'To get rid of this warning, remove those pillows from {}'. format(environment.paths.app_processes_yml))) return good_pillows
def get_github_credentials(repo_name, repo_is_private, require_write_permissions): global GITHUB_TOKEN token, found_in_legacy_location = get_github_credentials_no_prompt() if found_in_legacy_location: print(color_notice(f"[Deprecation Warning] Config file has moved.")) print( color_notice( f"New location is {PROJECT_ROOT}/config.py or else use the " f"'GITHUB_TOKEN' environment variable.")) print( color_notice( f"\nYou can move the config to the new location as follows:")) print( color_notice( f" $ mv {PROJECT_ROOT}/fab/config.py {PROJECT_ROOT}/config.py\n" )) if token is None: print(color_warning("Github credentials not found!")) private = "private " if repo_is_private else "" print(f"Github token is required for {private}repository {repo_name}.") if require_write_permissions: print( "The token must have write permissions to the repository to create release tags." ) print( "\nYou can add a config file to automate this step:\n" f" $ cp {PROJECT_ROOT}/config.example.py {PROJECT_ROOT}/config.py\n" f"Then edit {PROJECT_ROOT}/config.py") print( color_notice( "To generate a GitHub access token, follow these instructions: https://github.com/blog/1509-personal-api-tokens\n" "For permissions choose repo > public_repo")) token = getpass('Github Token: ') os.environ["GITHUB_TOKEN"] = token # set in env for access by subprocesses GITHUB_TOKEN = token return token or None
def run(self, args, unknown_args): config = get_config(args.config) keys_to_update = args.update_key or UPDATE_KEYS initialize_datadog(config) remote_monitor_api = RemoteMonitorAPI(filtered_ids=args.monitors) local_monitor_api = LocalMonitorAPI(config, filtered_ids=args.monitors) local_monitors = local_monitor_api.get_filtered() remote_monitors = remote_monitor_api.get_filtered() only_remote = { id: remote_monitors[id] for id in set(remote_monitors) - set(local_monitors) } only_local = { id: local_monitors[id] for id in set(local_monitors) - set(remote_monitors) } shared_local_remote_monitors = { id: (local_monitors[id], remote_monitors[id]) for id in set(local_monitors) & set(remote_monitors) } monitors_with_diffs = {} any_diffs = False if only_local: for id, monitor in only_local.items(): puts( color_warning( "\nMonitor missing from datadog: {} ({})\n".format( monitor['name'], id))) for id, (expected, actual) in shared_local_remote_monitors.items(): diff = list( _unidiff_output( dump_monitor_yaml( get_data_to_update(actual, keys_to_update)), dump_monitor_yaml( get_data_to_update(expected, keys_to_update)))) any_diffs |= bool(diff) if diff: puts(color_notice("\nDiff for '{}'".format(expected['name']))) puts(local_monitor_api.get_filename_for_monitor( expected['id'])) with indent(): print_diff(diff) monitors_with_diffs[id] = expected if any_diffs: if ask("Do you want to push these changes to Datadog?"): for id, expected in monitors_with_diffs.items(): print("Updating '{}'".format(expected['name'])) remote_monitor_api.update( id, get_data_to_update(expected, keys_to_update)) if only_remote: puts( color_warning("FYI you also have some untracked monitors. " "No change will be applied for these:")) for id, missing_monitor in sorted(only_remote.items()): puts( " - Untracked monitor {} '{}' (no change will be applied)" .format(id, missing_monitor['name'])) if ask("And BTW do you want to dump all untracked monitors as a starting point?" ): for id, missing_monitor in sorted(only_remote.items()): local_monitor_api.create(id, missing_monitor)