def run(self, args, unknown_args):
check_branch(args)
environment = get_environment(args.env_name)
deploy_revs = self._confirm_deploy_revs(environment, args, quiet=args.quiet)
deploy_component = args.component
if deploy_component == None:
deploy_component = 'both' if environment.meta_config.always_deploy_formplayer else 'commcare'
if deploy_component in ['commcare', 'both']:
print(color_summary("You are about to deploy the following code:"))
with indent():
for name, rev in deploy_revs:
print(color_summary("{}: {}".format(name, rev)))
if ask('Continue with deploy?', quiet=args.quiet):
if deploy_component != 'both':
_warn_no_formplayer()
self.deploy_commcare(environment, deploy_revs, args, unknown_args)
if deploy_component in ['formplayer', 'both']:
if deploy_component != 'both':
if args.commcare_rev:
print(color_warning('--commcare-rev does not apply to a formplayer deploy and will be ignored'))
if args.fab_settings:
print(color_warning('--set does not apply to a formplayer deploy and will be ignored'))
self._announce_formplayer_deploy_start(environment)
self.deploy_formplayer(environment, args, unknown_args)
def run(self, args, unknown_args):
check_branch(args)
environment = get_environment(args.env_name)
deploy_component = args.component
if deploy_component is None:
deploy_component = 'both' if environment.meta_config.always_deploy_formplayer else 'commcare'
if deploy_component in ['commcare', 'both']:
if deploy_component != 'both':
_warn_no_formplayer()
return deploy_commcare(environment, args, unknown_args)
if deploy_component in ['formplayer', 'both']:
if deploy_component != 'both':
if args.commcare_rev:
print(
color_warning(
'--commcare-rev does not apply to a formplayer deploy and will be ignored'
))
if args.fab_settings:
print(
color_warning(
'--set does not apply to a formplayer deploy and will be ignored'
))
return deploy_formplayer(environment, args)
def run(self, args, unknown_args):
check_branch(args)
environment = get_environment(args.env_name)
if args.resume:
try:
# use cached env to ensure consistency with last deploy
cached_fab_env = retrieve_cached_deploy_env(environment.deploy_env)
except Exception:
print(color_error('Unable to resume deploy, please start anew'))
else:
environment = cached_fab_env.ccc_environment
deploy_component = args.component
if not deploy_component:
deploy_component = ['commcare']
if environment.meta_config.always_deploy_formplayer:
deploy_component.append('formplayer')
rc = 0
if 'commcare' in deploy_component:
if 'formplayer' not in deploy_component:
_warn_no_formplayer()
rc = deploy_commcare(environment, args, unknown_args)
if 'formplayer' in deploy_component:
if 'commcare' not in deploy_component:
if args.commcare_rev:
print(color_warning('--commcare-rev does not apply to a formplayer deploy and will be ignored'))
if args.fab_settings:
print(color_warning('--set does not apply to a formplayer deploy and will be ignored'))
if rc:
print(color_error("Skipping formplayer because commcare failed"))
else:
rc = deploy_formplayer(environment, args)
return rc
def get_diff_context(self):
context = {
"new_version_details": self.new_version_details,
"user": get_default_username(),
"LABELS_TO_EXPAND": LABELS_TO_EXPAND,
"errors": [],
"warnings": []
}
if self.deployed_commit_matches_latest_commit:
context["errors"].append(
"Versions are identical. No changes since last deploy.")
return context
if not (self.current_commit and self.deploy_commit):
context["warnings"].append("Insufficient info to get deploy diff.")
return context
context["compare_url"] = self.url
if not self.generate_diff:
disabled_msg = "Deploy diffs disabled for this environment."
print(color_warning(disabled_msg))
context["warnings"].append(disabled_msg)
return context
if not self.repo.permissions:
# using unauthenticated API calls, skip diff creation to avoid hitting rate limits
print(
color_warning(
"Diff generation skipped. Supply a Github token to see deploy diffs."
))
context["warnings"].append("Diff omitted.")
return context
try:
pr_numbers = self._get_pr_numbers()
except GithubException as e:
print(color_error(f"Error getting diff commits: {e}"))
context["warnings"].append(
"There was an error fetching the PRs since the last deploy.")
return context
if len(pr_numbers) > 500:
context["warnings"].append("There are too many PRs to display.")
return context
elif not pr_numbers:
context["warnings"].append("No PRs merged since last release.")
return context
pool = Pool(5)
pr_infos = [_f for _f in pool.map(self._get_pr_info, pr_numbers) if _f]
context["pr_infos"] = pr_infos
prs_by_label = self._get_prs_by_label(pr_infos)
context["prs_by_label"] = prs_by_label
return context
def clean(migration, ansible_context, skip_check, limit):
diff_with_db = diff_plan(migration)
if diff_with_db:
puts(color_warning("Current plan differs with database:\n"))
puts("{}\n\n".format(diff_with_db))
puts(
color_notice(
"This could mean that the plan hasn't been committed yet\n"
"or that the plan was re-generated.\n"
"Performing the 'clean' operation is still safe but may\n"
"not have the outcome you are expecting.\n"))
if not ask("Do you wish to continue?"):
puts(color_error('Abort.'))
return 0
alloc_docs_by_db = get_db_allocations(migration.target_couch_config)
puts(color_summary("Checking shards on disk vs DB. Please wait."))
if not assert_files(migration, alloc_docs_by_db, ansible_context):
puts(color_error("Not all couch files are accounted for. Aborting."))
return 1
nodes = generate_shard_prune_playbook(migration)
if nodes:
return run_ansible_playbook(migration.target_environment,
migration.prune_playbook_path,
ansible_context,
skip_check=skip_check,
limit=limit)
def wrap(cls, data):
if 'encrypted' in data:
puts(
color_warning(
'Warning! The "encrypted" option on block_device is experimental '
'and not well-integrated into provisioning scripts.'))
return super(BlockDevice, cls).wrap(data)
def get_latest_formplayer_version(env_name):
"""Get version info of latest available version. This fetches
meta files from S3 and parses them to get the data.
"""
def get_url_content(url):
res = requests.get(url)
res.raise_for_status()
return res.text
def extract_vals_from_property_data(data, mapping):
out = {}
for line in data.splitlines(keepends=False):
if not line.strip():
continue
key, value = line.strip().split("=")
if key in mapping:
out[mapping[key]] = strip_escapes(value)
return out
git_info_url, build_info_url = get_info_urls(env_name)
try:
git_info = get_url_content(git_info_url)
build_info = get_url_content(build_info_url)
except RequestException as e:
print(color_warning(f"Error getting latest formplayer version: {e}"))
return
git_data = extract_vals_from_property_data(git_info, {
"git.commit.id": "commit",
"git.commit.message.short": "message",
"git.commit.time": "time"
})
build_data = extract_vals_from_property_data(build_info, {"build.time": "build_time"})
return VersionInfo(**git_data, **build_data)
def notify_slack_deploy_end(environment, context, is_success):
try:
client = SlackClient(environment)
except SlackException:
return
try:
client.send_deploy_end_message(context, is_success)
except SlackException as e:
puts(color_warning(f"Error sending Slack notification: {e}"))
def patch_environ():
if 'ANSIBLE_CONFIG' not in os.environ:
from commcare_cloud.environment.paths import ANSIBLE_DIR
constants_module = 'ansible.constants'
if constants_module in sys.modules:
puts(color_warning(
"\nSettings in 'ansible.cfg' have not been applied. "
"'ANSIBLE_CONFIG' environment variable must be set before the '{}' module is imported.\n"
).format(constants_module))
os.environ['ANSIBLE_CONFIG'] = os.path.join(ANSIBLE_DIR, 'ansible.cfg')
def get_current_formplayer_version(environment):
"""Get version of currently deployed Formplayer by querying
the Formplayer management endpoint to get the build info.
"""
formplayer0 = environment.groups["formplayer"][0]
try:
res = requests.get(f"http://{formplayer0}:8081/info", timeout=5)
res.raise_for_status()
except RequestException as e:
print(color_warning(f"Error getting current formplayer version: {e}"))
return
info = res.json()
return info.get("git", {}).get("commit", {}).get("id", None)
def run(self, args, unknown_args):
check_branch(args)
environment = get_environment(args.env_name)
commcare_rev = self._confirm_commcare_rev(environment, args.commcare_rev, quiet=args.quiet)
deploy_component = args.component
if deploy_component == None:
deploy_component = 'both' if environment.meta_config.always_deploy_formplayer else 'commcare'
if deploy_component in ['commcare', 'both']:
print(color_summary("You are about to deploy commcare from {}".format(commcare_rev)))
if ask('Deploy commcare?', quiet=args.quiet):
if deploy_component != 'both':
_warn_no_formplayer()
self.deploy_commcare(environment, commcare_rev, args, unknown_args)
if deploy_component in ['formplayer', 'both']:
if deploy_component != 'both':
if args.commcare_rev:
print(color_warning('--commcare-rev does not apply to a formplayer deploy and will be ignored'))
if args.fab_settings:
print(color_warning('--set does not apply to a formplayer deploy and will be ignored'))
self._announce_formplayer_deploy_start(environment)
self.deploy_formplayer(environment, args, unknown_args)
def filter_out_deprecated_pillows(environment, pillows):
deprecated_pillows = ['GeographyFluffPillow', 'FarmerRecordFluffPillow']
good_pillows = {}
bad_pillows = set()
for host, pillow_configs in pillows.items():
good_pillows[host] = {}
for pillow_name, pillow_config in pillow_configs.items():
if pillow_name not in deprecated_pillows:
good_pillows[host][pillow_name] = pillow_config
else:
bad_pillows.add(pillow_name)
if bad_pillows:
puts(
color_warning(
'This environment references deprecated pillow(s):\n'))
with indent():
for pillow_name in sorted(bad_pillows):
puts(color_warning('- {}'.format(pillow_name)))
puts(
color_warning(
'\nThis pillows are unused and no longer needed.\n'
'To get rid of this warning, remove those pillows from {}'.
format(environment.paths.app_processes_yml)))
return good_pillows
def get_github_credentials(repo_name, repo_is_private,
require_write_permissions):
global GITHUB_TOKEN
token, found_in_legacy_location = get_github_credentials_no_prompt()
if found_in_legacy_location:
print(color_notice(f"[Deprecation Warning] Config file has moved."))
print(
color_notice(
f"New location is {PROJECT_ROOT}/config.py or else use the "
f"'GITHUB_TOKEN' environment variable."))
print(
color_notice(
f"\nYou can move the config to the new location as follows:"))
print(
color_notice(
f" $ mv {PROJECT_ROOT}/fab/config.py {PROJECT_ROOT}/config.py\n"
))
if token is None:
print(color_warning("Github credentials not found!"))
private = "private " if repo_is_private else ""
print(f"Github token is required for {private}repository {repo_name}.")
if require_write_permissions:
print(
"The token must have write permissions to the repository to create release tags."
)
print(
"\nYou can add a config file to automate this step:\n"
f" $ cp {PROJECT_ROOT}/config.example.py {PROJECT_ROOT}/config.py\n"
f"Then edit {PROJECT_ROOT}/config.py")
print(
color_notice(
"To generate a GitHub access token, follow these instructions: https://github.com/blog/1509-personal-api-tokens\n"
"For permissions choose repo > public_repo"))
token = getpass('Github Token: ')
os.environ["GITHUB_TOKEN"] = token # set in env for access by subprocesses
GITHUB_TOKEN = token
return token or None
def run(self, args, unknown_args):
config = get_config(args.config)
keys_to_update = args.update_key or UPDATE_KEYS
initialize_datadog(config)
remote_monitor_api = RemoteMonitorAPI(filtered_ids=args.monitors)
local_monitor_api = LocalMonitorAPI(config, filtered_ids=args.monitors)
local_monitors = local_monitor_api.get_filtered()
remote_monitors = remote_monitor_api.get_filtered()
only_remote = {
id: remote_monitors[id]
for id in set(remote_monitors) - set(local_monitors)
}
only_local = {
id: local_monitors[id]
for id in set(local_monitors) - set(remote_monitors)
}
shared_local_remote_monitors = {
id: (local_monitors[id], remote_monitors[id])
for id in set(local_monitors) & set(remote_monitors)
}
monitors_with_diffs = {}
any_diffs = False
if only_local:
for id, monitor in only_local.items():
puts(
color_warning(
"\nMonitor missing from datadog: {} ({})\n".format(
monitor['name'], id)))
for id, (expected, actual) in shared_local_remote_monitors.items():
diff = list(
_unidiff_output(
dump_monitor_yaml(
get_data_to_update(actual, keys_to_update)),
dump_monitor_yaml(
get_data_to_update(expected, keys_to_update))))
any_diffs |= bool(diff)
if diff:
puts(color_notice("\nDiff for '{}'".format(expected['name'])))
puts(local_monitor_api.get_filename_for_monitor(
expected['id']))
with indent():
print_diff(diff)
monitors_with_diffs[id] = expected
if any_diffs:
if ask("Do you want to push these changes to Datadog?"):
for id, expected in monitors_with_diffs.items():
print("Updating '{}'".format(expected['name']))
remote_monitor_api.update(
id, get_data_to_update(expected, keys_to_update))
if only_remote:
puts(
color_warning("FYI you also have some untracked monitors. "
"No change will be applied for these:"))
for id, missing_monitor in sorted(only_remote.items()):
puts(
" - Untracked monitor {} '{}' (no change will be applied)"
.format(id, missing_monitor['name']))
if ask("And BTW do you want to dump all untracked monitors as a starting point?"
):
for id, missing_monitor in sorted(only_remote.items()):
local_monitor_api.create(id, missing_monitor)
