def test_bounce_status():
with asynctest.patch(
"paasta_tools.instance.kubernetes.kubernetes_tools", autospec=True
) as mock_kubernetes_tools:
mock_config = mock_kubernetes_tools.load_kubernetes_service_config.return_value
mock_kubernetes_tools.get_kubernetes_app_deploy_status.return_value = (
"deploy_status",
"message",
)
mock_kubernetes_tools.get_active_versions_for_service.return_value = [
(DeploymentVersion("aaa", None), "config_aaa"),
(DeploymentVersion("bbb", None), "config_bbb"),
(DeploymentVersion("ccc", "extrastuff"), "config_ccc"),
]
mock_settings = mock.Mock()
status = pik.bounce_status("fake_service", "fake_instance", mock_settings)
assert status == {
"expected_instance_count": mock_config.get_instances.return_value,
"desired_state": mock_config.get_desired_state.return_value,
"running_instance_count": mock_kubernetes_tools.get_kubernetes_app_by_name.return_value.status.ready_replicas,
"deploy_status": mock_kubernetes_tools.KubernetesDeployStatus.tostring.return_value,
"active_shas": [
("aaa", "config_aaa"),
("bbb", "config_bbb"),
("ccc", "config_ccc"),
],
"active_versions": [
("aaa", None, "config_aaa"),
("bbb", None, "config_bbb"),
("ccc", "extrastuff", "config_ccc"),
],
"app_count": 3,
}
def test_paasta_rollback_mark_for_deployment_simple_invocation(
mock_can_user_deploy_service,
mock_get_versions_for_service,
mock_mark_for_deployment,
mock_get_git_url,
mock_figure_out_service_name,
mock_list_deploy_groups,
mock_log_audit,
mock_get_currently_deployed_version,
):
fake_args, _ = parse_args([
"rollback", "-s", "fakeservice", "-k", "abcd" * 10, "-l",
"fake_deploy_group1"
])
mock_get_versions_for_service.return_value = {
DeploymentVersion(sha=fake_args.commit, image_version=None): (
"20170403T025512",
fake_args.deploy_groups,
),
DeploymentVersion(sha="dcba" * 10, image_version=None): (
"20161006T025416",
"fake_deploy_group2",
),
}
mock_get_git_url.return_value = "git://git.repo"
mock_figure_out_service_name.return_value = fake_args.service
mock_list_deploy_groups.return_value = [fake_args.deploy_groups]
mock_mark_for_deployment.return_value = 0
mock_get_currently_deployed_version.return_value = DeploymentVersion(
sha="1234" * 10, image_version=None)
assert paasta_rollback(fake_args) == 0
mock_mark_for_deployment.assert_called_once_with(
git_url=mock_get_git_url.return_value,
deploy_group=fake_args.deploy_groups,
service=mock_figure_out_service_name.return_value,
commit=fake_args.commit,
image_version=None,
)
# ensure that we logged each deploy group that was rolled back AND that we logged things correctly
mock_log_audit.call_count == len(fake_args.deploy_groups)
for call_args in mock_log_audit.call_args_list:
_, call_kwargs = call_args
assert call_kwargs["action"] == "rollback"
assert call_kwargs["action_details"]["rolled_back_from"] == str(
mock_get_currently_deployed_version.return_value)
assert call_kwargs["action_details"][
"rolled_back_to"] == fake_args.commit
assert (call_kwargs["action_details"]["rollback_type"] ==
RollbackTypes.USER_INITIATED_ROLLBACK.value)
assert call_kwargs["action_details"][
"deploy_group"] in fake_args.deploy_groups
assert call_kwargs["service"] == fake_args.service
def test_paasta_rollback_mark_for_deployment_no_deploy_group_arg(
mock_can_user_deploy_service,
mock_get_versions_for_service,
mock_mark_for_deployment,
mock_get_git_url,
mock_figure_out_service_name,
mock_list_deploy_groups,
mock_log_audit,
mock_get_currently_deployed_version,
):
fake_args, _ = parse_args(
["rollback", "-s", "fakeservice", "-k", "abcd" * 10])
mock_get_versions_for_service.return_value = {
DeploymentVersion(sha="fake_sha1", image_version=None): (
"20170403T025512",
"fake_deploy_group1",
),
DeploymentVersion(sha=fake_args.commit, image_version=None): (
"20161006T025416",
"fake_deploy_group2",
),
}
mock_get_git_url.return_value = "git://git.repo"
mock_figure_out_service_name.return_value = fake_args.service
mock_list_deploy_groups.return_value = [
"fake_deploy_group",
"fake_cluster.fake_instance",
]
mock_mark_for_deployment.return_value = 0
mock_get_currently_deployed_version.return_value = DeploymentVersion(
sha="1234" * 10, image_version=None)
assert paasta_rollback(fake_args) == 1
assert mock_mark_for_deployment.call_count == 0
mock_log_audit.call_count == len(fake_args.deploy_groups)
for call_args in mock_log_audit.call_args_list:
_, call_kwargs = call_args
assert call_kwargs["action"] == "rollback"
assert call_kwargs["action_details"]["rolled_back_from"] == str(
mock_get_currently_deployed_version.return_value)
assert call_kwargs["action_details"][
"rolled_back_to"] == fake_args.commit
assert (call_kwargs["action_details"]["rollback_type"] ==
RollbackTypes.USER_INITIATED_ROLLBACK.value)
assert (call_kwargs["action_details"]["deploy_group"]
in mock_list_deploy_groups.return_value)
assert call_kwargs["service"] == fake_args.service
def test_compose_timeout_message():
remaining_instances = {
"cluster1": ["instance1", "instance2"],
"cluster2": ["instance3"],
"cluster3": [],
}
message = mark_for_deployment.compose_timeout_message(
remaining_instances,
1,
"fake_group",
"someservice",
DeploymentVersion(sha="some_git_sha", image_version="extrastuff"),
)
assert (" paasta status -c cluster1 -s someservice -i instance1,instance2"
in message)
assert " paasta status -c cluster2 -s someservice -i instance3" in message
assert (
" paasta logs -c cluster1 -s someservice -i instance1,instance2 -C deploy -l 1000"
in message)
assert (
" paasta logs -c cluster2 -s someservice -i instance3 -C deploy -l 1000"
in message)
assert (
" paasta wait-for-deployment -s someservice -l fake_group -c some_git_sha --image-version extrastuff"
in message)
def get_versions_for_service(
service: str, deploy_groups: Collection[str],
soa_dir: str) -> Mapping[DeploymentVersion, Tuple[str, str]]:
"""Returns a dictionary of 2-tuples of the form (timestamp, deploy_group) for each version tuple of (deploy sha, image_version)"""
if service is None:
return {}
git_url = get_git_url(service=service, soa_dir=soa_dir)
all_deploy_groups = list_deploy_groups(service=service, soa_dir=soa_dir)
deploy_groups, _ = validate_given_deploy_groups(all_deploy_groups,
deploy_groups)
previously_deployed_versions: Dict[DeploymentVersion, Tuple[str, str]] = {}
for ref, sha in list_remote_refs(git_url).items():
regex_match = extract_tags(ref)
try:
deploy_group = regex_match["deploy_group"]
tstamp = regex_match["tstamp"]
image_version = regex_match["image_version"]
except KeyError:
pass
else:
# Now we filter and dedup by picking the most recent sha for a deploy group
# Note that all strings are greater than ''
if deploy_group in deploy_groups:
version = DeploymentVersion(sha=sha,
image_version=image_version)
tstamp_so_far = previously_deployed_versions.get(
version, ("all", ""))[1]
if tstamp > tstamp_so_far:
previously_deployed_versions[version] = (tstamp,
deploy_group)
return previously_deployed_versions
def test_validate_deploy_group_when_is_git_not_available(
mock_list_remote_refs, capsys):
test_error_message = "Git error"
mock_list_remote_refs.side_effect = LSRemoteException(test_error_message)
assert (validate_version_is_latest(
DeploymentVersion(sha="fake sha", image_version=None),
"fake_git_url",
"fake_group",
"fake_service",
) is None)
def get_latest_marked_version(
git_url: str, deploy_group: str) -> Optional[DeploymentVersion]:
"""Return the latest marked for deployment version or None"""
# TODO: correct this function for new tag format
refs = list_remote_refs(git_url)
_, sha, image_version = get_latest_deployment_tag(refs, deploy_group)
if sha:
return DeploymentVersion(sha=sha, image_version=image_version)
# We did not find a ref for this deploy group
return None
def paasta_wait_for_deployment(args):
"""Wrapping wait_for_deployment"""
if args.verbose:
log.setLevel(level=logging.DEBUG)
else:
log.setLevel(level=logging.INFO)
service = args.service
if service and service.startswith("services-"):
service = service.split("services-", 1)[1]
if args.git_url is None:
args.git_url = get_git_url(service=service, soa_dir=args.soa_dir)
args.commit = validate_git_sha(sha=args.commit, git_url=args.git_url)
version = DeploymentVersion(sha=args.commit,
image_version=args.image_version)
try:
validate_service_name(service, soa_dir=args.soa_dir)
validate_deploy_group(args.deploy_group, service, args.soa_dir)
validate_version_is_latest(version, args.git_url, args.deploy_group,
service)
except (VersionError, DeployGroupError, NoSuchService) as e:
print(PaastaColors.red(f"{e}"))
return 1
try:
asyncio.run(
wait_for_deployment(
service=service,
deploy_group=args.deploy_group,
git_sha=args.commit,
image_version=args.image_version,
soa_dir=args.soa_dir,
timeout=args.timeout,
polling_interval=args.polling_interval,
diagnosis_interval=args.diagnosis_interval,
time_before_first_diagnosis=args.time_before_first_diagnosis,
))
_log(
service=service,
component="deploy",
line=(f"Deployment of {version} for {args.deploy_group} complete"),
level="event",
)
except (KeyboardInterrupt, TimeoutError, NoSuchCluster):
report_waiting_aborted(service, args.deploy_group)
return 1
return 0
def test_get_latest_marked_version_good(mock_list_remote_refs):
mock_list_remote_refs.return_value = {
"refs/tags/paasta-fake_group1-20161129T203750-deploy":
"968b948b3fca457326718dc7b2e278f89ccc5c87",
"refs/tags/paasta-fake_group1-20161117T122449-deploy":
"eac9a6d7909d09ffec00538bbc43b64502aa2dc0",
"refs/tags/paasta-fake_group2-20161125T095651-deploy":
"a4911648beb2e53886658ba7ea7eb93d582d754c",
"refs/tags/paasta-fake_group1.everywhere-20161109T223959-deploy":
"71e97ec397a3f0e7c4ee46e8ea1e2982cbcb0b79",
}
assert get_latest_marked_version("", "fake_group1") == DeploymentVersion(
sha="968b948b3fca457326718dc7b2e278f89ccc5c87", image_version=None)
def test_check_if_instance_is_done(mock_get_paasta_oapi_client, mock__log,
side_effect, expected):
mock_paasta_api_client = Mock()
mock_paasta_api_client.api_error = ApiException
mock_paasta_api_client.service.bounce_status_instance.side_effect = side_effect
mock_get_paasta_oapi_client.return_value = mock_paasta_api_client
assert expected == mark_for_deployment.check_if_instance_is_done(
service="fake_service",
instance="fake_instance",
cluster="fake_cluster",
version=DeploymentVersion(sha="abc123", image_version=None),
instance_config=mock_marathon_instance_config("fake_instance"),
)
def test_paasta_rollback_git_sha_was_not_marked_before(
mock_can_user_deploy_service,
mock_get_versions_for_service,
mock_mark_for_deployment,
mock_get_git_url,
mock_figure_out_service_name,
mock_list_deploy_groups,
mock_log_audit,
):
fake_args, _ = parse_args([
"rollback", "-s", "fakeservice", "-k", "abcd" * 10, "-l",
"fake_deploy_group1"
])
mock_get_versions_for_service.return_value = {
DeploymentVersion(sha="fake_sha1", image_version="fake_image"): (
"20170403T025512",
"fake_deploy_group1",
),
DeploymentVersion(sha="fake_sha2", image_version="fake_image"): (
"20161006T025416",
"fake_deploy_group2",
),
DeploymentVersion(sha=fake_args.commit, image_version="fake_image"): (
"20161006T025416",
"fake_deploy_group2",
),
}
mock_get_git_url.return_value = "git://git.repo"
mock_figure_out_service_name.return_value = fake_args.service
mock_list_deploy_groups.return_value = [fake_args.deploy_groups]
mock_mark_for_deployment.return_value = 0
assert paasta_rollback(fake_args) == 1
assert not mock_mark_for_deployment.called
assert not mock_log_audit.called
def get_active_versions_for_marathon_apps(
marathon_apps_with_clients: List[Tuple[MarathonApp, MarathonClient]],
) -> Set[Tuple[DeploymentVersion, str]]:
ret = set()
for (app, client) in marathon_apps_with_clients:
git_sha = get_git_sha_from_dockerurl(app.container.docker.image,
long=True)
image_version = get_image_version_from_dockerurl(
app.container.docker.image)
_, _, _, config_sha = marathon_tools.deformat_job_id(app.id)
if config_sha.startswith("config"):
config_sha = config_sha[len("config"):]
ret.add((DeploymentVersion(sha=git_sha,
image_version=image_version), config_sha))
return ret
def test_get_currently_deployed_version(mock_load_v2_deployments_json, ):
mock_load_v2_deployments_json.return_value = DeploymentsJsonV2(
service="fake-service",
config_dict={
"controls": {},
"deployments": {
"everything": {
"git_sha": "abc",
"docker_image": "foo",
"image_version": "extrastuff",
}
},
},
)
actual = deployment_utils.get_currently_deployed_version(
service="service", deploy_group="everything")
assert actual == DeploymentVersion(sha="abc", image_version="extrastuff")
def paasta_rollback(args: argparse.Namespace) -> int:
"""Call mark_for_deployment with rollback parameters
:param args: contains all the arguments passed onto the script: service,
deploy groups and sha. These arguments will be verified and passed onto
mark_for_deployment.
"""
soa_dir = args.soa_dir
service = figure_out_service_name(args, soa_dir)
deploy_info = get_deploy_info(service=service, soa_dir=args.soa_dir)
if not can_user_deploy_service(deploy_info, service):
return 1
git_url = get_git_url(service, soa_dir)
if args.all_deploy_groups:
given_deploy_groups = list_deploy_groups(service=service,
soa_dir=soa_dir)
else:
given_deploy_groups = {
deploy_group
for deploy_group in args.deploy_groups.split(",") if deploy_group
}
all_deploy_groups = list_deploy_groups(service=service, soa_dir=soa_dir)
deploy_groups, invalid = validate_given_deploy_groups(
all_deploy_groups, given_deploy_groups)
if len(invalid) > 0:
print(
PaastaColors.yellow(
"These deploy groups are not valid and will be skipped: %s.\n"
% (",").join(invalid)))
if len(deploy_groups) == 0 and not args.all_deploy_groups:
print(
PaastaColors.red(
"ERROR: No valid deploy groups specified for %s.\n Use the flag -a to rollback all valid deploy groups for this service"
% (service)))
return 1
versions = get_versions_for_service(service, deploy_groups, soa_dir)
commit = args.commit
image_version = args.image_version
new_version = DeploymentVersion(sha=commit, image_version=image_version)
if not commit:
print("Please specify a commit to mark for rollback (-k, --commit).")
list_previous_versions(service, deploy_groups,
bool(given_deploy_groups), versions)
return 1
elif new_version not in versions and not args.force:
print(
PaastaColors.red(
f"This version {new_version} has never been deployed before."))
print(
"Please double check it or use --force to skip this verification.\n"
)
list_previous_versions(service, deploy_groups,
bool(given_deploy_groups), versions)
return 1
# TODO: Add similar check for when image_version is empty and no-commit redeploys is enforced for requested deploy_group
returncode = 0
for deploy_group in deploy_groups:
rolled_back_from = get_currently_deployed_version(
service, deploy_group)
returncode |= mark_for_deployment(
git_url=git_url,
service=service,
deploy_group=deploy_group,
commit=commit,
image_version=image_version,
)
# we could also gate this by the return code from m-f-d, but we probably care more about someone wanting to
# rollback than we care about if the underlying machinery was successfully able to complete the request
if rolled_back_from != new_version:
audit_action_details = {
"rolled_back_from": str(rolled_back_from),
"rolled_back_to": str(new_version),
"rollback_type": RollbackTypes.USER_INITIATED_ROLLBACK.value,
"deploy_group": deploy_group,
}
_log_audit(action="rollback",
action_details=audit_action_details,
service=service)
return returncode
def test_paasta_rollback_mark_for_deployment_multiple_deploy_group_args(
mock_can_user_deploy_service,
mock_get_versions_for_service,
mock_mark_for_deployment,
mock_get_git_url,
mock_figure_out_service_name,
mock_list_deploy_groups,
mock_log_audit,
mock_get_currently_deployed_version,
):
fake_args, _ = parse_args([
"rollback",
"-s",
"fakeservice",
"-k",
"abcd" * 10,
"-l",
"cluster.instance1,cluster.instance2",
])
fake_deploy_groups = fake_args.deploy_groups.split(",")
mock_get_versions_for_service.return_value = {
DeploymentVersion(sha="fake_sha1", image_version=None): (
"20170403T025512",
"fake_deploy_group1",
),
DeploymentVersion(sha=fake_args.commit, image_version=None): (
"20161006T025416",
"fake_deploy_group2",
),
}
mock_get_git_url.return_value = "git://git.repo"
mock_figure_out_service_name.return_value = fake_args.service
mock_list_deploy_groups.return_value = fake_deploy_groups
mock_mark_for_deployment.return_value = 0
mock_get_currently_deployed_version.return_value = DeploymentVersion(
sha="1234" * 10, image_version=None)
assert paasta_rollback(fake_args) == 0
expected = [
call(
git_url=mock_get_git_url.return_value,
service=mock_figure_out_service_name.return_value,
commit=fake_args.commit,
deploy_group=deploy_group,
image_version=None,
) for deploy_group in fake_deploy_groups
]
mock_mark_for_deployment.assert_has_calls(expected, any_order=True)
assert mock_mark_for_deployment.call_count == len(fake_deploy_groups)
mock_log_audit.call_count == len(fake_args.deploy_groups)
for call_args in mock_log_audit.call_args_list:
_, call_kwargs = call_args
assert call_kwargs["action"] == "rollback"
assert call_kwargs["action_details"]["rolled_back_from"] == str(
mock_get_currently_deployed_version.return_value)
assert call_kwargs["action_details"][
"rolled_back_to"] == fake_args.commit
assert (call_kwargs["action_details"]["rollback_type"] ==
RollbackTypes.USER_INITIATED_ROLLBACK.value)
assert (call_kwargs["action_details"]["deploy_group"]
in mock_list_deploy_groups.return_value)
assert call_kwargs["service"] == fake_args.service
def test_paasta_mark_for_deployment_with_good_rollback(
mock_get_metrics,
mock_load_system_paasta_config,
mock_list_deploy_groups,
mock_get_currently_deployed_version,
mock_do_wait_for_deployment,
mock_mark_for_deployment,
mock_validate_service_name,
mock_get_slack_client,
mock__log_audit,
mock_get_instance_configs,
mock_periodically_update_slack,
):
class FakeArgsRollback(FakeArgs):
auto_rollback = True
block = True
timeout = 600
warn = 80 # % of timeout to warn at
polling_interval = 15
diagnosis_interval = 15
time_before_first_diagnosis = 15
mock_list_deploy_groups.return_value = ["test_deploy_groups"]
config_mock = mock.Mock()
config_mock.get_default_push_groups.return_value = None
mock_load_system_paasta_config.return_value = config_mock
mock_get_instance_configs.return_value = {
"fake_cluster": [],
"fake_cluster2": []
}
mock_mark_for_deployment.return_value = 0
def do_wait_for_deployment_side_effect(self, target_commit,
target_image_version):
if (target_commit == FakeArgs.commit
and target_image_version == FakeArgs.image_version):
self.trigger("rollback_button_clicked")
else:
self.trigger("deploy_finished")
mock_do_wait_for_deployment.side_effect = do_wait_for_deployment_side_effect
def on_enter_rolled_back_side_effect(self):
self.trigger("abandon_button_clicked")
mock_get_currently_deployed_version.return_value = DeploymentVersion(
"old-sha", None)
with patch(
"paasta_tools.cli.cmds.mark_for_deployment.MarkForDeploymentProcess.on_enter_rolled_back",
autospec=True,
wraps=mark_for_deployment.MarkForDeploymentProcess.
on_enter_rolled_back,
side_effect=on_enter_rolled_back_side_effect,
):
assert mark_for_deployment.paasta_mark_for_deployment(
FakeArgsRollback) == 1
mock_mark_for_deployment.assert_any_call(
service="test_service",
deploy_group="test_deploy_group",
commit="d670460b4b4aece5915caf5c68d12f560a9fe3e4",
git_url="git://false.repo/services/test_services",
image_version="extrastuff",
)
mock_mark_for_deployment.assert_any_call(
service="test_service",
deploy_group="test_deploy_group",
commit="old-sha",
git_url="git://false.repo/services/test_services",
image_version=None,
)
assert mock_mark_for_deployment.call_count == 2
mock_do_wait_for_deployment.assert_any_call(
mock.ANY, "d670460b4b4aece5915caf5c68d12f560a9fe3e4", "extrastuff")
mock_do_wait_for_deployment.assert_any_call(mock.ANY, "old-sha", None)
assert mock_do_wait_for_deployment.call_count == 2
# in normal usage, this would also be called once per m-f-d, but we mock that out above
# so _log_audit is only called as part of handling the rollback
assert mock__log_audit.call_count == len(
mock_list_deploy_groups.return_value)
mock__log_audit.assert_called_once_with(
action="rollback",
action_details={
"deploy_group": "test_deploy_group",
"rolled_back_from":
"DeploymentVersion(sha=d670460b4b4aece5915caf5c68d12f560a9fe3e4, image_version=extrastuff)",
"rolled_back_to": "old-sha",
"rollback_type": "user_initiated_rollback",
},
service="test_service",
)
mock_get_metrics.assert_called_once_with("paasta.mark_for_deployment")
mock_get_metrics.return_value.create_timer.assert_called_once_with(
name="deploy_duration",
default_dimensions=dict(
paasta_service="test_service",
deploy_group="test_deploy_group",
old_version="old-sha",
new_version=
"DeploymentVersion(sha=d670460b4b4aece5915caf5c68d12f560a9fe3e4, image_version=extrastuff)",
deploy_timeout=600,
),
)
mock_timer = mock_get_metrics.return_value.create_timer.return_value
mock_timer.start.assert_called_once_with()
mock_timer.stop.assert_called_once_with(tmp_dimensions=dict(exit_status=1))
mock_emit_event = mock_get_metrics.return_value.emit_event
event_dimensions = dict(
paasta_service="test_service",
deploy_group="test_deploy_group",
rolled_back_from=
"DeploymentVersion(sha=d670460b4b4aece5915caf5c68d12f560a9fe3e4, image_version=extrastuff)",
rolled_back_to="old-sha",
rollback_type="user_initiated_rollback",
)
expected_calls = []
for cluster in mock_get_instance_configs.return_value.keys():
dims = dict(event_dimensions)
dims["paasta_cluster"] = cluster
exp_call = call(name="rollback", dimensions=dims)
expected_calls.append(exp_call)
mock_emit_event.assert_has_calls(expected_calls, any_order=True)
