def test_apply_args_filters_clusters_return_none_when_instance_not_in_deploy_group(
mock_figure_out_service_name,
mock_list_services,
mock_get_instance_configs_for_service,
):
args = StatusArgs(
service='fake_service',
soa_dir='/fake/soa/dir',
deploy_group='fake_deploy_group',
clusters=None,
instances='instance5',
owner=None,
registration=None,
verbose=False,
)
mock_figure_out_service_name.return_value = 'fake_service'
mock_list_services.return_value = ['fake_service']
mock_get_instance_configs_for_service.return_value = [
make_fake_instance_conf('cluster1', 'fake_service', 'instance1',
'other_fake_deploy_group'),
make_fake_instance_conf('cluster1', 'fake_service', 'instance2',
'other_fake_deploy_group'),
make_fake_instance_conf('cluster2', 'fake_service', 'instance3',
'other_fake_deploy_group'),
]
assert len(apply_args_filters(args)) == 0
def test_apply_args_filters_clusters_and_instances(
mock_figure_out_service_name,
mock_list_services,
mock_get_instance_configs_for_service,
):
args = StatusArgs(
service='fake_service',
soa_dir='/fake/soa/dir',
deploy_group=None,
clusters='cluster1',
instances='instance1,instance3',
owner=None,
registration=None,
verbose=False,
)
mock_figure_out_service_name.return_value = 'fake_service'
mock_list_services.return_value = ['fake_service']
mock_inst1 = make_fake_instance_conf('cluster1', 'fake_service',
'instance1', 'fake_deploy_group')
mock_inst2 = make_fake_instance_conf('cluster1', 'fake_service',
'instance2', 'fake_deploy_group')
mock_inst3 = make_fake_instance_conf('cluster1', 'fake_service',
'instance3', 'fake_deploy_group')
mock_get_instance_configs_for_service.return_value = [
mock_inst1,
mock_inst2,
mock_inst3,
]
pargs = apply_args_filters(args)
assert sorted(pargs.keys()) == ['cluster1']
assert pargs['cluster1']['fake_service'] == {
'instance1': mock_inst1.__class__,
'instance3': mock_inst3.__class__
}
def test_apply_args_filters_clusters_return_none_when_instance_not_in_deploy_group(
mock_figure_out_service_name,
mock_list_services,
mock_get_instance_configs_for_service,
):
PaastaArgs = namedtuple('PaastaArgs', [
'service', 'soa_dir', 'clusters', 'instances', 'deploy_group', 'owner'
])
args = PaastaArgs(
service='fake_service',
soa_dir='/fake/soa/dir',
deploy_group='fake_deploy_group',
clusters=None,
instances='instance5',
owner=None,
)
mock_figure_out_service_name.return_value = 'fake_service'
mock_list_services.return_value = ['fake_service']
mock_get_instance_configs_for_service.return_value = [
make_fake_instance_conf('cluster1', 'fake_service', 'instance1',
'other_fake_deploy_group'),
make_fake_instance_conf('cluster1', 'fake_service', 'instance2',
'other_fake_deploy_group'),
make_fake_instance_conf('cluster2', 'fake_service', 'instance3',
'other_fake_deploy_group'),
]
assert len(apply_args_filters(args)) == 0
def test_apply_args_filters_clusters_and_instances(
mock_figure_out_service_name,
mock_list_services,
mock_get_instance_configs_for_service,
):
PaastaArgs = namedtuple('PaastaArgs', [
'service', 'soa_dir', 'clusters', 'instances', 'deploy_group', 'owner'
])
args = PaastaArgs(
service='fake_service',
soa_dir='/fake/soa/dir',
deploy_group=None,
clusters='cluster1',
instances='instance1,instance3',
owner=None,
)
mock_figure_out_service_name.return_value = 'fake_service'
mock_list_services.return_value = ['fake_service']
mock_get_instance_configs_for_service.return_value = [
make_fake_instance_conf('cluster1', 'fake_service', 'instance1',
'fake_deploy_group'),
make_fake_instance_conf('cluster1', 'fake_service', 'instance2',
'fake_deploy_group'),
make_fake_instance_conf('cluster1', 'fake_service', 'instance3',
'fake_deploy_group'),
]
pargs = apply_args_filters(args)
assert sorted(pargs.keys()) == ['cluster1']
assert pargs['cluster1']['fake_service'] == {'instance1', 'instance3'}
def test_apply_args_filters_no_instances_found(
mock_figure_out_service_name, mock_list_services, mock_get_instance_configs_for_service,
mock_list_all_instances_for_service, capfd,
):
args = StatusArgs(
service='fake_service',
soa_dir='/fake/soa/dir',
deploy_group=None,
clusters='cluster1',
instances='instance4,instance5',
owner=None,
registration=None,
verbose=False,
)
mock_figure_out_service_name.return_value = 'fake_service'
mock_list_services.return_value = ['fake_service']
mock_get_instance_configs_for_service.return_value = [
make_fake_instance_conf('cluster1', 'fake_service', 'instance1', 'fake_deploy_group'),
make_fake_instance_conf('cluster1', 'fake_service', 'instance2', 'fake_deploy_group'),
make_fake_instance_conf('cluster1', 'fake_service', 'instance3', 'fake_deploy_group'),
]
mock_list_all_instances_for_service.return_value = ['instance1', 'instance2', 'instance3']
pargs = apply_args_filters(args)
output, _ = capfd.readouterr()
assert len(pargs.keys()) == 0
assert "fake_service doesn't have any instances matching instance4, instance5 on cluster1." in output
assert "Did you mean any of these?" in output
for i in ["instance1", "instance2", "instance3"]:
assert i in output
def test_apply_args_filters_clusters_uses_deploy_group_when_no_clusters_and_instances(
mock_figure_out_service_name, mock_list_services, mock_get_instance_configs_for_service,
):
args = StatusArgs(
service='fake_service',
soa_dir='/fake/soa/dir',
deploy_group='fake_deploy_group',
clusters=None,
instances=None,
owner=None,
registration=None,
verbose=False,
)
mock_figure_out_service_name.return_value = 'fake_service'
mock_list_services.return_value = ['fake_service']
mock_get_instance_configs_for_service.return_value = [
make_fake_instance_conf('cluster1', 'fake_service', 'instance1', 'fake_deploy_group'),
make_fake_instance_conf('cluster1', 'fake_service', 'instance2', 'fake_deploy_group'),
make_fake_instance_conf('cluster2', 'fake_service', 'instance3', 'fake_deploy_group'),
]
pargs = apply_args_filters(args)
assert sorted(pargs.keys()) == ['cluster1', 'cluster2']
assert pargs['cluster1']['fake_service'] == {'instance1', 'instance2'}
assert pargs['cluster2']['fake_service'] == {'instance3'}
def paasta_restart(args):
pargs = apply_args_filters(args)
soa_dir = args.soa_dir
affected_flinks = []
affected_non_flinks = []
for cluster, service_instances in pargs.items():
for service, instances in service_instances.items():
for instance in instances.keys():
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
if isinstance(service_config, FlinkDeploymentConfig):
affected_flinks.append(service_config)
else:
affected_non_flinks.append(service_config)
if affected_flinks:
flinks_info = ", ".join(
[f"{f.service}.{f.instance}" for f in affected_flinks])
print(
f"WARN: paasta restart is currently unsupported for Flink instances ({flinks_info})."
)
print("To restart, please run:", end="\n\n")
for flink in affected_flinks:
print(
f"paasta stop -s {flink.service} -i {flink.instance} -c {flink.cluster}"
)
print(
f"paasta start -s {flink.service} -i {flink.instance} -c {flink.cluster}",
end="\n\n",
)
if not affected_non_flinks:
return 1
non_flinks_info = ", ".join(
[f"{f.service}.{f.instance}" for f in affected_non_flinks])
proceed = choice.Binary(
f"Would you like to restart the other instances ({non_flinks_info}) anyway?",
False,
).ask()
if not proceed:
return 1
return paasta_start(args)
def test_apply_args_filters_bad_service_name(mock_list_services, capfd):
args = StatusArgs(
service='fake-service',
soa_dir='/fake/soa/dir',
deploy_group=None,
clusters='cluster1',
instances='instance4,instance5',
owner=None,
registration=None,
verbose=False,
)
mock_list_services.return_value = ['fake_service']
pargs = apply_args_filters(args)
output, _ = capfd.readouterr()
assert len(pargs) == 0
assert 'The service "fake-service" does not exist.' in output
assert 'Did you mean any of these?' in output
assert ' fake_service' in output
def test_apply_args_filters_bad_service_name(mock_list_services, capfd):
PaastaArgs = namedtuple('PaastaArgs', [
'service', 'soa_dir', 'clusters', 'instances', 'deploy_group', 'owner'
])
args = PaastaArgs(
service='fake-service',
soa_dir='/fake/soa/dir',
deploy_group=None,
clusters='cluster1',
instances='instance4,instance5',
owner=None,
)
mock_list_services.return_value = ['fake_service']
pargs = apply_args_filters(args)
output, _ = capfd.readouterr()
assert len(pargs) == 0
assert 'The service "fake-service" does not exist.' in output
assert 'Did you mean any of these?' in output
assert ' fake_service' in output
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
soa_dir = args.soa_dir
pargs = apply_args_filters(args)
if len(pargs) == 0:
return 1
affected_services = {s for service_list in pargs.values() for s in service_list.keys()}
if len(affected_services) > 1:
paasta_print(PaastaColors.red("Warning: trying to start/stop/restart multiple services:"))
for cluster, services_instances in pargs.items():
paasta_print("Cluster %s:" % cluster)
for service, instances in services_instances.items():
paasta_print(" Service %s:" % service)
paasta_print(" Instances %s" % ",".join(instances))
if sys.stdin.isatty():
confirm = choice.Binary('Are you sure you want to continue?', False).ask()
else:
confirm = False
if not confirm:
paasta_print()
paasta_print("exiting")
return 1
invalid_deploy_groups = []
marathon_message_printed, chronos_message_printed = False, False
for cluster, services_instances in pargs.items():
for service, instances in services_instances.items():
try:
remote_refs = remote_git.list_remote_refs(utils.get_git_url(service, soa_dir))
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment."
) % str(e)
paasta_print(msg)
return 1
for instance in instances:
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
deploy_group = service_config.get_deploy_group()
(deploy_tag, _) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(datetime.datetime.utcnow())
if isinstance(service_config, MarathonServiceConfig) and not marathon_message_printed:
print_marathon_message(desired_state)
marathon_message_printed = True
elif isinstance(service_config, ChronosJobConfig) and not chronos_message_printed:
print_chronos_message(desired_state)
chronos_message_printed = True
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
return_val = 0
if invalid_deploy_groups:
paasta_print("No branches found for %s in %s." %
(", ".join(invalid_deploy_groups), remote_refs))
paasta_print("Has %s been deployed there yet?" % service)
return_val = 1
return return_val
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
soa_dir = args.soa_dir
pargs = apply_args_filters(args)
if len(pargs) == 0:
return 1
affected_services = {
s
for service_list in pargs.values() for s in service_list.keys()
}
if len(affected_services) > 1:
print(
PaastaColors.red(
"Warning: trying to start/stop/restart multiple services:"))
for cluster, services_instances in pargs.items():
print("Cluster %s:" % cluster)
for service, instances in services_instances.items():
print(" Service %s:" % service)
print(" Instances %s" % ",".join(instances.keys()))
if sys.stdin.isatty():
confirm = choice.Binary("Are you sure you want to continue?",
False).ask()
else:
confirm = False
if not confirm:
print()
print("exiting")
return 1
invalid_deploy_groups = []
marathon_message_printed = False
affected_flinks = []
if args.clusters is None or args.instances is None:
if confirm_to_continue(pargs.items(), desired_state) is False:
print()
print("exiting")
return 1
for cluster, services_instances in pargs.items():
for service, instances in services_instances.items():
for instance in instances.keys():
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
if isinstance(service_config, FlinkDeploymentConfig):
affected_flinks.append(service_config)
continue
try:
remote_refs = get_remote_refs(service, soa_dir)
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment.") % str(e)
print(msg)
return 1
deploy_group = service_config.get_deploy_group()
(deploy_tag,
_) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(
datetime.datetime.utcnow())
if (isinstance(service_config, MarathonServiceConfig)
and not marathon_message_printed):
print_marathon_message(desired_state)
marathon_message_printed = True
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
return_val = 0
# TODO: Refactor to discover if set_state is available for given
# instance_type in API
if affected_flinks:
print_flink_message(desired_state)
csi = defaultdict(lambda: defaultdict(list))
for service_config in affected_flinks:
csi[service_config.cluster][service_config.service].append(
service_config.instance)
system_paasta_config = load_system_paasta_config()
for cluster, services_instances in csi.items():
client = get_paasta_api_client(cluster, system_paasta_config)
if not client:
print("Cannot get a paasta-api client")
exit(1)
for service, instances in services_instances.items():
for instance in instances:
try:
client.service.instance_set_state(
service=service,
instance=instance,
desired_state=desired_state,
).result()
except HTTPError as exc:
print(exc.response.text)
return exc.status_code
return_val = 0
if invalid_deploy_groups:
print(f"No deploy tags found for {', '.join(invalid_deploy_groups)}.")
print(f"Has {service} been deployed there yet?")
return_val = 1
return return_val
def paasta_start_or_stop(args, desired_state):
"""Requests a change of state to start or stop given branches of a service."""
soa_dir = args.soa_dir
pargs = apply_args_filters(args)
if len(pargs) == 0:
return 1
affected_services = {
s
for service_list in pargs.values() for s in service_list.keys()
}
if len(affected_services) > 1:
paasta_print(
PaastaColors.red(
"Warning: trying to start/stop/restart multiple services:"))
for cluster, services_instances in pargs.items():
paasta_print("Cluster %s:" % cluster)
for service, instances in services_instances.items():
paasta_print(" Service %s:" % service)
paasta_print(" Instances %s" %
",".join(instances.keys()))
if sys.stdin.isatty():
confirm = choice.Binary('Are you sure you want to continue?',
False).ask()
else:
confirm = False
if not confirm:
paasta_print()
paasta_print("exiting")
return 1
invalid_deploy_groups = []
marathon_message_printed = False
chronos_message_printed = False
affected_flinkclusters = []
if args.clusters is None or args.instances is None:
if confirm_to_continue(pargs.items(), desired_state) is False:
paasta_print()
paasta_print("exiting")
return 1
for cluster, services_instances in pargs.items():
for service, instances in services_instances.items():
for instance in instances.keys():
service_config = get_instance_config(
service=service,
cluster=cluster,
instance=instance,
soa_dir=soa_dir,
load_deployments=False,
)
if isinstance(service_config, FlinkClusterConfig):
affected_flinkclusters.append(service_config)
continue
try:
remote_refs = get_remote_refs(service, soa_dir)
except remote_git.LSRemoteException as e:
msg = (
"Error talking to the git server: %s\n"
"This PaaSTA command requires access to the git server to operate.\n"
"The git server may be down or not reachable from here.\n"
"Try again from somewhere where the git server can be reached, "
"like your developer environment.") % str(e)
paasta_print(msg)
return 1
deploy_group = service_config.get_deploy_group()
(deploy_tag,
_) = get_latest_deployment_tag(remote_refs, deploy_group)
if deploy_tag not in remote_refs:
invalid_deploy_groups.append(deploy_group)
else:
force_bounce = utils.format_timestamp(
datetime.datetime.utcnow())
if isinstance(service_config, MarathonServiceConfig
) and not marathon_message_printed:
print_marathon_message(desired_state)
marathon_message_printed = True
elif isinstance(
service_config,
ChronosJobConfig) and not chronos_message_printed:
print_chronos_message(desired_state)
chronos_message_printed = True
issue_state_change_for_service(
service_config=service_config,
force_bounce=force_bounce,
desired_state=desired_state,
)
return_val = 0
if affected_flinkclusters:
if os.environ.get('ON_PAASTA_MASTER'):
print_flinkcluster_message(desired_state)
kube_client = KubeClient()
for service_config in affected_flinkclusters:
set_flinkcluster_desired_state(
kube_client=kube_client,
service=service_config.service,
instance=service_config.instance,
desired_state=dict(start='running',
stop='stopped')[desired_state],
)
else:
csi = defaultdict(lambda: defaultdict(list))
for service_config in affected_flinkclusters:
csi[service_config.cluster][service_config.service].append(
service_config.instance)
system_paasta_config = load_system_paasta_config()
for cluster, services_instances in csi.items():
for service, instances in services_instances.items():
cmd_parts = [
'ON_PAASTA_MASTER=1',
'paasta',
desired_state,
'-c',
cluster,
'-s',
service,
'-i',
','.join(instances),
]
return_val, _ = run_on_master(
cluster=cluster,
system_paasta_config=system_paasta_config,
cmd_parts=cmd_parts,
graceful_exit=True,
)
if invalid_deploy_groups:
paasta_print("No branches found for %s in %s." %
(", ".join(invalid_deploy_groups), remote_refs))
paasta_print("Has %s been deployed there yet?" % service)
return_val = 1
return return_val
