def get_paasta_oapi_client(
cluster: str = None,
system_paasta_config: SystemPaastaConfig = None,
http_res: bool = False,
) -> Optional[PaastaOApiClient]:
if not system_paasta_config:
system_paasta_config = load_system_paasta_config()
if not cluster:
cluster = system_paasta_config.get_cluster()
api_endpoints = system_paasta_config.get_api_endpoints()
if cluster not in api_endpoints:
log.error("Cluster %s not in paasta-api endpoints config", cluster)
return None
parsed = urlparse(api_endpoints[cluster])
cert_file = key_file = ssl_ca_cert = None
if parsed.scheme == "https":
opts = get_paasta_ssl_opts(cluster, system_paasta_config)
if opts:
cert_file = opts["cert"]
key_file = opts["key"]
ssl_ca_cert = opts["ca"]
return get_paasta_oapi_client_by_url(parsed, cert_file, key_file,
ssl_ca_cert)
def get_sidecar_containers(
self,
system_paasta_config: SystemPaastaConfig) -> List[V1Container]:
registrations = " ".join(self.get_registrations())
# s_m_j currently asserts that services are healthy in smartstack before
# continuing a bounce. this readiness check lets us achieve the same thing
readiness_probe: Optional[V1Probe]
if system_paasta_config.get_enable_nerve_readiness_check():
readiness_probe = V1Probe(
_exec=V1ExecAction(command=[
system_paasta_config.get_nerve_readiness_check_script(),
] + self.get_registrations(), ),
initial_delay_seconds=10,
period_seconds=10,
)
else:
readiness_probe = None
hacheck_sidecar = V1Container(
image=system_paasta_config.get_hacheck_sidecar_image_url(),
lifecycle=V1Lifecycle(pre_stop=V1Handler(_exec=V1ExecAction(
command=[
"/bin/sh",
"-c",
f"/usr/bin/hadown {registrations}; sleep 31",
], ), ), ),
name="hacheck",
env=self.get_kubernetes_environment(),
ports=[
V1ContainerPort(container_port=6666, ),
],
readiness_probe=readiness_probe,
)
return [hacheck_sidecar]
def _get_spark_ports(system_paasta_config: SystemPaastaConfig) -> Dict[str, int]:
return {
"spark.ui.port": system_paasta_config.get_spark_ui_port(),
"spark.driver.port": system_paasta_config.get_spark_driver_port(),
"spark.blockManager.port": system_paasta_config.get_spark_blockmanager_port(),
"spark.driver.blockManager.port": system_paasta_config.get_spark_blockmanager_port(),
}
def __init__(self, scheme: str, cluster: str,
system_paasta_config: SystemPaastaConfig) -> None:
if scheme == "https":
ca_cert_path = None
if system_paasta_config.get_enable_client_cert_auth():
ecosystem = system_paasta_config.get_vault_cluster_config(
)[cluster]
paasta_dir = os.path.expanduser("~/.paasta/pki")
if (not os.path.isfile(f"{paasta_dir}/{ecosystem}.crt")
or not os.path.isfile(f"{paasta_dir}/{ecosystem}.key")
or
not os.path.isfile(f"{paasta_dir}/{ecosystem}_ca.crt")
):
renew_issue_cert(system_paasta_config=system_paasta_config,
cluster=cluster)
ca_cert_path = f"{paasta_dir}/{ecosystem}_ca.crt"
ssl_cert = (
f"{paasta_dir}/{ecosystem}.crt",
f"{paasta_dir}/{ecosystem}.key",
)
else:
ssl_cert = None
super().__init__(ssl_verify=True, ssl_cert=ssl_cert)
if ca_cert_path:
self.session.verify = ca_cert_path
else:
super().__init__()
def __init__(
self,
system_paasta_config: SystemPaastaConfig,
) -> None:
self._synapse_port = system_paasta_config.get_synapse_port()
self._synapse_haproxy_url_format = system_paasta_config.get_synapse_haproxy_url_format(
)
self._system_paasta_config = system_paasta_config
self._cache: Dict[str, Dict[str, int]] = {}
def extract_args(args):
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
paasta_print(
PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates"
"PaaSTA is not configured locally on this host, and remote-run may not behave"
"the same way it would behave on a server configured for PaaSTA.",
),
sep='\n',
)
system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
cluster = args.cluster or system_paasta_config.get_local_run_config().get('default_cluster', None)
if not cluster:
paasta_print(
PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster."
"Please pass one using '-c'.",
),
sep='\n',
file=sys.stderr,
)
sys.exit(1)
soa_dir = args.yelpsoa_config_root
instance = args.instance
if instance is None:
instance_type = 'adhoc'
instance = 'remote'
else:
instance_type = validate_service_instance(
service, instance, cluster, soa_dir,
)
if instance_type != 'adhoc':
paasta_print(
PaastaColors.red(
(
"Please use instance declared in adhoc.yaml for use "
"with remote-run, {} is declared as {}"
).format(instance, instance_type),
),
)
sys.exit(1)
return (
system_paasta_config,
service,
cluster,
soa_dir,
instance,
instance_type,
)
def __init__(
self,
mesos_slaves: List[_MesosSlaveDict],
system_paasta_config: SystemPaastaConfig,
) -> None:
self._mesos_slaves = mesos_slaves
self._synapse_port = system_paasta_config.get_synapse_port()
self._synapse_haproxy_url_format = system_paasta_config.get_synapse_haproxy_url_format(
)
self._system_paasta_config = system_paasta_config
self._cache: Dict[str, Dict[str, int]] = {}
def test_run_synapse_check():
module = 'paasta_tools.monitoring.check_synapse_replication'
parse_method = module + '.parse_synapse_check_options'
replication_method = module + '.get_replication_for_services'
check_replication_method = module + '.check_replication'
mock_parse_options = mock.Mock()
mock_parse_options.synapse_host_port = 'foo'
mock_parse_options.services = ['wat', 'service', 'is']
mock_replication = {'wat': 2, 'service': 1, 'is': 0}
for return_value in range(0, 4):
with nested(
mock.patch(parse_method,
return_value=mock_parse_options,
autospec=True),
mock.patch(replication_method,
return_value=mock_replication,
autospec=True),
mock.patch(check_replication_method,
return_value=(return_value, 'CHECK'),
autospec=True),
mock.patch('paasta_tools.utils.load_system_paasta_config',
autospec=True,
return_value=SystemPaastaConfig({},
'/fake/config')),
):
with pytest.raises(SystemExit) as error:
run_synapse_check()
assert error.value.code == return_value
# Mock check results for those services
mock_service_status = {'wat': 0, 'service': 1, 'is': 2}
def mock_check(name, replication, warn, crit):
return mock_service_status[name], 'CHECK'
with nested(
mock.patch(parse_method,
return_value=mock_parse_options,
autospec=True),
mock.patch(replication_method,
return_value=mock_replication,
autospec=True),
mock.patch(check_replication_method, new=mock_check,
autospec=None),
mock.patch('paasta_tools.utils.load_system_paasta_config',
autospec=True,
return_value=SystemPaastaConfig({}, '/fake/config')),
):
with pytest.raises(SystemExit) as error:
run_synapse_check()
assert error.value.code == 2
def test_get_smartstack_replication_for_attribute():
fake_namespace = 'fake_main'
fake_service = 'fake_service'
mock_filtered_slaves = [
{
'hostname': 'hostone',
'attributes': {
'fake_attribute': 'foo',
},
},
{
'hostname': 'hostone',
'attributes': {
'fake_attribute': 'bar',
},
},
]
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
with mock.patch(
'paasta_tools.mesos_tools.get_all_slaves_for_blacklist_whitelist',
return_value=mock_filtered_slaves,
autospec=True,
) as mock_get_all_slaves_for_blacklist_whitelist, mock.patch(
'paasta_tools.smartstack_tools.get_replication_for_services',
return_value={},
autospec=True,
) as mock_get_replication_for_services:
expected = {
'foo': {},
'bar': {},
}
actual = smartstack_tools.get_smartstack_replication_for_attribute(
attribute='fake_attribute',
service=fake_service,
namespace=fake_namespace,
blacklist=[],
system_paasta_config=fake_system_paasta_config,
)
mock_get_all_slaves_for_blacklist_whitelist.assert_called_once_with(
blacklist=[],
whitelist=None,
)
assert actual == expected
assert mock_get_replication_for_services.call_count == 2
mock_get_replication_for_services.assert_any_call(
synapse_host='hostone',
synapse_port=fake_system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=fake_system_paasta_config.
get_synapse_haproxy_url_format(),
services=['fake_service.fake_main'],
)
def test_get_smartstack_replication_for_attribute():
fake_namespace = 'fake_main'
fake_service = 'fake_service'
mock_filtered_slaves = [
{
'hostname': 'hostone',
'attributes': {
'fake_attribute': 'foo'
}
},
{
'hostname': 'hostone',
'attributes': {
'fake_attribute': 'bar'
}
}
]
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
with contextlib.nested(
mock.patch('paasta_tools.mesos_tools.get_all_slaves_for_blacklist_whitelist',
return_value=mock_filtered_slaves, autospec=True),
mock.patch('paasta_tools.smartstack_tools.get_replication_for_services',
return_value={}, autospec=True),
) as (
mock_get_all_slaves_for_blacklist_whitelist,
mock_get_replication_for_services,
):
expected = {
'foo': {},
'bar': {}
}
actual = smartstack_tools.get_smartstack_replication_for_attribute(
attribute='fake_attribute',
service=fake_service,
namespace=fake_namespace,
blacklist=[],
system_paasta_config=fake_system_paasta_config,
)
mock_get_all_slaves_for_blacklist_whitelist.assert_called_once_with(
blacklist=[],
whitelist=[]
)
assert actual == expected
assert mock_get_replication_for_services.call_count == 2
mock_get_replication_for_services.assert_any_call(
synapse_host='hostone',
synapse_port=fake_system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=fake_system_paasta_config.get_synapse_haproxy_url_format(),
services=['fake_service.fake_main'],
)
def get_paasta_api_client(
cluster: str = None,
system_paasta_config: SystemPaastaConfig = None,
http_res: bool = False,
) -> Any:
if not system_paasta_config:
system_paasta_config = load_system_paasta_config()
if not cluster:
cluster = system_paasta_config.get_cluster()
api_endpoints = system_paasta_config.get_api_endpoints()
if cluster not in api_endpoints:
log.error("Cluster %s not in paasta-api endpoints config", cluster)
return None
url = str(api_endpoints[cluster])
parsed = urlparse(url)
if not parsed:
log.error("Unsupported paasta-api url %s", url)
return None
api_server = parsed.netloc
# Get swagger spec from file system instead of the api server
paasta_api_path = os.path.dirname(paasta_tools.api.__file__)
swagger_file = os.path.join(paasta_api_path, "api_docs/swagger.json")
if not os.path.isfile(swagger_file):
log.error("paasta-api swagger spec %s does not exist", swagger_file)
return None
with open(swagger_file) as f:
spec_dict = json.load(f)
# replace localhost in swagger.json with actual api server
spec_dict["host"] = api_server
spec_dict["schemes"] = [parsed.scheme]
# sometimes we want the status code
requests_client = PaastaRequestsClient(
scheme=parsed.scheme,
cluster=cluster,
system_paasta_config=system_paasta_config)
if http_res:
config = {"also_return_response": True}
c = SwaggerClient.from_spec(spec_dict=spec_dict,
config=config,
http_client=requests_client)
else:
c = SwaggerClient.from_spec(spec_dict=spec_dict,
http_client=requests_client)
return paasta_tools.api.auth_decorator.AuthClientDecorator(
c, cluster_name=cluster)
def renew_issue_cert(system_paasta_config: SystemPaastaConfig, cluster: str) -> None:
secret_provider_kwargs = {
"vault_cluster_config": system_paasta_config.get_vault_cluster_config()
}
sp = get_secret_provider(
secret_provider_name=system_paasta_config.get_secret_provider_name(),
cluster_names=[cluster],
secret_provider_kwargs=secret_provider_kwargs,
soa_dir=None,
service_name=None,
)
sp.renew_issue_cert(
pki_backend=system_paasta_config.get_pki_backend(),
ttl=system_paasta_config.get_auth_certificate_ttl(),
)
def test_execute_paasta_serviceinit_status_on_remote_master_happy_path(
mock_run_paasta_serviceinit,
mock_find_connectable_master,
mock_calculate_remote_masters,
):
cluster = 'fake_cluster_name'
service = 'fake_service'
instancename = 'fake_instance'
remote_masters = (
'fake_master1',
'fake_master2',
'fake_master3',
)
mock_calculate_remote_masters.return_value = (remote_masters, None)
mock_find_connectable_master.return_value = ('fake_connectable_master', None)
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
actual = utils.execute_paasta_serviceinit_on_remote_master('status', cluster, service, instancename,
fake_system_paasta_config)
mock_calculate_remote_masters.assert_called_once_with(cluster, fake_system_paasta_config)
mock_find_connectable_master.assert_called_once_with(remote_masters)
mock_run_paasta_serviceinit.assert_called_once_with(
'status',
'fake_connectable_master',
service,
instancename,
cluster,
False
)
assert actual == mock_run_paasta_serviceinit.return_value
def test_perform_command_handles_no_docker_and_doesnt_raise():
fake_service = 'fake_service'
fake_instance = 'fake_instance'
fake_cluster = 'fake_cluster'
soa_dir = 'fake_soa_dir'
with contextlib.nested(
mock.patch(
'paasta_tools.marathon_serviceinit.marathon_tools.load_marathon_config',
autospec=True),
mock.patch(
'paasta_tools.marathon_serviceinit.marathon_tools.load_marathon_service_config',
autospec=True,
return_value=mock.Mock(format_marathon_app_dict=mock.Mock(
side_effect=NoDockerImageError))),
mock.patch(
'paasta_tools.marathon_serviceinit.load_system_paasta_config',
autospec=True,
return_value=SystemPaastaConfig({}, "/fake/config")),
) as (
mock_load_marathon_config,
mock_load_marathon_service_config,
mock_load_system_paasta_config,
):
actual = marathon_serviceinit.perform_command('start', fake_service,
fake_instance,
fake_cluster, False,
soa_dir)
assert actual == 1
def test_execute_paasta_metastatus_on_remote_master(
mock_run_paasta_metastatus,
mock_find_connectable_master,
mock_calculate_remote_masters,
):
cluster = 'fake_cluster_name'
remote_masters = [
'fake_master1',
'fake_master2',
'fake_master3',
]
mock_run_paasta_metastatus.return_value = (
mock.sentinel.paasta_metastatus_return_code, mock.sentinel.paasta_metastatus_output,
)
mock_calculate_remote_masters.return_value = (remote_masters, None)
mock_find_connectable_master.return_value = ('fake_connectable_master', None)
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
return_code, actual = utils.execute_paasta_metastatus_on_remote_master(
cluster, fake_system_paasta_config, False, [], 0, False,
)
mock_calculate_remote_masters.assert_called_once_with(cluster, fake_system_paasta_config)
mock_find_connectable_master.assert_called_once_with(remote_masters)
mock_run_paasta_metastatus.assert_called_once_with('fake_connectable_master', False, [], 0, False)
assert return_code == mock.sentinel.paasta_metastatus_return_code
assert actual == mock.sentinel.paasta_metastatus_output
def test_bad_calculate_remote_master(mock_get_by_hostname):
mock_get_by_hostname.side_effect = gaierror('foo', 'bar')
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
ips, output = utils.calculate_remote_masters('myhost',
fake_system_paasta_config)
assert ips == []
assert 'ERROR while doing DNS lookup of paasta-myhost.yelp:\nbar\n' in output
def test_classic_replication_check():
base_module = 'paasta_tools.monitoring'
check_classic_module = base_module + '.check_classic_service_replication'
read_config_method = check_classic_module + '.read_services_configuration'
replication_method = check_classic_module + '.get_replication_for_services'
extract_method = check_classic_module + '.extract_replication_info'
check_method = check_classic_module + '.do_replication_check'
load_system_paasta_config_module = check_classic_module + '.load_system_paasta_config'
mock_service_config = {'pow': {'wat': 1}}
mock_replication = {'pow': -1}
mock_monitoring = {'pow': 'bar'}
mock_check = {'team': 'testing', 'output': 'testing looks good'}
with nested(
mock.patch(read_config_method, return_value=mock_service_config),
mock.patch(replication_method, return_value=mock_replication),
mock.patch(extract_method, return_value=(True, mock_monitoring)),
mock.patch(check_method, return_value=mock_check),
mock.patch('pysensu_yelp.send_event'),
mock.patch.object(sys, 'argv', ['check_classic_service_replication.py']),
mock.patch(load_system_paasta_config_module, return_value=SystemPaastaConfig({}, '/fake/config'))
) as (_, _, _, mcheck, _, _, _):
with pytest.raises(SystemExit) as error:
check = ClassicServiceReplicationCheck()
check.run()
assert mcheck.assert_called_with('pow', {'wat': 1}, -1)
assert error.value.code == 0
def test_check_service_replication_for_non_smartstack():
service = 'test_service'
instance = 'worker'
cluster = 'fake_cluster'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
with mock.patch(
'paasta_tools.marathon_tools.get_proxy_port_for_instance',
autospec=True,
return_value=None,
), mock.patch(
'paasta_tools.marathon_tools.get_expected_instance_count_for_namespace',
autospec=True,
return_value=100,
), mock.patch(
'paasta_tools.check_marathon_services_replication.check_healthy_marathon_tasks_for_service_instance',
autospec=True,
) as mock_check_healthy_marathon_tasks:
mock_client = mock.Mock()
check_marathon_services_replication.check_service_replication(
client=mock_client,
service=service,
instance=instance,
cluster=cluster,
soa_dir=None,
system_paasta_config=fake_system_paasta_config)
mock_check_healthy_marathon_tasks.assert_called_once_with(
client=mock_client,
service=service,
instance=instance,
cluster=cluster,
soa_dir=None,
expected_count=100,
)
def test_perform_command_handles_no_docker_and_doesnt_raise():
fake_service = "fake_service"
fake_instance = "fake_instance"
fake_cluster = "fake_cluster"
soa_dir = "fake_soa_dir"
with mock.patch.object(
fake_marathon_job_config,
"format_marathon_app_dict",
autospec=True,
side_effect=NoDockerImageError,
), mock.patch(
"paasta_tools.marathon_serviceinit.load_system_paasta_config",
autospec=True,
return_value=SystemPaastaConfig({}, "/fake/config"),
):
actual = marathon_serviceinit.perform_command(
"start",
fake_service,
fake_instance,
fake_cluster,
False,
soa_dir,
job_config=fake_marathon_job_config,
clients=mock.Mock(),
)
assert actual == 1
def test_configure_and_run_explicit_cluster(
mock_run_docker_container,
mock_get_instance_config,
mock_load_system_paasta_config,
):
mock_load_system_paasta_config.return_value = SystemPaastaConfig(
{
'cluster': 'fake_cluster_that_will_be_overriden',
'volumes': []
}, '/fake_dir/')
mock_docker_client = mock.MagicMock(spec_set=docker.Client)
fake_service = 'fake_service'
docker_hash = '8' * 40
args = mock.MagicMock()
args.cmd = 'fake_command'
args.service = fake_service
args.healthcheck = False
args.instance = 'fake_instance'
args.interactive = False
args.cluster = 'fake_cluster'
args.yelpsoa_config_root = 'fakesoa-configs/'
assert configure_and_run_docker_container(mock_docker_client, docker_hash,
fake_service, args) is None
mock_get_instance_config.assert_called_once_with(
service=fake_service,
instance=args.instance,
cluster=args.cluster,
soa_dir=args.yelpsoa_config_root)
def test_ok_remote_masters(mock_get_by_hostname):
mock_get_by_hostname.return_value = ('myhost', [], ['1.2.3.4', '1.2.3.5'])
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
ips, output = utils.calculate_remote_masters('myhost',
fake_system_paasta_config)
assert output is None
assert ips == ['1.2.3.4', '1.2.3.5']
def test_configure_and_run_uses_bash_by_default_when_interactive(
mock_get_instance_config,
mock_load_system_paasta_config,
mock_run_docker_container,
):
mock_load_system_paasta_config.return_value = SystemPaastaConfig(
{
'cluster': 'fake_cluster',
'volumes': []
}, '/fake_dir/')
mock_docker_client = mock.MagicMock(spec_set=docker.Client)
fake_service = 'fake_service'
docker_hash = '8' * 40
args = mock.MagicMock()
args.cmd = None
args.service = fake_service
args.healthcheck = False
args.healthcheck_only = False
args.instance = 'fake_instance'
args.interactive = True
args.cluster = 'fake_cluster'
configure_and_run_docker_container(mock_docker_client, docker_hash,
fake_service, args) is None
mock_run_docker_container.assert_called_once_with(
docker_client=mock_docker_client,
service=fake_service,
instance=args.instance,
docker_hash=docker_hash,
volumes=[],
interactive=args.interactive,
command='bash',
healthcheck=args.healthcheck,
healthcheck_only=args.healthcheck_only,
instance_config=mock_get_instance_config.return_value)
def test_paasta_status():
fake_dict = {
"git_sha": "fake_git_sha",
"instance": "fake_instance",
"service": "fake_service"
}
fake_dict2 = {
"error_message": None,
"desired_state": "start",
"app_id": "fake_app_id",
"app_count": 1,
"running_instance_count": 2,
"expected_instance_count": 2,
"deploy_status": "Running",
"bounce_method": "crossover"
}
fake_status_obj = Struct(**fake_dict)
fake_status_obj.marathon = Struct(**fake_dict2)
system_paasta_config = SystemPaastaConfig(
{
'api_endpoints': {
'fake_cluster': "http://fake_cluster:5054"
},
'cluster': 'fake_cluster'
}, 'fake_directory')
with mock.patch('bravado.http_future.HttpFuture.result',
autospec=True) as mock_result:
mock_result.return_value = fake_status_obj
paasta_status_on_api_endpoint('fake_cluster',
'fake_service',
'fake_instance',
system_paasta_config,
verbose=False)
def test_check_service_replication_for_namespace_with_no_deployments():
service = 'test_service'
instance = 'worker'
cluster = 'fake_cluster'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
with contextlib.nested(
mock.patch(
'paasta_tools.marathon_tools.get_proxy_port_for_instance',
autospec=True,
return_value=None),
mock.patch(
'paasta_tools.marathon_tools.get_expected_instance_count_for_namespace',
autospec=True),
) as (
mock_get_proxy_port_for_instance,
mock_get_expected_count,
):
mock_client = mock.Mock()
mock_get_expected_count.side_effect = check_marathon_services_replication.NoDeploymentsAvailable
check_marathon_services_replication.check_service_replication(
client=mock_client,
service=service,
instance=instance,
cluster=cluster,
soa_dir=None,
system_paasta_config=fake_system_paasta_config)
assert mock_get_proxy_port_for_instance.call_count == 0
def test_check_smartstack_replication_for_instance_crit_when_zero_replication(
):
service = 'test'
instance = 'zero_running'
cluster = 'fake_cluster'
available = {
'fake_region': {
'test.zero_running': 0,
'test.main': 8,
'test.fully_replicated': 8
}
}
expected_replication_count = 8
soa_dir = 'test_dir'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
crit = 90
with contextlib.nested(
mock.patch(
'paasta_tools.check_marathon_services_replication.send_event',
autospec=True),
mock.patch(
'paasta_tools.marathon_tools.read_registration_for_service_instance',
autospec=True,
return_value=compose_job_id(service, instance)),
mock.patch(
'paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service',
autospec=True),
mock.patch(
'paasta_tools.marathon_tools.load_marathon_service_config',
autospec=True)) as (
mock_send_event,
mock_read_registration_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_load_smartstack_info_for_service.return_value = available
mock_service_job_config = mock.MagicMock(
spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
check_marathon_services_replication.check_smartstack_replication_for_instance(
service,
instance,
cluster,
soa_dir,
expected_replication_count,
fake_system_paasta_config,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert 'Service test.zero_running has 0 out of 8 expected instances in fake_region' in alert_output
assert "paasta status -s test -i zero_running -c fake_cluster -vv" in alert_output
def step_impl_when_fsm_auto(context):
service = "new_paasta_service"
fake_args = mock.Mock(
yelpsoa_config_root=context.fake_yelpsoa_configs,
srvname=service,
auto=True,
port=None,
team='paasta',
description=None,
external_link=None,
)
with contextlib.nested(
mock.patch('paasta_tools.cli.cmds.fsm.load_system_paasta_config'),
) as (mock_load_system_paasta_config, ):
mock_load_system_paasta_config.return_value = SystemPaastaConfig(
config={
'fsm_cluster_map': {
'pnw-stagea': 'STAGE',
'norcal-stageb': 'STAGE',
'norcal-devb': 'DEV',
'norcal-devc': 'DEV',
'norcal-prod': 'PROD',
'nova-prod': 'PROD'
},
},
directory=context.fake_yelpsoa_configs,
)
paasta_fsm(fake_args)
_load_yelpsoa_configs(context, service)
def test_check_service_replication_for_normal_smartstack():
service = 'test_service'
instance = 'test_instance'
cluster = 'fake_cluster'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
with contextlib.nested(
mock.patch(
'paasta_tools.marathon_tools.get_proxy_port_for_instance',
autospec=True,
return_value=666),
mock.patch(
'paasta_tools.marathon_tools.get_expected_instance_count_for_namespace',
autospec=True,
return_value=100),
mock.patch(
'paasta_tools.check_marathon_services_replication.check_smartstack_replication_for_instance',
autospec=True),
) as (mock_get_proxy_port_for_instance, mock_get_expected_count,
mock_check_smartstack_replication_for_service):
mock_client = mock.Mock()
check_marathon_services_replication.check_service_replication(
client=mock_client,
service=service,
instance=instance,
cluster=cluster,
soa_dir=None,
system_paasta_config=fake_system_paasta_config)
mock_check_smartstack_replication_for_service.assert_called_once_with(
service=service,
instance=instance,
cluster=cluster,
soa_dir=None,
expected_count=100,
system_paasta_config=fake_system_paasta_config,
)
def test_execute_chronos_rerun_on_remote_master(test_case):
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
with contextlib.nested(
patch('paasta_tools.cli.utils.calculate_remote_masters', autospec=True),
patch('paasta_tools.cli.utils.find_connectable_master', autospec=True),
patch('paasta_tools.cli.utils.run_chronos_rerun', autospec=True),
) as (
mock_calculate_remote_masters,
mock_find_connectable_master,
mock_run_chronos_rerun,
):
(mock_calculate_remote_masters.return_value,
mock_find_connectable_master.return_value,
mock_run_chronos_rerun.return_value) = test_case
outcome = utils.execute_chronos_rerun_on_remote_master(
'service',
'instance',
'cluster',
fake_system_paasta_config,
verbose=1,
)
# Always return an (rc, output) tuple
assert type(outcome) == tuple and \
len(outcome) == 2 and \
type(outcome[0]) == int and \
type(outcome[1]) == str
assert bool(mock_find_connectable_master.return_value) == mock_find_connectable_master.called
assert bool(mock_run_chronos_rerun.return_value) == mock_run_chronos_rerun.called
def test_report_cluster_status(mock_stdout, mock_load_system_paasta_config):
cluster = 'fake_cluster'
fake_system_paasta_config = SystemPaastaConfig(
{
'dashboard_links': {
'fake_cluster': {
'URL': 'http://paasta-fake_cluster.yelp:5050',
},
},
}, 'fake_directory')
mock_load_system_paasta_config.return_value = fake_system_paasta_config
thing_to_patch = 'paasta_tools.cli.cmds.metastatus.execute_paasta_metastatus_on_remote_master'
with mock.patch(
thing_to_patch) as mock_execute_paasta_metastatus_on_remote_master:
mock_execute_paasta_metastatus_on_remote_master.return_value = 'mock_status'
metastatus.print_cluster_status(cluster, fake_system_paasta_config)
mock_execute_paasta_metastatus_on_remote_master.assert_called_once_with(
cluster,
fake_system_paasta_config,
verbose=False,
)
actual = mock_stdout.getvalue()
assert 'Cluster: %s' % cluster in actual
assert 'mock_status' in actual
def test_get_replication_for_instance(
mock_get_replication_for_all_services,
mock_load_service_namespace_config,
):
mock_mesos_slaves = [
{
'hostname': 'host1',
'attributes': {
'region': 'fake_region1'
}
},
{
'hostname': 'host2',
'attributes': {
'region': 'fake_region1'
}
},
]
mock_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
instance_config = mock.Mock(service='fake_service',
instance='fake_instance')
instance_config.get_monitoring_blacklist.return_value = []
mock_get_replication_for_all_services.return_value = \
{'fake_service.fake_instance': 20}
mock_load_service_namespace_config.return_value.get_discover.return_value = 'region'
checker = smartstack_tools.SmartstackReplicationChecker(
mesos_slaves=mock_mesos_slaves,
system_paasta_config=mock_system_paasta_config,
)
assert checker.get_replication_for_instance(instance_config) == \
{'fake_region1': {'fake_service.fake_instance': 20}}
def get_pod_template_spec(
self,
code_sha: str,
system_paasta_config: SystemPaastaConfig,
) -> V1PodTemplateSpec:
service_namespace_config = load_service_namespace_config(
service=self.service,
namespace=self.get_nerve_namespace(),
)
docker_volumes = self.get_volumes(
system_volumes=system_paasta_config.get_volumes())
return V1PodTemplateSpec(
metadata=V1ObjectMeta(labels={
"service": self.get_service(),
"instance": self.get_instance(),
"git_sha": code_sha,
}, ),
spec=V1PodSpec(
containers=self.get_kubernetes_containers(
docker_volumes=docker_volumes,
aws_ebs_volumes=self.get_aws_ebs_volumes(),
system_paasta_config=system_paasta_config,
service_namespace_config=service_namespace_config,
),
restart_policy="Always",
volumes=self.get_pod_volumes(
docker_volumes=docker_volumes,
aws_ebs_volumes=self.get_aws_ebs_volumes(),
),
),
)
def test_get_smartstack_replication_for_attribute():
fake_namespace = 'fake_main'
fake_service = 'fake_service'
fake_values_and_hosts = {
'fake_value_1': ['fake_host_1', 'fake_host_3'],
'fake_other_value': ['fake_host_4'],
}
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
with contextlib.nested(
mock.patch('paasta_tools.mesos_tools.get_mesos_slaves_grouped_by_attribute',
return_value=fake_values_and_hosts),
mock.patch('paasta_tools.monitoring.replication_utils.get_replication_for_services',
return_value={}, autospec=True),
) as (
mock_get_mesos_slaves_grouped_by_attribute,
mock_get_replication_for_services,
):
expected = {
'fake_value_1': {},
'fake_other_value': {}
}
actual = check_marathon_services_replication.get_smartstack_replication_for_attribute(
attribute='fake_attribute',
service=fake_service,
namespace=fake_namespace,
blacklist=[],
system_paasta_config=fake_system_paasta_config,
)
assert actual == expected
assert mock_get_replication_for_services.call_count == 2
mock_get_mesos_slaves_grouped_by_attribute.assert_called_once_with(
attribute='fake_attribute',
blacklist=[],
)
mock_get_replication_for_services.assert_any_call(
synapse_host='fake_host_1',
synapse_port=fake_system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=fake_system_paasta_config.get_synapse_haproxy_url_format(),
services=['fake_service.fake_main'],
)
def configure_and_run_docker_container(
docker_client,
docker_hash,
service,
instance,
cluster,
args,
pull_image=False,
dry_run=False
):
"""
Run Docker container by image hash with args set in command line.
Function prints the output of run command in stdout.
"""
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
sys.stdout.write(PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates\n"
"PaaSTA is not configured locally on this host, and local-run may not behave\n"
"the same way it would behave on a server configured for PaaSTA.\n"
))
system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')
soa_dir = args.yelpsoa_config_root
volumes = list()
instance_type = validate_service_instance(service, instance, cluster, soa_dir)
try:
instance_config = get_instance_config(
service=service,
instance=instance,
cluster=cluster,
load_deployments=pull_image,
soa_dir=soa_dir,
)
except NoDeploymentsAvailable:
sys.stderr.write(PaastaColors.red(
"Error: No deployments.json found in %(soa_dir)s/%(service)s.\n"
"You can generate this by running:\n"
"generate_deployments_for_service -d %(soa_dir)s -s %(service)s\n" % {
'soa_dir': soa_dir, 'service': service}))
return
if pull_image:
try:
docker_url = get_docker_url(
system_paasta_config.get_docker_registry(), instance_config.get_docker_image())
except NoDockerImageError:
sys.stderr.write(PaastaColors.red(
"Error: No sha has been marked for deployment for the %s deploy group.\n"
"Please ensure this service has either run through a jenkins pipeline "
"or paasta mark-for-deployment has been run for %s" % (instance_config.get_deploy_group(), service)))
return
docker_hash = docker_url
docker_pull_image(docker_url)
# if only one volume specified, extra_volumes should be converted to a list
extra_volumes = instance_config.get_extra_volumes()
if type(extra_volumes) == dict:
extra_volumes = [extra_volumes]
for volume in system_paasta_config.get_volumes() + extra_volumes:
volumes.append('%s:%s:%s' % (volume['hostPath'], volume['containerPath'], volume['mode'].lower()))
if args.interactive is True and args.cmd is None:
command = ['bash']
elif args.cmd:
command = shlex.split(args.cmd, posix=False)
else:
command_from_config = instance_config.get_cmd()
if command_from_config:
command_modifier = command_function_for_framework(instance_type)
command = shlex.split(command_modifier(command_from_config), posix=False)
else:
command = instance_config.get_args()
hostname = socket.getfqdn()
run_docker_container(
docker_client=docker_client,
service=service,
instance=instance,
docker_hash=docker_hash,
volumes=volumes,
interactive=args.interactive,
command=command,
hostname=hostname,
healthcheck=args.healthcheck,
healthcheck_only=args.healthcheck_only,
instance_config=instance_config,
soa_dir=args.yelpsoa_config_root,
dry_run=dry_run,
)
def configure_and_run_docker_container(docker_client, docker_hash, service, instance, cluster, args, pull_image=False):
"""
Run Docker container by image hash with args set in command line.
Function prints the output of run command in stdout.
"""
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
sys.stdout.write(PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates\n"
"PaaSTA is not configured locally on this host, and local-run may not behave\n"
"the same way it would behave on a server configured for PaaSTA.\n"
))
system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')
volumes = list()
instance_config = get_instance_config(
service=service,
instance=instance,
cluster=cluster,
load_deployments=pull_image,
soa_dir=args.yelpsoa_config_root,
)
if pull_image:
docker_url = get_docker_url(
system_paasta_config.get_docker_registry(), instance_config.get_docker_image())
docker_pull_image(docker_url)
docker_hash = docker_url
# if only one volume specified, extra_volumes should be converted to a list
extra_volumes = instance_config.get_extra_volumes()
if type(extra_volumes) == dict:
extra_volumes = [extra_volumes]
for volume in system_paasta_config.get_volumes() + extra_volumes:
volumes.append('%s:%s:%s' % (volume['hostPath'], volume['containerPath'], volume['mode'].lower()))
if args.interactive is True and args.cmd is None:
command = ['bash']
elif args.cmd:
command = shlex.split(args.cmd)
else:
command_from_config = instance_config.get_cmd()
if command_from_config:
command = shlex.split(command_from_config)
else:
command = instance_config.get_args()
hostname = socket.getfqdn()
run_docker_container(
docker_client=docker_client,
service=service,
instance=instance,
docker_hash=docker_hash,
volumes=volumes,
interactive=args.interactive,
command=command,
hostname=hostname,
healthcheck=args.healthcheck,
healthcheck_only=args.healthcheck_only,
instance_config=instance_config,
soa_dir=args.yelpsoa_config_root,
)
def paasta_local_run(args):
if args.action == 'build' and not makefile_responds_to('cook-image'):
sys.stderr.write("A local Makefile with a 'cook-image' target is required for --build\n")
sys.stderr.write("If you meant to pull the docker image from the registry, explicitly pass --pull\n")
return 1
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
sys.stdout.write(PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates\n"
"PaaSTA is not configured locally on this host, and local-run may not behave\n"
"the same way it would behave on a server configured for PaaSTA.\n"
))
system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')
local_run_config = system_paasta_config.get_local_run_config()
service = figure_out_service_name(args, soa_dir=args.yelpsoa_config_root)
if args.cluster:
cluster = args.cluster
else:
try:
cluster = local_run_config['default_cluster']
except KeyError:
sys.stderr.write(PaastaColors.red(
"PaaSTA on this machine has not been configured with a default cluster.\n"
"Please pass one to local-run using '-c'.\n"))
return 1
instance = args.instance
docker_client = get_docker_client()
if args.action == 'build':
default_tag = 'paasta-local-run-%s-%s' % (service, get_username())
tag = os.environ.get('DOCKER_TAG', default_tag)
os.environ['DOCKER_TAG'] = tag
pull_image = False
cook_return = paasta_cook_image(args=None, service=service, soa_dir=args.yelpsoa_config_root)
if cook_return != 0:
return cook_return
elif args.action == 'dry_run':
pull_image = False
tag = None
else:
pull_image = True
tag = None
try:
configure_and_run_docker_container(
docker_client=docker_client,
docker_hash=tag,
service=service,
instance=instance,
cluster=cluster,
args=args,
pull_image=pull_image,
system_paasta_config=system_paasta_config,
dry_run=args.action == 'dry_run',
)
except errors.APIError as e:
sys.stderr.write('Can\'t run Docker container. Error: %s\n' % str(e))
return 1
def configure_and_run_docker_container(docker_client, docker_hash, service, args):
"""
Run Docker container by image hash with args set in command line.
Function prints the output of run command in stdout.
"""
try:
system_paasta_config = load_system_paasta_config()
except PaastaNotConfiguredError:
sys.stdout.write(PaastaColors.yellow(
"Warning: Couldn't load config files from '/etc/paasta'. This indicates\n"
"PaaSTA is not configured locally on this host, and local-run may not behave\n"
"the same way it would behave on a server configured for PaaSTA.\n"
))
system_paasta_config = SystemPaastaConfig({"volumes": []}, '/etc/paasta')
volumes = list()
if args.cluster:
cluster = args.cluster
else:
try:
cluster = get_default_cluster_for_service(service)
except NoConfigurationForServiceError:
sys.stdout.write(PaastaColors.red(
'Could not automatically detect cluster to emulate. Please specify one with the --cluster option.\n'))
sys.exit(2)
sys.stdout.write(PaastaColors.yellow(
'Using cluster configuration for %s. To override, use the --cluster option.\n\n' % cluster))
instance_config = get_instance_config(
service=service,
instance=args.instance,
cluster=cluster,
soa_dir=args.yelpsoa_config_root,
)
# if only one volume specified, extra_volumes should be converted to a list
extra_volumes = instance_config.get_extra_volumes()
if type(extra_volumes) == dict:
extra_volumes = [extra_volumes]
for volume in system_paasta_config.get_volumes() + extra_volumes:
volumes.append('%s:%s:%s' % (volume['hostPath'], volume['containerPath'], volume['mode'].lower()))
if args.cmd:
command = shlex.split(args.cmd)
else:
command_from_config = instance_config.get_cmd()
if command_from_config:
command = shlex.split(command_from_config)
else:
command = instance_config.get_args()
run_docker_container(
docker_client=docker_client,
service=service,
instance=args.instance,
docker_hash=docker_hash,
volumes=volumes,
interactive=args.interactive,
command=command,
healthcheck=args.healthcheck,
healthcheck_only=args.healthcheck_only,
instance_config=instance_config,
)
