def test_get_mesos_leader_cli_mesosmasterconnectionerror(mock_get_mesos_config):
with mock.patch(
'paasta_tools.mesos.master.MesosMaster.resolve',
side_effect=mesos.exceptions.MasterNotAvailableException, autospec=True,
):
with raises(mesos.exceptions.MasterNotAvailableException):
mesos_tools.get_mesos_leader()
def test_get_mesos_leader_no_hostname():
fake_url = 'localhost:5050'
with contextlib.nested(
mock.patch('paasta_tools.mesos_tools.master.CURRENT'), ) as (
mock_CURRENT, ):
mock_CURRENT.host = fake_url
with raises(ValueError):
mesos_tools.get_mesos_leader()
def test_get_mesos_leader_no_hostname():
fake_url = 'localhost:5050'
with mock.patch('paasta_tools.mesos_tools.get_mesos_master', autospec=True) as mock_get_master:
mock_master = mock.Mock()
mock_master.host = fake_url
mock_get_master.return_value = mock_master
with raises(ValueError):
mesos_tools.get_mesos_leader()
def test_get_mesos_leader_cli_mesosmasterconnectionerror():
with contextlib.nested(
mock.patch('mesos.cli.master.MesosMaster.resolve', side_effect=mesos_tools.MesosMasterConnectionError),
) as (
mock_resolve,
):
with raises(mesos_tools.MesosMasterConnectionError):
mesos_tools.get_mesos_leader()
def test_get_mesos_leader_connection_error():
fake_master = 'false.authority.yelpcorp.com'
with mock.patch(
'requests.get',
autospec=True,
side_effect=requests.exceptions.ConnectionError,
):
with raises(mesos_tools.MesosMasterConnectionError):
mesos_tools.get_mesos_leader(fake_master)
def test_get_mesos_leader_connection_error():
fake_master = 'false.authority.yelpcorp.com'
with mock.patch(
'requests.get',
autospec=True,
side_effect=requests.exceptions.ConnectionError,
):
with raises(mesos_tools.MesosMasterConnectionError):
mesos_tools.get_mesos_leader(fake_master)
def test_get_mesos_leader_cli_mesosmasterconnectionerror(mock_get_mesos_config):
with contextlib.nested(
mock.patch('paasta_tools.mesos.master.MesosMaster.resolve',
side_effect=mesos.exceptions.MasterNotAvailableException, autospec=True),
) as (
mock_resolve,
):
with raises(mesos.exceptions.MasterNotAvailableException):
mesos_tools.get_mesos_leader()
def test_get_mesos_leader_no_hostname():
fake_url = 'localhost:5050'
with contextlib.nested(
mock.patch('paasta_tools.mesos_tools.master.CURRENT'),
) as (
mock_CURRENT,
):
mock_CURRENT.host = fake_url
with raises(ValueError):
mesos_tools.get_mesos_leader()
def test_get_mesos_leader_no_hostname():
fake_url = 'localhost:5050'
with contextlib.nested(
mock.patch('paasta_tools.mesos_tools.get_mesos_master', autospec=True),
) as (
mock_get_master,
):
mock_master = mock.Mock()
mock_master.host = fake_url
mock_get_master.return_value = mock_master
with raises(ValueError):
mesos_tools.get_mesos_leader()
def test_get_mesos_leader_socket_eroror():
fake_url = 'http://93.184.216.34:5050'
with contextlib.nested(
mock.patch('paasta_tools.mesos_tools.master.CURRENT'),
mock.patch('paasta_tools.mesos_tools.socket.gethostbyaddr', side_effect=socket.error),
) as (
mock_CURRENT,
mock_gethostbyaddr,
):
mock_CURRENT.host = fake_url
with raises(socket.error):
mesos_tools.get_mesos_leader()
def test_get_mesos_leader_socket_error():
fake_url = 'http://93.184.216.34:5050'
with mock.patch(
'paasta_tools.mesos_tools.get_mesos_master', autospec=True,
) as mock_get_master, mock.patch(
'paasta_tools.mesos_tools.socket.gethostbyaddr', side_effect=socket.error, autospec=True,
):
mock_master = mock.Mock()
mock_master.host = fake_url
mock_get_master.return_value = mock_master
with raises(socket.error):
mesos_tools.get_mesos_leader()
def test_get_mesos_leader_socket_error():
fake_url = 'http://93.184.216.34:5050'
with contextlib.nested(
mock.patch('paasta_tools.mesos_tools.get_mesos_master', autospec=True),
mock.patch('paasta_tools.mesos_tools.socket.gethostbyaddr', side_effect=socket.error, autospec=True),
) as (
mock_get_master,
mock_gethostbyaddr,
):
mock_master = mock.Mock()
mock_master.host = fake_url
mock_get_master.return_value = mock_master
with raises(socket.error):
mesos_tools.get_mesos_leader()
def create_driver(framework_name,
scheduler,
system_paasta_config,
implicit_acks=False):
master_uri = "{}:{}".format(mesos_tools.get_mesos_leader(),
mesos_tools.MESOS_MASTER_PORT)
framework = {
"user": getpass.getuser(),
"name": framework_name,
"failover_timeout": 604800,
"id": {
"value": find_existing_id_if_exists_or_gen_new(framework_name)
},
"checkpoint": True,
"principal":
system_paasta_config.get_paasta_native_config()["principal"],
}
driver = MesosSchedulerDriver(
sched=scheduler,
framework=framework,
master_uri=master_uri,
use_addict=True,
implicit_acknowledgements=implicit_acks,
principal=system_paasta_config.get_paasta_native_config()["principal"],
secret=system_paasta_config.get_paasta_native_config()["secret"],
)
return driver
def create_driver(framework_name,
scheduler,
system_paasta_config,
implicit_acks=False):
master_uri = '{}:{}'.format(mesos_tools.get_mesos_leader(),
mesos_tools.MESOS_MASTER_PORT)
framework = Dict(
user=getpass.getuser(),
name=framework_name,
failover_timeout=604800,
id=Dict(value=find_existing_id_if_exists_or_gen_new(framework_name)),
checkpoint=True,
principal=system_paasta_config.get_paasta_native_config()['principal'])
driver = MesosSchedulerDriver(
sched=scheduler,
framework=framework,
master_uri=master_uri,
use_addict=True,
implicit_acknowledgements=implicit_acks,
principal=system_paasta_config.get_paasta_native_config()['principal'],
secret=system_paasta_config.get_paasta_native_config()['secret'],
)
return driver
def base_api():
"""Helper function for making all API requests
:returns: a function that can be callecd to make a request
"""
leader = get_mesos_leader()
def execute_request(method, endpoint, **kwargs):
url = "http://%s:%d%s" % (leader, MESOS_MASTER_PORT, endpoint)
timeout = 15
s = Session()
s.auth = load_credentials()
req = Request(method, url, **kwargs)
prepared = s.prepare_request(req)
try:
resp = s.send(
prepared,
timeout=timeout,
)
resp.raise_for_status()
return resp
except HTTPError as e:
e.msg = "Error executing API request calling %s. Got error: %s" % (url, e.msg)
raise
return execute_request
def base_api():
"""Helper function for making all API requests
:returns: a function that can be callecd to make a request
"""
leader = get_mesos_leader()
def execute_request(method, endpoint, **kwargs):
url = "http://%s:%d%s" % (leader, MESOS_MASTER_PORT, endpoint)
timeout = 15
s = Session()
s.auth = (get_principal(), get_secret())
req = Request(method, url, **kwargs)
prepared = s.prepare_request(req)
try:
resp = s.send(
prepared,
timeout=timeout,
)
resp.raise_for_status()
return resp
except HTTPError:
raise HTTPError("Error executing API request calling %s." % url)
return execute_request
def create_driver(framework_name,
scheduler,
system_paasta_config,
implicit_acks=False):
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = framework_name
framework.failover_timeout = 604800
framework.id.value = find_existing_id_if_exists_or_gen_new(framework.name)
framework.checkpoint = True
credential = mesos_pb2.Credential()
credential.principal = system_paasta_config.get_paasta_native_config(
)['principal']
credential.secret = system_paasta_config.get_paasta_native_config(
)['secret']
framework.principal = system_paasta_config.get_paasta_native_config(
)['principal']
driver = MesosSchedulerDriver(
scheduler, framework, '%s:%d' %
(mesos_tools.get_mesos_leader(), mesos_tools.MESOS_MASTER_PORT),
implicit_acks, credential)
return driver
def base_api():
"""Helper function for making all API requests
:returns: a function that can be callecd to make a request
"""
leader = get_mesos_leader()
def execute_request(method, endpoint, **kwargs):
url = "http://%s:%d%s" % (leader, MESOS_MASTER_PORT, endpoint)
timeout = 15
s = Session()
s.auth = load_credentials()
req = Request(method, url, **kwargs)
prepared = s.prepare_request(req)
try:
resp = s.send(
prepared,
timeout=timeout,
)
resp.raise_for_status()
return resp
except HTTPError as e:
e.msg = "Error executing API request calling %s. Got error: %s" % (url, e.msg)
raise
return execute_request
def create_driver(
framework_name,
scheduler,
system_paasta_config,
implicit_acks=False,
):
master_uri = '{}:{}'.format(
mesos_tools.get_mesos_leader(),
mesos_tools.MESOS_MASTER_PORT,
)
framework = {
'user': getpass.getuser(),
'name': framework_name,
'failover_timeout': 604800,
'id': {
'value': find_existing_id_if_exists_or_gen_new(framework_name)
},
'checkpoint': True,
'principal':
system_paasta_config.get_paasta_native_config()['principal'],
}
driver = MesosSchedulerDriver(
sched=scheduler,
framework=framework,
master_uri=master_uri,
use_addict=True,
implicit_acknowledgements=implicit_acks,
principal=system_paasta_config.get_paasta_native_config()['principal'],
secret=system_paasta_config.get_paasta_native_config()['secret'],
)
return driver
def test_get_mesos_leader():
expected = "mesos.master.yelpcorp.com"
fake_master = "false.authority.yelpcorp.com"
with mock.patch("requests.get", autospec=True) as mock_requests_get:
mock_requests_get.return_value = mock_response = mock.Mock()
mock_response.return_code = 307
mock_response.url = "http://%s:999" % expected
assert mesos_tools.get_mesos_leader(fake_master) == expected
mock_requests_get.assert_called_once_with("http://%s:5050/redirect" % fake_master, timeout=10)
def test_get_mesos_leader():
expected = 'mesos.master.yelpcorp.com'
fake_master = 'false.authority.yelpcorp.com'
with mock.patch('requests.get', autospec=True) as mock_requests_get:
mock_requests_get.return_value = mock_response = mock.Mock()
mock_response.return_code = 307
mock_response.url = 'http://%s:999' % expected
assert mesos_tools.get_mesos_leader(fake_master) == expected
mock_requests_get.assert_called_once_with('http://%s:5050/redirect' %
fake_master,
timeout=10)
def test_get_mesos_leader():
fake_url = "http://93.184.216.34:5050"
with asynctest.patch(
"paasta_tools.mesos_tools.get_mesos_master",
autospec=True) as mock_get_master, asynctest.patch(
"paasta_tools.mesos_tools.socket.gethostbyaddr",
autospec=True) as mock_gethostbyaddr, asynctest.patch(
"paasta_tools.mesos_tools.socket.getfqdn",
autospec=True) as mock_getfqdn:
mock_master = mock.Mock()
mock_master.host = fake_url
mock_get_master.return_value = mock_master
mock_gethostbyaddr.return_value = "example.org"
mock_getfqdn.return_value = "example.org"
assert mesos_tools.get_mesos_leader() == "example.org"
def test_get_mesos_leader():
fake_url = 'http://93.184.216.34:5050'
with mock.patch(
'paasta_tools.mesos_tools.get_mesos_master', autospec=True,
) as mock_get_master, mock.patch(
'paasta_tools.mesos_tools.socket.gethostbyaddr', autospec=True,
) as mock_gethostbyaddr, mock.patch(
'paasta_tools.mesos_tools.socket.getfqdn', autospec=True,
) as mock_getfqdn:
mock_master = mock.Mock()
mock_master.host = fake_url
mock_get_master.return_value = mock_master
mock_gethostbyaddr.return_value = 'example.org'
mock_getfqdn.return_value = 'example.org'
assert mesos_tools.get_mesos_leader() == 'example.org'
def test_get_mesos_leader():
fake_url = 'http://93.184.216.34:5050'
with contextlib.nested(
mock.patch('paasta_tools.mesos_tools.master.CURRENT'),
mock.patch('paasta_tools.mesos_tools.socket.gethostbyaddr'),
mock.patch('paasta_tools.mesos_tools.socket.getfqdn'),
) as (
mock_CURRENT,
mock_gethostbyaddr,
mock_getfqdn,
):
mock_CURRENT.host = fake_url
mock_gethostbyaddr.return_value = 'example.org'
mock_getfqdn.return_value = 'example.org'
assert mesos_tools.get_mesos_leader() == 'example.org'
def test_get_mesos_leader():
fake_url = 'http://93.184.216.34:5050'
with contextlib.nested(
mock.patch('paasta_tools.mesos_tools.get_mesos_master', autospec=True),
mock.patch('paasta_tools.mesos_tools.socket.gethostbyaddr', autospec=True),
mock.patch('paasta_tools.mesos_tools.socket.getfqdn', autospec=True),
) as (
mock_get_master,
mock_gethostbyaddr,
mock_getfqdn,
):
mock_master = mock.Mock()
mock_master.host = fake_url
mock_get_master.return_value = mock_master
mock_gethostbyaddr.return_value = 'example.org'
mock_getfqdn.return_value = 'example.org'
assert mesos_tools.get_mesos_leader() == 'example.org'
def build_executor_stack(
service,
instance,
run_id, # TODO: move run_id into task identifier?
system_paasta_config,
framework_staging_timeout):
mesos_address = '{}:{}'.format(mesos_tools.get_mesos_leader(),
mesos_tools.MESOS_MASTER_PORT)
taskproc_config = system_paasta_config.get('taskproc')
# TODO: implement DryRunExecutor?
mesos_executor = MesosExecutor(
role=taskproc_config.get('role', taskproc_config['principal']),
principal=taskproc_config['principal'],
secret=taskproc_config['secret'],
mesos_address=mesos_address,
framework_name="paasta-remote %s %s %s" %
(compose_job_id(service, instance),
datetime.utcnow().strftime('%Y%m%d%H%M%S%f'), run_id),
framework_staging_timeout=framework_staging_timeout)
retrying_executor = RetryingExecutor(mesos_executor)
return retrying_executor
def create_driver(service, instance, scheduler, system_paasta_config):
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "paasta %s" % compose_job_id(service, instance)
framework.failover_timeout = 604800
framework.id.value = find_existing_id_if_exists_or_gen_new(framework.name)
framework.checkpoint = True
credential = mesos_pb2.Credential()
credential.principal = system_paasta_config.get_paasta_native_config()['principal']
credential.secret = system_paasta_config.get_paasta_native_config()['secret']
framework.principal = system_paasta_config.get_paasta_native_config()['principal']
implicitAcknowledgements = False
driver = MesosSchedulerDriver(
scheduler,
framework,
'%s:%d' % (mesos_tools.get_mesos_leader(), mesos_tools.MESOS_MASTER_PORT),
implicitAcknowledgements,
credential
)
return driver
def base_api():
"""Helper function for making all API requests
:returns: a function that can be callecd to make a request
"""
leader = get_mesos_leader()
def execute_request(method, endpoint, **kwargs):
url = "http://%s:%d%s" % (leader, MESOS_MASTER_PORT, endpoint)
timeout = 15
s = Session()
s.auth = (get_principal(), get_secret())
req = Request(method, url, **kwargs)
prepared = s.prepare_request(req)
try:
resp = s.send(
prepared,
timeout=timeout,
)
resp.raise_for_status()
return resp
except HTTPError:
raise HTTPError("Error executing API request calling %s." % url)
return execute_request
def main():
print(get_mesos_leader())
def main():
paasta_print(get_mesos_leader())
def print_output(argv: Optional[Sequence[str]] = None) -> None:
mesos_available = is_mesos_available()
kube_available = is_kubernetes_available()
args = parse_args(argv)
system_paasta_config = load_system_paasta_config()
if mesos_available:
master_kwargs = {}
# we don't want to be passing False to not override a possible True
# value from system config
if args.use_mesos_cache:
master_kwargs["use_mesos_cache"] = True
master = get_mesos_master(**master_kwargs)
marathon_servers = get_marathon_servers(system_paasta_config)
marathon_clients = all_marathon_clients(
get_marathon_clients(marathon_servers))
try:
mesos_state = a_sync.block(master.state)
all_mesos_results = _run_mesos_checks(mesos_master=master,
mesos_state=mesos_state)
except MasterNotAvailableException as e:
# if we can't connect to master at all,
# then bomb out early
paasta_print(PaastaColors.red("CRITICAL: %s" % "\n".join(e.args)))
raise FatalError(2)
marathon_results = _run_marathon_checks(marathon_clients)
else:
marathon_results = [
metastatus_lib.HealthCheckResult(
message="Marathon is not configured to run here", healthy=True)
]
all_mesos_results = [
metastatus_lib.HealthCheckResult(
message="Mesos is not configured to run here", healthy=True)
]
if kube_available:
kube_client = KubeClient()
kube_results = _run_kube_checks(kube_client)
else:
kube_results = [
metastatus_lib.HealthCheckResult(
message="Kubernetes is not configured to run here",
healthy=True)
]
mesos_ok = all(metastatus_lib.status_for_results(all_mesos_results))
marathon_ok = all(metastatus_lib.status_for_results(marathon_results))
kube_ok = all(metastatus_lib.status_for_results(kube_results))
mesos_summary = metastatus_lib.generate_summary_for_check(
"Mesos", mesos_ok)
marathon_summary = metastatus_lib.generate_summary_for_check(
"Marathon", marathon_ok)
kube_summary = metastatus_lib.generate_summary_for_check(
"Kubernetes", kube_ok)
healthy_exit = True if all([mesos_ok, marathon_ok]) else False
paasta_print(f"Master paasta_tools version: {__version__}")
paasta_print("Mesos leader: %s" % get_mesos_leader())
metastatus_lib.print_results_for_healthchecks(mesos_summary, mesos_ok,
all_mesos_results,
args.verbose)
if args.verbose > 1 and mesos_available:
print_with_indent(
"Resources Grouped by %s" % ", ".join(args.groupings), 2)
all_rows, healthy_exit = utilization_table_by_grouping_from_mesos_state(
groupings=args.groupings,
threshold=args.threshold,
mesos_state=mesos_state)
for line in format_table(all_rows):
print_with_indent(line, 4)
if args.autoscaling_info:
print_with_indent("Autoscaling resources:", 2)
headers = [
field.replace("_", " ").capitalize()
for field in AutoscalingInfo._fields
]
table = [headers] + [[
str(x) for x in asi
] for asi in get_autoscaling_info_for_all_resources(mesos_state)]
for line in format_table(table):
print_with_indent(line, 4)
if args.verbose >= 3:
print_with_indent("Per Slave Utilization", 2)
cluster = system_paasta_config.get_cluster()
service_instance_stats = get_service_instance_stats(
args.service, args.instance, cluster)
if service_instance_stats:
print_with_indent(
"Service-Instance stats:" + str(service_instance_stats), 2)
# print info about slaves here. Note that we don't make modifications to
# the healthy_exit variable here, because we don't care about a single slave
# having high usage.
all_rows, _ = utilization_table_by_grouping_from_mesos_state(
groupings=args.groupings + ["hostname"],
threshold=args.threshold,
mesos_state=mesos_state,
service_instance_stats=service_instance_stats,
)
# The last column from utilization_table_by_grouping_from_mesos_state is "Agent count", which will always be
# 1 for per-slave resources, so delete it.
for row in all_rows:
row.pop()
for line in format_table(all_rows):
print_with_indent(line, 4)
metastatus_lib.print_results_for_healthchecks(marathon_summary,
marathon_ok,
marathon_results,
args.verbose)
metastatus_lib.print_results_for_healthchecks(kube_summary, kube_ok,
kube_results, args.verbose)
if args.verbose > 1 and kube_available:
print_with_indent(
"Resources Grouped by %s" % ", ".join(args.groupings), 2)
all_rows, healthy_exit = utilization_table_by_grouping_from_kube(
groupings=args.groupings,
threshold=args.threshold,
kube_client=kube_client)
for line in format_table(all_rows):
print_with_indent(line, 4)
if args.autoscaling_info:
print_with_indent("No autoscaling resources for Kubernetes", 2)
if args.verbose >= 3:
print_with_indent("Per Node Utilization", 2)
cluster = system_paasta_config.get_cluster()
service_instance_stats = get_service_instance_stats(
args.service, args.instance, cluster)
if service_instance_stats:
print_with_indent(
"Service-Instance stats:" + str(service_instance_stats), 2)
# print info about nodes here. Note that we don't make
# modifications to the healthy_exit variable here, because we don't
# care about a single node having high usage.
all_rows, _ = utilization_table_by_grouping_from_kube(
groupings=args.groupings + ["hostname"],
threshold=args.threshold,
kube_client=kube_client,
service_instance_stats=service_instance_stats,
)
# The last column from utilization_table_by_grouping_from_kube is "Agent count", which will always be
# 1 for per-node resources, so delete it.
for row in all_rows:
row.pop()
for line in format_table(all_rows):
print_with_indent(line, 4)
if not healthy_exit:
raise FatalError(2)
def build_executor_stack(
# TODO: rename to registry?
processor,
service,
instance,
cluster,
pool,
# TODO: move run_id into task identifier?
run_id,
system_paasta_config,
framework_staging_timeout,
):
mesos_address = '{}:{}'.format(
mesos_tools.get_mesos_leader(),
mesos_tools.MESOS_MASTER_PORT,
)
# TODO: implement DryRunExecutor?
taskproc_config = system_paasta_config.get_taskproc()
MesosExecutor = processor.executor_cls('mesos')
mesos_executor = MesosExecutor(
role=taskproc_config.get('role', taskproc_config.get('principal')),
pool=pool,
principal=taskproc_config.get('principal'),
secret=taskproc_config.get('secret'),
mesos_address=mesos_address,
framework_name="paasta-remote %s %s %s" % (
compose_job_id(service, instance),
datetime.utcnow().strftime('%Y%m%d%H%M%S%f'),
run_id,
),
framework_staging_timeout=framework_staging_timeout,
initial_decline_delay=0.5,
)
task_logging_executor = processor.executor_from_config(
provider='logging',
provider_config={
'downstream_executor': mesos_executor,
},
)
credentials_file = taskproc_config.get('boto_credential_file')
if credentials_file:
with open(credentials_file) as f:
credentials = json.loads(f.read())
else:
raise ValueError("Required aws credentials")
region = taskproc_config.get('aws_region')
endpoint = taskproc_config.get('dynamodb_endpoint')
session = Session(
region_name=region,
aws_access_key_id=credentials['accessKeyId'],
aws_secret_access_key=credentials['secretAccessKey'],
)
StatefulExecutor = processor.executor_cls(provider='stateful')
stateful_executor = StatefulExecutor(
downstream_executor=task_logging_executor,
persister=DynamoDBPersister(
table_name="taskproc_events_%s" % cluster,
session=session,
endpoint_url=endpoint,
),
)
return stateful_executor
