def test_get_mesos_leader_cli_mesosmasterconnectionerror(mock_get_mesos_config): with mock.patch( 'paasta_tools.mesos.master.MesosMaster.resolve', side_effect=mesos.exceptions.MasterNotAvailableException, autospec=True, ): with raises(mesos.exceptions.MasterNotAvailableException): mesos_tools.get_mesos_leader()
def test_get_mesos_leader_no_hostname(): fake_url = 'localhost:5050' with contextlib.nested( mock.patch('paasta_tools.mesos_tools.master.CURRENT'), ) as ( mock_CURRENT, ): mock_CURRENT.host = fake_url with raises(ValueError): mesos_tools.get_mesos_leader()
def test_get_mesos_leader_no_hostname(): fake_url = 'localhost:5050' with mock.patch('paasta_tools.mesos_tools.get_mesos_master', autospec=True) as mock_get_master: mock_master = mock.Mock() mock_master.host = fake_url mock_get_master.return_value = mock_master with raises(ValueError): mesos_tools.get_mesos_leader()
def test_get_mesos_leader_cli_mesosmasterconnectionerror(): with contextlib.nested( mock.patch('mesos.cli.master.MesosMaster.resolve', side_effect=mesos_tools.MesosMasterConnectionError), ) as ( mock_resolve, ): with raises(mesos_tools.MesosMasterConnectionError): mesos_tools.get_mesos_leader()
def test_get_mesos_leader_connection_error(): fake_master = 'false.authority.yelpcorp.com' with mock.patch( 'requests.get', autospec=True, side_effect=requests.exceptions.ConnectionError, ): with raises(mesos_tools.MesosMasterConnectionError): mesos_tools.get_mesos_leader(fake_master)
def test_get_mesos_leader_cli_mesosmasterconnectionerror(mock_get_mesos_config): with contextlib.nested( mock.patch('paasta_tools.mesos.master.MesosMaster.resolve', side_effect=mesos.exceptions.MasterNotAvailableException, autospec=True), ) as ( mock_resolve, ): with raises(mesos.exceptions.MasterNotAvailableException): mesos_tools.get_mesos_leader()
def test_get_mesos_leader_no_hostname(): fake_url = 'localhost:5050' with contextlib.nested( mock.patch('paasta_tools.mesos_tools.get_mesos_master', autospec=True), ) as ( mock_get_master, ): mock_master = mock.Mock() mock_master.host = fake_url mock_get_master.return_value = mock_master with raises(ValueError): mesos_tools.get_mesos_leader()
def test_get_mesos_leader_socket_eroror(): fake_url = 'http://93.184.216.34:5050' with contextlib.nested( mock.patch('paasta_tools.mesos_tools.master.CURRENT'), mock.patch('paasta_tools.mesos_tools.socket.gethostbyaddr', side_effect=socket.error), ) as ( mock_CURRENT, mock_gethostbyaddr, ): mock_CURRENT.host = fake_url with raises(socket.error): mesos_tools.get_mesos_leader()
def test_get_mesos_leader_socket_error(): fake_url = 'http://93.184.216.34:5050' with mock.patch( 'paasta_tools.mesos_tools.get_mesos_master', autospec=True, ) as mock_get_master, mock.patch( 'paasta_tools.mesos_tools.socket.gethostbyaddr', side_effect=socket.error, autospec=True, ): mock_master = mock.Mock() mock_master.host = fake_url mock_get_master.return_value = mock_master with raises(socket.error): mesos_tools.get_mesos_leader()
def test_get_mesos_leader_socket_error(): fake_url = 'http://93.184.216.34:5050' with contextlib.nested( mock.patch('paasta_tools.mesos_tools.get_mesos_master', autospec=True), mock.patch('paasta_tools.mesos_tools.socket.gethostbyaddr', side_effect=socket.error, autospec=True), ) as ( mock_get_master, mock_gethostbyaddr, ): mock_master = mock.Mock() mock_master.host = fake_url mock_get_master.return_value = mock_master with raises(socket.error): mesos_tools.get_mesos_leader()
def create_driver(framework_name, scheduler, system_paasta_config, implicit_acks=False): master_uri = "{}:{}".format(mesos_tools.get_mesos_leader(), mesos_tools.MESOS_MASTER_PORT) framework = { "user": getpass.getuser(), "name": framework_name, "failover_timeout": 604800, "id": { "value": find_existing_id_if_exists_or_gen_new(framework_name) }, "checkpoint": True, "principal": system_paasta_config.get_paasta_native_config()["principal"], } driver = MesosSchedulerDriver( sched=scheduler, framework=framework, master_uri=master_uri, use_addict=True, implicit_acknowledgements=implicit_acks, principal=system_paasta_config.get_paasta_native_config()["principal"], secret=system_paasta_config.get_paasta_native_config()["secret"], ) return driver
def create_driver(framework_name, scheduler, system_paasta_config, implicit_acks=False): master_uri = '{}:{}'.format(mesos_tools.get_mesos_leader(), mesos_tools.MESOS_MASTER_PORT) framework = Dict( user=getpass.getuser(), name=framework_name, failover_timeout=604800, id=Dict(value=find_existing_id_if_exists_or_gen_new(framework_name)), checkpoint=True, principal=system_paasta_config.get_paasta_native_config()['principal']) driver = MesosSchedulerDriver( sched=scheduler, framework=framework, master_uri=master_uri, use_addict=True, implicit_acknowledgements=implicit_acks, principal=system_paasta_config.get_paasta_native_config()['principal'], secret=system_paasta_config.get_paasta_native_config()['secret'], ) return driver
def base_api(): """Helper function for making all API requests :returns: a function that can be callecd to make a request """ leader = get_mesos_leader() def execute_request(method, endpoint, **kwargs): url = "http://%s:%d%s" % (leader, MESOS_MASTER_PORT, endpoint) timeout = 15 s = Session() s.auth = load_credentials() req = Request(method, url, **kwargs) prepared = s.prepare_request(req) try: resp = s.send( prepared, timeout=timeout, ) resp.raise_for_status() return resp except HTTPError as e: e.msg = "Error executing API request calling %s. Got error: %s" % (url, e.msg) raise return execute_request
def base_api(): """Helper function for making all API requests :returns: a function that can be callecd to make a request """ leader = get_mesos_leader() def execute_request(method, endpoint, **kwargs): url = "http://%s:%d%s" % (leader, MESOS_MASTER_PORT, endpoint) timeout = 15 s = Session() s.auth = (get_principal(), get_secret()) req = Request(method, url, **kwargs) prepared = s.prepare_request(req) try: resp = s.send( prepared, timeout=timeout, ) resp.raise_for_status() return resp except HTTPError: raise HTTPError("Error executing API request calling %s." % url) return execute_request
def create_driver(framework_name, scheduler, system_paasta_config, implicit_acks=False): framework = mesos_pb2.FrameworkInfo() framework.user = "" # Have Mesos fill in the current user. framework.name = framework_name framework.failover_timeout = 604800 framework.id.value = find_existing_id_if_exists_or_gen_new(framework.name) framework.checkpoint = True credential = mesos_pb2.Credential() credential.principal = system_paasta_config.get_paasta_native_config( )['principal'] credential.secret = system_paasta_config.get_paasta_native_config( )['secret'] framework.principal = system_paasta_config.get_paasta_native_config( )['principal'] driver = MesosSchedulerDriver( scheduler, framework, '%s:%d' % (mesos_tools.get_mesos_leader(), mesos_tools.MESOS_MASTER_PORT), implicit_acks, credential) return driver
def create_driver( framework_name, scheduler, system_paasta_config, implicit_acks=False, ): master_uri = '{}:{}'.format( mesos_tools.get_mesos_leader(), mesos_tools.MESOS_MASTER_PORT, ) framework = { 'user': getpass.getuser(), 'name': framework_name, 'failover_timeout': 604800, 'id': { 'value': find_existing_id_if_exists_or_gen_new(framework_name) }, 'checkpoint': True, 'principal': system_paasta_config.get_paasta_native_config()['principal'], } driver = MesosSchedulerDriver( sched=scheduler, framework=framework, master_uri=master_uri, use_addict=True, implicit_acknowledgements=implicit_acks, principal=system_paasta_config.get_paasta_native_config()['principal'], secret=system_paasta_config.get_paasta_native_config()['secret'], ) return driver
def test_get_mesos_leader(): expected = "mesos.master.yelpcorp.com" fake_master = "false.authority.yelpcorp.com" with mock.patch("requests.get", autospec=True) as mock_requests_get: mock_requests_get.return_value = mock_response = mock.Mock() mock_response.return_code = 307 mock_response.url = "http://%s:999" % expected assert mesos_tools.get_mesos_leader(fake_master) == expected mock_requests_get.assert_called_once_with("http://%s:5050/redirect" % fake_master, timeout=10)
def test_get_mesos_leader(): expected = 'mesos.master.yelpcorp.com' fake_master = 'false.authority.yelpcorp.com' with mock.patch('requests.get', autospec=True) as mock_requests_get: mock_requests_get.return_value = mock_response = mock.Mock() mock_response.return_code = 307 mock_response.url = 'http://%s:999' % expected assert mesos_tools.get_mesos_leader(fake_master) == expected mock_requests_get.assert_called_once_with('http://%s:5050/redirect' % fake_master, timeout=10)
def test_get_mesos_leader(): fake_url = "http://93.184.216.34:5050" with asynctest.patch( "paasta_tools.mesos_tools.get_mesos_master", autospec=True) as mock_get_master, asynctest.patch( "paasta_tools.mesos_tools.socket.gethostbyaddr", autospec=True) as mock_gethostbyaddr, asynctest.patch( "paasta_tools.mesos_tools.socket.getfqdn", autospec=True) as mock_getfqdn: mock_master = mock.Mock() mock_master.host = fake_url mock_get_master.return_value = mock_master mock_gethostbyaddr.return_value = "example.org" mock_getfqdn.return_value = "example.org" assert mesos_tools.get_mesos_leader() == "example.org"
def test_get_mesos_leader(): fake_url = 'http://93.184.216.34:5050' with mock.patch( 'paasta_tools.mesos_tools.get_mesos_master', autospec=True, ) as mock_get_master, mock.patch( 'paasta_tools.mesos_tools.socket.gethostbyaddr', autospec=True, ) as mock_gethostbyaddr, mock.patch( 'paasta_tools.mesos_tools.socket.getfqdn', autospec=True, ) as mock_getfqdn: mock_master = mock.Mock() mock_master.host = fake_url mock_get_master.return_value = mock_master mock_gethostbyaddr.return_value = 'example.org' mock_getfqdn.return_value = 'example.org' assert mesos_tools.get_mesos_leader() == 'example.org'
def test_get_mesos_leader(): fake_url = 'http://93.184.216.34:5050' with contextlib.nested( mock.patch('paasta_tools.mesos_tools.master.CURRENT'), mock.patch('paasta_tools.mesos_tools.socket.gethostbyaddr'), mock.patch('paasta_tools.mesos_tools.socket.getfqdn'), ) as ( mock_CURRENT, mock_gethostbyaddr, mock_getfqdn, ): mock_CURRENT.host = fake_url mock_gethostbyaddr.return_value = 'example.org' mock_getfqdn.return_value = 'example.org' assert mesos_tools.get_mesos_leader() == 'example.org'
def test_get_mesos_leader(): fake_url = 'http://93.184.216.34:5050' with contextlib.nested( mock.patch('paasta_tools.mesos_tools.get_mesos_master', autospec=True), mock.patch('paasta_tools.mesos_tools.socket.gethostbyaddr', autospec=True), mock.patch('paasta_tools.mesos_tools.socket.getfqdn', autospec=True), ) as ( mock_get_master, mock_gethostbyaddr, mock_getfqdn, ): mock_master = mock.Mock() mock_master.host = fake_url mock_get_master.return_value = mock_master mock_gethostbyaddr.return_value = 'example.org' mock_getfqdn.return_value = 'example.org' assert mesos_tools.get_mesos_leader() == 'example.org'
def build_executor_stack( service, instance, run_id, # TODO: move run_id into task identifier? system_paasta_config, framework_staging_timeout): mesos_address = '{}:{}'.format(mesos_tools.get_mesos_leader(), mesos_tools.MESOS_MASTER_PORT) taskproc_config = system_paasta_config.get('taskproc') # TODO: implement DryRunExecutor? mesos_executor = MesosExecutor( role=taskproc_config.get('role', taskproc_config['principal']), principal=taskproc_config['principal'], secret=taskproc_config['secret'], mesos_address=mesos_address, framework_name="paasta-remote %s %s %s" % (compose_job_id(service, instance), datetime.utcnow().strftime('%Y%m%d%H%M%S%f'), run_id), framework_staging_timeout=framework_staging_timeout) retrying_executor = RetryingExecutor(mesos_executor) return retrying_executor
def create_driver(service, instance, scheduler, system_paasta_config): framework = mesos_pb2.FrameworkInfo() framework.user = "" # Have Mesos fill in the current user. framework.name = "paasta %s" % compose_job_id(service, instance) framework.failover_timeout = 604800 framework.id.value = find_existing_id_if_exists_or_gen_new(framework.name) framework.checkpoint = True credential = mesos_pb2.Credential() credential.principal = system_paasta_config.get_paasta_native_config()['principal'] credential.secret = system_paasta_config.get_paasta_native_config()['secret'] framework.principal = system_paasta_config.get_paasta_native_config()['principal'] implicitAcknowledgements = False driver = MesosSchedulerDriver( scheduler, framework, '%s:%d' % (mesos_tools.get_mesos_leader(), mesos_tools.MESOS_MASTER_PORT), implicitAcknowledgements, credential ) return driver
def main(): print(get_mesos_leader())
def main(): paasta_print(get_mesos_leader())
def print_output(argv: Optional[Sequence[str]] = None) -> None: mesos_available = is_mesos_available() kube_available = is_kubernetes_available() args = parse_args(argv) system_paasta_config = load_system_paasta_config() if mesos_available: master_kwargs = {} # we don't want to be passing False to not override a possible True # value from system config if args.use_mesos_cache: master_kwargs["use_mesos_cache"] = True master = get_mesos_master(**master_kwargs) marathon_servers = get_marathon_servers(system_paasta_config) marathon_clients = all_marathon_clients( get_marathon_clients(marathon_servers)) try: mesos_state = a_sync.block(master.state) all_mesos_results = _run_mesos_checks(mesos_master=master, mesos_state=mesos_state) except MasterNotAvailableException as e: # if we can't connect to master at all, # then bomb out early paasta_print(PaastaColors.red("CRITICAL: %s" % "\n".join(e.args))) raise FatalError(2) marathon_results = _run_marathon_checks(marathon_clients) else: marathon_results = [ metastatus_lib.HealthCheckResult( message="Marathon is not configured to run here", healthy=True) ] all_mesos_results = [ metastatus_lib.HealthCheckResult( message="Mesos is not configured to run here", healthy=True) ] if kube_available: kube_client = KubeClient() kube_results = _run_kube_checks(kube_client) else: kube_results = [ metastatus_lib.HealthCheckResult( message="Kubernetes is not configured to run here", healthy=True) ] mesos_ok = all(metastatus_lib.status_for_results(all_mesos_results)) marathon_ok = all(metastatus_lib.status_for_results(marathon_results)) kube_ok = all(metastatus_lib.status_for_results(kube_results)) mesos_summary = metastatus_lib.generate_summary_for_check( "Mesos", mesos_ok) marathon_summary = metastatus_lib.generate_summary_for_check( "Marathon", marathon_ok) kube_summary = metastatus_lib.generate_summary_for_check( "Kubernetes", kube_ok) healthy_exit = True if all([mesos_ok, marathon_ok]) else False paasta_print(f"Master paasta_tools version: {__version__}") paasta_print("Mesos leader: %s" % get_mesos_leader()) metastatus_lib.print_results_for_healthchecks(mesos_summary, mesos_ok, all_mesos_results, args.verbose) if args.verbose > 1 and mesos_available: print_with_indent( "Resources Grouped by %s" % ", ".join(args.groupings), 2) all_rows, healthy_exit = utilization_table_by_grouping_from_mesos_state( groupings=args.groupings, threshold=args.threshold, mesos_state=mesos_state) for line in format_table(all_rows): print_with_indent(line, 4) if args.autoscaling_info: print_with_indent("Autoscaling resources:", 2) headers = [ field.replace("_", " ").capitalize() for field in AutoscalingInfo._fields ] table = [headers] + [[ str(x) for x in asi ] for asi in get_autoscaling_info_for_all_resources(mesos_state)] for line in format_table(table): print_with_indent(line, 4) if args.verbose >= 3: print_with_indent("Per Slave Utilization", 2) cluster = system_paasta_config.get_cluster() service_instance_stats = get_service_instance_stats( args.service, args.instance, cluster) if service_instance_stats: print_with_indent( "Service-Instance stats:" + str(service_instance_stats), 2) # print info about slaves here. Note that we don't make modifications to # the healthy_exit variable here, because we don't care about a single slave # having high usage. all_rows, _ = utilization_table_by_grouping_from_mesos_state( groupings=args.groupings + ["hostname"], threshold=args.threshold, mesos_state=mesos_state, service_instance_stats=service_instance_stats, ) # The last column from utilization_table_by_grouping_from_mesos_state is "Agent count", which will always be # 1 for per-slave resources, so delete it. for row in all_rows: row.pop() for line in format_table(all_rows): print_with_indent(line, 4) metastatus_lib.print_results_for_healthchecks(marathon_summary, marathon_ok, marathon_results, args.verbose) metastatus_lib.print_results_for_healthchecks(kube_summary, kube_ok, kube_results, args.verbose) if args.verbose > 1 and kube_available: print_with_indent( "Resources Grouped by %s" % ", ".join(args.groupings), 2) all_rows, healthy_exit = utilization_table_by_grouping_from_kube( groupings=args.groupings, threshold=args.threshold, kube_client=kube_client) for line in format_table(all_rows): print_with_indent(line, 4) if args.autoscaling_info: print_with_indent("No autoscaling resources for Kubernetes", 2) if args.verbose >= 3: print_with_indent("Per Node Utilization", 2) cluster = system_paasta_config.get_cluster() service_instance_stats = get_service_instance_stats( args.service, args.instance, cluster) if service_instance_stats: print_with_indent( "Service-Instance stats:" + str(service_instance_stats), 2) # print info about nodes here. Note that we don't make # modifications to the healthy_exit variable here, because we don't # care about a single node having high usage. all_rows, _ = utilization_table_by_grouping_from_kube( groupings=args.groupings + ["hostname"], threshold=args.threshold, kube_client=kube_client, service_instance_stats=service_instance_stats, ) # The last column from utilization_table_by_grouping_from_kube is "Agent count", which will always be # 1 for per-node resources, so delete it. for row in all_rows: row.pop() for line in format_table(all_rows): print_with_indent(line, 4) if not healthy_exit: raise FatalError(2)
def build_executor_stack( # TODO: rename to registry? processor, service, instance, cluster, pool, # TODO: move run_id into task identifier? run_id, system_paasta_config, framework_staging_timeout, ): mesos_address = '{}:{}'.format( mesos_tools.get_mesos_leader(), mesos_tools.MESOS_MASTER_PORT, ) # TODO: implement DryRunExecutor? taskproc_config = system_paasta_config.get_taskproc() MesosExecutor = processor.executor_cls('mesos') mesos_executor = MesosExecutor( role=taskproc_config.get('role', taskproc_config.get('principal')), pool=pool, principal=taskproc_config.get('principal'), secret=taskproc_config.get('secret'), mesos_address=mesos_address, framework_name="paasta-remote %s %s %s" % ( compose_job_id(service, instance), datetime.utcnow().strftime('%Y%m%d%H%M%S%f'), run_id, ), framework_staging_timeout=framework_staging_timeout, initial_decline_delay=0.5, ) task_logging_executor = processor.executor_from_config( provider='logging', provider_config={ 'downstream_executor': mesos_executor, }, ) credentials_file = taskproc_config.get('boto_credential_file') if credentials_file: with open(credentials_file) as f: credentials = json.loads(f.read()) else: raise ValueError("Required aws credentials") region = taskproc_config.get('aws_region') endpoint = taskproc_config.get('dynamodb_endpoint') session = Session( region_name=region, aws_access_key_id=credentials['accessKeyId'], aws_secret_access_key=credentials['secretAccessKey'], ) StatefulExecutor = processor.executor_cls(provider='stateful') stateful_executor = StatefulExecutor( downstream_executor=task_logging_executor, persister=DynamoDBPersister( table_name="taskproc_events_%s" % cluster, session=session, endpoint_url=endpoint, ), ) return stateful_executor