def test_incompatible_resource_role(self): scheduler1 = MysosScheduler( self._state, self._state_provider, self._framework_user, "./executor.pex", "cmd.sh", self._zk_client, self._zk_url, Amount(5, Time.SECONDS), "/etc/mysos/admin_keyfile.yml", gen_encryption_key(), framework_role='mysos') # Require 'mysos' but the resources are in '*'. RootMetrics().register_observable('scheduler', scheduler1) scheduler1.registered(self._driver, self._framework_id, object()) scheduler1.create_cluster("cluster1", "mysql_user", 3) scheduler1.resourceOffers(self._driver, [self._offer]) assert "declineOffer" in self._driver.method_calls assert len(self._driver.method_calls["declineOffer"]) == 1 # [0][0][1]: [First declineOffer call][The positional args][The first positional arg], which is # a 'Filters' object. assert (self._driver.method_calls["declineOffer"][0][0][1].refuse_seconds == INCOMPATIBLE_ROLE_OFFER_REFUSE_DURATION.as_(Time.SECONDS)) sample = RootMetrics().sample() assert sample['scheduler.offers_incompatible_role'] == 1
def register_build_properties(): if not HAS_PEX: return rm = RootMetrics().scope('build') try: build_properties = PEX().info.build_properties except PEX.NotFound: return for key, value in build_properties.items(): rm.register(Label(str(key), str(value)))
def test_basic_scoping(): lb = Label('ping', 'pong') rm = RootMetrics() rm.register(lb) rm.scope('bing').register(lb) assert rm.sample() == {'ping': 'pong', 'bing.ping': 'pong'} rm.clear()
def test_bad_scope_names(): rm = RootMetrics() my_scope = rm.scope("my_scope") with pytest.raises(TypeError): my_scope.scope(None) with pytest.raises(TypeError): my_scope.scope({}) with pytest.raises(TypeError): my_scope.scope(123) with pytest.raises(TypeError): my_scope.scope(RootMetrics)
def test_bad_scope_names(): rm = RootMetrics() my_scope = rm.scope('my_scope') with pytest.raises(TypeError): my_scope.scope(None) with pytest.raises(TypeError): my_scope.scope({}) with pytest.raises(TypeError): my_scope.scope(123) with pytest.raises(TypeError): my_scope.scope(RootMetrics)
def test_basic_scoping(): lb = Label('ping', 'pong') rm = RootMetrics() rm.register(lb) rm.scope('bing').register(lb) assert rm.sample() == { 'ping': 'pong', 'bing.ping': 'pong' } rm.clear()
def test_basic_scoping(): lb = Label("ping", "pong") rm = RootMetrics() rm.register(lb) rm.scope("bing").register(lb) assert rm.sample() == {"ping": "pong", "bing.ping": "pong"} rm.clear()
def test_basic_registration_and_clear(): lb = Label('ping', 'pong') rm = RootMetrics() rm.register(lb) assert rm.sample() == {'ping': 'pong'} rm.clear() assert rm.sample() == {}
def test_filtering_vars_filter_enabled_and_not_requested(self): rm = RootMetrics() zone = NamedGauge('alpha', "wont_be_visible") alpha = NamedGauge('zone', "smf1") rm.register(zone) rm.register(alpha) metrics = RootMetrics() vars_subsystem = VarsSubsystem() regex = vars_subsystem.compile_stats_filters(["alpha", "beta.*"]) endpoint = VarsEndpoint(period=Amount(60000, Time.MILLISECONDS), stats_filter=regex) metrics_returned = endpoint.handle_vars_json() assert "zone" in metrics_returned assert "alpha" in metrics_returned request.GET.replace('filtered', None)
def __init__(self, period=None): self._metrics = RootMetrics() if period is not None: self._monitor = MetricSampler(self._metrics, period) else: self._monitor = MetricSampler(self._metrics) self._monitor.start()
def test_scoped_registration_uses_references(): mg = MutatorGauge('name', 'brian') rm = RootMetrics() rm.scope('earth').register(mg) rm.scope('pluto').register(mg) assert rm.sample() == {'earth.name': 'brian', 'pluto.name': 'brian'} mg.write('zargon') assert rm.sample() == {'earth.name': 'zargon', 'pluto.name': 'zargon'} rm.clear()
def test_basic_registration_and_clear(): lb = Label("ping", "pong") rm = RootMetrics() rm.register(lb) assert rm.sample() == {"ping": "pong"} rm.clear() assert rm.sample() == {}
def test_register_string(): rm = RootMetrics() hello_gauge = rm.register('hello') assert rm.sample() == {'hello': None} hello_gauge.write('poop') assert rm.sample() == {'hello': 'poop'} rm.clear()
def main(args, options): pingpong = PingPongServer(options.target_host, options.target_port) RootServer().mount_routes(pingpong) RootMetrics().register_observable('pingpong', pingpong) try: time.sleep(2**20) except KeyboardInterrupt: log.info('Shutting down.')
def test_nontrivial_gauges(): for label_value in ['a', 0, 2.5, [1,2,"3"], {'a': 'b'}, {'c': None}, False]: lb = Label('ping', label_value) rm = RootMetrics() rm.register(lb) assert rm.sample() == {'ping': label_value} rm.clear() assert rm.sample() == {}
def test_nontrivial_gauges(): for label_value in ["a", 0, 2.5, [1, 2, "3"], {"a": "b"}, {"c": None}, False]: lb = Label("ping", label_value) rm = RootMetrics() rm.register(lb) assert rm.sample() == {"ping": label_value} rm.clear() assert rm.sample() == {}
def configure_server(task_observer): bottle_wrapper = BottleObserver(task_observer) root_metrics = RootMetrics() server = HttpServer() server.mount_routes(bottle_wrapper) server.mount_routes(DiagnosticsEndpoints()) server.mount_routes(VarsEndpoint()) register_build_properties(root_metrics) register_diagnostics(root_metrics) return server
def test_scoped_registration_uses_references(): mg = MutatorGauge('name', 'brian') rm = RootMetrics() rm.scope('earth').register(mg) rm.scope('pluto').register(mg) assert rm.sample() == { 'earth.name': 'brian', 'pluto.name': 'brian' } mg.write('zargon') assert rm.sample() == { 'earth.name': 'zargon', 'pluto.name': 'zargon' } rm.clear()
def test_scoped_registration_uses_references(): mg = MutatorGauge("name", "brian") rm = RootMetrics() rm.scope("earth").register(mg) rm.scope("pluto").register(mg) assert rm.sample() == {"earth.name": "brian", "pluto.name": "brian"} mg.write("zargon") assert rm.sample() == {"earth.name": "zargon", "pluto.name": "zargon"} rm.clear()
def test_register_string(): rm = RootMetrics() hello_gauge = rm.register('hello') assert rm.sample() == { 'hello': 'None' } hello_gauge.write('poop') assert rm.sample() == { 'hello': 'poop' } rm.clear()
def test_register_string(): rm = RootMetrics() hello_gauge = rm.register("hello") assert rm.sample() == {"hello": None} hello_gauge.write("poop") assert rm.sample() == {"hello": "poop"} rm.clear()
def test_nontrivial_gauges(): for label_value in [ 'a', 0, 2.5, [1, 2, "3"], { 'a': 'b' }, { 'c': None }, False ]: lb = Label('ping', label_value) rm = RootMetrics() rm.register(lb) assert rm.sample() == {'ping': label_value} rm.clear() assert rm.sample() == {}
def test_scheduler_metrics(self): scheduler_key = gen_encryption_key() scheduler = MysosScheduler( self._state, self._state_provider, self._framework_user, "./executor.pex", "cmd.sh", self._zk_client, self._zk_url, Amount(5, Time.SECONDS), "/etc/mysos/admin_keyfile.yml", scheduler_key) RootMetrics().register_observable('scheduler', scheduler) scheduler.registered(self._driver, self._framework_id, object()) sample = RootMetrics().sample() assert sample['scheduler.framework_registered'] == 1 scheduler.create_cluster( "cluster1", "mysql_user", 3, cluster_password='******') sample = RootMetrics().sample() assert sample['scheduler.cluster_count'] == 1 assert sample['scheduler.total_requested_mem_mb'] == DEFAULT_TASK_MEM.as_(Data.MB) * 3 assert sample['scheduler.total_requested_disk_mb'] == DEFAULT_TASK_DISK.as_(Data.MB) * 3 assert sample['scheduler.total_requested_cpus'] == DEFAULT_TASK_CPUS * 3 scheduler.resourceOffers(self._driver, [self._offer]) sample = RootMetrics().sample() assert sample['scheduler.resource_offers'] == 1 assert sample['scheduler.tasks_launched'] == 1 status = mesos_pb2.TaskStatus() status.state = mesos_pb2.TASK_RUNNING status.slave_id.value = self._offer.slave_id.value status.task_id.value = 'mysos-cluster1-0' scheduler.statusUpdate(self._driver, status) status.state = mesos_pb2.TASK_FAILED scheduler.statusUpdate(self._driver, status) sample = RootMetrics().sample() assert sample['scheduler.tasks_failed'] == 1 scheduler.delete_cluster("cluster1", 'test_password') sample = RootMetrics().sample() assert sample['scheduler.cluster_count'] == 0 assert sample['scheduler.total_requested_mem_mb'] == 0 assert sample['scheduler.total_requested_disk_mb'] == 0 assert sample['scheduler.total_requested_cpus'] == 0
def test_scheduler_metrics(self): scheduler_key = gen_encryption_key() scheduler = MysosScheduler(self._state, self._state_provider, self._framework_user, "./executor.pex", "cmd.sh", self._zk_client, self._zk_url, Amount(5, Time.SECONDS), "/etc/mysos/admin_keyfile.yml", scheduler_key) RootMetrics().register_observable('scheduler', scheduler) scheduler.registered(self._driver, self._framework_id, object()) scheduler.create_cluster("cluster1", "mysql_user", 3, cluster_password='******') sample = RootMetrics().sample() assert sample['scheduler.cluster_count'] == 1 assert sample[ 'scheduler.total_requested_mem_mb'] == DEFAULT_TASK_MEM.as_( Data.MB) * 3 assert sample[ 'scheduler.total_requested_disk_mb'] == DEFAULT_TASK_DISK.as_( Data.MB) * 3 assert sample[ 'scheduler.total_requested_cpus'] == DEFAULT_TASK_CPUS * 3 scheduler.delete_cluster("cluster1", 'test_password') sample = RootMetrics().sample() assert sample['scheduler.cluster_count'] == 0 assert sample['scheduler.total_requested_mem_mb'] == 0 assert sample['scheduler.total_requested_disk_mb'] == 0 assert sample['scheduler.total_requested_cpus'] == 0
def setup_function(self): options = app.get_options() rs = RootServer() if rs: varz = VarsEndpoint(period=Amount( options.twitter_common_metrics_vars_sampling_delay_ms, Time.MILLISECONDS)) rs.mount_routes(varz) register_diagnostics() register_build_properties() if options.twitter_common_app_modules_varz_trace_endpoints: plugin = EndpointTracePlugin() rs.install(plugin) RootMetrics().register_observable( options.twitter_common_app_modules_varz_trace_namespace, plugin)
def test_root_metrics_singleton(): rm = RootMetrics() rm2 = RootMetrics() assert id(rm) == id(rm2)
def test_nested_scopes(): rm = RootMetrics() mg = rm.scope("a").scope("b").scope("c").register("123") mg.write(Amount(1, Time.MILLISECONDS)) assert rm.sample() == {"a.b.c.123": "1 ms"} rm.clear()
def _cleared_rm(): rm = RootMetrics() rm.clear() return rm
def test_nested_scopes(): rm = RootMetrics() mg = rm.scope('a').scope('b').scope('c').register('123') mg.write(Amount(1, Time.MILLISECONDS)) assert rm.sample() == {'a.b.c.123': '1 ms'} rm.clear()
def test_scheduler_runs(): """ Verifies that the scheduler successfully launches 3 "no-op" MySQL tasks. NOTE: Due to the limitation of zake the scheduler's ZK operations are not propagated to executors in separate processes but they are unit-tested separately. """ import mesos.native # Make sure fake_mysos_executor.pex is available to be fetched by Mesos slave. assert os.path.isfile('dist/fake_mysos_executor.pex') storage = FakeStorage(SequentialThreadingHandler()) zk_client = FakeClient(storage=storage) zk_client.start() zk_url = "zk://fake_host/home/mysos/clusters" cluster_name = "test_cluster" num_nodes = 3 state_provider = LocalStateProvider(safe_mkdtemp()) framework_info = FrameworkInfo(user=getpass.getuser(), name="mysos", checkpoint=False) state = Scheduler(framework_info) scheduler = MysosScheduler(state, state_provider, getpass.getuser(), os.path.abspath("dist/fake_mysos_executor.pex"), "./fake_mysos_executor.pex", zk_client, zk_url, Amount(40, Time.SECONDS), "/fakepath", gen_encryption_key()) RootMetrics().register_observable('scheduler', scheduler) scheduler_driver = mesos.native.MesosSchedulerDriver( scheduler, framework_info, "local") scheduler_driver.start() # Wait until the scheduler is connected and becomes available. assert scheduler.connected.wait(30) scheduler.create_cluster(cluster_name, "mysql_user", num_nodes, cluster_password="******") # A slave is promoted to be the master. deadline( lambda: wait_for_master( get_cluster_path(posixpath.join(zk_url, 'discover'), cluster_name), zk_client), Amount(40, Time.SECONDS)) scheduler.delete_cluster(cluster_name, password="******") # The cluster is deleted from ZooKeeper. deadline( lambda: wait_for_termination( get_cluster_path(posixpath.join(zk_url, 'discover'), cluster_name), zk_client), Amount(40, Time.SECONDS)) sample = RootMetrics().sample() assert sample['scheduler.tasks_killed'] == 1 assert scheduler_driver.stop() == DRIVER_STOPPED
def register_diagnostics(): import os, sys, time rm = RootMetrics().scope('sys') now = time.time() rm.register(LambdaGauge('uptime', lambda: time.time() - now)) rm.register(Label('argv', repr(sys.argv))) rm.register(Label('path', repr(sys.path))) rm.register(LambdaGauge('modules', lambda: ', '.join(sys.modules.keys()))) rm.register(Label('version', sys.version)) rm.register(Label('platform', sys.platform)) rm.register(Label('executable', sys.executable)) rm.register(Label('prefix', sys.prefix)) rm.register(Label('exec_prefix', sys.exec_prefix)) rm.register(Label('uname', ' '.join(os.uname())))
def setUp(self): self._scheduler = FakeScheduler() self._app = TestApp( MysosServer(self._scheduler, self.web_assets_dir, MetricSampler(RootMetrics())).app)
def register_diagnostics(): rm = RootMetrics().scope('sys') now = time.time() rm.register(LambdaGauge('uptime', lambda: time.time() - now)) rm.register(Label('argv', repr(sys.argv))) rm.register(Label('path', repr(sys.path))) rm.register(Label('version', sys.version)) rm.register(Label('platform', sys.platform)) rm.register(Label('executable', sys.executable)) rm.register(Label('prefix', sys.prefix)) rm.register(Label('exec_prefix', sys.exec_prefix)) rm.register(Label('uname', ' '.join(os.uname())))