def test_atomic_gauge():
ag = AtomicGauge('a')
assert ag.name() == 'a'
assert ag.read() == 0
assert ag.add(-2) == -2
ag = AtomicGauge('a')
assert ag.decrement() == -1
def test_named_gauge_types():
with pytest.raises(TypeError):
ag = AtomicGauge(0)
with pytest.raises(TypeError):
ag = AtomicGauge(None)
with pytest.raises(TypeError):
lb = Label(None, 3)
with pytest.raises(TypeError):
mg = MutatorGauge({})
def _init_metrics(self):
self._session_expirations = AtomicGauge('session_expirations')
self._connection_losses = AtomicGauge('connection_losses')
self.metrics.register(self._session_expirations)
self.metrics.register(self._connection_losses)
self.metrics.register(
LambdaGauge('session_id', lambda: self.session_id))
self.metrics.register(
LambdaGauge('live', lambda: int(self._live.is_set())))
def test_atomic_gauge_types():
with pytest.raises(TypeError):
ag = AtomicGauge('a', None)
with pytest.raises(TypeError):
ag = AtomicGauge('a', 'hello')
ag = AtomicGauge('a', 23)
with pytest.raises(TypeError):
ag.add(None)
with pytest.raises(TypeError):
ag.add('hello')
def __init__(self, *args, **kw):
super(TwitterKazooClient, self).__init__(*args, **kw)
self.connecting = threading.Event()
self.__session_expirations = AtomicGauge('session_expirations')
self.__connection_losses = AtomicGauge('connection_losses')
self.__session_id = LambdaGauge('session_id', lambda:
(self._session_id or 0))
self.metrics.register(self.__session_expirations)
self.metrics.register(self.__connection_losses)
self.metrics.register(self.__session_id)
self.add_listener(self._observable_listener)
def __init__(self, *args, **kw):
if 'connection_retry' not in kw:
# The default backoff delay limit in kazoo is 3600 seconds, which is generally
# too conservative for our use cases. If not supplied by the caller, provide
# a backoff that will truncate earlier.
kw['connection_retry'] = KazooRetry(
max_delay=DEFAULT_RETRY_MAX_DELAY_SECS, **DEFAULT_RETRY_DICT)
super(TwitterKazooClient, self).__init__(*args, **kw)
self.connecting = threading.Event()
self.__session_expirations = AtomicGauge('session_expirations')
self.__connection_losses = AtomicGauge('connection_losses')
self.__session_id = LambdaGauge('session_id', lambda: (self._session_id or 0))
self.metrics.register(self.__session_expirations)
self.metrics.register(self.__connection_losses)
self.metrics.register(self.__session_id)
self.add_listener(self._observable_listener)
def _setup_metrics(self):
self._metrics = self.Metrics()
self._metrics.cluster_count = self.metrics.register(
AtomicGauge('cluster_count', 0))
# Total resources requested by the scheduler's clients. When a cluster is created its resources
# are added to the total; when it's deleted its resources are subtracted from the total.
# NOTE: These are 'requested' resources that are independent of resources offered by Mesos or
# allocated to or used by Mysos tasks running on Mesos cluster.
self._metrics.total_requested_cpus = self.metrics.register(
MutatorGauge('total_requested_cpus', 0.))
self._metrics.total_requested_mem_mb = self.metrics.register(
MutatorGauge('total_requested_mem_mb', 0.))
self._metrics.total_requested_disk_mb = self.metrics.register(
MutatorGauge('total_requested_disk_mb', 0.))
# 1: registered; 0: not registered.
self._metrics.framework_registered = self.metrics.register(
MutatorGauge('framework_registered', 0))
self._startup_time = datetime.utcnow()
self._metrics.uptime = self.metrics.register(
LambdaGauge(
'uptime', lambda:
(datetime.utcnow() - self._startup_time).total_seconds()))
# Counters for tasks in terminal states.
self._metrics.tasks_lost = self.metrics.register(
AtomicGauge('tasks_lost', 0))
self._metrics.tasks_finished = self.metrics.register(
AtomicGauge('tasks_finished', 0))
self._metrics.tasks_failed = self.metrics.register(
AtomicGauge('tasks_failed', 0))
self._metrics.tasks_killed = self.metrics.register(
AtomicGauge('tasks_killed', 0))
self._metrics.resource_offers = self.metrics.register(
AtomicGauge('resource_offers', 0))
self._metrics.offers_incompatible_role = self.metrics.register(
AtomicGauge('offers_incompatible_role', 0))
self._metrics.tasks_launched = self.metrics.register(
AtomicGauge('tasks_launched', 0))
# 'offers_unused' are due to idle scheduler or resources don't fit, i.e.,
# 'resource_offers' - 'tasks_launched' - 'offers_incompatible_role'.
self._metrics.offers_unused = self.metrics.register(
AtomicGauge('offers_unused', 0))
def __init__(self):
self._count = AtomicGauge('count')
self._ns = AtomicGauge('total_ns')
self.metrics.register(self._count)
self.metrics.register(self._ns)
def __init__(self, target_host, target_port, clock=time):
self._clock = clock
self._target = (target_host, target_port)
self._pings = AtomicGauge('pings')
self.metrics.register(self._pings)
def _init_stats(self):
self._gauge_session_expirations = AtomicGauge('session-expirations')
self._gauge_connection_losses = AtomicGauge('connection-losses')
def __init__(self,
state,
state_provider,
framework_user,
executor_uri,
executor_cmd,
kazoo,
zk_url,
election_timeout,
admin_keypath,
scheduler_key,
installer_args=None,
backup_store_args=None,
executor_environ=None,
executor_source_prefix=None,
framework_role='*'):
"""
:param state: The Scheduler object.
:param state_provider: The StateProvider instance that the scheduler should use to
restore/persist states.
:param framework_user: See flags.
:param executor_uri: See flags.
:param executor_cmd: See flags.
:param framework_role: See flags.
:param election_timeout: See flags.
:param admin_keypath: See flags.
:param scheduler_key: Scheduler uses it to encrypt cluster passwords.
:param installer_args: See flags.
:param backup_store_args: See flags.
:param executor_environ: See flags.
:param executor_source_prefix: See flags.
:param kazoo: The Kazoo client for communicating MySQL cluster information between the
scheduler and the executors.
:param zk_url: ZooKeeper URL for used by the scheduler and the executors to access ZooKeeper.
"""
self._lock = threading.Lock()
if not isinstance(state, Scheduler):
raise TypeError("'state' should be an instance of Scheduler")
self._state = state
if not isinstance(state_provider, StateProvider):
raise TypeError(
"'state_provider' should be an instance of StateProvider")
self._state_provider = state_provider
self._framework_user = framework_user
self._executor_uri = executor_uri
self._executor_cmd = executor_cmd
self._framework_role = framework_role
self._election_timeout = election_timeout
self._admin_keypath = admin_keypath
self._installer_args = installer_args
self._backup_store_args = backup_store_args
self._executor_environ = executor_environ
self._executor_source_prefix = executor_source_prefix
self._driver = None # Will be set by registered().
# Use a subdir to avoid name collision with the state storage.
self._discover_zk_url = posixpath.join(zk_url, "discover")
self._kazoo = kazoo
self._scheduler_key = scheduler_key
self._password_box = PasswordBox(scheduler_key)
self._tasks = {} # {Task ID: cluster name} mappings.
self._launchers = OrderedDict(
) # Order-preserving {cluster name : MySQLClusterLauncher}
# mappings so cluster requests are fulfilled on a first come,
# first serve (FCFS) basis.
self.stopped = threading.Event(
) # An event set when the scheduler is stopped.
self.connected = threading.Event(
) # An event set when the scheduler is first connected to
# Mesos. The scheduler tolerates later disconnections.
self._cluster_count = self.metrics.register(
AtomicGauge('cluster_count', 0))
# Total resources requested by the scheduler's clients. When a cluster is created its resources
# are added to the total; when it's deleted its resources are subtracted from the total.
# NOTE: These are 'requested' resources that are independent of resources offered by Mesos or
# allocated to or used by Mysos tasks running on Mesos cluster.
self._total_requested_cpus = self.metrics.register(
MutatorGauge('total_requested_cpus', 0.))
self._total_requested_mem_mb = self.metrics.register(
MutatorGauge('total_requested_mem_mb', 0.))
self._total_requested_disk_mb = self.metrics.register(
MutatorGauge('total_requested_disk_mb', 0.))
