def setUp(self):
self.ringbuffer = MagicMock()
self.ringbuffer.tail_sequence.return_value = ImmediateFuture(0)
client = MagicMock()
client.get_ringbuffer.return_value = self.ringbuffer
context = MagicMock(client=client)
self.topic = ReliableTopic("", "", context)
def get_or_connect(self, address, authenticator=None):
"""
Gets the existing connection for a given address. If it does not exist, the system will try to connect
asynchronously. In this case, it returns a Future. When the connection is established at some point in time, it
can be retrieved by using the get_connection(:class:`~hazelcast.core.Address`) or from Future.
:param address: (:class:`~hazelcast.core.Address`), the address to connect to.
:param authenticator: (Function), function to be used for authentication (optional).
:return: (:class:`~hazelcast.connection.Connection`), the existing connection or it returns a Future which includes asynchronously.
"""
if address in self.connections:
return ImmediateFuture(self.connections[address])
else:
with self._new_connection_mutex:
if address in self._pending_connections:
return self._pending_connections[address]
else:
authenticator = authenticator or self._cluster_authenticator
try:
translated_address = self._address_translator.translate(address)
if translated_address is None:
raise ValueError("Address translator could not translate address: {}".format(address))
connection = self._new_connection_func(translated_address,
self._client.config.network_config.connection_timeout,
self._client.config.network_config.socket_options,
connection_closed_callback=self._connection_closed,
message_callback=self._client.invoker._handle_client_message,
network_config=self._client.config.network_config)
except IOError:
return ImmediateExceptionFuture(sys.exc_info()[1], sys.exc_info()[2])
future = authenticator(connection).continue_with(self.on_auth, connection, address)
if not future.done():
self._pending_connections[address] = future
return future
def put_all(self, map):
check_not_none(map, "map can't be None")
if not map:
return ImmediateFuture(None)
partition_service = self._client.partition_service
partition_map = {}
for key, value in map.iteritems():
check_not_none(key, "key can't be None")
check_not_none(value, "value can't be None")
entry = (self._to_data(key), self._to_data(value))
partition_id = partition_service.get_partition_id(entry[0])
try:
partition_map[partition_id].append(entry)
except KeyError:
partition_map[partition_id] = [entry]
futures = []
for partition_id, entry_list in partition_map.iteritems():
future = self._encode_invoke_on_partition(map_put_all_codec,
partition_id,
entries=dict(entry_list))
futures.append(future)
return combine_futures(*futures)
def mock(*_, **__):
d = {
"session_id": self.session_id,
"ttl_millis": 1000,
"heartbeat_millis": 100,
}
return ImmediateFuture(d)
def get_all(self, keys):
check_not_none(keys, "keys can't be None")
if not keys:
return ImmediateFuture({})
partition_service = self._client.partition_service
partition_to_keys = {}
for key in keys:
check_not_none(key, "key can't be None")
key_data = self._to_data(key)
partition_id = partition_service.get_partition_id(key_data)
try:
partition_to_keys[partition_id].append(key_data)
except KeyError:
partition_to_keys[partition_id] = [key_data]
futures = []
for partition_id, key_list in partition_to_keys.iteritems():
future = self._encode_invoke_on_partition(map_get_all_codec,
partition_id,
keys=key_list)
futures.append(future)
def merge(f):
return dict(itertools.chain(*f.result()))
return combine_futures(*futures).continue_with(merge)
def _get_internal(self, key_data):
try:
value = self._near_cache[key_data]
return ImmediateFuture(value)
except KeyError:
future = super(MapFeatNearCache, self)._get_internal(key_data)
return future.continue_with(self._update_cache, key_data)
def send_state_to_cluster_fn():
counter.add(1)
if counter.get() == 5:
# Let's pretend it succeeds at some point
return ImmediateFuture(None)
raise RuntimeError("expected")
def deregister_listener(self, user_registration_id):
check_not_none(user_registration_id, "None user_registration_id is not allowed!")
with self._registration_lock:
listener_registration = self._active_registrations.pop(user_registration_id, None)
if not listener_registration:
return ImmediateFuture(False)
for connection, event_registration in six.iteritems(
listener_registration.connection_registrations
):
# Remove local handler
self.remove_event_handler(event_registration.correlation_id)
# The rest is for deleting the remote registration
server_registration_id = event_registration.server_registration_id
deregister_request = listener_registration.encode_deregister_request(
server_registration_id
)
if deregister_request is None:
# None means no remote registration (e.g. for backup acks)
continue
invocation = Invocation(
deregister_request, connection=connection, timeout=six.MAXSIZE, urgent=True
)
self._invocation_service.invoke(invocation)
def handler(f, connection=connection):
e = f.exception()
if e:
if isinstance(
e, (HazelcastClientNotActiveError, IOError, TargetDisconnectedError)
):
return
_logger.warning(
"Deregistration of listener with ID %s has failed for address %s",
user_registration_id,
connection.remote_address,
)
invocation.future.add_done_callback(handler)
listener_registration.connection_registrations.clear()
return ImmediateFuture(True)
def remove_listener(self, registration_id: str) -> Future[bool]:
"""Stops receiving messages for the given message listener.
If the given listener already removed, this method does nothing.
Args:
registration_id: ID of listener registration.
Returns:
``True`` if registration is removed, ``False`` otherwise.
"""
check_not_none(registration_id, "Registration id cannot be None")
runner = self._runners.get(registration_id, None)
if not runner:
return ImmediateFuture(False)
runner.cancel()
return ImmediateFuture(True)
def capacity(self):
if not self._capacity:
def cache_capacity(f):
self._capacity = f.result()
return f.result()
return self._encode_invoke(
ringbuffer_capacity_codec).continue_with(cache_capacity)
return ImmediateFuture(self._capacity)
def _get_or_connect_to_address(self, address):
for connection in list(six.itervalues(self.active_connections)):
if connection.remote_address == address:
return ImmediateFuture(connection)
try:
translated = self._translate(address)
connection = self._create_connection(translated)
return self._authenticate(connection).continue_with(self._on_auth, connection)
except Exception as e:
return ImmediateExceptionFuture(e)
def _get_or_connect_to_member(self, member):
connection = self.active_connections.get(member.uuid, None)
if connection:
return ImmediateFuture(connection)
try:
translated = self._translate(member.address)
connection = self._create_connection(translated)
return self._authenticate(connection).continue_with(self._on_auth, connection)
except Exception as e:
return ImmediateExceptionFuture(e)
def test_shutdown(self):
self.set_session(self.prepare_state())
self.set_thread_id(123)
self.manager._mutexes[self.raft_group_id] = object()
m = MagicMock(return_value=ImmediateFuture(True))
self.manager._request_close_session = m
self.manager.shutdown().result()
m.assert_called_once_with(self.raft_group_id, self.session_id)
self.assertEqual(0, len(self.manager._sessions))
self.assertEqual(0, len(self.manager._mutexes))
self.assertEqual(0, len(self.manager._thread_ids))
def _has_next(self):
"""Returns a Future indicating whether there are more rows
left to iterate.
Returns:
hazelcast.future.Future:
"""
if self.position == self.row_count:
# We exhausted the current page.
if self.is_last:
# This was the last page, no row left
# on the server side.
return ImmediateFuture(False)
# It seems that there are some rows left on the server.
# Fetch them, and then return.
return self.fetch_fn().continue_with(self._fetch_continuation)
# There are some elements left in the current page.
return ImmediateFuture(True)
def _get_or_create_session(self, group_id):
with self._lock:
if self._shutdown:
error = HazelcastClientNotActiveError("Session manager is already shut down!")
return ImmediateExceptionFuture(error)
session = self._sessions.get(group_id, None)
if session is None or not session.is_valid():
with self._mutex(group_id):
session = self._sessions.get(group_id)
if session is None or not session.is_valid():
return self._create_new_session(group_id)
return ImmediateFuture(session)
def send_all_schemas(self) -> Future:
schemas = self._compact_serializer.get_schemas()
if not schemas:
_logger.debug("There is no schema to send to the cluster.")
return ImmediateFuture(None)
_logger.debug("Sending the following schemas to the cluster: %s",
schemas)
request = client_send_all_schemas_codec.encode_request(schemas)
invocation = Invocation(request, urgent=True)
self._invocation_service.invoke(invocation)
return invocation.future
def _get_or_connect(self, address):
connection = self.get_connection_from_address(address)
if connection:
return ImmediateFuture(connection)
with self._lock:
connection = self.get_connection_from_address(address)
if connection:
return ImmediateFuture(connection)
else:
pending = self._pending_connections.get(address, None)
if pending:
return pending
else:
try:
translated = self._address_provider.translate(address)
if not translated:
error = ValueError(
"Address translator could not translate address %s"
% address)
return ImmediateExceptionFuture(error)
factory = self._reactor.connection_factory
connection = factory(
self,
self._connection_id_generator.get_and_increment(),
translated,
self._client.config,
self._invocation_service.handle_client_message,
)
except IOError:
error = sys.exc_info()
return ImmediateExceptionFuture(error[1], error[2])
future = self._authenticate(connection).continue_with(
self._on_auth, connection, address)
self._pending_connections[address] = future
return future
def get_or_create_unique_thread_id(self, group_id):
with self._lock:
if self._shutdown:
error = HazelcastClientNotActiveError(
"Session manager is already shut down!")
return ImmediateExceptionFuture(error)
key = (group_id, thread_id())
global_thread_id = self._thread_ids.get(key)
if global_thread_id:
return ImmediateFuture(global_thread_id)
return self._request_generate_thread_id(group_id).continue_with(
lambda t_id: self._thread_ids.setdefault(key, t_id.result()))
def close(self):
"""Release the resources associated with the query result.
The query engine delivers the rows asynchronously. The query may
become inactive even before all rows are consumed. The invocation
of this command will cancel the execution of the query on all members
if the query is still active. Otherwise it is no-op. For a result
with an update count it is always no-op.
The returned Future results with:
- :class:`HazelcastSqlError`: In case there is an error closing the
result.
Returns:
Future[None]:
"""
with self._lock:
if self._closed:
# Do nothing if the result is already closed.
return ImmediateFuture(None)
error = HazelcastSqlError(
self._sql_service.get_client_id(),
_SqlErrorCode.CANCELLED_BY_USER,
"Query was cancelled by the user",
None,
)
if not self._fetch_future:
# Make sure that all subsequent fetches will fail.
self._fetch_future = Future()
self._on_fetch_error(error)
def wrap_error_on_failure(f):
# If the close request is failed somehow,
# wrap it in a HazelcastSqlError.
try:
return f.result()
except Exception as e:
raise self._sql_service.re_raise(e, self._connection)
self._closed = True
# Send the close request
return self._sql_service.close(
self._connection,
self._query_id).continue_with(wrap_error_on_failure)
def prepare_acquire_session(self, session_id, err=None):
if err:
val = ImmediateExceptionFuture(err)
else:
val = ImmediateFuture(session_id)
acquire_mock = MagicMock(return_value=val)
release_mock = MagicMock()
invalidate_mock = MagicMock()
self.session_manager.acquire_session = acquire_mock
self.session_manager.release_session = release_mock
self.session_manager.invalidate_session = invalidate_mock
self.acquire_session = acquire_mock
self.release_session = release_mock
self.invalidate_session = invalidate_mock
def capacity(self):
"""
Returns the capacity of this Ringbuffer.
:return: (long), the capacity of Ringbuffer.
"""
if not self._capacity:
def cache_capacity(f):
self._capacity = f.result()
return f.result()
return self._encode_invoke(
ringbuffer_capacity_codec).continue_with(cache_capacity)
return ImmediateFuture(self._capacity)
def _get_all_internal(self, partition_to_keys, futures=None):
if futures is None:
futures = []
for key_dic in partition_to_keys.itervalues():
for key in key_dic.keys():
try:
key_data = key_dic[key]
value = self._near_cache[key_data]
future = ImmediateFuture((key, value))
futures.append(future)
del key_dic[key]
except KeyError:
pass
return super(MapFeatNearCache,
self)._get_all_internal(partition_to_keys, futures)
def test_heartbeat(self):
reactor = self.mock_reactor()
self.mock_request_new_session()
r = MagicMock(return_value=ImmediateFuture(None))
self.manager._request_heartbeat = r
self.manager.acquire_session(self.raft_group_id, 1).result()
time.sleep(2)
self.manager.shutdown()
reactor.shutdown()
self.assertGreater(self.context.reactor.add_timer.call_count,
1) # assert that the heartbeat task is executed
r.assert_called()
r.assert_called_with(self.raft_group_id, self.session_id)
self.assertEqual(1, len(self.manager._sessions))
def capacity(self):
"""Returns the capacity of this Ringbuffer.
Returns:
hazelcast.future.Future[int]: The capacity of Ringbuffer.
"""
if not self._capacity:
def handler(message):
self._capacity = ringbuffer_capacity_codec.decode_response(
message)
return self._capacity
request = ringbuffer_capacity_codec.encode_request(self.name)
return self._invoke(request, handler)
return ImmediateFuture(self._capacity)
def test_heartbeat_when_session_is_released(self):
reactor = self.mock_reactor()
self.mock_request_new_session()
r = MagicMock(return_value=ImmediateFuture(None))
self.manager._request_heartbeat = r
self.manager.acquire_session(self.raft_group_id, 1).add_done_callback(
lambda _: self.manager.release_session(self.raft_group_id, self.session_id, 1)
)
time.sleep(2)
self.manager.shutdown()
reactor.shutdown()
# assert that the heartbeat task is executed
self.assertGreater(self.context.reactor.add_timer.call_count, 1)
r.assert_not_called()
self.assertEqual(1, len(self.manager._sessions))
def get_all(self, keys):
check_not_none(keys, "keys can't be None")
if not keys:
return ImmediateFuture({})
partition_service = self._client.partition_service
partition_to_keys = {}
for key in keys:
check_not_none(key, "key can't be None")
key_data = self._to_data(key)
partition_id = partition_service.get_partition_id(key_data)
try:
partition_to_keys[partition_id][key] = key_data
except KeyError:
partition_to_keys[partition_id] = {key: key_data}
return self._get_all_internal(partition_to_keys)
def increase_permits(self, increase: int) -> Future[None]:
"""Increases the number of available permits by the indicated amount.
If there are some callers waiting for permits to become available, they
will be notified. Moreover, if the caller has acquired some permits,
they are not released with this call.
Args:
increase: The number of permits to increase.
Raises:
AssertionError: If ``increase`` is negative.
"""
check_not_negative(increase, "Increase must be non-negative")
if increase == 0:
return ImmediateFuture(None)
return self._do_change_permits(increase)
def new_id(self):
while True:
block = self._block
next_id = block.next_id()
if next_id is not None:
return ImmediateFuture(next_id)
with self._lock:
# new block was assigned in the meantime
if block is not self._block:
continue
future = Future()
self._id_queue.append(future)
if not self._request_in_air:
self._request_in_air = True
self._request_new_batch()
return future
def reduce_permits(self, reduction: int) -> Future[None]:
"""Reduces the number of available permits by the indicated amount.
This method differs from ``acquire`` as it does not block until permits
become available. Similarly, if the caller has acquired some permits,
they are not released with this call.
Args:
reduction: The number of permits to reduce.
Raises:
AssertionError: If the ``reduction`` is negative.
"""
check_not_negative(reduction, "Reduction must be non-negative")
if reduction == 0:
return ImmediateFuture(None)
return self._do_change_permits(-reduction)
def shutdown(self):
with self._lock:
if self._shutdown:
return ImmediateFuture(None)
self._shutdown = True
if self._heartbeat_timer:
self._heartbeat_timer.cancel()
futures = []
for session in list(self._sessions.values()):
future = self._request_close_session(session.group_id, session.id)
futures.append(future)
def clear(_):
self._sessions.clear()
self._mutexes.clear()
self._thread_ids.clear()
return combine_futures(futures).continue_with(clear)
class ImmediateFutureTest(unittest.TestCase):
f = None
def setUp(self):
self.f = ImmediateFuture("done")
def test_result(self):
self.assertEqual("done", self.f.result())
def test_exception(self):
self.assertIsNone(self.f.exception())
def test_traceback(self):
self.assertIsNone(self.f.traceback())
def test_set_result(self):
with self.assertRaises(NotImplementedError):
self.f.set_result("done")
def test_set_exception(self):
with self.assertRaises(NotImplementedError):
self.f.set_exception(RuntimeError())
def test_is_succcess(self):
self.assertTrue(self.f.is_success())
def test_is_done(self):
self.assertTrue(self.f.done())
def test_is_not_running(self):
self.assertFalse(self.f.running())
def test_callback(self):
n = [0]
def callback(f):
self.assertEqual(f, self.f)
n[0] += 1
self.f.add_done_callback(callback)
self.assertEqual(n[0], 1)
def setUp(self):
self.f = ImmediateFuture("done")
