def get_pending_messages(self, max=None):
"""Get any pending messages that aren't being held, up to max."""
accepted_types = self.get_accepted_types()
server_api = self.get_server_api()
messages = []
for filename in self._walk_pending_messages():
if max is not None and len(messages) >= max:
break
data = read_binary_file(self._message_dir(filename))
try:
# don't reinterpret messages that are meant to be sent out
message = bpickle.loads(data, as_is=True)
except ValueError as e:
logging.exception(e)
self._add_flags(filename, BROKEN)
else:
if u"type" not in message:
# Special case to decode keys for messages which were
# serialized by py27 prior to py3 upgrade, and having
# implicit byte message keys. Message may still get
# rejected by the server, but it won't block the client
# broker. (lp: #1718689)
message = {
(k if isinstance(k, str) else k.decode("ascii")): v
for k, v in message.items()}
message[u"type"] = message[u"type"].decode("ascii")
unknown_type = message["type"] not in accepted_types
unknown_api = not is_version_higher(server_api, message["api"])
if unknown_type or unknown_api:
self._add_flags(filename, HELD)
else:
messages.append(message)
return messages
def add(self, message):
"""Queue a message for delivery.
@param message: a C{dict} with a C{type} key and other keys conforming
to the L{Message} schema for that specific message type.
@return: message_id, which is an identifier for the added
message or C{None} if the message was rejected.
"""
assert "type" in message
if self._persist.get("blackhole-messages"):
logging.debug("Dropped message, awaiting resync.")
return
server_api = self.get_server_api()
if "api" not in message:
message["api"] = server_api
# We apply the schema with the highest API version that is greater
# or equal to the API version the message is tagged with.
schemas = self._schemas[message["type"]]
for api in sort_versions(schemas.keys()):
if is_version_higher(server_api, api):
schema = schemas[api]
break
message = schema.coerce(message)
message_data = bpickle.dumps(message)
filename = self._get_next_message_filename()
temp_path = filename + ".tmp"
create_file(temp_path, message_data)
os.rename(temp_path, filename)
if not self.accepts(message["type"]):
filename = self._set_flags(filename, HELD)
# For now we use the inode as the message id, as it will work
# correctly even faced with holding/unholding. It will break
# if the store is copied over for some reason, but this shouldn't
# present an issue given the current uses. In the future we
# should have a nice transactional storage (e.g. sqlite) which
# will offer a more strong primary key.
message_id = os.stat(filename).st_ino
return message_id
def get_pending_messages(self, max=None):
"""Get any pending messages that aren't being held, up to max."""
accepted_types = self.get_accepted_types()
server_api = self.get_server_api()
messages = []
for filename in self._walk_pending_messages():
if max is not None and len(messages) >= max:
break
data = self._get_content(self._message_dir(filename))
try:
message = bpickle.loads(data)
except ValueError, e:
logging.exception(e)
self._add_flags(filename, BROKEN)
else:
unknown_type = message["type"] not in accepted_types
unknown_api = not is_version_higher(server_api, message["api"])
if unknown_type or unknown_api:
self._add_flags(filename, HELD)
else:
messages.append(message)
def _handle_result(self, payload, result):
"""Handle a response from the server.
Called by L{exchange} after a batch of messages has been
successfully delivered to the server.
If the C{server_uuid} changed, a C{"server-uuid-changed"} event
will be fired.
Call L{handle_message} for each message in C{result}.
@param payload: The payload that was sent to the server.
@param result: The response got in reply to the C{payload}.
"""
message_store = self._message_store
self._client_accepted_types_hash = result.get(
"client-accepted-types-hash")
next_expected = result.get("next-expected-sequence")
old_sequence = message_store.get_sequence()
if next_expected is None:
# If the server doesn't specify anything for the next-expected
# value, just assume that it processed all messages that we sent
# fine.
next_expected = message_store.get_sequence()
next_expected += len(payload["messages"])
message_store_state = got_next_expected(message_store, next_expected)
if message_store_state == ANCIENT:
# The server has probably lost some data we sent it. The
# slate has been wiped clean (by got_next_expected), now
# let's fire an event to tell all the plugins that they
# ought to generate new messages so the server gets some
# up-to-date data.
logging.info("Server asked for ancient data: resynchronizing all "
"state with the server.")
self.send({"type": "resynchronize"})
self._reactor.fire("resynchronize-clients")
# Save the exchange token that the server has sent us. We will provide
# it at the next exchange to prove that we're still the same client.
# See also landscape.broker.transport.
message_store.set_exchange_token(result.get("next-exchange-token"))
old_uuid = message_store.get_server_uuid()
new_uuid = result.get("server-uuid")
if new_uuid != old_uuid:
logging.info("Server UUID changed (old=%s, new=%s)." %
(old_uuid, new_uuid))
self._reactor.fire("server-uuid-changed", old_uuid, new_uuid)
message_store.set_server_uuid(new_uuid)
# Extract the server API from the payload. If it's not there it must
# be 3.2, because it's the one that didn't have this field.
server_api = result.get("server-api", "3.2")
if is_version_higher(server_api, message_store.get_server_api()):
# The server can handle a message API that is higher than the one
# we're currently using. If the highest server API is greater than
# our one, so let's use our own, which is the most recent we can
# speak. Otherwise if the highest server API is less than or equal
# than ours, let's use the server one, because is the most recent
# common one.
lowest_server_api = sort_versions([server_api, self._api])[-1]
message_store.set_server_api(lowest_server_api)
message_store.commit()
sequence = message_store.get_server_sequence()
for message in result.get("messages", ()):
self.handle_message(message)
sequence += 1
message_store.set_server_sequence(sequence)
message_store.commit()
if message_store.get_pending_messages(1):
logging.info("Pending messages remain after the last exchange.")
# Either the server asked us for old messages, or we
# otherwise have more messages even after transferring
# what we could.
if next_expected != old_sequence:
self.schedule_exchange(urgent=True)
def test_greater(self):
"""
The C{is_version_higher} function returns C{True} if the first
version is greater than the second.
"""
self.assertTrue(is_version_higher(b"3.2", b"3.1"))
def test_lower(self):
"""
The C{is_version_higher} function returns C{False} if the first
version is lower than the second.
"""
self.assertFalse(is_version_higher(b"3.1", b"3.2"))
def test_equal(self):
"""
The C{is_version_higher} function returns C{True} if the first
version is the same as the second.
"""
self.assertTrue(is_version_higher(b"3.1", b"3.1"))
def _handle_pre_exchange(self):
"""
An exchange is about to happen. If we don't have a secure id already
set, and we have the needed information available, queue a registration
message with the server.
"""
# The point of storing this flag is that if we should *not* register
# now, and then after the exchange we *should*, we schedule an urgent
# exchange again. Without this flag we would just spin trying to
# connect to the server when something is clearly preventing the
# registration.
self._should_register = self.should_register()
if not self._should_register:
return
# These are just to shorten the code.
identity = self._identity
account_name = identity.account_name
if not account_name:
self._reactor.fire("registration-failed", reason="unknown-account")
return
tags = identity.tags
group = identity.access_group
registration_key = identity.registration_key
self._message_store.delete_all_messages()
if not is_valid_tag_list(tags):
tags = None
logging.error("Invalid tags provided for registration.")
message = {
"type": "register",
"hostname": get_fqdn(),
"account_name": account_name,
"computer_title": identity.computer_title,
"registration_password": identity.registration_key,
"tags": tags,
"container-info": get_container_info(),
"vm-info": get_vm_info()
}
if self._clone_secure_id:
# We use the secure id here because the registration is encrypted
# and the insecure id has been already exposed to unencrypted
# http from the ping server. In addition it's more straightforward
# to get the computer from the server through it than the insecure
message["clone_secure_id"] = self._clone_secure_id
if group:
message["access_group"] = group
server_api = self._message_store.get_server_api()
# If we have juju data to send and if the server is recent enough to
# know how to handle juju data, then we include it in the registration
# message. We want to trigger the 3.3 server handler because client
# version 14.01 has a different format for the juju-info field,
# so this makes sure that the correct schema is used by the server
# when validating our message.
if self._juju_data and is_version_higher(server_api, b"3.3"):
message["juju-info"] = {
"environment-uuid": self._juju_data["environment-uuid"],
"api-addresses": self._juju_data["api-addresses"],
"machine-id": self._juju_data["machine-id"]
}
# The computer is a normal computer, possibly a container.
with_word = "with" if bool(registration_key) else "without"
with_tags = "and tags %s " % tags if tags else ""
with_group = "in access group '%s' " % group if group else ""
logging.info(u"Queueing message to register with account %r %s%s"
"%s a password." %
(account_name, with_group, with_tags, with_word))
self._exchange.send(message)