def get_pending_messages(self, max=None): """Get any pending messages that aren't being held, up to max.""" accepted_types = self.get_accepted_types() server_api = self.get_server_api() messages = [] for filename in self._walk_pending_messages(): if max is not None and len(messages) >= max: break data = read_binary_file(self._message_dir(filename)) try: # don't reinterpret messages that are meant to be sent out message = bpickle.loads(data, as_is=True) except ValueError as e: logging.exception(e) self._add_flags(filename, BROKEN) else: if u"type" not in message: # Special case to decode keys for messages which were # serialized by py27 prior to py3 upgrade, and having # implicit byte message keys. Message may still get # rejected by the server, but it won't block the client # broker. (lp: #1718689) message = { (k if isinstance(k, str) else k.decode("ascii")): v for k, v in message.items()} message[u"type"] = message[u"type"].decode("ascii") unknown_type = message["type"] not in accepted_types unknown_api = not is_version_higher(server_api, message["api"]) if unknown_type or unknown_api: self._add_flags(filename, HELD) else: messages.append(message) return messages
def add(self, message): """Queue a message for delivery. @param message: a C{dict} with a C{type} key and other keys conforming to the L{Message} schema for that specific message type. @return: message_id, which is an identifier for the added message or C{None} if the message was rejected. """ assert "type" in message if self._persist.get("blackhole-messages"): logging.debug("Dropped message, awaiting resync.") return server_api = self.get_server_api() if "api" not in message: message["api"] = server_api # We apply the schema with the highest API version that is greater # or equal to the API version the message is tagged with. schemas = self._schemas[message["type"]] for api in sort_versions(schemas.keys()): if is_version_higher(server_api, api): schema = schemas[api] break message = schema.coerce(message) message_data = bpickle.dumps(message) filename = self._get_next_message_filename() temp_path = filename + ".tmp" create_file(temp_path, message_data) os.rename(temp_path, filename) if not self.accepts(message["type"]): filename = self._set_flags(filename, HELD) # For now we use the inode as the message id, as it will work # correctly even faced with holding/unholding. It will break # if the store is copied over for some reason, but this shouldn't # present an issue given the current uses. In the future we # should have a nice transactional storage (e.g. sqlite) which # will offer a more strong primary key. message_id = os.stat(filename).st_ino return message_id
def get_pending_messages(self, max=None): """Get any pending messages that aren't being held, up to max.""" accepted_types = self.get_accepted_types() server_api = self.get_server_api() messages = [] for filename in self._walk_pending_messages(): if max is not None and len(messages) >= max: break data = self._get_content(self._message_dir(filename)) try: message = bpickle.loads(data) except ValueError, e: logging.exception(e) self._add_flags(filename, BROKEN) else: unknown_type = message["type"] not in accepted_types unknown_api = not is_version_higher(server_api, message["api"]) if unknown_type or unknown_api: self._add_flags(filename, HELD) else: messages.append(message)
def _handle_result(self, payload, result): """Handle a response from the server. Called by L{exchange} after a batch of messages has been successfully delivered to the server. If the C{server_uuid} changed, a C{"server-uuid-changed"} event will be fired. Call L{handle_message} for each message in C{result}. @param payload: The payload that was sent to the server. @param result: The response got in reply to the C{payload}. """ message_store = self._message_store self._client_accepted_types_hash = result.get( "client-accepted-types-hash") next_expected = result.get("next-expected-sequence") old_sequence = message_store.get_sequence() if next_expected is None: # If the server doesn't specify anything for the next-expected # value, just assume that it processed all messages that we sent # fine. next_expected = message_store.get_sequence() next_expected += len(payload["messages"]) message_store_state = got_next_expected(message_store, next_expected) if message_store_state == ANCIENT: # The server has probably lost some data we sent it. The # slate has been wiped clean (by got_next_expected), now # let's fire an event to tell all the plugins that they # ought to generate new messages so the server gets some # up-to-date data. logging.info("Server asked for ancient data: resynchronizing all " "state with the server.") self.send({"type": "resynchronize"}) self._reactor.fire("resynchronize-clients") # Save the exchange token that the server has sent us. We will provide # it at the next exchange to prove that we're still the same client. # See also landscape.broker.transport. message_store.set_exchange_token(result.get("next-exchange-token")) old_uuid = message_store.get_server_uuid() new_uuid = result.get("server-uuid") if new_uuid != old_uuid: logging.info("Server UUID changed (old=%s, new=%s)." % (old_uuid, new_uuid)) self._reactor.fire("server-uuid-changed", old_uuid, new_uuid) message_store.set_server_uuid(new_uuid) # Extract the server API from the payload. If it's not there it must # be 3.2, because it's the one that didn't have this field. server_api = result.get("server-api", "3.2") if is_version_higher(server_api, message_store.get_server_api()): # The server can handle a message API that is higher than the one # we're currently using. If the highest server API is greater than # our one, so let's use our own, which is the most recent we can # speak. Otherwise if the highest server API is less than or equal # than ours, let's use the server one, because is the most recent # common one. lowest_server_api = sort_versions([server_api, self._api])[-1] message_store.set_server_api(lowest_server_api) message_store.commit() sequence = message_store.get_server_sequence() for message in result.get("messages", ()): self.handle_message(message) sequence += 1 message_store.set_server_sequence(sequence) message_store.commit() if message_store.get_pending_messages(1): logging.info("Pending messages remain after the last exchange.") # Either the server asked us for old messages, or we # otherwise have more messages even after transferring # what we could. if next_expected != old_sequence: self.schedule_exchange(urgent=True)
def test_greater(self): """ The C{is_version_higher} function returns C{True} if the first version is greater than the second. """ self.assertTrue(is_version_higher(b"3.2", b"3.1"))
def test_lower(self): """ The C{is_version_higher} function returns C{False} if the first version is lower than the second. """ self.assertFalse(is_version_higher(b"3.1", b"3.2"))
def test_equal(self): """ The C{is_version_higher} function returns C{True} if the first version is the same as the second. """ self.assertTrue(is_version_higher(b"3.1", b"3.1"))
def _handle_pre_exchange(self): """ An exchange is about to happen. If we don't have a secure id already set, and we have the needed information available, queue a registration message with the server. """ # The point of storing this flag is that if we should *not* register # now, and then after the exchange we *should*, we schedule an urgent # exchange again. Without this flag we would just spin trying to # connect to the server when something is clearly preventing the # registration. self._should_register = self.should_register() if not self._should_register: return # These are just to shorten the code. identity = self._identity account_name = identity.account_name if not account_name: self._reactor.fire("registration-failed", reason="unknown-account") return tags = identity.tags group = identity.access_group registration_key = identity.registration_key self._message_store.delete_all_messages() if not is_valid_tag_list(tags): tags = None logging.error("Invalid tags provided for registration.") message = { "type": "register", "hostname": get_fqdn(), "account_name": account_name, "computer_title": identity.computer_title, "registration_password": identity.registration_key, "tags": tags, "container-info": get_container_info(), "vm-info": get_vm_info() } if self._clone_secure_id: # We use the secure id here because the registration is encrypted # and the insecure id has been already exposed to unencrypted # http from the ping server. In addition it's more straightforward # to get the computer from the server through it than the insecure message["clone_secure_id"] = self._clone_secure_id if group: message["access_group"] = group server_api = self._message_store.get_server_api() # If we have juju data to send and if the server is recent enough to # know how to handle juju data, then we include it in the registration # message. We want to trigger the 3.3 server handler because client # version 14.01 has a different format for the juju-info field, # so this makes sure that the correct schema is used by the server # when validating our message. if self._juju_data and is_version_higher(server_api, b"3.3"): message["juju-info"] = { "environment-uuid": self._juju_data["environment-uuid"], "api-addresses": self._juju_data["api-addresses"], "machine-id": self._juju_data["machine-id"] } # The computer is a normal computer, possibly a container. with_word = "with" if bool(registration_key) else "without" with_tags = "and tags %s " % tags if tags else "" with_group = "in access group '%s' " % group if group else "" logging.info(u"Queueing message to register with account %r %s%s" "%s a password." % (account_name, with_group, with_tags, with_word)) self._exchange.send(message)