def get_pending_messages(self, max=None): """Get any pending messages that aren't being held, up to max.""" accepted_types = self.get_accepted_types() server_api = self.get_server_api() messages = [] for filename in self._walk_pending_messages(): if max is not None and len(messages) >= max: break data = read_binary_file(self._message_dir(filename)) try: # don't reinterpret messages that are meant to be sent out message = bpickle.loads(data, as_is=True) except ValueError as e: logging.exception(e) self._add_flags(filename, BROKEN) else: if u"type" not in message: # Special case to decode keys for messages which were # serialized by py27 prior to py3 upgrade, and having # implicit byte message keys. Message may still get # rejected by the server, but it won't block the client # broker. (lp: #1718689) message = { (k if isinstance(k, str) else k.decode("ascii")): v for k, v in message.items()} message[u"type"] = message[u"type"].decode("ascii") unknown_type = message["type"] not in accepted_types unknown_api = not is_version_higher(server_api, message["api"]) if unknown_type or unknown_api: self._add_flags(filename, HELD) else: messages.append(message) return messages
def _reprocess_holding(self): """ Unhold accepted messages left behind, and hold unaccepted pending messages. """ offset = 0 pending_offset = self.get_pending_offset() accepted_types = self.get_accepted_types() for old_filename in self._walk_messages(): flags = self._get_flags(old_filename) try: message = bpickle.loads(self._get_content(old_filename)) except ValueError, e: logging.exception(e) if HELD not in flags: offset += 1 else: accepted = message["type"] in accepted_types if HELD in flags: if accepted: new_filename = self._get_next_message_filename() os.rename(old_filename, new_filename) self._set_flags(new_filename, set(flags) - set(HELD)) else: if not accepted and offset >= pending_offset: self._set_flags(old_filename, set(flags) | set(HELD)) offset += 1
def send_pending_messages(self): """ As the last callback of L{PackageReporter}, sends messages stored. """ if self.global_store_filename is None: self.global_store_filename = os.environ["FAKE_PACKAGE_STORE"] if not os.path.exists(self.global_store_filename): return succeed(None) message_sent = set(self._store.get_message_ids()) global_store = FakePackageStore(self.global_store_filename) all_message_ids = set(global_store.get_message_ids()) not_sent = all_message_ids - message_sent deferred = succeed(None) got_type = set() if not_sent: messages = global_store.get_messages_by_ids(not_sent) sent = [] for message_id, message in messages: message = bpickle.loads(message) if message["type"] not in got_type: got_type.add(message["type"]) sent.append(message_id) deferred.addCallback( lambda x, message=message: self.send_message(message)) self._store.save_message_ids(sent) return deferred
def _got_result(self, webtext): """ Given a response that came from a ping server, return True if the response indicates that their are messages waiting for this computer, False otherwise. """ if loads(webtext) == {"messages": True}: return True
def test_dict_bytes_keys(self): """Check loading dict bytes keys without reinterpreting.""" # Happens in amp and broker. Since those messages are meant to be # forwarded to the server without changing schema, keys shouldn't be # decoded in this case. initial_data = {b"hello": True} data = bpickle.dumps(initial_data) result = bpickle.loads(data, as_is=True) self.assertEqual(initial_data, result)
def got_result(ignored): try: get_header = resource.request.requestHeaders.getRawHeaders except AttributeError: # For backwards compatibility with Twisted versions # without requestHeaders def get_header(header): return [resource.request.received_headers[header]] self.assertEqual(get_header("x-computer-id"), ["34"]) self.assertEqual(get_header("user-agent"), ["landscape-client/%s" % (VERSION, )]) self.assertEqual(get_header("x-message-api"), ["X.Y"]) self.assertEqual(bpickle.loads(resource.content), "HI")
def __init__(self, db, id): self._db = db self.id = id cursor = db.cursor() try: cursor.execute("SELECT queue, timestamp, data FROM task " "WHERE id=?", (id,)) row = cursor.fetchone() finally: cursor.close() self.queue = row[0] self.timestamp = row[1] self.data = bpickle.loads(bytes(row[2]))
def exchange(self, payload, computer_id=None, exchange_token=None, message_api=SERVER_API): """Exchange message data with the server. @param payload: The object to send, it must be L{bpickle}-compatible. @param computer_id: The computer ID to send the message as (see also L{Identity}). @param exchange_token: The token that the server has given us at the last exchange. It's used to prove that we are still the same client. @type: C{dict} @return: The server's response to sent message or C{None} in case of error. @note: This code is thread safe (HOPEFULLY). """ spayload = bpickle.dumps(payload) start_time = time.time() if logging.getLogger().getEffectiveLevel() <= logging.DEBUG: logging.debug("Sending payload:\n%s", pprint.pformat(payload)) try: curly, data = self._curl(spayload, computer_id, exchange_token, message_api) except Exception: logging.exception("Error contacting the server at %s." % self._url) raise else: logging.info("Sent %d bytes and received %d bytes in %s.", len(spayload), len(data), format_delta(time.time() - start_time)) try: response = bpickle.loads(data) except Exception: logging.exception("Server returned invalid data: %r" % data) return None else: if logging.getLogger().getEffectiveLevel() <= logging.DEBUG: logging.debug("Received payload:\n%s", pprint.pformat(response)) return response
def receive_method_call(self, sequence, method, arguments): """Call an object's method with the given arguments. If a connected client sends a L{MethodCall} for method C{foo_bar}, then the actual method C{foo_bar} of the object associated with the protocol will be called with the given C{args} and C{kwargs} and its return value delivered back to the client as response to the command. @param sequence: The integer that uniquely identifies the L{MethodCall} being received. @param method: The name of the object's method to call. @param arguments: A bpickle'd binary tuple of (args, kwargs) to be passed to the method. In case this L{MethodCall} has been preceded by one or more L{MethodCallChunk}s, C{arguments} is the last chunk of data. """ chunks = self._pending_chunks.pop(sequence, None) if chunks is not None: # We got some L{MethodCallChunk}s before, this is the last. chunks.append(arguments) arguments = b"".join(chunks) # Pass the the arguments as-is without reinterpreting strings. args, kwargs = bpickle.loads(arguments, as_is=True) # We encoded the method name in `send_method_call` and have to decode # it here again. method = method.decode("utf-8") if method not in self._methods: raise MethodCallError("Forbidden method '%s'" % method) method_func = getattr(self._object, method) def handle_result(result): return {"result": self._check_result(result)} def handle_failure(failure): raise MethodCallError(failure.value) deferred = maybeDeferred(method_func, *args, **kwargs) deferred.addCallback(handle_result) deferred.addErrback(handle_failure) return deferred
def get_pending_messages(self, max=None): """Get any pending messages that aren't being held, up to max.""" accepted_types = self.get_accepted_types() server_api = self.get_server_api() messages = [] for filename in self._walk_pending_messages(): if max is not None and len(messages) >= max: break data = self._get_content(self._message_dir(filename)) try: message = bpickle.loads(data) except ValueError, e: logging.exception(e) self._add_flags(filename, BROKEN) else: unknown_type = message["type"] not in accepted_types unknown_api = not is_version_higher(server_api, message["api"]) if unknown_type or unknown_api: self._add_flags(filename, HELD) else: messages.append(message)
def hashes(self, cursor): cursor.execute("SELECT hashes FROM hash_id_request WHERE id=?", (self.id,)) return bpickle.loads(bytes(cursor.fetchone()[0]))
def test_float(self): self.assertAlmostEquals(bpickle.loads(bpickle.dumps(2.3)), 2.3)
def test_int(self): self.assertEqual(bpickle.loads(bpickle.dumps(1)), 1)
def test_long(self): long = 99999999999999999999999999999 self.assertEqual(bpickle.loads(bpickle.dumps(long)), long)
def message_system(request): secure_id = request.META.get('HTTP_X_COMPUTER_ID') data = loads(request.raw_post_data) received_msgs = data.pop('messages', []) if len(received_msgs) != data['total-messages']: raise return_msgs = [] computer = company = None # Get company and computer objects if we do have a secure_id if secure_id: try: computer = Computer.objects.select_related('company')\ .get(secure_id=secure_id) company = computer.company if not all((computer.next_client_sequence, computer.next_server_sequence)): computer.next_client_sequence = data['sequence'] computer.next_server_sequence = data['next-expected-sequence'] computer.save() except Computer.DoesNotExist: return render_messages([{'type': 'unkown-id'}]) # Special case registration, as we could get a request with nothing, # and without accepting register, we'll never get a registration. if computer is not None and computer.confirmed: accepted_types = company.activated_plugins.values_list( 'identifier', flat=True).order_by('identifier') accepted_types_hash = company.activated_plugins_hash.decode('hex') else: accepted_types = ['register'] accepted_types_hash = hash_types(['register']) # Check if sequence numbers match if computer is not None and computer.confirmed and \ ((data['sequence'] != computer.next_client_sequence) or (data['next-expected-sequence'] != computer.next_server_sequence)): raise # Determine whether we need to notify the client about new/delete types if data.get('accepted-types') != accepted_types_hash: return_msgs.append({ 'type': 'accepted-types', 'types': list(accepted_types) }) for msg in received_msgs: if computer is None and msg['type'] != 'register': continue message_logger.debug('Received message with data:\n%s' % pformat(msg)) ret_ = message_available.send(sender=MessageType(msg['type']), computer=computer, request_data=data, msg_data=msg) ret = itertools.chain(*map(operator.itemgetter(1), ret_)) return_msgs.extend(ret) messages = Message.objects.filter(computer=computer) if computer is not None: if messages.count(): for msg in messages: return_msgs.append(simplejson.loads(msg.message)) messages.delete() computer.next_client_sequence += data['total-messages'] computer.next_server_sequence += len(return_msgs) # prevent sending of unneeded data Computer.objects.filter(pk=computer.pk).update( next_client_sequence = \ F('next_client_sequence') + data['total-messages'], next_server_sequence = \ F('next_server_sequence') + len(return_msgs)) return render_messages(return_msgs, computer=computer)
def test_float_scientific_notation(self): number = 0.00005 self.assertTrue("e" in repr(number)) self.assertAlmostEquals(bpickle.loads(bpickle.dumps(number)), number)
def test_bool(self): self.assertEqual(bpickle.loads(bpickle.dumps(True)), True)
def test_tuple(self): data = bpickle.dumps((1, [], 2, 'hello', 3.0)) self.assertEqual(bpickle.loads(data), (1, [], 2, 'hello', 3.0))
def test_list(self): self.assertEqual(bpickle.loads(bpickle.dumps([1, 2, 'hello', 3.0])), [1, 2, 'hello', 3.0])
def test_string(self): self.assertEqual(bpickle.loads(bpickle.dumps('foo')), 'foo')
def test_bytes(self): self.assertEqual(bpickle.loads(bpickle.dumps(b'foo')), b'foo')
def fromString(self, inString): """Unserialize an argument.""" return loads(inString)
def test_none(self): self.assertEqual(bpickle.loads(bpickle.dumps(None)), None)
def test_dict(self): dumped_tostr = bpickle.dumps({True: "hello"}) self.assertEqual(bpickle.loads(dumped_tostr), {True: "hello"}) dumped_tobool = bpickle.dumps({True: False}) self.assertEqual(bpickle.loads(dumped_tobool), {True: False})
def test_unicode(self): self.assertEqual(bpickle.loads(bpickle.dumps(u'\xc0')), u'\xc0')
def message_system(request): secure_id = request.META.get("HTTP_X_COMPUTER_ID") data = loads(request.raw_post_data) received_msgs = data.pop("messages", []) if len(received_msgs) != data["total-messages"]: raise return_msgs = [] computer = company = None # Get company and computer objects if we do have a secure_id if secure_id: try: computer = Computer.objects.select_related("company").get(secure_id=secure_id) company = computer.company if not all((computer.next_client_sequence, computer.next_server_sequence)): computer.next_client_sequence = data["sequence"] computer.next_server_sequence = data["next-expected-sequence"] computer.save() except Computer.DoesNotExist: return render_messages([{"type": "unkown-id"}]) # Special case registration, as we could get a request with nothing, # and without accepting register, we'll never get a registration. if computer is not None and computer.confirmed: accepted_types = company.activated_plugins.values_list("identifier", flat=True).order_by("identifier") accepted_types_hash = company.activated_plugins_hash.decode("hex") else: accepted_types = ["register"] accepted_types_hash = hash_types(["register"]) # Check if sequence numbers match if ( computer is not None and computer.confirmed and ( (data["sequence"] != computer.next_client_sequence) or (data["next-expected-sequence"] != computer.next_server_sequence) ) ): raise # Determine whether we need to notify the client about new/delete types if data.get("accepted-types") != accepted_types_hash: return_msgs.append({"type": "accepted-types", "types": list(accepted_types)}) for msg in received_msgs: if computer is None and msg["type"] != "register": continue message_logger.debug("Received message with data:\n%s" % pformat(msg)) ret_ = message_available.send( sender=MessageType(msg["type"]), computer=computer, request_data=data, msg_data=msg ) ret = itertools.chain(*map(operator.itemgetter(1), ret_)) return_msgs.extend(ret) messages = Message.objects.filter(computer=computer) if computer is not None: if messages.count(): for msg in messages: return_msgs.append(simplejson.loads(msg.message)) messages.delete() computer.next_client_sequence += data["total-messages"] computer.next_server_sequence += len(return_msgs) # prevent sending of unneeded data Computer.objects.filter(pk=computer.pk).update( next_client_sequence=F("next_client_sequence") + data["total-messages"], next_server_sequence=F("next_server_sequence") + len(return_msgs), ) return render_messages(return_msgs, computer=computer)