def get_pending_messages(self, max=None):
"""Get any pending messages that aren't being held, up to max."""
accepted_types = self.get_accepted_types()
server_api = self.get_server_api()
messages = []
for filename in self._walk_pending_messages():
if max is not None and len(messages) >= max:
break
data = read_binary_file(self._message_dir(filename))
try:
# don't reinterpret messages that are meant to be sent out
message = bpickle.loads(data, as_is=True)
except ValueError as e:
logging.exception(e)
self._add_flags(filename, BROKEN)
else:
if u"type" not in message:
# Special case to decode keys for messages which were
# serialized by py27 prior to py3 upgrade, and having
# implicit byte message keys. Message may still get
# rejected by the server, but it won't block the client
# broker. (lp: #1718689)
message = {
(k if isinstance(k, str) else k.decode("ascii")): v
for k, v in message.items()}
message[u"type"] = message[u"type"].decode("ascii")
unknown_type = message["type"] not in accepted_types
unknown_api = not is_version_higher(server_api, message["api"])
if unknown_type or unknown_api:
self._add_flags(filename, HELD)
else:
messages.append(message)
return messages
def _reprocess_holding(self):
"""
Unhold accepted messages left behind, and hold unaccepted
pending messages.
"""
offset = 0
pending_offset = self.get_pending_offset()
accepted_types = self.get_accepted_types()
for old_filename in self._walk_messages():
flags = self._get_flags(old_filename)
try:
message = bpickle.loads(self._get_content(old_filename))
except ValueError, e:
logging.exception(e)
if HELD not in flags:
offset += 1
else:
accepted = message["type"] in accepted_types
if HELD in flags:
if accepted:
new_filename = self._get_next_message_filename()
os.rename(old_filename, new_filename)
self._set_flags(new_filename, set(flags) - set(HELD))
else:
if not accepted and offset >= pending_offset:
self._set_flags(old_filename, set(flags) | set(HELD))
offset += 1
def send_pending_messages(self):
"""
As the last callback of L{PackageReporter}, sends messages stored.
"""
if self.global_store_filename is None:
self.global_store_filename = os.environ["FAKE_PACKAGE_STORE"]
if not os.path.exists(self.global_store_filename):
return succeed(None)
message_sent = set(self._store.get_message_ids())
global_store = FakePackageStore(self.global_store_filename)
all_message_ids = set(global_store.get_message_ids())
not_sent = all_message_ids - message_sent
deferred = succeed(None)
got_type = set()
if not_sent:
messages = global_store.get_messages_by_ids(not_sent)
sent = []
for message_id, message in messages:
message = bpickle.loads(message)
if message["type"] not in got_type:
got_type.add(message["type"])
sent.append(message_id)
deferred.addCallback(
lambda x, message=message: self.send_message(message))
self._store.save_message_ids(sent)
return deferred
def _got_result(self, webtext):
"""
Given a response that came from a ping server, return True if
the response indicates that their are messages waiting for
this computer, False otherwise.
"""
if loads(webtext) == {"messages": True}:
return True
def test_dict_bytes_keys(self):
"""Check loading dict bytes keys without reinterpreting."""
# Happens in amp and broker. Since those messages are meant to be
# forwarded to the server without changing schema, keys shouldn't be
# decoded in this case.
initial_data = {b"hello": True}
data = bpickle.dumps(initial_data)
result = bpickle.loads(data, as_is=True)
self.assertEqual(initial_data, result)
def got_result(ignored):
try:
get_header = resource.request.requestHeaders.getRawHeaders
except AttributeError:
# For backwards compatibility with Twisted versions
# without requestHeaders
def get_header(header):
return [resource.request.received_headers[header]]
self.assertEqual(get_header("x-computer-id"), ["34"])
self.assertEqual(get_header("user-agent"),
["landscape-client/%s" % (VERSION, )])
self.assertEqual(get_header("x-message-api"), ["X.Y"])
self.assertEqual(bpickle.loads(resource.content), "HI")
def __init__(self, db, id):
self._db = db
self.id = id
cursor = db.cursor()
try:
cursor.execute("SELECT queue, timestamp, data FROM task "
"WHERE id=?", (id,))
row = cursor.fetchone()
finally:
cursor.close()
self.queue = row[0]
self.timestamp = row[1]
self.data = bpickle.loads(bytes(row[2]))
def exchange(self,
payload,
computer_id=None,
exchange_token=None,
message_api=SERVER_API):
"""Exchange message data with the server.
@param payload: The object to send, it must be L{bpickle}-compatible.
@param computer_id: The computer ID to send the message as (see
also L{Identity}).
@param exchange_token: The token that the server has given us at the
last exchange. It's used to prove that we are still the same
client.
@type: C{dict}
@return: The server's response to sent message or C{None} in case
of error.
@note: This code is thread safe (HOPEFULLY).
"""
spayload = bpickle.dumps(payload)
start_time = time.time()
if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
logging.debug("Sending payload:\n%s", pprint.pformat(payload))
try:
curly, data = self._curl(spayload, computer_id, exchange_token,
message_api)
except Exception:
logging.exception("Error contacting the server at %s." % self._url)
raise
else:
logging.info("Sent %d bytes and received %d bytes in %s.",
len(spayload), len(data),
format_delta(time.time() - start_time))
try:
response = bpickle.loads(data)
except Exception:
logging.exception("Server returned invalid data: %r" % data)
return None
else:
if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
logging.debug("Received payload:\n%s",
pprint.pformat(response))
return response
def receive_method_call(self, sequence, method, arguments):
"""Call an object's method with the given arguments.
If a connected client sends a L{MethodCall} for method C{foo_bar}, then
the actual method C{foo_bar} of the object associated with the protocol
will be called with the given C{args} and C{kwargs} and its return
value delivered back to the client as response to the command.
@param sequence: The integer that uniquely identifies the L{MethodCall}
being received.
@param method: The name of the object's method to call.
@param arguments: A bpickle'd binary tuple of (args, kwargs) to be
passed to the method. In case this L{MethodCall} has been preceded
by one or more L{MethodCallChunk}s, C{arguments} is the last chunk
of data.
"""
chunks = self._pending_chunks.pop(sequence, None)
if chunks is not None:
# We got some L{MethodCallChunk}s before, this is the last.
chunks.append(arguments)
arguments = b"".join(chunks)
# Pass the the arguments as-is without reinterpreting strings.
args, kwargs = bpickle.loads(arguments, as_is=True)
# We encoded the method name in `send_method_call` and have to decode
# it here again.
method = method.decode("utf-8")
if method not in self._methods:
raise MethodCallError("Forbidden method '%s'" % method)
method_func = getattr(self._object, method)
def handle_result(result):
return {"result": self._check_result(result)}
def handle_failure(failure):
raise MethodCallError(failure.value)
deferred = maybeDeferred(method_func, *args, **kwargs)
deferred.addCallback(handle_result)
deferred.addErrback(handle_failure)
return deferred
def get_pending_messages(self, max=None):
"""Get any pending messages that aren't being held, up to max."""
accepted_types = self.get_accepted_types()
server_api = self.get_server_api()
messages = []
for filename in self._walk_pending_messages():
if max is not None and len(messages) >= max:
break
data = self._get_content(self._message_dir(filename))
try:
message = bpickle.loads(data)
except ValueError, e:
logging.exception(e)
self._add_flags(filename, BROKEN)
else:
unknown_type = message["type"] not in accepted_types
unknown_api = not is_version_higher(server_api, message["api"])
if unknown_type or unknown_api:
self._add_flags(filename, HELD)
else:
messages.append(message)
def hashes(self, cursor):
cursor.execute("SELECT hashes FROM hash_id_request WHERE id=?",
(self.id,))
return bpickle.loads(bytes(cursor.fetchone()[0]))
def test_float(self):
self.assertAlmostEquals(bpickle.loads(bpickle.dumps(2.3)), 2.3)
def test_int(self):
self.assertEqual(bpickle.loads(bpickle.dumps(1)), 1)
def test_long(self):
long = 99999999999999999999999999999
self.assertEqual(bpickle.loads(bpickle.dumps(long)), long)
def message_system(request):
secure_id = request.META.get('HTTP_X_COMPUTER_ID')
data = loads(request.raw_post_data)
received_msgs = data.pop('messages', [])
if len(received_msgs) != data['total-messages']:
raise
return_msgs = []
computer = company = None
# Get company and computer objects if we do have a secure_id
if secure_id:
try:
computer = Computer.objects.select_related('company')\
.get(secure_id=secure_id)
company = computer.company
if not all((computer.next_client_sequence,
computer.next_server_sequence)):
computer.next_client_sequence = data['sequence']
computer.next_server_sequence = data['next-expected-sequence']
computer.save()
except Computer.DoesNotExist:
return render_messages([{'type': 'unkown-id'}])
# Special case registration, as we could get a request with nothing,
# and without accepting register, we'll never get a registration.
if computer is not None and computer.confirmed:
accepted_types = company.activated_plugins.values_list(
'identifier', flat=True).order_by('identifier')
accepted_types_hash = company.activated_plugins_hash.decode('hex')
else:
accepted_types = ['register']
accepted_types_hash = hash_types(['register'])
# Check if sequence numbers match
if computer is not None and computer.confirmed and \
((data['sequence'] != computer.next_client_sequence) or
(data['next-expected-sequence'] != computer.next_server_sequence)):
raise
# Determine whether we need to notify the client about new/delete types
if data.get('accepted-types') != accepted_types_hash:
return_msgs.append({
'type': 'accepted-types',
'types': list(accepted_types)
})
for msg in received_msgs:
if computer is None and msg['type'] != 'register':
continue
message_logger.debug('Received message with data:\n%s' % pformat(msg))
ret_ = message_available.send(sender=MessageType(msg['type']),
computer=computer,
request_data=data,
msg_data=msg)
ret = itertools.chain(*map(operator.itemgetter(1), ret_))
return_msgs.extend(ret)
messages = Message.objects.filter(computer=computer)
if computer is not None:
if messages.count():
for msg in messages:
return_msgs.append(simplejson.loads(msg.message))
messages.delete()
computer.next_client_sequence += data['total-messages']
computer.next_server_sequence += len(return_msgs)
# prevent sending of unneeded data
Computer.objects.filter(pk=computer.pk).update(
next_client_sequence = \
F('next_client_sequence') + data['total-messages'],
next_server_sequence = \
F('next_server_sequence') + len(return_msgs))
return render_messages(return_msgs, computer=computer)
def test_float_scientific_notation(self):
number = 0.00005
self.assertTrue("e" in repr(number))
self.assertAlmostEquals(bpickle.loads(bpickle.dumps(number)), number)
def test_bool(self):
self.assertEqual(bpickle.loads(bpickle.dumps(True)), True)
def test_tuple(self):
data = bpickle.dumps((1, [], 2, 'hello', 3.0))
self.assertEqual(bpickle.loads(data), (1, [], 2, 'hello', 3.0))
def test_list(self):
self.assertEqual(bpickle.loads(bpickle.dumps([1, 2, 'hello', 3.0])),
[1, 2, 'hello', 3.0])
def test_string(self):
self.assertEqual(bpickle.loads(bpickle.dumps('foo')), 'foo')
def test_bytes(self):
self.assertEqual(bpickle.loads(bpickle.dumps(b'foo')), b'foo')
def fromString(self, inString):
"""Unserialize an argument."""
return loads(inString)
def test_none(self):
self.assertEqual(bpickle.loads(bpickle.dumps(None)), None)
def test_dict(self):
dumped_tostr = bpickle.dumps({True: "hello"})
self.assertEqual(bpickle.loads(dumped_tostr), {True: "hello"})
dumped_tobool = bpickle.dumps({True: False})
self.assertEqual(bpickle.loads(dumped_tobool), {True: False})
def test_unicode(self):
self.assertEqual(bpickle.loads(bpickle.dumps(u'\xc0')), u'\xc0')
def message_system(request):
secure_id = request.META.get("HTTP_X_COMPUTER_ID")
data = loads(request.raw_post_data)
received_msgs = data.pop("messages", [])
if len(received_msgs) != data["total-messages"]:
raise
return_msgs = []
computer = company = None
# Get company and computer objects if we do have a secure_id
if secure_id:
try:
computer = Computer.objects.select_related("company").get(secure_id=secure_id)
company = computer.company
if not all((computer.next_client_sequence, computer.next_server_sequence)):
computer.next_client_sequence = data["sequence"]
computer.next_server_sequence = data["next-expected-sequence"]
computer.save()
except Computer.DoesNotExist:
return render_messages([{"type": "unkown-id"}])
# Special case registration, as we could get a request with nothing,
# and without accepting register, we'll never get a registration.
if computer is not None and computer.confirmed:
accepted_types = company.activated_plugins.values_list("identifier", flat=True).order_by("identifier")
accepted_types_hash = company.activated_plugins_hash.decode("hex")
else:
accepted_types = ["register"]
accepted_types_hash = hash_types(["register"])
# Check if sequence numbers match
if (
computer is not None
and computer.confirmed
and (
(data["sequence"] != computer.next_client_sequence)
or (data["next-expected-sequence"] != computer.next_server_sequence)
)
):
raise
# Determine whether we need to notify the client about new/delete types
if data.get("accepted-types") != accepted_types_hash:
return_msgs.append({"type": "accepted-types", "types": list(accepted_types)})
for msg in received_msgs:
if computer is None and msg["type"] != "register":
continue
message_logger.debug("Received message with data:\n%s" % pformat(msg))
ret_ = message_available.send(
sender=MessageType(msg["type"]), computer=computer, request_data=data, msg_data=msg
)
ret = itertools.chain(*map(operator.itemgetter(1), ret_))
return_msgs.extend(ret)
messages = Message.objects.filter(computer=computer)
if computer is not None:
if messages.count():
for msg in messages:
return_msgs.append(simplejson.loads(msg.message))
messages.delete()
computer.next_client_sequence += data["total-messages"]
computer.next_server_sequence += len(return_msgs)
# prevent sending of unneeded data
Computer.objects.filter(pk=computer.pk).update(
next_client_sequence=F("next_client_sequence") + data["total-messages"],
next_server_sequence=F("next_server_sequence") + len(return_msgs),
)
return render_messages(return_msgs, computer=computer)
