def test_format_vumi_date(self):
self.assertEqual(
format_vumi_date(
datetime(2015, 1, 2, 23, 14, 11, microsecond=456000)),
'2015-01-02 23:14:11.456000')
self.assertEqual(
format_vumi_date(datetime(2015, 1, 2, 23, 14, 11, microsecond=0)),
'2015-01-02 23:14:11.000000')
def test_format_vumi_date(self):
self.assertEqual(
format_vumi_date(
datetime(2015, 1, 2, 23, 14, 11, microsecond=456000)),
'2015-01-02 23:14:11.456000')
self.assertEqual(
format_vumi_date(
datetime(2015, 1, 2, 23, 14, 11, microsecond=0)),
'2015-01-02 23:14:11.000000')
def from_reverse_timestamp(reverse_timestamp):
"""
Turn a reverse timestamp string (from `to_reverse_timestamp()`) into a
vumi_date-formatted string.
"""
timestamp = 0xffffffffff - int(reverse_timestamp, 16)
return format_vumi_date(datetime.utcfromtimestamp(timestamp))
def create_ack_event_sequence(self, event_count=5, delay_seconds=1):
"""
Generate a sequence of events in a new batch with a test message and
add them to the backend.
:param event_count:
The number of events to create
:param delay_seconds:
The delay in seconds between the timestamps of sequential events
:returns:
The batch id of the new batch, the message id of the test message,
and a list of tuples describing each event in the form
(key, timestamp, status).
"""
batch_id = yield self._backend.batch_start()
msg = self._msg_helper.make_outbound("pears")
all_keys = []
start = datetime.utcnow().replace(microsecond=0)
for i in xrange(event_count):
timestamp = start - timedelta(seconds=i * delay_seconds)
ack = self._msg_helper.make_ack(msg, timestamp=timestamp)
yield self._backend.add_event(ack, batch_ids=[batch_id])
all_keys.append((
ack["event_id"], # Key
format_vumi_date(timestamp), # Timestamp
ack["event_type"])) # Status
returnValue((batch_id, msg["message_id"], all_keys))
def create_outbound_message_sequence(self, msg_count=5, delay_seconds=1):
"""
Generate a sequence of outbound messages in a new batch and add them to
the backend.
:param msg_count:
The number of messages to create
:param delay_seconds:
The delay in seconds between the timestamps of sequential messages
:returns:
The batch id of the new batch and a list of tuples describing each
message in the form (key, timestamp, address)
"""
batch_id = yield self._backend.batch_start()
all_keys = []
start = datetime.utcnow().replace(microsecond=0)
for i in xrange(msg_count):
timestamp = start - timedelta(seconds=i * delay_seconds)
addr = "addr%s" % (i, )
msg = self._msg_helper.make_inbound("Message %s" % (i, ),
timestamp=timestamp,
to_addr=addr)
yield self._backend.add_outbound_message(msg, batch_ids=[batch_id])
all_keys.append((
msg["message_id"], # Key
format_vumi_date(timestamp), # Timestamp
addr)) # Address
returnValue((batch_id, all_keys))
def test_add_ack_event_to_new_batch(self):
"""
When an existing event is added with a new batch identifier, it belongs
to the new batch as well as batches it already belonged to.
"""
events = self.manager.proxy(Event)
msg = self.msg_helper.make_outbound("apples")
ack = self.msg_helper.make_ack(msg)
yield self.backend.add_event(ack, batch_ids=["mybatch"])
yield self.backend.add_event(ack, batch_ids=["yourbatch"])
stored_event = yield events.load(ack["event_id"])
self.assertEqual(stored_event.event, ack)
self.assertEqual(sorted(stored_event.batches.keys()),
["mybatch", "yourbatch"])
# Make sure we're writing the right indexes.
timestamp = format_vumi_date(msg['timestamp'])
reverse_ts = to_reverse_timestamp(timestamp)
self.assertEqual(
stored_event._riak_object.get_indexes(),
set([
("message_bin", ack["user_message_id"]),
("batches_bin", "mybatch"),
('batches_bin', "yourbatch"),
("message_with_status_bin", "%s$%s$%s" %
(ack["user_message_id"], ack["timestamp"], "ack")),
("batches_with_statuses_reverse_bin",
"%s$%s$%s" % ("mybatch", reverse_ts, "ack")),
("batches_with_statuses_reverse_bin",
"%s$%s$%s" % ("yourbatch", reverse_ts, "ack")),
]))
def test_add_outbound_message_to_new_batch(self):
"""
When an existing outbound message is added with a new batch identifier,
it belongs to the new batch as well as batches it already belonged to.
"""
outbound_messages = self.manager.proxy(OutboundMessage)
msg = self.msg_helper.make_outbound("apples")
yield self.backend.add_outbound_message(msg, batch_ids=["mybatch"])
yield self.backend.add_outbound_message(msg, batch_ids=["yourbatch"])
stored_msg = yield outbound_messages.load(msg["message_id"])
self.assertEqual(stored_msg.msg, msg)
self.assertEqual(sorted(stored_msg.batches.keys()),
["mybatch", "yourbatch"])
# Make sure we're writing the right indexes.
timestamp = format_vumi_date(msg['timestamp'])
reverse_ts = to_reverse_timestamp(timestamp)
self.assertEqual(
stored_msg._riak_object.get_indexes(),
set([
('batches_bin', "mybatch"),
('batches_bin', "yourbatch"),
('batches_with_addresses_bin',
"%s$%s$%s" % ("mybatch", timestamp, msg['to_addr'])),
('batches_with_addresses_bin',
"%s$%s$%s" % ("yourbatch", timestamp, msg['to_addr'])),
('batches_with_addresses_reverse_bin',
"%s$%s$%s" % ("mybatch", reverse_ts, msg['to_addr'])),
('batches_with_addresses_reverse_bin',
"%s$%s$%s" % ("yourbatch", reverse_ts, msg['to_addr'])),
]))
def test_add_inbound_message_with_multiple_batch_ids(self):
"""
When an inbound message is added with multiple batch identifiers, it
belongs to all the specified batches.
"""
inbound_messages = self.manager.proxy(InboundMessage)
msg = self.msg_helper.make_inbound("apples")
yield self.backend.add_inbound_message(
msg, batch_ids=["mybatch", "yourbatch"])
stored_msg = yield inbound_messages.load(msg["message_id"])
self.assertEqual(stored_msg.msg, msg)
self.assertEqual(sorted(stored_msg.batches.keys()),
["mybatch", "yourbatch"])
# Make sure we're writing the right indexes.
timestamp = format_vumi_date(msg['timestamp'])
reverse_ts = to_reverse_timestamp(timestamp)
self.assertEqual(
stored_msg._riak_object.get_indexes(),
set([
('batches_bin', "mybatch"),
('batches_bin', "yourbatch"),
('batches_with_addresses_bin',
"%s$%s$%s" % ("mybatch", timestamp, msg['from_addr'])),
('batches_with_addresses_bin',
"%s$%s$%s" % ("yourbatch", timestamp, msg['from_addr'])),
('batches_with_addresses_reverse_bin',
"%s$%s$%s" % ("mybatch", reverse_ts, msg['from_addr'])),
('batches_with_addresses_reverse_bin',
"%s$%s$%s" % ("yourbatch", reverse_ts, msg['from_addr'])),
]))
def from_reverse_timestamp(reverse_timestamp):
"""
Turn a reverse timestamp string (from `to_reverse_timestamp()`) into a
vumi_date-formatted string.
"""
timestamp = 0xffffffffff - int(reverse_timestamp, 16)
return format_vumi_date(datetime.utcfromtimestamp(timestamp))
def _extract_date_arg(self, request, argname):
arg = self._extract_arg(request, argname)
if arg is None:
return None
try:
timestamp = iso8601.parse_date(arg)
return format_vumi_date(timestamp)
except iso8601.ParseError as e:
raise ParameterError(
"Invalid '%s' parameter: %s" % (argname, str(e)))
def save(self):
# We override this method to set our index fields before saving.
batches_with_timestamps = []
batches_with_addresses = []
timestamp = format_vumi_date(self.msg['timestamp'])
for batch_id in self.batches.keys():
batches_with_timestamps.append(u"%s$%s" % (batch_id, timestamp))
batches_with_addresses.append(
u"%s$%s$%s" % (batch_id, timestamp, self.msg['to_addr']))
self.batches_with_timestamps = batches_with_timestamps
self.batches_with_addresses = batches_with_addresses
return super(OutboundMessageV3, self).save()
def save(self):
# We override this method to set our index fields before saving.
batches_with_timestamps = []
batches_with_addresses = []
timestamp = format_vumi_date(self.msg['timestamp'])
for batch_id in self.batches.keys():
batches_with_timestamps.append(u"%s$%s" % (batch_id, timestamp))
batches_with_addresses.append(
u"%s$%s$%s" % (batch_id, timestamp, self.msg['to_addr']))
self.batches_with_timestamps = batches_with_timestamps
self.batches_with_addresses = batches_with_addresses
return super(OutboundMessageV3, self).save()
def reconcile_cache(self, batch_id, start_timestamp=None):
"""
Rebuild the cache for the given batch.
The ``start_timestamp`` parameter is used for testing only.
"""
if start_timestamp is None:
start_timestamp = format_vumi_date(datetime.utcnow())
yield self.cache.clear_batch(batch_id)
yield self.cache.batch_start(batch_id)
yield self.reconcile_outbound_cache(batch_id, start_timestamp)
yield self.reconcile_inbound_cache(batch_id, start_timestamp)
def reconcile_cache(self, batch_id, start_timestamp=None):
"""
Rebuild the cache for the given batch.
The ``start_timestamp`` parameter is used for testing only.
"""
if start_timestamp is None:
start_timestamp = format_vumi_date(datetime.utcnow())
yield self.cache.clear_batch(batch_id)
yield self.cache.batch_start(batch_id)
yield self.reconcile_outbound_cache(batch_id, start_timestamp)
yield self.reconcile_inbound_cache(batch_id, start_timestamp)
def _extract_date_arg(self, request, argname):
if argname not in request.args:
return None
if len(request.args[argname]) > 1:
raise ParameterError(
"Invalid '%s' parameter: Too many values" % (argname,))
[value] = request.args[argname]
try:
timestamp = iso8601.parse_date(value)
return format_vumi_date(timestamp)
except iso8601.ParseError as e:
raise ParameterError(
"Invalid '%s' parameter: %s" % (argname, str(e)))
def _extract_date_arg(self, request, argname):
if argname not in request.args:
return None
if len(request.args[argname]) > 1:
raise ParameterError("Invalid '%s' parameter: Too many values" %
(argname, ))
[value] = request.args[argname]
try:
timestamp = iso8601.parse_date(value)
return format_vumi_date(timestamp)
except iso8601.ParseError as e:
raise ParameterError("Invalid '%s' parameter: %s" %
(argname, str(e)))
def save(self):
# We override this method to set our index fields before saving.
self.batches_with_addresses = []
self.batches_with_addresses_reverse = []
timestamp = self.msg['timestamp']
if not isinstance(timestamp, basestring):
timestamp = format_vumi_date(timestamp)
reverse_ts = to_reverse_timestamp(timestamp)
for batch_id in self.batches.keys():
self.batches_with_addresses.append(
u"%s$%s$%s" % (batch_id, timestamp, self.msg['from_addr']))
self.batches_with_addresses_reverse.append(
u"%s$%s$%s" % (batch_id, reverse_ts, self.msg['from_addr']))
return super(InboundMessage, self).save()
def save(self):
# We override this method to set our index fields before saving.
self.batches_with_addresses = []
self.batches_with_addresses_reverse = []
timestamp = self.msg['timestamp']
if not isinstance(timestamp, basestring):
timestamp = format_vumi_date(timestamp)
reverse_ts = to_reverse_timestamp(timestamp)
for batch_id in self.batches.keys():
self.batches_with_addresses.append(
u"%s$%s$%s" % (batch_id, timestamp, self.msg['from_addr']))
self.batches_with_addresses_reverse.append(
u"%s$%s$%s" % (batch_id, reverse_ts, self.msg['from_addr']))
return super(InboundMessage, self).save()
def save(self):
# We override this method to set our index fields before saving.
timestamp = self.event['timestamp']
if not isinstance(timestamp, basestring):
timestamp = format_vumi_date(timestamp)
status = self.event.status()
self.message_with_status = u"%s$%s$%s" % (
self.message.key, timestamp, status)
self.batches_with_statuses_reverse = []
reverse_ts = to_reverse_timestamp(timestamp)
for batch_id in self.batches.keys():
self.batches_with_statuses_reverse.append(
u"%s$%s$%s" % (batch_id, reverse_ts, status))
return super(Event, self).save()
def test_add_outbound_message_with_batch_id(self):
"""
When an outbound message is added with a batch identifier, that batch
identifier is stored with it and indexed.
"""
outbound_messages = self.manager.proxy(OutboundMessage)
msg = self.msg_helper.make_outbound("apples")
yield self.backend.add_outbound_message(msg, batch_ids=["mybatch"])
stored_msg = yield outbound_messages.load(msg["message_id"])
self.assertEqual(stored_msg.msg, msg)
self.assertEqual(stored_msg.batches.keys(), ["mybatch"])
# Make sure we're writing the right indexes.
timestamp = format_vumi_date(msg['timestamp'])
reverse_ts = to_reverse_timestamp(timestamp)
self.assertEqual(
stored_msg._riak_object.get_indexes(),
set([
('batches_bin', "mybatch"),
('batches_with_addresses_bin',
"%s$%s$%s" % ("mybatch", timestamp, msg['to_addr'])),
('batches_with_addresses_reverse_bin',
"%s$%s$%s" % ("mybatch", reverse_ts, msg['to_addr'])),
]))
def _timestamp_to_json(self, dt):
return format_vumi_date(dt)
def custom_to_riak(self, value):
return format_vumi_date(value)
def bwa_out_value(batch_id, msg):
return "%s$%s$%s" % (
batch_id, format_vumi_date(msg['timestamp']), msg['to_addr'])
def _timestamp_to_json(self, dt):
return format_vumi_date(dt)
def bwar_out_value(batch_id, msg):
reverse_ts = to_reverse_timestamp(format_vumi_date(msg['timestamp']))
return "%s$%s$%s" % (batch_id, reverse_ts, msg['to_addr'])
def mws_value(msg_id, event, status):
return "%s$%s$%s" % (msg_id, format_vumi_date(event['timestamp']), status)
def bwsr_value(batch_id, event, status):
reverse_ts = to_reverse_timestamp(format_vumi_date(event['timestamp']))
return "%s$%s$%s" % (batch_id, reverse_ts, status)
def bwt_value(batch_id, msg):
return "%s$%s" % (batch_id, format_vumi_date(msg['timestamp']))
def mws_value(msg_id, event, status):
return "%s$%s$%s" % (msg_id, format_vumi_date(event['timestamp']), status)
def bwt_value(batch_id, msg):
return "%s$%s" % (batch_id, format_vumi_date(msg['timestamp']))
def bwsr_value(batch_id, event, status):
reverse_ts = to_reverse_timestamp(format_vumi_date(event['timestamp']))
return "%s$%s$%s" % (batch_id, reverse_ts, status)
def bwar_out_value(batch_id, msg):
reverse_ts = to_reverse_timestamp(format_vumi_date(msg['timestamp']))
return "%s$%s$%s" % (batch_id, reverse_ts, msg['to_addr'])
def custom_to_riak(self, value):
return format_vumi_date(value)
def bwa_out_value(batch_id, msg):
return "%s$%s$%s" % (
batch_id, format_vumi_date(msg['timestamp']), msg['to_addr'])
