def test_deletes_txlogs_slice(self):
"""Delete a txlog slice by date and quantity."""
owner = self.make_user_without_txlog()
right_now = now()
# Old enough
timestamp_limit = right_now - timedelta(days=7)
quantity_limit = 2
txlogs = [
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner,
timestamp=timestamp_limit),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner,
timestamp=timestamp_limit),
self.factory.make_transaction_log(owner=owner),
]
removed = utils.delete_txlogs_slice(date=right_now.date(),
quantity_limit=quantity_limit)
self.assertEqual(removed, quantity_limit)
txlist = utils.get_txn_recs(num_recs=len(txlogs), last_id=0)
self.assertEqual(len(txlist), 3)
self.assertEqual(removed, quantity_limit)
ids = [int(txdict['txn_id']) for txdict in txlist]
expected = [txlogs[1], txlogs[3], txlogs[4]]
self.assertItemsEqual(ids, [t.id for t in expected])
def test_maintains_newish_txlogs_when_purging(self):
"""Test that txnlogs not old enough are maintained, instead of being
deleted."""
owner = self.make_user_without_txlog()
right_now = now()
limit_datetime = right_now - timedelta(days=7)
# Not so old
old_datetime = limit_datetime + timedelta(seconds=1)
expected = [
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(
owner=owner, timestamp=old_datetime),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(
owner=owner, timestamp=old_datetime),
]
removed = utils.delete_old_txlogs(timestamp_limit=limit_datetime)
self.assertEqual(removed, 0)
txlist = utils.get_txn_recs(num_recs=4, last_id=0)
ids = [int(txdict['txn_id']) for txdict in txlist]
self.assertItemsEqual(ids, [t.id for t in expected])
def test_deletes_old_txlogs_within_quantity_limit(self):
"""Test that txnlogs old enough are deleted and are within the quantity
limit given."""
owner = self.make_user_without_txlog()
right_now = now()
timestamp_limit = right_now - timedelta(days=7)
# Old enough
old_datetime = timestamp_limit
quantity_limit = 2
txlogs = [
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner,
timestamp=old_datetime),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner,
timestamp=old_datetime),
self.factory.make_transaction_log(owner=owner,
timestamp=old_datetime),
]
expected = [
txlogs[0],
txlogs[2],
txlogs[4],
]
removed = utils.delete_old_txlogs(timestamp_limit=timestamp_limit,
quantity_limit=quantity_limit)
txlist = utils.get_txn_recs(num_recs=len(txlogs), last_id=0)
self.assertEqual(len(txlist), 3)
self.assertEqual(removed, quantity_limit)
ids = [int(txdict['txn_id']) for txdict in txlist]
self.assertItemsEqual(ids, [t.id for t in expected])
def test_deletes_old_txlogs_within_quantity_limit(self):
"""Test that txnlogs old enough are deleted and are within the quantity
limit given."""
owner = self.make_user_without_txlog()
right_now = now()
timestamp_limit = right_now - timedelta(days=7)
# Old enough
old_datetime = timestamp_limit
quantity_limit = 2
txlogs = [
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(
owner=owner, timestamp=old_datetime),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(
owner=owner, timestamp=old_datetime),
self.factory.make_transaction_log(
owner=owner, timestamp=old_datetime),
]
expected = [txlogs[0], txlogs[2], txlogs[4], ]
removed = utils.delete_old_txlogs(timestamp_limit=timestamp_limit,
quantity_limit=quantity_limit)
txlist = utils.get_txn_recs(num_recs=len(txlogs), last_id=0)
self.assertEqual(len(txlist), 3)
self.assertEqual(removed, quantity_limit)
ids = [int(txdict['txn_id']) for txdict in txlist]
self.assertItemsEqual(ids, [t.id for t in expected])
def test_get_txn_recs_for_partition(self):
"""Get txlogs for the provided partition ID.
When owner_id % num_partitions == partition_id, the txlog is added to
the result set, so that it matches the filter by partition. Also, any
txlog that is related to sharing is also returned, no matter what the
owner_id is.
"""
owner = self.make_user_without_txlog()
other = self.make_user_without_txlog()
num_partitions = 8
partition_id = owner.id % num_partitions
t1 = self.factory.make_transaction_log(owner=owner)
self.factory.make_transaction_log(owner=other) # Different one
t3 = self.factory.make_transaction_log(owner=owner)
# Share txlogs, but with a different owner, are also returned.
t4 = self.factory.make_transaction_log(
owner=other, op_type=TransactionLog.OP_SHARE_ACCEPTED)
t5 = self.factory.make_transaction_log(
owner=other, op_type=TransactionLog.OP_SHARE_DELETED)
txlogs = [t1, t3, t4, t5]
txlist = utils.get_txn_recs(num_recs=5,
last_id=0,
num_partitions=num_partitions,
partition_id=partition_id)
self.assertEqual([t.as_dict() for t in txlogs], txlist)
def test_maintains_newish_txlogs_when_purging(self):
"""Test that txnlogs not old enough are maintained, instead of being
deleted."""
owner = self.make_user_without_txlog()
right_now = now()
limit_datetime = right_now - timedelta(days=7)
# Not so old
old_datetime = limit_datetime + timedelta(seconds=1)
expected = [
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner,
timestamp=old_datetime),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner,
timestamp=old_datetime),
]
removed = utils.delete_old_txlogs(timestamp_limit=limit_datetime)
self.assertEqual(removed, 0)
txlist = utils.get_txn_recs(num_recs=4, last_id=0)
ids = [int(txdict['txn_id']) for txdict in txlist]
self.assertItemsEqual(ids, [t.id for t in expected])
def test_get_txn_recs_for_partition(self):
"""Get txlogs for the provided partition ID.
When owner_id % num_partitions == partition_id, the txlog is added to
the result set, so that it matches the filter by partition. Also, any
txlog that is related to sharing is also returned, no matter what the
owner_id is.
"""
owner = self.make_user_without_txlog()
other = self.make_user_without_txlog()
num_partitions = 8
partition_id = owner.id % num_partitions
t1 = self.factory.make_transaction_log(owner=owner)
self.factory.make_transaction_log(owner=other) # Different one
t3 = self.factory.make_transaction_log(owner=owner)
# Share txlogs, but with a different owner, are also returned.
t4 = self.factory.make_transaction_log(
owner=other, op_type=TransactionLog.OP_SHARE_ACCEPTED)
t5 = self.factory.make_transaction_log(
owner=other, op_type=TransactionLog.OP_SHARE_DELETED)
txlogs = [t1, t3, t4, t5]
txlist = utils.get_txn_recs(
num_recs=5, last_id=0,
num_partitions=num_partitions, partition_id=partition_id
)
self.assertEqual([t.as_dict() for t in txlogs], txlist)
def test_deletes_txlogs_slice(self):
"""Delete a txlog slice by date and quantity."""
owner = self.make_user_without_txlog()
right_now = now()
# Old enough
timestamp_limit = right_now - timedelta(days=7)
quantity_limit = 2
txlogs = [
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(
owner=owner, timestamp=timestamp_limit),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(
owner=owner, timestamp=timestamp_limit),
self.factory.make_transaction_log(owner=owner),
]
removed = utils.delete_txlogs_slice(date=right_now.date(),
quantity_limit=quantity_limit)
self.assertEqual(removed, quantity_limit)
txlist = utils.get_txn_recs(num_recs=len(txlogs), last_id=0)
self.assertEqual(len(txlist), 3)
self.assertEqual(removed, quantity_limit)
ids = [int(txdict['txn_id']) for txdict in txlist]
expected = [txlogs[1], txlogs[3], txlogs[4]]
self.assertItemsEqual(ids, [t.id for t in expected])
def test_get_txn_recs_respects_order(self):
"""Test that transaction log entries are returned in order."""
owner = self.make_user_without_txlog()
txlogs = [self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner)]
txlist = utils.get_txn_recs(num_recs=3, last_id=0)
self.assertEqual([t.as_dict() for t in txlogs], txlist)
def test_get_txn_recs_unseen(self):
"""Getting transactions with unseen ids records those as unseen.
Querying again returns unseen transactions if they are now present.
"""
owner = self.make_user_without_txlog()
txlogs = [self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner)]
worker_id = self.factory.get_unique_unicode()
txlist = utils.get_txn_recs(num_recs=3, worker_id=worker_id)
self.assertEqual([t.as_dict() for t in txlogs], txlist)
unseen = self.factory.make_transaction_log(owner=owner)
txlist = utils.get_txn_recs(
num_recs=3, last_id=txlogs[1].id, worker_id=worker_id)
self.assertEqual([unseen.as_dict()], txlist)
def test_get_txn_recs_retry_list_no_new_or_retry(self):
"""Test getting a batch of transactions when there are unseen ids
records those as unseen. Querying again when unseen isn't available yet
returns nothing.
"""
owner = self.make_user_without_txlog()
t1 = self.factory.make_transaction_log(owner=owner)
t2 = self.factory.make_transaction_log(owner=owner)
t3 = self.factory.make_transaction_log(owner=owner)
t2.delete()
worker_id = self.factory.get_unique_unicode()
txlist = utils.get_txn_recs(num_recs=3, worker_id=worker_id)
self.assertEqual([t1.as_dict(), t3.as_dict()], txlist)
txlist = utils.get_txn_recs(
num_recs=3, last_id=t3.id, worker_id=worker_id)
self.assertEqual([], txlist)
def test_get_txn_recs_retry_list_no_new_or_retry(self):
"""Test getting a batch of transactions when there are unseen ids
records those as unseen. Querying again when unseen isn't available yet
returns nothing.
"""
owner = self.make_user_without_txlog()
t1 = self.factory.make_transaction_log(owner=owner)
t2 = self.factory.make_transaction_log(owner=owner)
t3 = self.factory.make_transaction_log(owner=owner)
t2.delete()
worker_id = self.factory.get_unique_string()
txlist = utils.get_txn_recs(num_recs=3, worker_id=worker_id)
self.assertEqual([t1.as_dict(), t3.as_dict()], txlist)
txlist = utils.get_txn_recs(num_recs=3,
last_id=t3.id,
worker_id=worker_id)
self.assertEqual([], txlist)
def test_get_txn_recs_previous_no_new(self):
"""Test getting a batch of transactions when we have previously
processed rows and there are no newer rows in the transaction_log
table.
"""
owner = self.make_user_without_txlog()
self.factory.make_transaction_log(owner=owner)
log = self.factory.make_transaction_log(owner=owner)
txlist = utils.get_txn_recs(num_recs=1, last_id=log.id)
self.assertEqual([], txlist)
def test_get_txn_recs_no_previous_exact_result_set(self):
"""Test getting a batch of transactions when we have not previously
processed any rows and the number of rows in the transaction_log table
is exactly the number requested.
"""
owner = self.make_user_without_txlog()
txlogs = [self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner)]
txlist = utils.get_txn_recs(num_recs=2, last_id=0)
self.assertEqual([t.as_dict() for t in txlogs], txlist)
def test_get_txn_recs_previous_no_new(self):
"""Test getting a batch of transactions when we have previously
processed rows and there are no newer rows in the transaction_log
table.
"""
owner = self.make_user_without_txlog()
self.factory.make_transaction_log(owner=owner)
log = self.factory.make_transaction_log(owner=owner)
txlist = utils.get_txn_recs(num_recs=1, last_id=log.id)
self.assertEqual([], txlist)
def test_get_txn_recs_respects_order(self):
"""Test that transaction log entries are returned in order."""
owner = self.make_user_without_txlog()
txlogs = [
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner)
]
txlist = utils.get_txn_recs(num_recs=3, last_id=0)
self.assertEqual([t.as_dict() for t in txlogs], txlist)
def test_get_txn_recs_unseen(self):
"""Getting transactions with unseen ids records those as unseen.
Querying again returns unseen transactions if they are now present.
"""
owner = self.make_user_without_txlog()
txlogs = [
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner)
]
worker_id = self.factory.get_unique_string()
txlist = utils.get_txn_recs(num_recs=3, worker_id=worker_id)
self.assertEqual([t.as_dict() for t in txlogs], txlist)
unseen = self.factory.make_transaction_log(owner=owner)
txlist = utils.get_txn_recs(num_recs=3,
last_id=txlogs[1].id,
worker_id=worker_id)
self.assertEqual([unseen.as_dict()], txlist)
def test_get_txn_recs_previous_exact_new(self):
"""Test getting a batch of transactions when we have previously
processed rows and there are the exact number of newer rows in the
transaction_log table that we requested.
"""
owner = self.make_user_without_txlog()
txlogs = [self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner)]
txlist = utils.get_txn_recs(num_recs=1, last_id=txlogs[0].id)
expected = [txlogs[1].as_dict()]
self.assertEqual(expected, txlist)
def test_get_txn_recs_no_previous_exact_result_set(self):
"""Test getting a batch of transactions when we have not previously
processed any rows and the number of rows in the transaction_log table
is exactly the number requested.
"""
owner = self.make_user_without_txlog()
txlogs = [
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner)
]
txlist = utils.get_txn_recs(num_recs=2, last_id=0)
self.assertEqual([t.as_dict() for t in txlogs], txlist)
def test_get_txn_recs_previous_exact_new(self):
"""Test getting a batch of transactions when we have previously
processed rows and there are the exact number of newer rows in the
transaction_log table that we requested.
"""
owner = self.make_user_without_txlog()
txlogs = [
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner)
]
txlist = utils.get_txn_recs(num_recs=1, last_id=txlogs[0].id)
expected = [txlogs[1].as_dict()]
self.assertEqual(expected, txlist)
def test_get_txn_recs_previous_small_new(self):
"""Test getting a batch of transactions when we have previously
processed rows and there are fewer newer rows in the transaction_log
table than we requested.
"""
owner = self.make_user_without_txlog()
t1 = self.factory.make_transaction_log(owner=owner)
txlogs = [self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner)]
txlist = utils.get_txn_recs(num_recs=5, last_id=t1.id)
self.assertEqual([t.as_dict() for t in txlogs], txlist)
def test_get_txn_recs_no_previous_large_result_set(self):
"""Test getting a batch of transactions when we have not previously
processed any rows and the number of rows in the transaction_log table
is larger than the number requested.
"""
owner = self.make_user_without_txlog()
txlogs = [self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner)]
txlist = utils.get_txn_recs(num_recs=1, last_id=0)
expected = [txlogs[0].as_dict()]
self.assertEqual(expected, txlist)
def test_get_txn_recs_previous_small_new(self):
"""Test getting a batch of transactions when we have previously
processed rows and there are fewer newer rows in the transaction_log
table than we requested.
"""
owner = self.make_user_without_txlog()
t1 = self.factory.make_transaction_log(owner=owner)
txlogs = [
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner)
]
txlist = utils.get_txn_recs(num_recs=5, last_id=t1.id)
self.assertEqual([t.as_dict() for t in txlogs], txlist)
def test_get_txn_recs_no_previous_large_result_set(self):
"""Test getting a batch of transactions when we have not previously
processed any rows and the number of rows in the transaction_log table
is larger than the number requested.
"""
owner = self.make_user_without_txlog()
txlogs = [
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner),
self.factory.make_transaction_log(owner=owner)
]
txlist = utils.get_txn_recs(num_recs=1, last_id=0)
expected = [txlogs[0].as_dict()]
self.assertEqual(expected, txlist)
def test_get_txn_recs_no_previous_no_txns(self):
"""Test getting a batch of transactions when we have not previously
processed any rows and the transaction_log table is empty.
"""
txlist = utils.get_txn_recs(num_recs=5, last_id=0)
self.assertEqual([], txlist)
def test_get_txn_recs_no_previous_no_txns(self):
"""Test getting a batch of transactions when we have not previously
processed any rows and the transaction_log table is empty.
"""
txlist = utils.get_txn_recs(num_recs=5, last_id=0)
self.assertEqual([], txlist)
