def test_get_txn_recs_retry_list_no_new_or_retry(self): """Test getting a batch of transactions when there are unseen ids records those as unseen. Querying again when unseen isn't available yet returns nothing. """ txlogs = [self.obj_factory.make_transaction_log(tx_id=1), self.obj_factory.make_transaction_log(tx_id=3)] worker_id = self.obj_factory.get_unique_unicode() txlist = utils.get_txn_recs(num_recs=3, worker_id=worker_id) self.assertEqual( self._convert_txlogs_to_dicts([txlogs[0], txlogs[1]]), txlist) txlist = utils.get_txn_recs( num_recs=3, last_id=txlogs[1].id, worker_id=worker_id) self.assertEqual(list(), txlist)
def test_deletes_txlogs_slice(self): """Delete a txlog slice by date and quantity.""" now = datetime.datetime.utcnow() timestamp_limit = now - datetime.timedelta(days=7) # Old enough old_dt = timestamp_limit quantity_limit = 2 txlogs = [ self.obj_factory.make_transaction_log(tx_id=1), self.obj_factory.make_transaction_log(tx_id=2, timestamp=old_dt), self.obj_factory.make_transaction_log(tx_id=3), self.obj_factory.make_transaction_log(tx_id=4, timestamp=old_dt), self.obj_factory.make_transaction_log(tx_id=5), ] self.store.commit() removed = utils.delete_txlogs_slice(self.obj_factory.sstore_name, date=now.date(), quantity_limit=quantity_limit) self.store.rollback() # Shouldn't affect the deletion result txlist = utils.get_txn_recs( self.obj_factory.sstore_name, num_recs=len(txlogs), last_id=0) self.assertEqual(len(txlist), 3) self.assertEqual(removed, quantity_limit) ids = sorted(int(txdict['txn_id']) for txdict in txlist) self.assertIn(2, ids) self.assertIn(4, ids)
def test_deletes_txlogs_slice(self): """Delete a txlog slice by date and quantity.""" now = datetime.datetime.utcnow() timestamp_limit = now - datetime.timedelta(days=7) # Old enough old_dt = timestamp_limit quantity_limit = 2 txlogs = [ self.obj_factory.make_transaction_log(tx_id=1), self.obj_factory.make_transaction_log(tx_id=2, timestamp=old_dt), self.obj_factory.make_transaction_log(tx_id=3), self.obj_factory.make_transaction_log(tx_id=4, timestamp=old_dt), self.obj_factory.make_transaction_log(tx_id=5), ] self.store.commit() removed = utils.delete_txlogs_slice(self.obj_factory.sstore_name, date=now.date(), quantity_limit=quantity_limit) self.store.rollback() # Shouldn't affect the deletion result txlist = utils.get_txn_recs(self.obj_factory.sstore_name, num_recs=len(txlogs), last_id=0) self.assertEqual(len(txlist), 3) self.assertEqual(removed, quantity_limit) ids = sorted(int(txdict['txn_id']) for txdict in txlist) self.assertIn(2, ids) self.assertIn(4, ids)
def test_get_txn_recs_for_partition(self): """Get txlogs for the provided partition ID. When owner_id % num_partitions == partition_id, the txlog is added to the result set, so that it matches the filter by partition. Also, any txlog that is related to sharing is also returned, no matter what the owner_id is. """ owner_id = 1 num_partitions = 8 partition_id = owner_id % num_partitions txlogs = [ self.obj_factory.make_transaction_log(), self.obj_factory.make_transaction_log(owner_id=2), # Different one self.obj_factory.make_transaction_log(), # Share txlogs, but with a different owner, are also returned. self.obj_factory.make_transaction_log( owner_id=2, op_type=TransactionLog.OP_SHARE_ACCEPTED), self.obj_factory.make_transaction_log( owner_id=2, op_type=TransactionLog.OP_SHARE_DELETED), ] expected_txlogs = [txlogs[0], txlogs[2], txlogs[3], txlogs[4]] txlist = utils.get_txn_recs(self.obj_factory.sstore_name, num_recs=5, last_id=0, num_partitions=num_partitions, partition_id=partition_id) self.assertEqual(self._convert_txlogs_to_dicts(expected_txlogs), txlist)
def test_maintains_newish_txlogs_when_purging(self): """Test that txnlogs not old enough are maintained, instead of being deleted.""" now = datetime.datetime.utcnow() limit_datetime = now - datetime.timedelta(days=7) # Not so old old_datetime = limit_datetime + datetime.timedelta(seconds=1) self.obj_factory.make_transaction_log(tx_id=1) self.obj_factory.make_transaction_log(tx_id=2, timestamp=old_datetime) self.obj_factory.make_transaction_log(tx_id=3) self.obj_factory.make_transaction_log(tx_id=4, timestamp=old_datetime) self.store.commit() removed = utils.delete_old_txlogs(self.obj_factory.sstore_name, timestamp_limit=limit_datetime) self.store.rollback() # Shouldn't affect the deletion result txlist = utils.get_txn_recs( self.obj_factory.sstore_name, num_recs=4, last_id=0) self.assertEqual(len(txlist), 4) self.assertEqual(removed, 0) ids = sorted(int(txdict['txn_id']) for txdict in txlist) self.assertEqual(ids, [1, 2, 3, 4])
def test_deletes_old_txlogs_within_quantity_limit(self): """Test that txnlogs old enough are deleted and are within the quantity limit given.""" now = datetime.datetime.utcnow() timestamp_limit = now - datetime.timedelta(days=7) # Old enough old_datetime = timestamp_limit quantity_limit = 2 txlogs = [ self.obj_factory.make_transaction_log(tx_id=1), self.obj_factory.make_transaction_log( tx_id=2, timestamp=old_datetime), self.obj_factory.make_transaction_log(tx_id=3), self.obj_factory.make_transaction_log( tx_id=4, timestamp=old_datetime), self.obj_factory.make_transaction_log( tx_id=5, timestamp=old_datetime), ] self.store.commit() removed = utils.delete_old_txlogs(self.obj_factory.sstore_name, timestamp_limit=timestamp_limit, quantity_limit=quantity_limit) self.store.rollback() # Shouldn't affect the deletion result txlist = utils.get_txn_recs( self.obj_factory.sstore_name, num_recs=len(txlogs), last_id=0) self.assertEqual(len(txlist), 3) self.assertEqual(removed, quantity_limit) ids = sorted(int(txdict['txn_id']) for txdict in txlist) self.assertEqual(ids, [1, 3, 5])
def test_maintains_newish_txlogs_when_purging(self): """Test that txnlogs not old enough are maintained, instead of being deleted.""" now = datetime.datetime.utcnow() limit_datetime = now - datetime.timedelta(days=7) # Not so old old_datetime = limit_datetime + datetime.timedelta(seconds=1) self.obj_factory.make_transaction_log(tx_id=1) self.obj_factory.make_transaction_log(tx_id=2, timestamp=old_datetime) self.obj_factory.make_transaction_log(tx_id=3) self.obj_factory.make_transaction_log(tx_id=4, timestamp=old_datetime) self.store.commit() removed = utils.delete_old_txlogs(self.obj_factory.sstore_name, timestamp_limit=limit_datetime) self.store.rollback() # Shouldn't affect the deletion result txlist = utils.get_txn_recs(self.obj_factory.sstore_name, num_recs=4, last_id=0) self.assertEqual(len(txlist), 4) self.assertEqual(removed, 0) ids = sorted(int(txdict['txn_id']) for txdict in txlist) self.assertEqual(ids, [1, 2, 3, 4])
def test_get_txn_recs_for_partition(self): """Get txlogs for the provided partition ID. When owner_id % num_partitions == partition_id, the txlog is added to the result set, so that it matches the filter by partition. Also, any txlog that is related to sharing is also returned, no matter what the owner_id is. """ owner_id = 1 num_partitions = 8 partition_id = owner_id % num_partitions txlogs = [ self.obj_factory.make_transaction_log(), self.obj_factory.make_transaction_log(owner_id=2), # Different one self.obj_factory.make_transaction_log(), # Share txlogs, but with a different owner, are also returned. self.obj_factory.make_transaction_log( owner_id=2, op_type=TransactionLog.OP_SHARE_ACCEPTED), self.obj_factory.make_transaction_log( owner_id=2, op_type=TransactionLog.OP_SHARE_DELETED), ] expected_txlogs = [txlogs[0], txlogs[2], txlogs[3], txlogs[4]] txlist = utils.get_txn_recs( self.obj_factory.sstore_name, num_recs=5, last_id=0, num_partitions=num_partitions, partition_id=partition_id ) self.assertEqual(self._convert_txlogs_to_dicts(expected_txlogs), txlist)
def test_get_txn_recs_no_previous_no_txns(self): """Test getting a batch of transactions when we have not previously processed any rows and the transaction_log table is empty. """ txlist = utils.get_txn_recs( self.obj_factory.sstore_name, num_recs=5, last_id=0) self.assertEqual([], txlist)
def test_deletes_old_enough_txlogs(self): """Test that txnlogs old enough are deleted.""" now = datetime.datetime.utcnow() timestamp_limit = now - datetime.timedelta(days=7) # Old enough old_datetime = timestamp_limit txlogs = [ self.obj_factory.make_transaction_log(tx_id=1), self.obj_factory.make_transaction_log( tx_id=2, timestamp=old_datetime), self.obj_factory.make_transaction_log(tx_id=3), self.obj_factory.make_transaction_log( tx_id=4, timestamp=old_datetime), ] self.store.commit() removed = utils.delete_old_txlogs(timestamp_limit=timestamp_limit) self.store.rollback() # Shouldn't affect the deletion result txlist = utils.get_txn_recs(num_recs=len(txlogs), last_id=0) self.assertEqual(len(txlist), 2) self.assertEqual(removed, 2) ids = sorted(int(txdict['txn_id']) for txdict in txlist) self.assertEqual(ids, [1, 3])
def test_deletes_old_txlogs_within_quantity_limit(self): """Test that txnlogs old enough are deleted and are within the quantity limit given.""" now = datetime.datetime.utcnow() timestamp_limit = now - datetime.timedelta(days=7) # Old enough old_datetime = timestamp_limit quantity_limit = 2 txlogs = [ self.obj_factory.make_transaction_log(tx_id=1), self.obj_factory.make_transaction_log(tx_id=2, timestamp=old_datetime), self.obj_factory.make_transaction_log(tx_id=3), self.obj_factory.make_transaction_log(tx_id=4, timestamp=old_datetime), self.obj_factory.make_transaction_log(tx_id=5, timestamp=old_datetime), ] self.store.commit() removed = utils.delete_old_txlogs(self.obj_factory.sstore_name, timestamp_limit=timestamp_limit, quantity_limit=quantity_limit) self.store.rollback() # Shouldn't affect the deletion result txlist = utils.get_txn_recs(self.obj_factory.sstore_name, num_recs=len(txlogs), last_id=0) self.assertEqual(len(txlist), 3) self.assertEqual(removed, quantity_limit) ids = sorted(int(txdict['txn_id']) for txdict in txlist) self.assertEqual(ids, [1, 3, 5])
def test_get_txn_recs_unseen(self): """Test getting a batch of transactions when there are unseen ids records those as unseen. Querying again returns unseen transactions if they are now present. """ txlogs = [self.obj_factory.make_transaction_log(tx_id=1), self.obj_factory.make_transaction_log(tx_id=3)] worker_id = self.obj_factory.get_unique_unicode() txlist = utils.get_txn_recs(num_recs=3, worker_id=worker_id) expected = self._convert_txlogs_to_dicts([txlogs[0], txlogs[1]]) self.assertEqual(expected, txlist) unseen = self.obj_factory.make_transaction_log(tx_id=2) txlist = utils.get_txn_recs( num_recs=3, last_id=txlogs[1].id, worker_id=worker_id) self.assertEqual( self._convert_txlogs_to_dicts([unseen]), txlist)
def test_get_txn_recs_no_previous_no_txns(self): """Test getting a batch of transactions when we have not previously processed any rows and the transaction_log table is empty. """ txlist = utils.get_txn_recs(self.obj_factory.sstore_name, num_recs=5, last_id=0) self.assertEqual([], txlist)
def test_get_txn_recs_respects_order(self): """Test that transaction log entries are returned in order.""" txlogs = [self.obj_factory.make_transaction_log(tx_id=3), self.obj_factory.make_transaction_log(tx_id=2), self.obj_factory.make_transaction_log(tx_id=1)] txlist = utils.get_txn_recs(num_recs=3, last_id=0) txlogs.sort(key=lambda x: x.id) self.assertEqual(self._convert_txlogs_to_dicts(txlogs), txlist)
def test_get_txn_recs_no_previous_large_result_set(self): """Test getting a batch of transactions when we have not previously processed any rows and the number of rows in the transaction_log table is larger than the number requested. """ txlogs = [self.obj_factory.make_transaction_log(), self.obj_factory.make_transaction_log()] txlist = utils.get_txn_recs(num_recs=1, last_id=0) self.assertEqual(self._convert_txlogs_to_dicts(txlogs[:1]), txlist)
def test_get_txn_recs_previous_no_new(self): """Test getting a batch of transactions when we have previously processed rows and there are no newer rows in the transaction_log table. """ self.obj_factory.make_transaction_log() log = self.obj_factory.make_transaction_log() txlist = utils.get_txn_recs(num_recs=1, last_id=log.id) self.assertEqual([], txlist)
def test_get_txn_recs_previous_exact_new(self): """Test getting a batch of transactions when we have previously processed rows and there are the exact number of newer rows in the transaction_log table that we requested. """ txlogs = [self.obj_factory.make_transaction_log(), self.obj_factory.make_transaction_log()] txlist = utils.get_txn_recs(num_recs=1, last_id=txlogs[0].id) self.assertEqual(self._convert_txlogs_to_dicts(txlogs[1:]), txlist)
def test_get_txn_recs_previous_small_new(self): """Test getting a batch of transactions when we have previously processed rows and there are fewer newer rows in the transaction_log table than we requested. """ txlogs = [self.obj_factory.make_transaction_log(), self.obj_factory.make_transaction_log()] txlist = utils.get_txn_recs( self.obj_factory.sstore_name, num_recs=5, last_id=1) self.assertEqual(self._convert_txlogs_to_dicts(txlogs[1:]), txlist)
def test_get_txn_recs_retry_list_no_new_or_retry(self): """Test getting a batch of transactions when there are unseen ids records those as unseen. Querying again when unseen isn't available yet returns nothing. """ txlogs = [ self.obj_factory.make_transaction_log(tx_id=1), self.obj_factory.make_transaction_log(tx_id=3) ] worker_id = self.obj_factory.get_unique_unicode() txlist = utils.get_txn_recs(self.obj_factory.sstore_name, num_recs=3, worker_id=worker_id) self.assertEqual(self._convert_txlogs_to_dicts([txlogs[0], txlogs[1]]), txlist) txlist = utils.get_txn_recs(self.obj_factory.sstore_name, num_recs=3, last_id=txlogs[1].id, worker_id=worker_id) self.assertEqual(list(), txlist)
def test_get_txn_recs_previous_no_new(self): """Test getting a batch of transactions when we have previously processed rows and there are no newer rows in the transaction_log table. """ self.obj_factory.make_transaction_log() self.obj_factory.make_transaction_log() txlist = utils.get_txn_recs(self.obj_factory.sstore_name, num_recs=1, last_id=2) self.assertEqual([], txlist)
def test_get_txn_recs_unseen(self): """Test getting a batch of transactions when there are unseen ids records those as unseen. Querying again returns unseen transactions if they are now present. """ txlogs = [ self.obj_factory.make_transaction_log(tx_id=1), self.obj_factory.make_transaction_log(tx_id=3) ] worker_id = self.obj_factory.get_unique_unicode() txlist = utils.get_txn_recs(self.obj_factory.sstore_name, num_recs=3, worker_id=worker_id) self.assertEqual(self._convert_txlogs_to_dicts([txlogs[0], txlogs[1]]), txlist) unseen = self.obj_factory.make_transaction_log(tx_id=2) txlist = utils.get_txn_recs(self.obj_factory.sstore_name, num_recs=3, last_id=txlogs[1].id, worker_id=worker_id) self.assertEqual(self._convert_txlogs_to_dicts([unseen]), txlist)
def test_get_txn_recs_respects_order(self): """Test that transaction log entries are returned in order.""" txlogs = [ self.obj_factory.make_transaction_log(tx_id=3), self.obj_factory.make_transaction_log(tx_id=2), self.obj_factory.make_transaction_log(tx_id=1) ] txlist = utils.get_txn_recs(self.obj_factory.sstore_name, num_recs=3, last_id=0) txlogs.sort(key=lambda x: x.id) self.assertEqual(self._convert_txlogs_to_dicts(txlogs), txlist)
def test_get_txn_recs_previous_exact_new(self): """Test getting a batch of transactions when we have previously processed rows and there are the exact number of newer rows in the transaction_log table that we requested. """ txlogs = [ self.obj_factory.make_transaction_log(), self.obj_factory.make_transaction_log() ] txlist = utils.get_txn_recs(self.obj_factory.sstore_name, num_recs=1, last_id=1) self.assertEqual(self._convert_txlogs_to_dicts(txlogs[1:]), txlist)
def test_get_txn_recs_no_previous_large_result_set(self): """Test getting a batch of transactions when we have not previously processed any rows and the number of rows in the transaction_log table is larger than the number requested. """ txlogs = [ self.obj_factory.make_transaction_log(), self.obj_factory.make_transaction_log() ] txlist = utils.get_txn_recs(self.obj_factory.sstore_name, num_recs=1, last_id=0) self.assertEqual(self._convert_txlogs_to_dicts(txlogs[:1]), txlist)