def test_transaction_deletion_dry_run(self, now, db, default_namespace):
shard_id = default_namespace.id >> 48
query = "SELECT count(id) FROM transaction WHERE namespace_id={}".format(
default_namespace.id
)
all_transactions = db.session.execute(query).scalar()
# Ensure no transactions are deleted during a dry run
purge_transactions(shard_id, days_ago=30, dry_run=True, now=now)
assert db.session.execute(query).scalar() == all_transactions
def purge_old_transactions(host, days_ago, limit, throttle, dry_run):
while True:
for shard in host["SHARDS"]:
# Ensure shard is explicitly not marked as disabled
if "DISABLED" in shard and not shard["DISABLED"]:
log.info("Spawning transaction purge process for shard",
shard_id=shard["ID"])
purge_transactions(shard["ID"], days_ago, limit, throttle,
dry_run)
else:
log.info("Will not spawn process for disabled shard",
shard_id=shard["ID"])
gevent.sleep(600)
def test_transaction_deletion_30_days(self, now, db, default_namespace):
shard_id = default_namespace.id >> 48
query = "SELECT count(id) FROM transaction WHERE namespace_id={}".format(
default_namespace.id
)
all_transactions = db.session.execute(query).scalar()
date_query = (
"SELECT count(id) FROM transaction WHERE created_at < "
"DATE_SUB({}, INTERVAL 30 day)"
).format(format_datetime(now))
older_than_thirty_days = db.session.execute(date_query).scalar()
# Delete all transactions older than 30 days
purge_transactions(shard_id, days_ago=30, dry_run=False, now=now)
assert (
all_transactions - older_than_thirty_days
== db.session.execute(query).scalar()
)
def test_transaction_deletion_purges_redis(
self, now, transactions, db, default_namespace
):
def _get_redis_transactions():
return redis_txn.zrangebyscore(
TXN_REDIS_KEY, "-inf", "+inf", withscores=True, score_cast_func=int,
)
assert db.session.query(Transaction).count()
assert len(_get_redis_transactions())
shard_id = default_namespace.id >> 48
purge_transactions(
shard_id, days_ago=1, dry_run=False, now=now + timedelta(days=30)
)
assert not db.session.query(Transaction).count()
assert not len(_get_redis_transactions())
def test_transaction_deletion_one_day(
self, now, transactions, db, default_namespace
):
shard_id = default_namespace.id >> 48
query = "SELECT count(id) FROM transaction WHERE namespace_id={}".format(
default_namespace.id
)
all_transactions = db.session.execute(query).scalar()
date_query = (
"SELECT count(id) FROM transaction WHERE created_at < "
"DATE_SUB({}, INTERVAL 1 day)"
).format(format_datetime(now))
older_than_one_day = db.session.execute(date_query).scalar()
# Delete all transactions older than 1 day
purge_transactions(shard_id, days_ago=1, dry_run=False, now=now)
assert (
all_transactions - older_than_one_day == db.session.execute(query).scalar()
)
latest_transaction = get_latest_transaction(db.session, default_namespace.id)
assert latest_transaction.id == transactions.id
def test_transaction_deletion(db, default_namespace):
# Test that transaction deletion respects the days_ago
# parameter. Arbitrarily chose 30 days for `days_ago`
now = datetime.now()
# Transactions created less than 30 days ago should not be deleted
t0 = create_transaction(db, now, default_namespace.id)
create_transaction(db, now - timedelta(days=29), default_namespace.id)
create_transaction(db, now - timedelta(days=30), default_namespace.id)
# Transactions older than 30 days should be deleted
for i in xrange(10):
create_transaction(db, now - timedelta(days=31 + i),
default_namespace.id)
shard_id = (default_namespace.id >> 48)
query = "SELECT count(id) FROM transaction WHERE namespace_id={}".\
format(default_namespace.id)
all_transactions = db.session.execute(query).scalar()
date_query = ("SELECT count(id) FROM transaction WHERE created_at < "
"DATE_SUB({}, INTERVAL 30 day)").format(format_datetime(now))
older_than_thirty_days = db.session.execute(date_query).scalar()
# Ensure no transactions are deleted during a dry run
purge_transactions(shard_id, days_ago=30, dry_run=True, now=now)
assert db.session.execute(query).scalar() == all_transactions
# Delete all transactions older than 30 days
purge_transactions(shard_id, days_ago=30, dry_run=False, now=now)
assert all_transactions - older_than_thirty_days == \
db.session.execute(query).scalar()
query = "SELECT count(id) FROM transaction WHERE namespace_id={}".\
format(default_namespace.id)
all_transactions = db.session.execute(query).scalar()
date_query = ("SELECT count(id) FROM transaction WHERE created_at < "
"DATE_SUB({}, INTERVAL 1 day)").format(format_datetime(now))
older_than_one_day = db.session.execute(date_query).scalar()
# Delete all transactions older than 1 day
purge_transactions(shard_id, days_ago=1, dry_run=False, now=now)
assert all_transactions - older_than_one_day == \
db.session.execute(query).scalar()
latest_transaction = get_latest_transaction(db.session,
default_namespace.id)
assert latest_transaction.id == t0.id