def test_duplicate_identifier_expire(self):
now = datetime(2017, 1, 1)
with patch('corehq.blobs.util._utcnow', return_value=now):
self.db.put(StringIO(u'content'),
self.identifier,
bucket=self.bucket,
timeout=60)
self.assertIsNotNone(self.db.get(self.identifier, self.bucket))
with patch('corehq.blobs.tasks._utcnow',
return_value=now + timedelta(minutes=61)):
delete_expired_blobs()
with self.assertRaises(NotFound):
self.db.get(self.identifier, self.bucket)
now = datetime(2018, 1, 1)
# Put another blob a year later with the same identifier
with patch('corehq.blobs.util._utcnow', return_value=now):
self.db.put(StringIO(u'content'),
self.identifier,
bucket=self.bucket,
timeout=60)
# Even though there is an old BlobExpiration record for that bucket/identifier,
# we should not delete delete the new blob
with patch('corehq.blobs.tasks._utcnow',
return_value=now + timedelta(minutes=0)):
delete_expired_blobs()
self.assertIsNotNone(self.db.get(self.identifier, self.bucket))
self.assertEqual(BlobExpiration.objects.all().count(), 2)
def test_blob_does_not_expire(self):
now = datetime(2017, 1, 1)
pre_expire_count = BlobExpiration.objects.all().count()
with patch('corehq.blobs.util._utcnow', return_value=now):
self.db.put(StringIO('content'), self.identifier, bucket=self.bucket, timeout=60)
self.assertIsNotNone(self.db.get(self.identifier, self.bucket))
with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=30)):
delete_expired_blobs()
self.assertIsNotNone(self.db.get(self.identifier, self.bucket))
self.assertEqual(BlobExpiration.objects.all().count(), pre_expire_count + 1)
def test_blob_does_not_expire(self):
now = datetime(2017, 1, 1)
manager = BlobMeta.objects.partitioned_query(self.args["parent_id"])
pre_expire_count = manager.all().count()
with patch('corehq.blobs.metadata._utcnow', return_value=now):
self.db.put(BytesIO(b'content'), timeout=60, **self.args)
self.assertIsNotNone(self.db.get(key=self.key, type_code=CODES.tempfile))
with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=30)):
delete_expired_blobs()
self.assertIsNotNone(self.db.get(key=self.key, type_code=CODES.tempfile))
self.assertEqual(manager.all().count(), pre_expire_count + 1)
def test_blob_does_not_expire(self):
now = datetime(2017, 1, 1)
shard = get_db_alias_for_partitioned_doc(self.args["parent_id"])
manager = BlobMeta.objects.using(shard)
pre_expire_count = manager.all().count()
with patch('corehq.blobs.metadata._utcnow', return_value=now):
self.db.put(BytesIO(b'content'), timeout=60, **self.args)
self.assertIsNotNone(self.db.get(key=self.key))
with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=30)):
delete_expired_blobs()
self.assertIsNotNone(self.db.get(key=self.key))
self.assertEqual(manager.all().count(), pre_expire_count + 1)
def test_blob_does_not_expire(self):
now = datetime(2017, 1, 1)
shard = get_db_alias_for_partitioned_doc(self.args["parent_id"])
manager = BlobMeta.objects.using(shard)
pre_expire_count = manager.all().count()
with patch('corehq.blobs.metadata._utcnow', return_value=now):
self.db.put(BytesIO(b'content'), timeout=60, **self.args)
self.assertIsNotNone(self.db.get(key=self.key))
with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=30)):
delete_expired_blobs()
self.assertIsNotNone(self.db.get(key=self.key))
self.assertEqual(manager.all().count(), pre_expire_count + 1)
def test_blob_expires(self):
now = datetime(2017, 1, 1)
manager = BlobMeta.objects.partitioned_query(self.args["parent_id"])
pre_expire_count = manager.count()
with capture_log_output(mod.__name__) as logs:
with patch('corehq.blobs.metadata._utcnow', return_value=now):
self.db.put(BytesIO(b'content'), timeout=60, **self.args)
self.assertIsNotNone(
self.db.get(key=self.key, type_code=CODES.tempfile))
with patch('corehq.blobs.tasks._utcnow',
return_value=now + timedelta(minutes=61)):
bytes_deleted = delete_expired_blobs()
self.assertEqual(bytes_deleted, len('content'))
with self.assertRaises(NotFound):
self.db.get(key=self.key, type_code=CODES.tempfile)
self.assertEqual(manager.all().count(), pre_expire_count)
self.assertRegex(
logs.get_output(),
r"deleted expired blobs: .+'blob-identifier'",
)
def test_blob_expires(self):
now = datetime(2017, 1, 1)
shard = get_db_alias_for_partitioned_doc(self.args["parent_id"])
manager = BlobMeta.objects.using(shard)
pre_expire_count = manager.count()
with capture_log_output(mod.__name__) as logs:
with patch('corehq.blobs.metadata._utcnow', return_value=now):
self.db.put(StringIO('content'), timeout=60, **self.args)
self.assertIsNotNone(self.db.get(key=self.key))
with patch('corehq.blobs.tasks._utcnow',
return_value=now + timedelta(minutes=61)):
bytes_deleted = delete_expired_blobs()
self.assertEqual(bytes_deleted, len('content'))
with self.assertRaises(NotFound):
self.db.get(key=self.key)
self.assertEqual(manager.all().count(), pre_expire_count)
self.assertRegexpMatches(
logs.get_output(),
r"deleted expired blobs: .+'blob-identifier'",
)
def test_expire(self):
now = datetime.utcnow()
meta = self.db.put(
BytesIO(b"content"),
domain="test",
parent_id="test",
type_code=CODES.tempfile,
)
with self.db.get(key=meta.key) as fh:
self.assertEqual(fh.read(), b"content")
self.db.expire("test", meta.key)
future_date = now + timedelta(minutes=120)
with temporary_blob_db(self.db), \
patch('corehq.blobs.tasks._utcnow', return_value=future_date):
delete_expired_blobs()
with self.assertRaises(mod.NotFound):
self.db.get(key=meta.key)
def test_expire(self):
now = datetime.utcnow()
meta = self.db.put(
BytesIO(b"content"),
domain="test",
parent_id="test",
type_code=CODES.tempfile,
)
with self.db.get(key=meta.key) as fh:
self.assertEqual(fh.read(), b"content")
self.db.expire("test", meta.key)
future_date = now + timedelta(minutes=120)
with temporary_blob_db(self.db), \
patch('corehq.blobs.tasks._utcnow', return_value=future_date):
delete_expired_blobs()
with self.assertRaises(mod.NotFound):
self.db.get(key=meta.key)
def test_blob_expires(self):
now = datetime(2017, 1, 1)
with patch('corehq.blobs.util._utcnow', return_value=now):
self.db.put(StringIO(u'content'),
self.identifier,
bucket=self.bucket,
timeout=60)
self.assertIsNotNone(self.db.get(self.identifier, self.bucket))
with patch('corehq.blobs.tasks._utcnow',
return_value=now + timedelta(minutes=61)):
bytes_deleted = delete_expired_blobs()
self.assertEqual(bytes_deleted, len('content'))
with self.assertRaises(NotFound):
self.db.get(self.identifier, self.bucket)
self.assertEqual(BlobExpiration.objects.all().count(), 1)
def test_blob_expires(self):
now = datetime(2017, 1, 1)
pre_expire_count = BlobExpiration.objects.all().count()
with capture_log_output(mod.__name__) as logs:
with patch('corehq.blobs.util._utcnow', return_value=now):
self.db.put(StringIO('content'), self.identifier, bucket=self.bucket, timeout=60)
self.assertIsNotNone(self.db.get(self.identifier, self.bucket))
with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=61)):
bytes_deleted = delete_expired_blobs()
self.assertEqual(bytes_deleted, len('content'))
with self.assertRaises(NotFound):
self.db.get(self.identifier, self.bucket)
self.assertEqual(BlobExpiration.objects.all().count(), pre_expire_count)
self.assertRegexpMatches(
logs.get_output(),
r"deleted expired blobs: .+/blob-bucket/blob-identifier'",
)
def test_legacy_blob_expires(self):
# this test can be removed when BlobExpiration is removed
now = datetime(2017, 1, 1)
pre_expire_count = BlobExpiration.objects.all().count()
with capture_log_output(mod.__name__) as logs:
args = self.args.copy()
args["key"] = blob_key = "bucket/" + self.key
meta = self.db.put(BytesIO(b'content'), **args)
self.assertFalse(meta.expires_on, meta.expires_on)
self.addCleanup(lambda: self.db.delete(key=blob_key))
# create legacy BlobExpiration object
expire = BlobExpiration(
bucket="bucket",
identifier=self.key,
expires_on=now + timedelta(minutes=60),
length=7,
)
expire.save()
self.addCleanup(BlobExpiration.objects.filter(id=expire.id).delete)
self.assertIsNotNone(self.db.get(key=blob_key))
with patch('corehq.blobs.tasks._utcnow',
return_value=now + timedelta(minutes=61)):
bytes_deleted = delete_expired_blobs()
self.assertEqual(bytes_deleted, len('content'))
with self.assertRaises(NotFound):
self.db.get(key=blob_key)
self.assertEqual(BlobExpiration.objects.all().count(),
pre_expire_count)
self.assertRegexpMatches(
logs.get_output(),
r"deleted expired blobs: .+'bucket/blob-identifier'",
)
def test_legacy_blob_expires(self):
# this test can be removed when BlobExpiration is removed
now = datetime(2017, 1, 1)
pre_expire_count = BlobExpiration.objects.all().count()
with capture_log_output(mod.__name__) as logs:
args = self.args.copy()
args["key"] = blob_key = "bucket/" + self.key
meta = self.db.put(BytesIO(b'content'), **args)
self.assertFalse(meta.expires_on, meta.expires_on)
self.addCleanup(lambda: self.db.delete(key=blob_key))
# create legacy BlobExpiration object
expire = BlobExpiration(
bucket="bucket",
identifier=self.key,
expires_on=now + timedelta(minutes=60),
length=7,
)
expire.save()
self.addCleanup(BlobExpiration.objects.filter(id=expire.id).delete)
self.assertIsNotNone(self.db.get(key=blob_key))
with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=61)):
bytes_deleted = delete_expired_blobs()
self.assertEqual(bytes_deleted, len('content'))
with self.assertRaises(NotFound):
self.db.get(key=blob_key)
self.assertEqual(BlobExpiration.objects.all().count(), pre_expire_count)
self.assertRegexpMatches(
logs.get_output(),
r"deleted expired blobs: .+'bucket/blob-identifier'",
)
def test_blob_expires(self):
now = datetime(2017, 1, 1)
shard = get_db_alias_for_partitioned_doc(self.args["parent_id"])
manager = BlobMeta.objects.using(shard)
pre_expire_count = manager.count()
with capture_log_output(mod.__name__) as logs:
with patch('corehq.blobs.metadata._utcnow', return_value=now):
self.db.put(BytesIO(b'content'), timeout=60, **self.args)
self.assertIsNotNone(self.db.get(key=self.key))
with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=61)):
bytes_deleted = delete_expired_blobs()
self.assertEqual(bytes_deleted, len('content'))
with self.assertRaises(NotFound):
self.db.get(key=self.key)
self.assertEqual(manager.all().count(), pre_expire_count)
self.assertRegexpMatches(
logs.get_output(),
r"deleted expired blobs: .+'blob-identifier'",
)
