def test_duplicate_identifier_expire(self): now = datetime(2017, 1, 1) with patch('corehq.blobs.util._utcnow', return_value=now): self.db.put(StringIO(u'content'), self.identifier, bucket=self.bucket, timeout=60) self.assertIsNotNone(self.db.get(self.identifier, self.bucket)) with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=61)): delete_expired_blobs() with self.assertRaises(NotFound): self.db.get(self.identifier, self.bucket) now = datetime(2018, 1, 1) # Put another blob a year later with the same identifier with patch('corehq.blobs.util._utcnow', return_value=now): self.db.put(StringIO(u'content'), self.identifier, bucket=self.bucket, timeout=60) # Even though there is an old BlobExpiration record for that bucket/identifier, # we should not delete delete the new blob with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=0)): delete_expired_blobs() self.assertIsNotNone(self.db.get(self.identifier, self.bucket)) self.assertEqual(BlobExpiration.objects.all().count(), 2)
def test_blob_does_not_expire(self): now = datetime(2017, 1, 1) pre_expire_count = BlobExpiration.objects.all().count() with patch('corehq.blobs.util._utcnow', return_value=now): self.db.put(StringIO('content'), self.identifier, bucket=self.bucket, timeout=60) self.assertIsNotNone(self.db.get(self.identifier, self.bucket)) with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=30)): delete_expired_blobs() self.assertIsNotNone(self.db.get(self.identifier, self.bucket)) self.assertEqual(BlobExpiration.objects.all().count(), pre_expire_count + 1)
def test_blob_does_not_expire(self): now = datetime(2017, 1, 1) manager = BlobMeta.objects.partitioned_query(self.args["parent_id"]) pre_expire_count = manager.all().count() with patch('corehq.blobs.metadata._utcnow', return_value=now): self.db.put(BytesIO(b'content'), timeout=60, **self.args) self.assertIsNotNone(self.db.get(key=self.key, type_code=CODES.tempfile)) with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=30)): delete_expired_blobs() self.assertIsNotNone(self.db.get(key=self.key, type_code=CODES.tempfile)) self.assertEqual(manager.all().count(), pre_expire_count + 1)
def test_blob_does_not_expire(self): now = datetime(2017, 1, 1) shard = get_db_alias_for_partitioned_doc(self.args["parent_id"]) manager = BlobMeta.objects.using(shard) pre_expire_count = manager.all().count() with patch('corehq.blobs.metadata._utcnow', return_value=now): self.db.put(BytesIO(b'content'), timeout=60, **self.args) self.assertIsNotNone(self.db.get(key=self.key)) with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=30)): delete_expired_blobs() self.assertIsNotNone(self.db.get(key=self.key)) self.assertEqual(manager.all().count(), pre_expire_count + 1)
def test_blob_does_not_expire(self): now = datetime(2017, 1, 1) shard = get_db_alias_for_partitioned_doc(self.args["parent_id"]) manager = BlobMeta.objects.using(shard) pre_expire_count = manager.all().count() with patch('corehq.blobs.metadata._utcnow', return_value=now): self.db.put(BytesIO(b'content'), timeout=60, **self.args) self.assertIsNotNone(self.db.get(key=self.key)) with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=30)): delete_expired_blobs() self.assertIsNotNone(self.db.get(key=self.key)) self.assertEqual(manager.all().count(), pre_expire_count + 1)
def test_blob_expires(self): now = datetime(2017, 1, 1) manager = BlobMeta.objects.partitioned_query(self.args["parent_id"]) pre_expire_count = manager.count() with capture_log_output(mod.__name__) as logs: with patch('corehq.blobs.metadata._utcnow', return_value=now): self.db.put(BytesIO(b'content'), timeout=60, **self.args) self.assertIsNotNone( self.db.get(key=self.key, type_code=CODES.tempfile)) with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=61)): bytes_deleted = delete_expired_blobs() self.assertEqual(bytes_deleted, len('content')) with self.assertRaises(NotFound): self.db.get(key=self.key, type_code=CODES.tempfile) self.assertEqual(manager.all().count(), pre_expire_count) self.assertRegex( logs.get_output(), r"deleted expired blobs: .+'blob-identifier'", )
def test_blob_expires(self): now = datetime(2017, 1, 1) shard = get_db_alias_for_partitioned_doc(self.args["parent_id"]) manager = BlobMeta.objects.using(shard) pre_expire_count = manager.count() with capture_log_output(mod.__name__) as logs: with patch('corehq.blobs.metadata._utcnow', return_value=now): self.db.put(StringIO('content'), timeout=60, **self.args) self.assertIsNotNone(self.db.get(key=self.key)) with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=61)): bytes_deleted = delete_expired_blobs() self.assertEqual(bytes_deleted, len('content')) with self.assertRaises(NotFound): self.db.get(key=self.key) self.assertEqual(manager.all().count(), pre_expire_count) self.assertRegexpMatches( logs.get_output(), r"deleted expired blobs: .+'blob-identifier'", )
def test_expire(self): now = datetime.utcnow() meta = self.db.put( BytesIO(b"content"), domain="test", parent_id="test", type_code=CODES.tempfile, ) with self.db.get(key=meta.key) as fh: self.assertEqual(fh.read(), b"content") self.db.expire("test", meta.key) future_date = now + timedelta(minutes=120) with temporary_blob_db(self.db), \ patch('corehq.blobs.tasks._utcnow', return_value=future_date): delete_expired_blobs() with self.assertRaises(mod.NotFound): self.db.get(key=meta.key)
def test_expire(self): now = datetime.utcnow() meta = self.db.put( BytesIO(b"content"), domain="test", parent_id="test", type_code=CODES.tempfile, ) with self.db.get(key=meta.key) as fh: self.assertEqual(fh.read(), b"content") self.db.expire("test", meta.key) future_date = now + timedelta(minutes=120) with temporary_blob_db(self.db), \ patch('corehq.blobs.tasks._utcnow', return_value=future_date): delete_expired_blobs() with self.assertRaises(mod.NotFound): self.db.get(key=meta.key)
def test_blob_expires(self): now = datetime(2017, 1, 1) with patch('corehq.blobs.util._utcnow', return_value=now): self.db.put(StringIO(u'content'), self.identifier, bucket=self.bucket, timeout=60) self.assertIsNotNone(self.db.get(self.identifier, self.bucket)) with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=61)): bytes_deleted = delete_expired_blobs() self.assertEqual(bytes_deleted, len('content')) with self.assertRaises(NotFound): self.db.get(self.identifier, self.bucket) self.assertEqual(BlobExpiration.objects.all().count(), 1)
def test_blob_expires(self): now = datetime(2017, 1, 1) pre_expire_count = BlobExpiration.objects.all().count() with capture_log_output(mod.__name__) as logs: with patch('corehq.blobs.util._utcnow', return_value=now): self.db.put(StringIO('content'), self.identifier, bucket=self.bucket, timeout=60) self.assertIsNotNone(self.db.get(self.identifier, self.bucket)) with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=61)): bytes_deleted = delete_expired_blobs() self.assertEqual(bytes_deleted, len('content')) with self.assertRaises(NotFound): self.db.get(self.identifier, self.bucket) self.assertEqual(BlobExpiration.objects.all().count(), pre_expire_count) self.assertRegexpMatches( logs.get_output(), r"deleted expired blobs: .+/blob-bucket/blob-identifier'", )
def test_legacy_blob_expires(self): # this test can be removed when BlobExpiration is removed now = datetime(2017, 1, 1) pre_expire_count = BlobExpiration.objects.all().count() with capture_log_output(mod.__name__) as logs: args = self.args.copy() args["key"] = blob_key = "bucket/" + self.key meta = self.db.put(BytesIO(b'content'), **args) self.assertFalse(meta.expires_on, meta.expires_on) self.addCleanup(lambda: self.db.delete(key=blob_key)) # create legacy BlobExpiration object expire = BlobExpiration( bucket="bucket", identifier=self.key, expires_on=now + timedelta(minutes=60), length=7, ) expire.save() self.addCleanup(BlobExpiration.objects.filter(id=expire.id).delete) self.assertIsNotNone(self.db.get(key=blob_key)) with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=61)): bytes_deleted = delete_expired_blobs() self.assertEqual(bytes_deleted, len('content')) with self.assertRaises(NotFound): self.db.get(key=blob_key) self.assertEqual(BlobExpiration.objects.all().count(), pre_expire_count) self.assertRegexpMatches( logs.get_output(), r"deleted expired blobs: .+'bucket/blob-identifier'", )
def test_legacy_blob_expires(self): # this test can be removed when BlobExpiration is removed now = datetime(2017, 1, 1) pre_expire_count = BlobExpiration.objects.all().count() with capture_log_output(mod.__name__) as logs: args = self.args.copy() args["key"] = blob_key = "bucket/" + self.key meta = self.db.put(BytesIO(b'content'), **args) self.assertFalse(meta.expires_on, meta.expires_on) self.addCleanup(lambda: self.db.delete(key=blob_key)) # create legacy BlobExpiration object expire = BlobExpiration( bucket="bucket", identifier=self.key, expires_on=now + timedelta(minutes=60), length=7, ) expire.save() self.addCleanup(BlobExpiration.objects.filter(id=expire.id).delete) self.assertIsNotNone(self.db.get(key=blob_key)) with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=61)): bytes_deleted = delete_expired_blobs() self.assertEqual(bytes_deleted, len('content')) with self.assertRaises(NotFound): self.db.get(key=blob_key) self.assertEqual(BlobExpiration.objects.all().count(), pre_expire_count) self.assertRegexpMatches( logs.get_output(), r"deleted expired blobs: .+'bucket/blob-identifier'", )
def test_blob_expires(self): now = datetime(2017, 1, 1) shard = get_db_alias_for_partitioned_doc(self.args["parent_id"]) manager = BlobMeta.objects.using(shard) pre_expire_count = manager.count() with capture_log_output(mod.__name__) as logs: with patch('corehq.blobs.metadata._utcnow', return_value=now): self.db.put(BytesIO(b'content'), timeout=60, **self.args) self.assertIsNotNone(self.db.get(key=self.key)) with patch('corehq.blobs.tasks._utcnow', return_value=now + timedelta(minutes=61)): bytes_deleted = delete_expired_blobs() self.assertEqual(bytes_deleted, len('content')) with self.assertRaises(NotFound): self.db.get(key=self.key) self.assertEqual(manager.all().count(), pre_expire_count) self.assertRegexpMatches( logs.get_output(), r"deleted expired blobs: .+'blob-identifier'", )