class BaseTestMixin:
@staticmethod
def bucket_name(name):
return bucket_name(name)
def minio_client(self):
minio_client = Minio(
endpoint=get_setting("MINIO_STORAGE_ENDPOINT"),
access_key=get_setting("MINIO_STORAGE_ACCESS_KEY"),
secret_key=get_setting("MINIO_STORAGE_SECRET_KEY"),
secure=get_setting("MINIO_STORAGE_USE_HTTPS"))
return minio_client
def setUp(self):
self.media_storage = MinioMediaStorage()
self.static_storage = MinioStaticStorage()
self.new_file = self.media_storage.save("test-file",
ContentFile(b"yep"))
self.second_file = self.media_storage.save("test-file",
ContentFile(b"nope"))
def tearDown(self):
client = self.minio_client()
def obliterate_bucket(name):
for obj in client.list_objects(name, "", True):
client.remove_object(name, obj.object_name)
for obj in client.list_incomplete_uploads(
name, ""): # pragma: no cover # noqa
client.remove_incomplete_upload(name, obj.objectname)
client.remove_bucket(name)
obliterate_bucket(self.bucket_name("tests-media"))
obliterate_bucket(self.bucket_name("tests-static"))
class BaseTestMixin:
@staticmethod
def bucket_name(name):
return bucket_name(name)
def minio_client(self):
minio_client = Minio(
endpoint=get_setting("MINIO_STORAGE_ENDPOINT"),
access_key=get_setting("MINIO_STORAGE_ACCESS_KEY"),
secret_key=get_setting("MINIO_STORAGE_SECRET_KEY"),
secure=get_setting("MINIO_STORAGE_USE_HTTPS"))
return minio_client
def setUp(self):
self.media_storage = MinioMediaStorage()
self.static_storage = MinioStaticStorage()
self.new_file = self.media_storage.save("test-file",
ContentFile(b"yep"))
self.second_file = self.media_storage.save("test-file",
ContentFile(b"nope"))
def tearDown(self):
client = self.minio_client()
def obliterate_bucket(name):
for obj in client.list_objects(name, "", True):
client.remove_object(name, obj.object_name)
for obj in client.list_incomplete_uploads(name, ""): # pragma: no cover # noqa
client.remove_incomplete_upload(name, obj.objectname)
client.remove_bucket(name)
obliterate_bucket(self.bucket_name("tests-media"))
obliterate_bucket(self.bucket_name("tests-static"))
class RetrieveTestsWithPublicBucket(BaseTestMixin, TestCase):
def setUp(self):
self.media_storage = MinioMediaStorage()
self.static_storage = MinioStaticStorage()
self.new_file = self.media_storage.save("test-file",
ContentFile(b"yep"))
self.media_storage.client.set_bucket_policy(
self.media_storage.bucket_name,
self.media_storage._policy("READ_WRITE"))
self.static_storage.client.set_bucket_policy(
self.static_storage.bucket_name,
self.static_storage._policy("READ_WRITE"))
def test_public_url_generation(self):
media_test_file_name = self.media_storage.save(
u"weird & ÜRΛ", ContentFile(b"irrelevant"))
url = self.media_storage.url(media_test_file_name)
res = requests.get(url)
self.assertEqual(res.content, b"irrelevant")
static_test_file_name = self.static_storage.save(
u"weird & ÜRΛ", ContentFile(b"irrelevant"))
url = self.static_storage.url(static_test_file_name)
res = requests.get(url)
self.assertEqual(res.content, b"irrelevant")
class RetrieveTestsWithPublicBucket(BaseTestMixin, TestCase):
def setUp(self):
self.media_storage = MinioMediaStorage()
self.static_storage = MinioStaticStorage()
self.new_file = self.media_storage.save("test-file",
ContentFile(b"yep"))
self.media_storage.client.set_bucket_policy(
self.media_storage.bucket_name,
self.media_storage._policy("READ_WRITE")
)
self.static_storage.client.set_bucket_policy(
self.static_storage.bucket_name,
self.static_storage._policy("READ_WRITE")
)
def test_public_url_generation(self):
media_test_file_name = self.media_storage.save(
u"weird & ÜRΛ", ContentFile(b"irrelevant"))
url = self.media_storage.url(media_test_file_name)
res = requests.get(url)
self.assertEqual(res.content, b"irrelevant")
static_test_file_name = self.static_storage.save(
u"weird & ÜRΛ", ContentFile(b"irrelevant"))
url = self.static_storage.url(static_test_file_name)
res = requests.get(url)
self.assertEqual(res.content, b"irrelevant")
def test_media_policy_write(self):
ms = MinioMediaStorage()
self.maxDiff = 50000
self.assertPolicyEqual(
Policy.write.bucket(ms.bucket_name, json_encode=False),
json.loads(ms.client.get_bucket_policy(ms.bucket_name)),
)
fn = ms.save("somefile", ContentFile(b"test"))
self.assertEqual(ms.open(fn).read(), b"test")
self.assertEqual(requests.get(ms.url(fn)).status_code, 403)
self.assertEqual(
requests.get(f"{ms.client._endpoint_url}/{ms.bucket_name}").status_code, 403
)
def test_media_policy_auto_true(self):
ms = MinioMediaStorage()
self.maxDiff = 50000
self.assertPolicyEqual(
Policy.get.bucket(ms.bucket_name, json_encode=False),
json.loads(ms.client.get_bucket_policy(ms.bucket_name)),
)
fn = ms.save("somefile", ContentFile(b"test"))
self.assertEqual(ms.open(fn).read(), b"test")
url = ms.url(fn)
self.assertEqual(requests.get(url).status_code, 200)
self.assertEqual(
requests.get("{}/{}".format(ms.client._endpoint_url, ms.bucket_name)).status_code, 403
)
def test_public_url_generation(self):
media_storage = MinioMediaStorage()
media_test_file_name = media_storage.save("weird & ÜRΛ",
ContentFile(b"irrelevant"))
url = media_storage.url(media_test_file_name)
res = requests.get(url)
self.assertEqual(res.content, b"irrelevant")
static_storage = MinioStaticStorage()
static_test_file_name = static_storage.save("weird & ÜRΛ",
ContentFile(b"irrelevant"))
url = static_storage.url(static_test_file_name)
res = requests.get(url)
self.assertEqual(res.content, b"irrelevant")
class MinioStorageTests(TestCase):
def setUp(self):
self.media_storage = MinioMediaStorage()
self.static_storage = MinioStaticStorage()
self.new_file = self.media_storage.save("a new & original file",
ContentFile(b"yep"))
@override_settings(
MINIO_STORAGE_MEDIA_BUCKET_NAME="inexistent",
MINIO_STORAGE_AUTO_CREATE_MEDIA_BUCKET=False,
)
def test_media_storage_cannot_be_initialized_without_bucket(self):
with self.assertRaises(IOError):
MinioMediaStorage()
@override_settings(
MINIO_STORAGE_STATIC_BUCKET_NAME="inexistent",
MINIO_STORAGE_AUTO_CREATE_STATIC_BUCKET=False,
)
def test_static_storage_cannot_be_initialized_without_bucket(self):
with self.assertRaises(IOError):
MinioStaticStorage()
def test_get_setting_throws_early(self):
with self.assertRaises(ImproperlyConfigured):
get_setting("INEXISTENT_SETTING")
def test_it_works(self):
self.assertEqual(2 + 2, 4)
def test_file_upload_does_not_throw(self):
self.media_storage.save("trivial.txt", ContentFile(b"12345"))
def test_two_files_with_the_same_name_can_be_uploaded(self):
ivan = self.media_storage.save("pelican.txt",
ContentFile(b"Ivan le Pelican"))
jean = self.media_storage.save("pelican.txt",
ContentFile(b"Jean le Pelican"))
self.assertNotEqual(jean, ivan)
@patch("minio.Minio.put_object", side_effect=ResponseError(MagicMock()))
def test_file_upload_throws_on_failure(self, mock):
with self.assertRaises(IOError):
self.media_storage.save("meh", ContentFile(b"meh"))
def test_file_removal(self):
test_file = self.media_storage.save("should_be_removed.txt",
ContentFile(b"meh"))
self.media_storage.delete(test_file)
self.assertFalse(self.media_storage.exists(test_file))
def test_delete_of_non_existent_throws(self):
with self.assertRaises(IOError):
self.media_storage.delete("i don't even exist")
def test_url_generation(self):
test_file = self.media_storage.save("weird & ÜRΛ",
ContentFile(b"irrelevant"))
url = self.media_storage.url(test_file)
res = requests.get(url)
self.assertEqual(res.content, b"irrelevant")
def test_files_from_filesystem_are_uploaded_properly(self):
f = File(io.open("watermelon-cat.jpg", "br"))
saved_file = self.media_storage.save("watermelon-cat.jpg", f)
res = requests.get(self.media_storage.url(saved_file))
self.assertAlmostEqual(round(res.content.__sizeof__() / 100),
round(f.size / 100))
def test_url_of_non_existent_throws(self):
with self.assertRaises(IOError):
self.media_storage.url("this does not exist")
def test_file_stat(self):
pass
def test_file_size(self):
test_file = self.media_storage.save("sizetest.txt",
ContentFile(b"1234"))
self.assertEqual(4, self.media_storage.size(test_file))
def test_size_of_non_existent_throws(self):
test_file = self.media_storage.save("sizetest.txt",
ContentFile(b"1234"))
self.media_storage.delete(test_file)
with self.assertRaises(IOError):
self.media_storage.size(test_file)
def test_modified_time(self):
self.assertIsInstance(self.media_storage.modified_time(self.new_file),
datetime.datetime)
def test_accessed_time(self):
self.assertIsInstance(self.media_storage.accessed_time(self.new_file),
datetime.datetime)
def test_created_time(self):
self.assertIsInstance(self.media_storage.created_time(self.new_file),
datetime.datetime)
def test_modified_time_of_non_existent_throws(self):
with self.assertRaises(IOError):
self.media_storage.modified_time("nonexistent.jpg")
def test_files_cannot_be_open_in_write_mode(self):
test_file = self.media_storage.save("iomodetest.txt",
ContentFile(b"should not change"))
with self.assertRaises(NotImplementedError):
self.media_storage.open(test_file, mode="bw")
def test_list_dir_base(self):
test_dir = self.media_storage.listdir("/")
files = [elem for elem in test_dir]
self.assertIsInstance(files, list)
self.assertGreaterEqual(len(files), 1)
@patch("minio.Minio.list_objects", side_effect=ResponseError(MagicMock()))
def test_list_dir_throws_on_failure(self, mock):
with self.assertRaises(IOError):
self.media_storage.listdir("")
def test_file_exists(self):
existent = self.media_storage.save("existent.txt", ContentFile(b"meh"))
self.assertTrue(self.media_storage.exists(existent))
def test_file_exists_failure(self):
self.assertFalse(self.media_storage.exists("nonexistent.txt"))
@patch("minio.Minio.stat_object", side_effect=ResponseError(MagicMock()))
def test_file_exists_can_throw(self, mock):
with self.assertRaises(IOError):
self.media_storage.exists("peu importe")
def test_opening_non_existing_file_raises_exception(self):
with self.assertRaises(IOError):
self.media_storage.open("this does not exist")
def test_file_names_are_properly_sanitized(self):
self.media_storage.save("./meh22222.txt", io.BytesIO(b"stuff"))
def test_upload_and_get_back_file_with_funky_name(self):
pass
def test_uploaded_and_downloaded_file_sizes_match(self):
pass
def test_uploaded_files_end_up_in_the_right_bucket(self):
pass
def test_static_files_end_up_in_the_right_bucket(self):
pass
def tearDown(self):
client = Minio(endpoint=get_setting("MINIO_STORAGE_ENDPOINT"),
access_key=get_setting("MINIO_STORAGE_ACCESS_KEY"),
secret_key=get_setting("MINIO_STORAGE_SECRET_KEY"),
secure=get_setting("MINIO_STORAGE_USE_HTTPS"))
def obliterate_bucket(name):
for obj in client.list_objects(name, ""):
client.remove_object(name, obj.object_name)
for obj in client.list_incomplete_uploads(name, ""): # pragma: no cover
client.remove_incomplete_upload(name, obj.objectname)
client.remove_bucket(name)
obliterate_bucket("tests-media")
obliterate_bucket("tests-static")
class MinioStorageTests(TestCase):
def setUp(self):
self.media_storage = MinioMediaStorage()
self.static_storage = MinioStaticStorage()
self.new_file = self.media_storage.save("a new & original file",
ContentFile(b"yep"))
@override_settings(
MINIO_STORAGE_MEDIA_BUCKET_NAME="inexistent",
MINIO_STORAGE_AUTO_CREATE_MEDIA_BUCKET=False,
)
def test_media_storage_cannot_be_initialized_without_bucket(self):
with self.assertRaises(IOError):
MinioMediaStorage()
@override_settings(
MINIO_STORAGE_STATIC_BUCKET_NAME="inexistent",
MINIO_STORAGE_AUTO_CREATE_STATIC_BUCKET=False,
)
def test_static_storage_cannot_be_initialized_without_bucket(self):
with self.assertRaises(IOError):
MinioStaticStorage()
def test_get_setting_throws_early(self):
with self.assertRaises(ImproperlyConfigured):
get_setting("INEXISTENT_SETTING")
def test_it_works(self):
self.assertEqual(2 + 2, 4)
def test_file_upload_does_not_throw(self):
self.media_storage.save("trivial.txt", ContentFile(b"12345"))
def test_two_files_with_the_same_name_can_be_uploaded(self):
ivan = self.media_storage.save("pelican.txt",
ContentFile(b"Ivan le Pelican"))
jean = self.media_storage.save("pelican.txt",
ContentFile(b"Jean le Pelican"))
self.assertNotEqual(jean, ivan)
@patch("minio.Minio.put_object", side_effect=ResponseError(MagicMock()))
def test_file_upload_throws_on_failure(self, mock):
with self.assertRaises(IOError):
self.media_storage.save("meh", ContentFile(b"meh"))
def test_file_removal(self):
test_file = self.media_storage.save("should_be_removed.txt",
ContentFile(b"meh"))
self.media_storage.delete(test_file)
self.assertFalse(self.media_storage.exists(test_file))
@patch("minio.Minio.remove_object", side_effect=ResponseError(MagicMock()))
def test_deletion_error(self, mock):
with self.assertRaises(IOError):
self.media_storage.delete("this is irrelevant")
def test_url_generation(self):
test_file_name = self.media_storage.save(u"weird & ÜRΛ",
ContentFile(b"irrelevant"))
url = self.media_storage.url(test_file_name)
res = requests.get(url)
self.assertEqual(res.content, b"irrelevant")
def test_files_from_filesystem_are_uploaded_properly(self):
f = File(io.open("watermelon-cat.jpg", "br"))
saved_file = self.media_storage.save("watermelon-cat.jpg", f)
res = requests.get(self.media_storage.url(saved_file))
self.assertAlmostEqual(round(res.content.__sizeof__() / 100),
round(f.size / 100))
def test_url_of_non_existent_throws(self):
with self.assertRaises(IOError):
self.media_storage.url("this does not exist")
def test_file_stat(self):
pass
def test_file_size(self):
test_file = self.media_storage.save("sizetest.txt",
ContentFile(b"1234"))
self.assertEqual(4, self.media_storage.size(test_file))
def test_size_of_non_existent_throws(self):
test_file = self.media_storage.save("sizetest.txt",
ContentFile(b"1234"))
self.media_storage.delete(test_file)
with self.assertRaises(IOError):
self.media_storage.size(test_file)
def test_modified_time(self):
self.assertIsInstance(self.media_storage.modified_time(self.new_file),
datetime.datetime)
def test_accessed_time(self):
self.assertIsInstance(self.media_storage.accessed_time(self.new_file),
datetime.datetime)
def test_created_time(self):
self.assertIsInstance(self.media_storage.created_time(self.new_file),
datetime.datetime)
def test_modified_time_of_non_existent_throws(self):
with self.assertRaises(IOError):
self.media_storage.modified_time("nonexistent.jpg")
def test_files_cannot_be_open_in_write_mode(self):
test_file = self.media_storage.save("iomodetest.txt",
ContentFile(b"should not change"))
with self.assertRaises(NotImplementedError):
self.media_storage.open(test_file, mode="bw")
def test_list_dir_base(self):
# Pre-condition
self.assertIsNotNone(self.new_file)
test_dir = self.media_storage.listdir("./")
files = [elem for elem in test_dir]
self.assertIsInstance(files, list)
self.assertGreaterEqual(len(files), 1)
@patch("minio.Minio.list_objects", side_effect=ResponseError(MagicMock()))
def test_list_dir_throws_on_failure(self, mock):
with self.assertRaises(IOError):
self.media_storage.listdir("")
def test_file_exists(self):
existent = self.media_storage.save("existent.txt", ContentFile(b"meh"))
self.assertTrue(self.media_storage.exists(existent))
def test_file_exists_failure(self):
self.assertFalse(self.media_storage.exists("nonexistent.txt"))
@patch("minio.Minio.stat_object", side_effect=ResponseError(MagicMock()))
def test_file_exists_can_throw(self, mock):
with self.assertRaises(IOError):
self.media_storage.exists("peu importe")
def test_opening_non_existing_file_raises_exception(self):
with self.assertRaises(IOError):
self.media_storage.open("this does not exist")
def test_file_names_are_properly_sanitized(self):
self.media_storage.save("./meh22222.txt", io.BytesIO(b"stuff"))
def test_upload_and_get_back_file_with_funky_name(self):
pass
def test_uploaded_and_downloaded_file_sizes_match(self):
pass
def test_uploaded_files_end_up_in_the_right_bucket(self):
pass
def test_static_files_end_up_in_the_right_bucket(self):
pass
def tearDown(self):
client = Minio(endpoint=get_setting("MINIO_STORAGE_ENDPOINT"),
access_key=get_setting("MINIO_STORAGE_ACCESS_KEY"),
secret_key=get_setting("MINIO_STORAGE_SECRET_KEY"),
secure=get_setting("MINIO_STORAGE_USE_HTTPS"))
def obliterate_bucket(name):
for obj in client.list_objects(name, ""):
client.remove_object(name, obj.object_name)
for obj in client.list_incomplete_uploads(name,
""): # pragma: no cover
client.remove_incomplete_upload(name, obj.objectname)
client.remove_bucket(name)
obliterate_bucket("tests-media")
obliterate_bucket("tests-static")
