Exemplo n.º 1
0
class RetrieveTestsWithPublicBucket(BaseTestMixin, TestCase):
    def setUp(self):
        self.media_storage = MinioMediaStorage()
        self.static_storage = MinioStaticStorage()
        self.new_file = self.media_storage.save("test-file",
                                                ContentFile(b"yep"))

        self.media_storage.client.set_bucket_policy(
            self.media_storage.bucket_name,
            self.media_storage._policy("READ_WRITE"))

        self.static_storage.client.set_bucket_policy(
            self.static_storage.bucket_name,
            self.static_storage._policy("READ_WRITE"))

    def test_public_url_generation(self):
        media_test_file_name = self.media_storage.save(
            u"weird & ÜRΛ", ContentFile(b"irrelevant"))
        url = self.media_storage.url(media_test_file_name)
        res = requests.get(url)
        self.assertEqual(res.content, b"irrelevant")

        static_test_file_name = self.static_storage.save(
            u"weird & ÜRΛ", ContentFile(b"irrelevant"))
        url = self.static_storage.url(static_test_file_name)
        res = requests.get(url)
        self.assertEqual(res.content, b"irrelevant")
Exemplo n.º 2
0
 def setUp(self):
     self.media_storage = MinioMediaStorage()
     self.static_storage = MinioStaticStorage()
     self.new_file = self.media_storage.save("test-file",
                                             ContentFile(b"yep"))
     self.second_file = self.media_storage.save("test-file",
                                                ContentFile(b"nope"))
Exemplo n.º 3
0
 def test_no_base_url_subpath(self):
     endpoint = os.getenv("MINIO_STORAGE_ENDPOINT", "minio:9000")
     assert endpoint != ""
     media_storage = MinioMediaStorage()
     name = "23/23/aaa/bbb/22"
     url = media_storage.url(name)
     self.assertEqual(url, f"http://{endpoint}/foo/23/23/aaa/bbb/22")
Exemplo n.º 4
0
class BaseTestMixin:
    @staticmethod
    def bucket_name(name):
        return bucket_name(name)

    def minio_client(self):
        minio_client = Minio(
            endpoint=get_setting("MINIO_STORAGE_ENDPOINT"),
            access_key=get_setting("MINIO_STORAGE_ACCESS_KEY"),
            secret_key=get_setting("MINIO_STORAGE_SECRET_KEY"),
            secure=get_setting("MINIO_STORAGE_USE_HTTPS"))
        return minio_client

    def setUp(self):
        self.media_storage = MinioMediaStorage()
        self.static_storage = MinioStaticStorage()
        self.new_file = self.media_storage.save("test-file",
                                                ContentFile(b"yep"))
        self.second_file = self.media_storage.save("test-file",
                                                   ContentFile(b"nope"))

    def tearDown(self):
        client = self.minio_client()

        def obliterate_bucket(name):
            for obj in client.list_objects(name, "", True):
                client.remove_object(name, obj.object_name)
            for obj in client.list_incomplete_uploads(
                    name, ""):  # pragma: no cover  # noqa
                client.remove_incomplete_upload(name, obj.objectname)
            client.remove_bucket(name)

        obliterate_bucket(self.bucket_name("tests-media"))
        obliterate_bucket(self.bucket_name("tests-static"))
Exemplo n.º 5
0
class BaseTestMixin:

    @staticmethod
    def bucket_name(name):
        return bucket_name(name)

    def minio_client(self):
        minio_client = Minio(
            endpoint=get_setting("MINIO_STORAGE_ENDPOINT"),
            access_key=get_setting("MINIO_STORAGE_ACCESS_KEY"),
            secret_key=get_setting("MINIO_STORAGE_SECRET_KEY"),
            secure=get_setting("MINIO_STORAGE_USE_HTTPS"))
        return minio_client

    def setUp(self):
        self.media_storage = MinioMediaStorage()
        self.static_storage = MinioStaticStorage()
        self.new_file = self.media_storage.save("test-file",
                                                ContentFile(b"yep"))
        self.second_file = self.media_storage.save("test-file",
                                                   ContentFile(b"nope"))

    def tearDown(self):
        client = self.minio_client()

        def obliterate_bucket(name):
            for obj in client.list_objects(name, "", True):
                client.remove_object(name, obj.object_name)
            for obj in client.list_incomplete_uploads(name, ""):  # pragma: no cover  # noqa
                client.remove_incomplete_upload(name, obj.objectname)
            client.remove_bucket(name)

        obliterate_bucket(self.bucket_name("tests-media"))
        obliterate_bucket(self.bucket_name("tests-static"))
class RetrieveTestsWithPublicBucket(BaseTestMixin, TestCase):

    def setUp(self):
        self.media_storage = MinioMediaStorage()
        self.static_storage = MinioStaticStorage()
        self.new_file = self.media_storage.save("test-file",
                                                ContentFile(b"yep"))

        self.media_storage.client.set_bucket_policy(
            self.media_storage.bucket_name,
            self.media_storage._policy("READ_WRITE")
        )

        self.static_storage.client.set_bucket_policy(
            self.static_storage.bucket_name,
            self.static_storage._policy("READ_WRITE")
        )

    def test_public_url_generation(self):
        media_test_file_name = self.media_storage.save(
            u"weird & ÜRΛ", ContentFile(b"irrelevant"))
        url = self.media_storage.url(media_test_file_name)
        res = requests.get(url)
        self.assertEqual(res.content, b"irrelevant")

        static_test_file_name = self.static_storage.save(
            u"weird & ÜRΛ", ContentFile(b"irrelevant"))
        url = self.static_storage.url(static_test_file_name)
        res = requests.get(url)
        self.assertEqual(res.content, b"irrelevant")
 def test_no_base_url_subpath(self):
     endpoint = os.getenv("MINIO_STORAGE_ENDPOINT", "minio:9000")
     assert endpoint != ""
     media_storage = MinioMediaStorage()
     name = "23/23/aaa/bbb/22"
     url = media_storage.url(name)
     self.assertEqual(
         url, 'http://{}/foo/23/23/aaa/bbb/22'.format(endpoint))
 def test_presigned_base_url_subpath(self):
     # The url generated here probably doenst work in a real situation
     media_storage = MinioMediaStorage()
     name = "1/555/666/777"
     url = media_storage.url(name)
     self.assertIn('X-Amz-Signature', url)
     self.assertIn("http://example11.com/foo", url)
     self.assertIn(name, url)
 def test_presigned_base_url_subpath(self):
     # The url generated here probably doenst work in a real situation
     media_storage = MinioMediaStorage()
     name = "1/555/666/777"
     url = media_storage.url(name)
     self.assertIn("X-Amz-Signature", url)
     self.assertIn("http://example11.com/foo", url)
     self.assertIn(name, url)
Exemplo n.º 10
0
 def setUp(self):
     self.media_storage = MinioMediaStorage()
     self.static_storage = MinioStaticStorage()
     self.new_file = self.media_storage.save("test-file",
                                             ContentFile(b"yep"))
     self.media_storage.client.set_bucket_policy(
         self.media_storage.bucket_name, '', Policy.READ_WRITE)
     self.static_storage.client.set_bucket_policy(
         self.static_storage.bucket_name, '', Policy.READ_WRITE)
Exemplo n.º 11
0
 def test_media_policy_write(self):
     ms = MinioMediaStorage()
     self.maxDiff = 50000
     self.assertPolicyEqual(
         Policy.write.bucket(ms.bucket_name, json_encode=False),
         json.loads(ms.client.get_bucket_policy(ms.bucket_name)),
     )
     fn = ms.save("somefile", ContentFile(b"test"))
     self.assertEqual(ms.open(fn).read(), b"test")
     self.assertEqual(requests.get(ms.url(fn)).status_code, 403)
     self.assertEqual(
         requests.get(f"{ms.client._endpoint_url}/{ms.bucket_name}").status_code, 403
     )
Exemplo n.º 12
0
 def test_media_policy_auto_true(self):
     ms = MinioMediaStorage()
     self.maxDiff = 50000
     self.assertPolicyEqual(
         Policy.get.bucket(ms.bucket_name, json_encode=False),
         json.loads(ms.client.get_bucket_policy(ms.bucket_name)),
     )
     fn = ms.save("somefile", ContentFile(b"test"))
     self.assertEqual(ms.open(fn).read(), b"test")
     url = ms.url(fn)
     self.assertEqual(requests.get(url).status_code, 200)
     self.assertEqual(
         requests.get("{}/{}".format(ms.client._endpoint_url, ms.bucket_name)).status_code, 403
     )
Exemplo n.º 13
0
    def test_public_url_generation(self):
        media_storage = MinioMediaStorage()
        media_test_file_name = media_storage.save("weird & ÜRΛ",
                                                  ContentFile(b"irrelevant"))
        url = media_storage.url(media_test_file_name)
        res = requests.get(url)
        self.assertEqual(res.content, b"irrelevant")

        static_storage = MinioStaticStorage()
        static_test_file_name = static_storage.save("weird & ÜRΛ",
                                                    ContentFile(b"irrelevant"))
        url = static_storage.url(static_test_file_name)
        res = requests.get(url)
        self.assertEqual(res.content, b"irrelevant")
Exemplo n.º 14
0
 def setUp(self):
     self.media_storage = MinioMediaStorage()
     self.static_storage = MinioStaticStorage()
     self.new_file = self.media_storage.save("test-file",
                                             ContentFile(b"yep"))
     self.second_file = self.media_storage.save("test-file",
                                                ContentFile(b"nope"))
Exemplo n.º 15
0
 def test_media_storage_ignore_bucket_check(self):
     try:
         MinioMediaStorage()
     except OSError:
         self.assertTrue(False)
     else:
         pass
Exemplo n.º 16
0
 def test_media_policy_rw(self):
     ms = MinioMediaStorage()
     self.maxDiff = 50000
     self.assertPolicyEqual(
         Policy.read_write.bucket(ms.bucket_name, json_encode=False),
         json.loads(ms.client.get_bucket_policy(ms.bucket_name)),
     )
Exemplo n.º 17
0
    def setUp(self):
        self.media_storage = MinioMediaStorage()
        self.static_storage = MinioStaticStorage()
        self.new_file = self.media_storage.save("test-file",
                                                ContentFile(b"yep"))

        self.media_storage.client.set_bucket_policy(
            self.media_storage.bucket_name,
            self.media_storage._policy("READ_WRITE")
        )

        self.static_storage.client.set_bucket_policy(
            self.static_storage.bucket_name,
            self.static_storage._policy("READ_WRITE")
        )
Exemplo n.º 18
0
class ZipDumpFile(models.Model):
    dump_file = models.ForeignKey(DumpFile, on_delete=models.CASCADE)
    file = models.FileField(upload_to=zipfile_upload_to, storage=MinioMediaStorage())

    @classmethod
    def create_from_dump_file(cls, dump_file: DumpFile, dump_file_path: str):

        with TempZipDumpFile(dump_file) as zip_file:
            zip_file.write(dump_file_path, arcname=f'{dump_file.file_name}.{dump_file.file_type}')
            zip_file.close()

            with open(zip_file.filename, 'rb') as f:
                return cls.objects.create(file=File(f),
                                          dump_file=dump_file)

    def delete(self, using=None, keep_parents=False):
        if self.file:
            minio = create_minio_client_from_settings()
            minio.remove_object(settings.MINIO_STORAGE_MEDIA_BUCKET_NAME, self.file.name)
        super(ZipDumpFile, self).delete(using, keep_parents)
Exemplo n.º 19
0
 def test_media_storage_cannot_be_initialized_without_bucket(self):
     with self.assertRaises(OSError):
         MinioMediaStorage()
Exemplo n.º 20
0
 def test_quote_base_url(self):
     media_storage = MinioMediaStorage()
     url = media_storage.url(self.NAME)
     self.assertEqual(url, f"{self.BASE}/{self.ENCODED}")
Exemplo n.º 21
0
 def test_base_url(self):
     media_storage = MinioMediaStorage()
     url = media_storage.url("1")
     self.assertEqual(url, 'https://example23.com/foo/1')
 def test_base_url_subpath(self):
     media_storage = MinioMediaStorage()
     url = media_storage.url("1/2/3/4")
     self.assertEqual(url, "https://example23.com/foo/1/2/3/4")
Exemplo n.º 23
0
 def setUp(self):
     self.media_storage = MinioMediaStorage()
     self.static_storage = MinioStaticStorage()
     self.new_file = self.media_storage.save("a new & original file",
                                             ContentFile(b"yep"))
Exemplo n.º 24
0
 def test_file_upload_fail_incorrect_keys(self):
     with self.assertRaises(InvalidAccessKeyId):
         MinioMediaStorage()
Exemplo n.º 25
0
class DumpFile(models.Model):

    file = models.FileField(upload_to=dumpfile_upload_to, storage=MinioMediaStorage())

    FILENAME_FULL = 'series-tiempo'
    FILENAME_VALUES = 'series-tiempo-valores'
    FILENAME_METADATA = 'series-tiempo-metadatos'
    FILENAME_SOURCES = 'series-tiempo-fuentes'
    FILENAME_CHOICES = (
        (FILENAME_FULL, 'Series de tiempo (valores + metadatos)'),
        (FILENAME_VALUES, 'Series de tiempo (valores)'),
        (FILENAME_METADATA, 'Series de tiempo (metadatos)'),
        (FILENAME_SOURCES, 'Series de tiempo (fuentes)'),
    )
    file_name = models.CharField(max_length=64, choices=FILENAME_CHOICES)

    TYPE_CSV = 'csv'
    TYPE_XLSX = 'xlsx'
    TYPE_ZIP = 'zip'
    TYPE_SQL = 'sqlite'
    TYPE_DTA = 'dta'

    TYPE_CHOICES = (
        (TYPE_CSV, 'CSV'),
        (TYPE_XLSX, 'XLSX'),
        (TYPE_ZIP, 'ZIP'),
        (TYPE_SQL, 'SQL'),
        (TYPE_DTA, 'DTA'),
    )

    ZIP_FILES = (
        (FILENAME_FULL, TYPE_CSV),
        (FILENAME_VALUES, TYPE_CSV),
        (FILENAME_FULL, TYPE_SQL),
        (FILENAME_VALUES, TYPE_DTA),
    )

    file_type = models.CharField(max_length=12, choices=TYPE_CHOICES, default=TYPE_CSV)

    task = models.ForeignKey(GenerateDumpTask, on_delete=models.CASCADE)
    node = models.ForeignKey(Node, on_delete=models.PROTECT, blank=True, null=True)

    def delete(self, using=None, keep_parents=False):
        if self.file:
            minio = create_minio_client_from_settings()
            minio.remove_object(settings.MINIO_STORAGE_MEDIA_BUCKET_NAME, self.file.name)
        super(DumpFile, self).delete(using, keep_parents)

    def get_path(self):
        directory = f'{self.node.catalog_id}/' if self.node else ''
        return f'{constants.DUMP_DIR}/{directory}{self.get_file_name()}'

    def get_file_name(self):
        return f'{self.file_name}.{self.file_type}'

    def __str__(self):
        return f'{self.get_file_name()} ({self.node or "global"})'

    @classmethod
    def get_from_path(cls, filename: str, node: str = None) -> 'DumpFile':
        """Devuelve la última instancia de clase que corresponda al archivo con filename de formato tipo
        <catalog>/<file_name>.<extension>. Lanza DoesNotExist si no se encuentra un dump correspondiente.
        """

        try:
            node = Node.objects.get(catalog_id=node) if node else None
        except Node.DoesNotExist:
            raise cls.DoesNotExist

        try:
            name, extension = filename.split('.')
        except ValueError:
            raise cls.DoesNotExist

        if extension == cls.TYPE_ZIP:
            split = name.rfind('-')
            orig_extension = name[split + 1:]
            name = name[:split]
            dump = cls.objects.filter(file_name=name, file_type=orig_extension, node=node).last()
            try:
                return ZipDumpFile.objects.get(dump_file=dump)
            except ZipDumpFile.DoesNotExist:
                raise cls.DoesNotExist

        dump = cls.objects.filter(file_name=name, file_type=extension, node=node).last()

        if dump is None:
            raise cls.DoesNotExist
        return dump

    @classmethod
    def get_last_of_type(cls, file_type: str, node: str = None) -> list:
        """Devuelve el último dump generado del formato file_type especificado.
        Si se pasa un parámetro node, devuelve los últimos dumps para ese node.
        Si no, se devuelve los últimos dumps globales.
        """

        dumps_qs = cls.objects.filter(node__catalog_id=node)

        dumps = []
        for dump_name, _ in cls.FILENAME_CHOICES:
            dump_file = dumps_qs.filter(file_name=dump_name, file_type=file_type).last()
            if dump_file is not None:
                dumps.append(dump_file)

        return dumps
 def test_quote_base_url(self):
     media_storage = MinioMediaStorage()
     url = media_storage.url(self.NAME)
     self.assertEqual(url, "{}/{}".format(self.BASE, self.ENCODED))
Exemplo n.º 27
0
 def test_presigned_no_base_url(self):
     endpoint = os.getenv("MINIO_STORAGE_ENDPOINT", "minio:9000")
     assert endpoint != ""
     media_storage = MinioMediaStorage()
     url = media_storage.url("22")
     self.assertRegex(url, rf"^http://{endpoint}/foo/22\?")
Exemplo n.º 28
0
 def test_base_url(self):
     media_storage = MinioMediaStorage()
     url = media_storage.url("1")
     self.assertEqual(url, 'https://example23.com/foo/1')
Exemplo n.º 29
0
 def test_presigned_base_url_subpath(self):
     media_storage = MinioMediaStorage()
     url = media_storage.url("1/555/666/777")
     self.assertIn("X-Amz-Signature", url)
     self.assertRegex(url, r"^http://example11.com/foo/1/555/666/777\?")
Exemplo n.º 30
0
 def test_media_storage_ignore_bucket_check(self):
     MinioMediaStorage()
Exemplo n.º 31
0
 def test_auto_create_no_policy(self):
     ms = MinioMediaStorage()
     with self.assertRaises(minio.error.NoSuchBucketPolicy):
         ms.client.get_bucket_policy(ms.bucket_name)
Exemplo n.º 32
0
class MinioStorageTests(TestCase):
    def setUp(self):
        self.media_storage = MinioMediaStorage()
        self.static_storage = MinioStaticStorage()
        self.new_file = self.media_storage.save("a new & original file",
                                                ContentFile(b"yep"))

    @override_settings(
        MINIO_STORAGE_MEDIA_BUCKET_NAME="inexistent",
        MINIO_STORAGE_AUTO_CREATE_MEDIA_BUCKET=False,
    )
    def test_media_storage_cannot_be_initialized_without_bucket(self):
        with self.assertRaises(IOError):
            MinioMediaStorage()

    @override_settings(
        MINIO_STORAGE_STATIC_BUCKET_NAME="inexistent",
        MINIO_STORAGE_AUTO_CREATE_STATIC_BUCKET=False,
    )
    def test_static_storage_cannot_be_initialized_without_bucket(self):
        with self.assertRaises(IOError):
            MinioStaticStorage()

    def test_get_setting_throws_early(self):
        with self.assertRaises(ImproperlyConfigured):
            get_setting("INEXISTENT_SETTING")

    def test_it_works(self):
        self.assertEqual(2 + 2, 4)

    def test_file_upload_does_not_throw(self):
        self.media_storage.save("trivial.txt", ContentFile(b"12345"))

    def test_two_files_with_the_same_name_can_be_uploaded(self):
        ivan = self.media_storage.save("pelican.txt",
                                       ContentFile(b"Ivan le Pelican"))
        jean = self.media_storage.save("pelican.txt",
                                       ContentFile(b"Jean le Pelican"))
        self.assertNotEqual(jean, ivan)

    @patch("minio.Minio.put_object", side_effect=ResponseError(MagicMock()))
    def test_file_upload_throws_on_failure(self, mock):
        with self.assertRaises(IOError):
            self.media_storage.save("meh", ContentFile(b"meh"))

    def test_file_removal(self):
        test_file = self.media_storage.save("should_be_removed.txt",
                                            ContentFile(b"meh"))
        self.media_storage.delete(test_file)
        self.assertFalse(self.media_storage.exists(test_file))

    @patch("minio.Minio.remove_object", side_effect=ResponseError(MagicMock()))
    def test_deletion_error(self, mock):
        with self.assertRaises(IOError):
            self.media_storage.delete("this is irrelevant")

    def test_url_generation(self):
        test_file_name = self.media_storage.save(u"weird & ÜRΛ",
                                                 ContentFile(b"irrelevant"))
        url = self.media_storage.url(test_file_name)
        res = requests.get(url)
        self.assertEqual(res.content, b"irrelevant")

    def test_files_from_filesystem_are_uploaded_properly(self):
        f = File(io.open("watermelon-cat.jpg", "br"))
        saved_file = self.media_storage.save("watermelon-cat.jpg", f)
        res = requests.get(self.media_storage.url(saved_file))
        self.assertAlmostEqual(round(res.content.__sizeof__() / 100),
                               round(f.size / 100))

    def test_url_of_non_existent_throws(self):
        with self.assertRaises(IOError):
            self.media_storage.url("this does not exist")

    def test_file_stat(self):
        pass

    def test_file_size(self):
        test_file = self.media_storage.save("sizetest.txt",
                                            ContentFile(b"1234"))
        self.assertEqual(4, self.media_storage.size(test_file))

    def test_size_of_non_existent_throws(self):
        test_file = self.media_storage.save("sizetest.txt",
                                            ContentFile(b"1234"))
        self.media_storage.delete(test_file)
        with self.assertRaises(IOError):
            self.media_storage.size(test_file)

    def test_modified_time(self):
        self.assertIsInstance(self.media_storage.modified_time(self.new_file),
                              datetime.datetime)

    def test_accessed_time(self):
        self.assertIsInstance(self.media_storage.accessed_time(self.new_file),
                              datetime.datetime)

    def test_created_time(self):
        self.assertIsInstance(self.media_storage.created_time(self.new_file),
                              datetime.datetime)

    def test_modified_time_of_non_existent_throws(self):
        with self.assertRaises(IOError):
            self.media_storage.modified_time("nonexistent.jpg")

    def test_files_cannot_be_open_in_write_mode(self):
        test_file = self.media_storage.save("iomodetest.txt",
                                            ContentFile(b"should not change"))
        with self.assertRaises(NotImplementedError):
            self.media_storage.open(test_file, mode="bw")

    def test_list_dir_base(self):
        # Pre-condition
        self.assertIsNotNone(self.new_file)

        test_dir = self.media_storage.listdir("./")
        files = [elem for elem in test_dir]
        self.assertIsInstance(files, list)
        self.assertGreaterEqual(len(files), 1)

    @patch("minio.Minio.list_objects", side_effect=ResponseError(MagicMock()))
    def test_list_dir_throws_on_failure(self, mock):
        with self.assertRaises(IOError):
            self.media_storage.listdir("")

    def test_file_exists(self):
        existent = self.media_storage.save("existent.txt", ContentFile(b"meh"))
        self.assertTrue(self.media_storage.exists(existent))

    def test_file_exists_failure(self):
        self.assertFalse(self.media_storage.exists("nonexistent.txt"))

    @patch("minio.Minio.stat_object", side_effect=ResponseError(MagicMock()))
    def test_file_exists_can_throw(self, mock):
        with self.assertRaises(IOError):
            self.media_storage.exists("peu importe")

    def test_opening_non_existing_file_raises_exception(self):
        with self.assertRaises(IOError):
            self.media_storage.open("this does not exist")

    def test_file_names_are_properly_sanitized(self):
        self.media_storage.save("./meh22222.txt", io.BytesIO(b"stuff"))

    def test_upload_and_get_back_file_with_funky_name(self):
        pass

    def test_uploaded_and_downloaded_file_sizes_match(self):
        pass

    def test_uploaded_files_end_up_in_the_right_bucket(self):
        pass

    def test_static_files_end_up_in_the_right_bucket(self):
        pass

    def tearDown(self):
        client = Minio(endpoint=get_setting("MINIO_STORAGE_ENDPOINT"),
                       access_key=get_setting("MINIO_STORAGE_ACCESS_KEY"),
                       secret_key=get_setting("MINIO_STORAGE_SECRET_KEY"),
                       secure=get_setting("MINIO_STORAGE_USE_HTTPS"))

        def obliterate_bucket(name):
            for obj in client.list_objects(name, ""):
                client.remove_object(name, obj.object_name)
            for obj in client.list_incomplete_uploads(name,
                                                      ""):  # pragma: no cover
                client.remove_incomplete_upload(name, obj.objectname)
            client.remove_bucket(name)

        obliterate_bucket("tests-media")
        obliterate_bucket("tests-static")
Exemplo n.º 33
0
class MinioStorageTests(TestCase):
    def setUp(self):
        self.media_storage = MinioMediaStorage()
        self.static_storage = MinioStaticStorage()
        self.new_file = self.media_storage.save("a new & original file",
                                                ContentFile(b"yep"))

    @override_settings(
        MINIO_STORAGE_MEDIA_BUCKET_NAME="inexistent",
        MINIO_STORAGE_AUTO_CREATE_MEDIA_BUCKET=False,
    )
    def test_media_storage_cannot_be_initialized_without_bucket(self):
        with self.assertRaises(IOError):
            MinioMediaStorage()

    @override_settings(
        MINIO_STORAGE_STATIC_BUCKET_NAME="inexistent",
        MINIO_STORAGE_AUTO_CREATE_STATIC_BUCKET=False,
    )
    def test_static_storage_cannot_be_initialized_without_bucket(self):
        with self.assertRaises(IOError):
            MinioStaticStorage()

    def test_get_setting_throws_early(self):
        with self.assertRaises(ImproperlyConfigured):
            get_setting("INEXISTENT_SETTING")

    def test_it_works(self):
        self.assertEqual(2 + 2, 4)

    def test_file_upload_does_not_throw(self):
        self.media_storage.save("trivial.txt", ContentFile(b"12345"))

    def test_two_files_with_the_same_name_can_be_uploaded(self):
        ivan = self.media_storage.save("pelican.txt",
                                       ContentFile(b"Ivan le Pelican"))
        jean = self.media_storage.save("pelican.txt",
                                       ContentFile(b"Jean le Pelican"))
        self.assertNotEqual(jean, ivan)


    @patch("minio.Minio.put_object", side_effect=ResponseError(MagicMock()))
    def test_file_upload_throws_on_failure(self, mock):
        with self.assertRaises(IOError):
            self.media_storage.save("meh", ContentFile(b"meh"))

    def test_file_removal(self):
        test_file = self.media_storage.save("should_be_removed.txt",
                                            ContentFile(b"meh"))
        self.media_storage.delete(test_file)
        self.assertFalse(self.media_storage.exists(test_file))

    def test_delete_of_non_existent_throws(self):
        with self.assertRaises(IOError):
            self.media_storage.delete("i don't even exist")

    def test_url_generation(self):
        test_file = self.media_storage.save("weird & ÜRΛ",
                                            ContentFile(b"irrelevant"))
        url = self.media_storage.url(test_file)
        res = requests.get(url)
        self.assertEqual(res.content, b"irrelevant")

    def test_files_from_filesystem_are_uploaded_properly(self):
        f = File(io.open("watermelon-cat.jpg", "br"))
        saved_file = self.media_storage.save("watermelon-cat.jpg", f)
        res = requests.get(self.media_storage.url(saved_file))
        self.assertAlmostEqual(round(res.content.__sizeof__() / 100),
                               round(f.size / 100))

    def test_url_of_non_existent_throws(self):
        with self.assertRaises(IOError):
            self.media_storage.url("this does not exist")

    def test_file_stat(self):
        pass

    def test_file_size(self):
        test_file = self.media_storage.save("sizetest.txt",
                                            ContentFile(b"1234"))
        self.assertEqual(4, self.media_storage.size(test_file))

    def test_size_of_non_existent_throws(self):
        test_file = self.media_storage.save("sizetest.txt",
                                            ContentFile(b"1234"))
        self.media_storage.delete(test_file)
        with self.assertRaises(IOError):
            self.media_storage.size(test_file)

    def test_modified_time(self):
        self.assertIsInstance(self.media_storage.modified_time(self.new_file),
                              datetime.datetime)

    def test_accessed_time(self):
        self.assertIsInstance(self.media_storage.accessed_time(self.new_file),
                              datetime.datetime)

    def test_created_time(self):
        self.assertIsInstance(self.media_storage.created_time(self.new_file),
                              datetime.datetime)

    def test_modified_time_of_non_existent_throws(self):
        with self.assertRaises(IOError):
            self.media_storage.modified_time("nonexistent.jpg")

    def test_files_cannot_be_open_in_write_mode(self):
        test_file = self.media_storage.save("iomodetest.txt",
                                            ContentFile(b"should not change"))
        with self.assertRaises(NotImplementedError):
            self.media_storage.open(test_file, mode="bw")

    def test_list_dir_base(self):
        test_dir = self.media_storage.listdir("/")
        files = [elem for elem in test_dir]
        self.assertIsInstance(files, list)
        self.assertGreaterEqual(len(files), 1)

    @patch("minio.Minio.list_objects", side_effect=ResponseError(MagicMock()))
    def test_list_dir_throws_on_failure(self, mock):
        with self.assertRaises(IOError):
            self.media_storage.listdir("")

    def test_file_exists(self):
        existent = self.media_storage.save("existent.txt", ContentFile(b"meh"))
        self.assertTrue(self.media_storage.exists(existent))

    def test_file_exists_failure(self):
        self.assertFalse(self.media_storage.exists("nonexistent.txt"))

    @patch("minio.Minio.stat_object", side_effect=ResponseError(MagicMock()))
    def test_file_exists_can_throw(self, mock):
        with self.assertRaises(IOError):
            self.media_storage.exists("peu importe")

    def test_opening_non_existing_file_raises_exception(self):
        with self.assertRaises(IOError):
            self.media_storage.open("this does not exist")

    def test_file_names_are_properly_sanitized(self):
        self.media_storage.save("./meh22222.txt", io.BytesIO(b"stuff"))

    def test_upload_and_get_back_file_with_funky_name(self):
        pass

    def test_uploaded_and_downloaded_file_sizes_match(self):
        pass

    def test_uploaded_files_end_up_in_the_right_bucket(self):
        pass

    def test_static_files_end_up_in_the_right_bucket(self):
        pass

    def tearDown(self):
        client = Minio(endpoint=get_setting("MINIO_STORAGE_ENDPOINT"),
                       access_key=get_setting("MINIO_STORAGE_ACCESS_KEY"),
                       secret_key=get_setting("MINIO_STORAGE_SECRET_KEY"),
                       secure=get_setting("MINIO_STORAGE_USE_HTTPS"))

        def obliterate_bucket(name):
            for obj in client.list_objects(name, ""):
                client.remove_object(name, obj.object_name)
            for obj in client.list_incomplete_uploads(name, ""):  # pragma: no cover
                client.remove_incomplete_upload(name, obj.objectname)
            client.remove_bucket(name)

        obliterate_bucket("tests-media")
        obliterate_bucket("tests-static")
Exemplo n.º 34
0
 def setUp(self):
     self.media_storage = MinioMediaStorage()
     self.static_storage = MinioStaticStorage()
     self.new_file = self.media_storage.save("a new & original file",
                                             ContentFile(b"yep"))
 def test_no_base_url(self):
     endpoint = os.getenv("MINIO_STORAGE_ENDPOINT", "minio:9000")
     assert endpoint != ""
     media_storage = MinioMediaStorage()
     url = media_storage.url("22")
     self.assertEqual(url, "http://{}/foo/22".format(endpoint))
Exemplo n.º 36
0
 def test_presigned_base_url(self):
     # The url generated here probably doenst work in a real situation
     media_storage = MinioMediaStorage()
     url = media_storage.url("1")
     self.assertIn('X-Amz-Signature', url)
     self.assertIn("http://example11.com/foo", url)
 def test_presigned_base_url(self):
     # The url generated here probably doenst work in a real situation
     media_storage = MinioMediaStorage()
     url = media_storage.url("1")
     self.assertIn("X-Amz-Signature", url)
     self.assertIn("http://example11.com/foo", url)
Exemplo n.º 38
0
 def test_base_url_subpath(self):
     media_storage = MinioMediaStorage()
     url = media_storage.url("1/2/3/4")
     self.assertEqual(url, 'https://example23.com/foo/1/2/3/4')
 def test_quote_url(self):
     media_storage = MinioMediaStorage()
     url = media_storage.url(self.NAME)
     self.assertTrue(url.endswith(self.ENCODED))
     self.assertTrue(len(url) > len(self.ENCODED))
Exemplo n.º 40
0
 def test_file_upload_fail_incorrect_keys(self):
     with self.assertRaises(ResponseError):
         MinioMediaStorage()
 def test_quote_base_url_presigned(self):
     media_storage = MinioMediaStorage()
     url = media_storage.url(self.NAME)
     prefix = "{}/{}".format(self.BASE, self.ENCODED)
     self.assertTrue(url.startswith(prefix))
     self.assertTrue(len(url) > len(prefix))
Exemplo n.º 42
0
 def test_no_base_url(self):
     endpoint = os.getenv("MINIO_STORAGE_ENDPOINT", "minio:9000")
     assert endpoint != ""
     media_storage = MinioMediaStorage()
     url = media_storage.url("22")
     self.assertEqual(url, 'http://{}/foo/22'.format(endpoint))