예제 #1
0
    def handle_noargs(self, **options):
        # ensure that project has required configuration
        if not os.path.exists(settings.MEDIA_ROOT):
            raise CommandError('Cannot migrate files from non existing MEDIA_ROOT (%s)' % settings.MEDIA_ROOT)
        if not hasattr(settings, 'AZURE_ACCOUNT_NAME'):
            raise CommandError('AZURE_ACCOUNT_NAME setting missing')
        if not hasattr(settings, 'AZURE_ACCOUNT_KEY'):
            raise CommandError('AZURE_ACCOUNT_KEY setting missing')
        if not hasattr(settings, 'AZURE_DEFAULT_CONTAINER'):
            raise CommandError('AZURE_DEFAULT_CONTAINER settings missing')

        # get service interface
        service = BlobService(settings.AZURE_ACCOUNT_NAME, settings.AZURE_ACCOUNT_KEY)

        self.stdout.write('Starting migration from "%s" to '
                          'Cloud Storage container "%s"' % (settings.MEDIA_ROOT, settings.AZURE_DEFAULT_CONTAINER))

        for root, dirs, files in os.walk(settings.MEDIA_ROOT):
            for file in files:
                path = os.path.join(root, file)
                blobname = os.path.relpath(path, settings.MEDIA_ROOT).replace('\\', '/')
                self.stdout.write(blobname + "...", ending='')
                try:
                    with open(path, 'rb') as f:
                        if service.create_blob(settings.AZURE_DEFAULT_CONTAINER, blobname, bytearray(f.read())):
                            self.stdout.write('ok')
                        else:
                            self.stdout.write('fail')
                except Exception as e:
                    self.stdout.write('fail')
                    traceback.print_exc()
                    self.stdout.write('aborted migration.')
                    return

        self.stdout.write('migration complete')
예제 #2
0
    def __init__(self, container = None, account_name = None, account_key = None):
        """
        Creates a new AzureStorage. The container is not automatically created and therefore must already exist.
        """
        if container is None:
            if hasattr(settings, 'AZURE_DEFAULT_CONTAINER'):
                self.container = settings.AZURE_DEFAULT_CONTAINER
            else:
                self.container = "$root"
        else:
            self.container = container

        if account_name and account_key:
            self.service = BlobService(account_name, account_key)
        else:
            self.service = BlobService(settings.AZURE_ACCOUNT_NAME, settings.AZURE_ACCOUNT_KEY)
예제 #3
0
    def setUp(self):
        # find azure credentials for testing. expects them in cwd
        if not os.path.exists(self.CREDENTIALS_PATH):
            raise Exception('Azure Credentials file at "%s" does not exist' %
                            os.path.abspath(self.CREDENTIALS_PATH))

        # read json formatted credentials ("account_name" and "account_key")
        with open(self.CREDENTIALS_PATH, "r") as file:
            self.credentials = json.load(file)

        # create the blob service
        self.service = BlobService(self.credentials['account_name'],
                                   self.credentials['account_key'])

        # generate names for the test containers
        self.container_names = [
            "%s-%d" % (self.CONTAINER_PREFIX, i) for i in range(5)
        ]
예제 #4
0
    def handle_noargs(self, **options):
        # ensure that project has required configuration
        if not os.path.exists(settings.MEDIA_ROOT):
            raise CommandError(
                'Cannot migrate files from non existing MEDIA_ROOT (%s)' %
                settings.MEDIA_ROOT)
        if not hasattr(settings, 'AZURE_ACCOUNT_NAME'):
            raise CommandError('AZURE_ACCOUNT_NAME setting missing')
        if not hasattr(settings, 'AZURE_ACCOUNT_KEY'):
            raise CommandError('AZURE_ACCOUNT_KEY setting missing')
        if not hasattr(settings, 'AZURE_DEFAULT_CONTAINER'):
            raise CommandError('AZURE_DEFAULT_CONTAINER settings missing')

        # get service interface
        service = BlobService(settings.AZURE_ACCOUNT_NAME,
                              settings.AZURE_ACCOUNT_KEY)

        self.stdout.write(
            'Starting migration from "%s" to '
            'Cloud Storage container "%s"' %
            (settings.MEDIA_ROOT, settings.AZURE_DEFAULT_CONTAINER))

        for root, dirs, files in os.walk(settings.MEDIA_ROOT):
            for file in files:
                path = os.path.join(root, file)
                blobname = os.path.relpath(path, settings.MEDIA_ROOT).replace(
                    '\\', '/')
                self.stdout.write(blobname + "...", ending='')
                try:
                    with open(path, 'rb') as f:
                        if service.create_blob(
                                settings.AZURE_DEFAULT_CONTAINER, blobname,
                                bytearray(f.read())):
                            self.stdout.write('ok')
                        else:
                            self.stdout.write('fail')
                except Exception as e:
                    self.stdout.write('fail')
                    traceback.print_exc()
                    self.stdout.write('aborted migration.')
                    return

        self.stdout.write('migration complete')
    def setUp(self):
        # find azure credentials for testing. expects them in cwd
        if not os.path.exists(self.CREDENTIALS_PATH):
            raise Exception('Azure Credentials file at "%s" does not exist' % os.path.abspath(self.CREDENTIALS_PATH))

        # read json formatted credentials ("account_name" and "account_key")
        with open(self.CREDENTIALS_PATH, "r") as file:
            self.credentials = json.load(file)

        # create the blob service
        self.service = BlobService(self.credentials['account_name'], self.credentials['account_key'])

        # generate names for the test containers
        self.container_names = ["%s-%d" % (self.CONTAINER_PREFIX, i) for i in range(5)]
예제 #6
0
class TestBlobService(TestCase):

    # expecting a JSON file with credentials ("account_name" and "account_key") in test directory
    CREDENTIALS_PATH = os.path.dirname(
        os.path.abspath(__file__)) + '/azurecredentials.json'

    CONTAINER_PREFIX = 'azurepython3-test'

    def setUp(self):
        # find azure credentials for testing. expects them in cwd
        if not os.path.exists(self.CREDENTIALS_PATH):
            raise Exception('Azure Credentials file at "%s" does not exist' %
                            os.path.abspath(self.CREDENTIALS_PATH))

        # read json formatted credentials ("account_name" and "account_key")
        with open(self.CREDENTIALS_PATH, "r") as file:
            self.credentials = json.load(file)

        # create the blob service
        self.service = BlobService(self.credentials['account_name'],
                                   self.credentials['account_key'])

        # generate names for the test containers
        self.container_names = [
            "%s-%d" % (self.CONTAINER_PREFIX, i) for i in range(5)
        ]

    def create_container(self, name):
        # create a test container
        try:
            self.service.create_container(name)
        except HTTPError as e:
            if e.response.status_code != 409:
                raise e

    def tearDown(self):
        # delete the test containers
        for name in self.container_names:
            self.service.delete_container(name)

    def list_containers(self):
        containers = self.service.list_containers()
        for name in self.container_names:
            self.assertIn(name, [x.name for x in containers])

    def test_create_and_list_containers(self):
        for name in self.container_names:
            self.assertTrue(self.service.create_container(name, 'container'))

        self.list_containers()

    def test_create_blob(self):
        container = '%s-test1' % self.CONTAINER_PREFIX

        try:
            self.assertTrue(
                self.service.create_container(container, 'container'))
        except HTTPError as e:
            # there will be a HTTP 409 error if the container already exists
            if e.response.status_code != 409:
                self.fail("Failed to create test container")

        # attempt to upload the file
        data = bytearray(b'test byte string 11.5.2.7.3.14.59.2013.08.12')
        self.assertTrue(
            self.service.create_blob(container, 'somefile.ext', data))

        # delete the container
        self.service.delete_container(container)

    def test_delete_blob(self):
        container = '%s-test2' % self.CONTAINER_PREFIX

        # create a test container
        self.create_container(container)

        # create a file
        self.service.create_blob(container, 'file-to-delete.ext',
                                 bytearray(b'THIS FILE SHOULD BE DELETED'))

        # delete the file
        self.assertTrue(
            self.service.delete_blob(container, 'file-to-delete.ext'))

        # delete the container again
        self.service.delete_container(container)

    def test_list_blobs(self):
        container = '%s-test3' % self.CONTAINER_PREFIX
        self.create_container(container)

        self.service.create_blob(container, 'folder1/file1.ext',
                                 bytearray(b'THIS FILE SHOULD BE LISTED'))
        self.service.create_blob(container, 'folder1/file2.ext',
                                 bytearray(b'THIS FILE SHOULD BE LISTED'))
        self.service.create_blob(container, 'folder2/file3.ext',
                                 bytearray(b'THIS FILE SHOULD BE LISTED'))
        self.service.create_blob(container, 'file4.ext',
                                 bytearray(b'THIS FILE SHOULD BE LISTED'))

        blobs1 = self.service.list_blobs(container)
        self.assertSetEqual(
            {
                'folder1/file1.ext', 'folder1/file2.ext', 'folder2/file3.ext',
                'file4.ext'
            }, set([blob.name for blob in blobs1]))

        blobs2 = self.service.list_blobs(container, prefix='folder1')
        self.assertSetEqual({'folder1/file1.ext', 'folder1/file2.ext'},
                            set([blob.name for blob in blobs2]))

        blobs3 = self.service.list_blobs(container, prefix='folder2')
        self.assertSetEqual({'folder2/file3.ext'},
                            set([blob.name for blob in blobs3]))

        self.service.delete_container(container)
class TestBlobService(TestCase):

    # expecting a JSON file with credentials ("account_name" and "account_key") in test directory
    CREDENTIALS_PATH =  os.path.dirname(os.path.abspath(__file__)) + '/azurecredentials.json'

    CONTAINER_PREFIX = 'azurepython3-test'

    def setUp(self):
        # find azure credentials for testing. expects them in cwd
        if not os.path.exists(self.CREDENTIALS_PATH):
            raise Exception('Azure Credentials file at "%s" does not exist' % os.path.abspath(self.CREDENTIALS_PATH))

        # read json formatted credentials ("account_name" and "account_key")
        with open(self.CREDENTIALS_PATH, "r") as file:
            self.credentials = json.load(file)

        # create the blob service
        self.service = BlobService(self.credentials['account_name'], self.credentials['account_key'])

        # generate names for the test containers
        self.container_names = ["%s-%d" % (self.CONTAINER_PREFIX, i) for i in range(5)]

    def create_container(self, name):
        # create a test container
        try:
            self.service.create_container(name)
        except HTTPError as e:
            if e.response.status_code != 409:
                raise e

    def tearDown(self):
        # delete the test containers
        for name in self.container_names:
            self.service.delete_container(name)

    def list_containers(self):
        containers = self.service.list_containers()
        for name in self.container_names:
            self.assertIn(name, [x.name for x in containers])

    def test_create_and_list_containers(self):
        for name in self.container_names:
            self.assertTrue(self.service.create_container(name, 'container'))

        self.list_containers()

    def test_create_blob(self):
        container = '%s-test1' % self.CONTAINER_PREFIX

        try:
            self.assertTrue(self.service.create_container(container, 'container'))
        except HTTPError as e:
            # there will be a HTTP 409 error if the container already exists
            if e.response.status_code != 409:
                self.fail("Failed to create test container")

        # attempt to upload the file
        data = bytearray(b'test byte string 11.5.2.7.3.14.59.2013.08.12')
        self.assertTrue(self.service.create_blob(container, 'somefile.ext', data))

        # delete the container
        self.service.delete_container(container)

    def test_delete_blob(self):
        container = '%s-test2' % self.CONTAINER_PREFIX

        # create a test container
        self.create_container(container)

        # create a file
        self.service.create_blob(container, 'file-to-delete.ext', bytearray(b'THIS FILE SHOULD BE DELETED'))

        # delete the file
        self.assertTrue(self.service.delete_blob(container, 'file-to-delete.ext'))

        # delete the container again
        self.service.delete_container(container)

    def test_list_blobs(self):
        container = '%s-test3' % self.CONTAINER_PREFIX
        self.create_container(container)

        self.service.create_blob(container, 'folder1/file1.ext', bytearray(b'THIS FILE SHOULD BE LISTED'))
        self.service.create_blob(container, 'folder1/file2.ext', bytearray(b'THIS FILE SHOULD BE LISTED'))
        self.service.create_blob(container, 'folder2/file3.ext', bytearray(b'THIS FILE SHOULD BE LISTED'))
        self.service.create_blob(container, 'file4.ext', bytearray(b'THIS FILE SHOULD BE LISTED'))

        blobs1 = self.service.list_blobs(container)
        self.assertSetEqual({'folder1/file1.ext', 'folder1/file2.ext', 'folder2/file3.ext', 'file4.ext'},
                             set([blob.name for blob in blobs1]))

        blobs2 = self.service.list_blobs(container, prefix = 'folder1')
        self.assertSetEqual({'folder1/file1.ext', 'folder1/file2.ext'}, set([blob.name for blob in blobs2]))

        blobs3 = self.service.list_blobs(container, prefix = 'folder2')
        self.assertSetEqual({'folder2/file3.ext'}, set([blob.name for blob in blobs3]))

        self.service.delete_container(container)
예제 #8
0
class AzureStorage(Storage):

    def __init__(self, container = None, account_name = None, account_key = None):
        """
        Creates a new AzureStorage. The container is not automatically created and therefore must already exist.
        """
        if container is None:
            if hasattr(settings, 'AZURE_DEFAULT_CONTAINER'):
                self.container = settings.AZURE_DEFAULT_CONTAINER
            else:
                self.container = "$root"
        else:
            self.container = container

        if account_name and account_key:
            self.service = BlobService(account_name, account_key)
        else:
            self.service = BlobService(settings.AZURE_ACCOUNT_NAME, settings.AZURE_ACCOUNT_KEY)

    def _transform_name(self, name):
        return name.replace("\\", "/")

    def _open(self, name, mode = 'rb') -> File:
        name = self._transform_name(name)
        content = self.service.get_blob_content(self.container, name)
        file = SpooledTemporaryFile()
        file.write(content)
        file.seek(0) # explicitly reset to allow reading from the beginning afterwards as-is

        return File(file)

    def _save(self, name, content):
        name = self._transform_name(name)
        content.open(mode='rb')
        data = bytearray(content.read())
        self.service.create_blob(self.container, name, data)
        return name

    def delete(self, name):
        name = self._transform_name(name)
        self.service.delete_blob(self.container, name)
        return name

    def exists(self, name):
        if not name:
            return False
        name = self._transform_name(name)
        return self.service.blob_exists(self.container, name)

    def listdir(self, path = None):
        path = self._transform_name(path)
        blobs = self.service.list_blobs(self.container, prefix = path)
        paths = [os.path.split(blob.name) for blob in blobs]
        dirs = [path[0] for path in paths]
        files = [path[1] for path in paths]
        return (dirs, files)

    def size(self, name):
        name = self._transform_name(name)
        blob = self.service.get_blob(self.container, name, with_content=False)
        return blob.content_length() if blob != None else 0

    def url(self, name):
        name = self._transform_name(name)
        return self.service.get_blob_url(self.container, name)

    def modified_time(self, name):
        name = self._transform_name(name)
        blob = self.service.get_blob(self.container, name, with_content=False)
        return datetime.strptime(blob.properties['Last-Modified'], "%a, %d %b %Y %H:%M:%S GMT")