def handle_noargs(self, **options):
# ensure that project has required configuration
if not os.path.exists(settings.MEDIA_ROOT):
raise CommandError('Cannot migrate files from non existing MEDIA_ROOT (%s)' % settings.MEDIA_ROOT)
if not hasattr(settings, 'AZURE_ACCOUNT_NAME'):
raise CommandError('AZURE_ACCOUNT_NAME setting missing')
if not hasattr(settings, 'AZURE_ACCOUNT_KEY'):
raise CommandError('AZURE_ACCOUNT_KEY setting missing')
if not hasattr(settings, 'AZURE_DEFAULT_CONTAINER'):
raise CommandError('AZURE_DEFAULT_CONTAINER settings missing')
# get service interface
service = BlobService(settings.AZURE_ACCOUNT_NAME, settings.AZURE_ACCOUNT_KEY)
self.stdout.write('Starting migration from "%s" to '
'Cloud Storage container "%s"' % (settings.MEDIA_ROOT, settings.AZURE_DEFAULT_CONTAINER))
for root, dirs, files in os.walk(settings.MEDIA_ROOT):
for file in files:
path = os.path.join(root, file)
blobname = os.path.relpath(path, settings.MEDIA_ROOT).replace('\\', '/')
self.stdout.write(blobname + "...", ending='')
try:
with open(path, 'rb') as f:
if service.create_blob(settings.AZURE_DEFAULT_CONTAINER, blobname, bytearray(f.read())):
self.stdout.write('ok')
else:
self.stdout.write('fail')
except Exception as e:
self.stdout.write('fail')
traceback.print_exc()
self.stdout.write('aborted migration.')
return
self.stdout.write('migration complete')
def __init__(self, container = None, account_name = None, account_key = None):
"""
Creates a new AzureStorage. The container is not automatically created and therefore must already exist.
"""
if container is None:
if hasattr(settings, 'AZURE_DEFAULT_CONTAINER'):
self.container = settings.AZURE_DEFAULT_CONTAINER
else:
self.container = "$root"
else:
self.container = container
if account_name and account_key:
self.service = BlobService(account_name, account_key)
else:
self.service = BlobService(settings.AZURE_ACCOUNT_NAME, settings.AZURE_ACCOUNT_KEY)
def setUp(self):
# find azure credentials for testing. expects them in cwd
if not os.path.exists(self.CREDENTIALS_PATH):
raise Exception('Azure Credentials file at "%s" does not exist' %
os.path.abspath(self.CREDENTIALS_PATH))
# read json formatted credentials ("account_name" and "account_key")
with open(self.CREDENTIALS_PATH, "r") as file:
self.credentials = json.load(file)
# create the blob service
self.service = BlobService(self.credentials['account_name'],
self.credentials['account_key'])
# generate names for the test containers
self.container_names = [
"%s-%d" % (self.CONTAINER_PREFIX, i) for i in range(5)
]
def handle_noargs(self, **options):
# ensure that project has required configuration
if not os.path.exists(settings.MEDIA_ROOT):
raise CommandError(
'Cannot migrate files from non existing MEDIA_ROOT (%s)' %
settings.MEDIA_ROOT)
if not hasattr(settings, 'AZURE_ACCOUNT_NAME'):
raise CommandError('AZURE_ACCOUNT_NAME setting missing')
if not hasattr(settings, 'AZURE_ACCOUNT_KEY'):
raise CommandError('AZURE_ACCOUNT_KEY setting missing')
if not hasattr(settings, 'AZURE_DEFAULT_CONTAINER'):
raise CommandError('AZURE_DEFAULT_CONTAINER settings missing')
# get service interface
service = BlobService(settings.AZURE_ACCOUNT_NAME,
settings.AZURE_ACCOUNT_KEY)
self.stdout.write(
'Starting migration from "%s" to '
'Cloud Storage container "%s"' %
(settings.MEDIA_ROOT, settings.AZURE_DEFAULT_CONTAINER))
for root, dirs, files in os.walk(settings.MEDIA_ROOT):
for file in files:
path = os.path.join(root, file)
blobname = os.path.relpath(path, settings.MEDIA_ROOT).replace(
'\\', '/')
self.stdout.write(blobname + "...", ending='')
try:
with open(path, 'rb') as f:
if service.create_blob(
settings.AZURE_DEFAULT_CONTAINER, blobname,
bytearray(f.read())):
self.stdout.write('ok')
else:
self.stdout.write('fail')
except Exception as e:
self.stdout.write('fail')
traceback.print_exc()
self.stdout.write('aborted migration.')
return
self.stdout.write('migration complete')
def setUp(self):
# find azure credentials for testing. expects them in cwd
if not os.path.exists(self.CREDENTIALS_PATH):
raise Exception('Azure Credentials file at "%s" does not exist' % os.path.abspath(self.CREDENTIALS_PATH))
# read json formatted credentials ("account_name" and "account_key")
with open(self.CREDENTIALS_PATH, "r") as file:
self.credentials = json.load(file)
# create the blob service
self.service = BlobService(self.credentials['account_name'], self.credentials['account_key'])
# generate names for the test containers
self.container_names = ["%s-%d" % (self.CONTAINER_PREFIX, i) for i in range(5)]
class TestBlobService(TestCase):
# expecting a JSON file with credentials ("account_name" and "account_key") in test directory
CREDENTIALS_PATH = os.path.dirname(
os.path.abspath(__file__)) + '/azurecredentials.json'
CONTAINER_PREFIX = 'azurepython3-test'
def setUp(self):
# find azure credentials for testing. expects them in cwd
if not os.path.exists(self.CREDENTIALS_PATH):
raise Exception('Azure Credentials file at "%s" does not exist' %
os.path.abspath(self.CREDENTIALS_PATH))
# read json formatted credentials ("account_name" and "account_key")
with open(self.CREDENTIALS_PATH, "r") as file:
self.credentials = json.load(file)
# create the blob service
self.service = BlobService(self.credentials['account_name'],
self.credentials['account_key'])
# generate names for the test containers
self.container_names = [
"%s-%d" % (self.CONTAINER_PREFIX, i) for i in range(5)
]
def create_container(self, name):
# create a test container
try:
self.service.create_container(name)
except HTTPError as e:
if e.response.status_code != 409:
raise e
def tearDown(self):
# delete the test containers
for name in self.container_names:
self.service.delete_container(name)
def list_containers(self):
containers = self.service.list_containers()
for name in self.container_names:
self.assertIn(name, [x.name for x in containers])
def test_create_and_list_containers(self):
for name in self.container_names:
self.assertTrue(self.service.create_container(name, 'container'))
self.list_containers()
def test_create_blob(self):
container = '%s-test1' % self.CONTAINER_PREFIX
try:
self.assertTrue(
self.service.create_container(container, 'container'))
except HTTPError as e:
# there will be a HTTP 409 error if the container already exists
if e.response.status_code != 409:
self.fail("Failed to create test container")
# attempt to upload the file
data = bytearray(b'test byte string 11.5.2.7.3.14.59.2013.08.12')
self.assertTrue(
self.service.create_blob(container, 'somefile.ext', data))
# delete the container
self.service.delete_container(container)
def test_delete_blob(self):
container = '%s-test2' % self.CONTAINER_PREFIX
# create a test container
self.create_container(container)
# create a file
self.service.create_blob(container, 'file-to-delete.ext',
bytearray(b'THIS FILE SHOULD BE DELETED'))
# delete the file
self.assertTrue(
self.service.delete_blob(container, 'file-to-delete.ext'))
# delete the container again
self.service.delete_container(container)
def test_list_blobs(self):
container = '%s-test3' % self.CONTAINER_PREFIX
self.create_container(container)
self.service.create_blob(container, 'folder1/file1.ext',
bytearray(b'THIS FILE SHOULD BE LISTED'))
self.service.create_blob(container, 'folder1/file2.ext',
bytearray(b'THIS FILE SHOULD BE LISTED'))
self.service.create_blob(container, 'folder2/file3.ext',
bytearray(b'THIS FILE SHOULD BE LISTED'))
self.service.create_blob(container, 'file4.ext',
bytearray(b'THIS FILE SHOULD BE LISTED'))
blobs1 = self.service.list_blobs(container)
self.assertSetEqual(
{
'folder1/file1.ext', 'folder1/file2.ext', 'folder2/file3.ext',
'file4.ext'
}, set([blob.name for blob in blobs1]))
blobs2 = self.service.list_blobs(container, prefix='folder1')
self.assertSetEqual({'folder1/file1.ext', 'folder1/file2.ext'},
set([blob.name for blob in blobs2]))
blobs3 = self.service.list_blobs(container, prefix='folder2')
self.assertSetEqual({'folder2/file3.ext'},
set([blob.name for blob in blobs3]))
self.service.delete_container(container)
class TestBlobService(TestCase):
# expecting a JSON file with credentials ("account_name" and "account_key") in test directory
CREDENTIALS_PATH = os.path.dirname(os.path.abspath(__file__)) + '/azurecredentials.json'
CONTAINER_PREFIX = 'azurepython3-test'
def setUp(self):
# find azure credentials for testing. expects them in cwd
if not os.path.exists(self.CREDENTIALS_PATH):
raise Exception('Azure Credentials file at "%s" does not exist' % os.path.abspath(self.CREDENTIALS_PATH))
# read json formatted credentials ("account_name" and "account_key")
with open(self.CREDENTIALS_PATH, "r") as file:
self.credentials = json.load(file)
# create the blob service
self.service = BlobService(self.credentials['account_name'], self.credentials['account_key'])
# generate names for the test containers
self.container_names = ["%s-%d" % (self.CONTAINER_PREFIX, i) for i in range(5)]
def create_container(self, name):
# create a test container
try:
self.service.create_container(name)
except HTTPError as e:
if e.response.status_code != 409:
raise e
def tearDown(self):
# delete the test containers
for name in self.container_names:
self.service.delete_container(name)
def list_containers(self):
containers = self.service.list_containers()
for name in self.container_names:
self.assertIn(name, [x.name for x in containers])
def test_create_and_list_containers(self):
for name in self.container_names:
self.assertTrue(self.service.create_container(name, 'container'))
self.list_containers()
def test_create_blob(self):
container = '%s-test1' % self.CONTAINER_PREFIX
try:
self.assertTrue(self.service.create_container(container, 'container'))
except HTTPError as e:
# there will be a HTTP 409 error if the container already exists
if e.response.status_code != 409:
self.fail("Failed to create test container")
# attempt to upload the file
data = bytearray(b'test byte string 11.5.2.7.3.14.59.2013.08.12')
self.assertTrue(self.service.create_blob(container, 'somefile.ext', data))
# delete the container
self.service.delete_container(container)
def test_delete_blob(self):
container = '%s-test2' % self.CONTAINER_PREFIX
# create a test container
self.create_container(container)
# create a file
self.service.create_blob(container, 'file-to-delete.ext', bytearray(b'THIS FILE SHOULD BE DELETED'))
# delete the file
self.assertTrue(self.service.delete_blob(container, 'file-to-delete.ext'))
# delete the container again
self.service.delete_container(container)
def test_list_blobs(self):
container = '%s-test3' % self.CONTAINER_PREFIX
self.create_container(container)
self.service.create_blob(container, 'folder1/file1.ext', bytearray(b'THIS FILE SHOULD BE LISTED'))
self.service.create_blob(container, 'folder1/file2.ext', bytearray(b'THIS FILE SHOULD BE LISTED'))
self.service.create_blob(container, 'folder2/file3.ext', bytearray(b'THIS FILE SHOULD BE LISTED'))
self.service.create_blob(container, 'file4.ext', bytearray(b'THIS FILE SHOULD BE LISTED'))
blobs1 = self.service.list_blobs(container)
self.assertSetEqual({'folder1/file1.ext', 'folder1/file2.ext', 'folder2/file3.ext', 'file4.ext'},
set([blob.name for blob in blobs1]))
blobs2 = self.service.list_blobs(container, prefix = 'folder1')
self.assertSetEqual({'folder1/file1.ext', 'folder1/file2.ext'}, set([blob.name for blob in blobs2]))
blobs3 = self.service.list_blobs(container, prefix = 'folder2')
self.assertSetEqual({'folder2/file3.ext'}, set([blob.name for blob in blobs3]))
self.service.delete_container(container)
class AzureStorage(Storage):
def __init__(self, container = None, account_name = None, account_key = None):
"""
Creates a new AzureStorage. The container is not automatically created and therefore must already exist.
"""
if container is None:
if hasattr(settings, 'AZURE_DEFAULT_CONTAINER'):
self.container = settings.AZURE_DEFAULT_CONTAINER
else:
self.container = "$root"
else:
self.container = container
if account_name and account_key:
self.service = BlobService(account_name, account_key)
else:
self.service = BlobService(settings.AZURE_ACCOUNT_NAME, settings.AZURE_ACCOUNT_KEY)
def _transform_name(self, name):
return name.replace("\\", "/")
def _open(self, name, mode = 'rb') -> File:
name = self._transform_name(name)
content = self.service.get_blob_content(self.container, name)
file = SpooledTemporaryFile()
file.write(content)
file.seek(0) # explicitly reset to allow reading from the beginning afterwards as-is
return File(file)
def _save(self, name, content):
name = self._transform_name(name)
content.open(mode='rb')
data = bytearray(content.read())
self.service.create_blob(self.container, name, data)
return name
def delete(self, name):
name = self._transform_name(name)
self.service.delete_blob(self.container, name)
return name
def exists(self, name):
if not name:
return False
name = self._transform_name(name)
return self.service.blob_exists(self.container, name)
def listdir(self, path = None):
path = self._transform_name(path)
blobs = self.service.list_blobs(self.container, prefix = path)
paths = [os.path.split(blob.name) for blob in blobs]
dirs = [path[0] for path in paths]
files = [path[1] for path in paths]
return (dirs, files)
def size(self, name):
name = self._transform_name(name)
blob = self.service.get_blob(self.container, name, with_content=False)
return blob.content_length() if blob != None else 0
def url(self, name):
name = self._transform_name(name)
return self.service.get_blob_url(self.container, name)
def modified_time(self, name):
name = self._transform_name(name)
blob = self.service.get_blob(self.container, name, with_content=False)
return datetime.strptime(blob.properties['Last-Modified'], "%a, %d %b %Y %H:%M:%S GMT")