def test_blobchunkworker_run(tmpdir):
lpath = str(tmpdir.join("test.tmp"))
with open(lpath, "wt") as f:
f.write(str(uuid.uuid4()))
args = MagicMock()
args.pageblob = True
args.autovhd = False
args.timeout = None
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount("mock", adapter)
exc_list = []
flock = threading.Lock()
sa_in_queue = queue.Queue()
sa_out_queue = queue.Queue()
with requests_mock.mock() as m:
m.put("mock://blobepcontainer/blob?saskey", status_code=200)
sbs = blobxfer.SasBlobService("mock://blobep", "saskey", None)
bcw = blobxfer.BlobChunkWorker(exc_list, sa_in_queue, sa_out_queue, args, sbs, True)
with pytest.raises(IOError):
bcw.putblobdata(lpath, "container", "blob", "blockid", 0, 4, flock, None)
args.pageblob = False
with requests_mock.mock() as m:
m.put("mock://blobepcontainer/blob?saskey", status_code=201)
sbs = blobxfer.SasBlobService("mock://blobep", "saskey", None)
bcw = blobxfer.BlobChunkWorker(exc_list, sa_in_queue, sa_out_queue, args, sbs, True)
try:
bcw.putblobdata(lpath, "container", "blob", "blockid", 0, 4, flock, None)
except Exception:
pytest.fail("unexpected Exception raised")
m.get("mock://blobepcontainer/blob?saskey", status_code=200)
try:
bcw.getblobrange(lpath, "container", "blob", 0, 4, flock, None)
except Exception:
pytest.fail("unexpected Exception raised")
# test zero-length putblob
bcw.putblobdata(lpath, "container", "blob", "blockid", 0, 0, flock, None)
bcw._pageblob = True
bcw.putblobdata(lpath, "container", "blob", "blockid", 0, 0, flock, None)
# test empty page
with open(lpath, "wb") as f:
f.write(b"\0" * 4 * 1024 * 1024)
bcw.putblobdata(lpath, "container", "blob", "blockid", 0, 4 * 1024 * 1024, flock, None)
with open(lpath, "wb") as f:
f.write(b"\0" * 4 * 1024)
bcw.putblobdata(lpath, "container", "blob", "blockid", 0, 4 * 1024, flock, None)
sa_in_queue.put((lpath, "container", "blob", "blockid", 0, 4, flock, None))
with requests_mock.mock() as m:
sbs = blobxfer.SasBlobService("mock://blobep", "saskey", None)
bcw = blobxfer.BlobChunkWorker(exc_list, sa_in_queue, sa_out_queue, args, sbs, False)
m.get("mock://blobepcontainer/blob?saskey", status_code=201)
bcw.run()
assert len(exc_list) > 0
def test_stream_build_config_app_creation__no_timeout_specified__timeout_defaults_to_60_Minutes(
self, mock_wait_attribute, mock_success_events):
mock_wait_attribute.return_value = True
self.mock_beanstalk.get_application_versions.return_value = self.app_version_raw_response
self.mock_codebuild.batch_get_builds.return_value = self.build_response
build_spec = MagicMock()
build_spec.timeout = None
buildspecops.stream_build_configuration_app_version_creation(
self.app_name, self.version_label, build_spec)
mock_wait_attribute.assert_called_with(self.app_name,
[self.version_label],
timeout=1)
self.mock_beanstalk.get_application_versions.assert_called_with(
self.app_name, version_labels=[self.version_label])
self.mock_codebuild.batch_get_builds.assert_called_with([
self.app_version_raw_response['ApplicationVersions'][0]['BuildArn']
])
timeout_error_message = ' '.join([
'The CodeBuild build timed out after 60 minute(s).',
"To increase the time limit, use the 'Timeout' option in the 'buildspec.yml' file."
])
mock_success_events.assert_called_with(
app_name='foo-app',
can_abort=False,
request_id=None,
timeout_error_message=timeout_error_message,
timeout_in_minutes=60,
version_label=self.version_label)
def test_stream_build_config_app_creation_no_build_arn_failed_to_create(
self, mock_wait_attribute, mock_success_events):
mock_wait_attribute.return_value = False
self.mock_beanstalk.get_application_versions.return_value = self.app_version_raw_response
mock_success_events.side_effect = ServiceError(
'Failed to create application version')
build_spec = MagicMock()
build_spec.timeout = 60
self.assertRaises(
ServiceError,
buildspecops.stream_build_configuration_app_version_creation,
self.app_name, self.version_label, build_spec)
mock_wait_attribute.assert_called_with(self.app_name,
[self.version_label],
timeout=1)
self.mock_beanstalk.get_application_versions.assert_called_with(
self.app_name, version_labels=[self.version_label])
self.mock_codebuild.batch_get_builds.assert_not_called()
timeout_error_message = ' '.join([
'The CodeBuild build timed out after 60 minute(s).',
"To increase the time limit, use the 'Timeout' option in the 'buildspec.yml' file."
])
mock_success_events.assert_called_with(
app_name='foo-app',
can_abort=False,
request_id=None,
timeout_error_message=timeout_error_message,
timeout_in_minutes=60,
version_label=self.version_label)
self.mock_beanstalk.delete_application_version.assert_called_with(
self.app_name, self.version_label)
def test_blobchunkworker_run(tmpdir):
lpath = str(tmpdir.join('test.tmp'))
with open(lpath, 'wt') as f:
f.write(str(uuid.uuid4()))
exc_list = []
sa_in_queue = queue.Queue()
sa_out_queue = queue.Queue()
flock = threading.Lock()
sa_in_queue.put((True, lpath, 'blobep', 'saskey', 'container',
'blob', 'blockid', 0, 4, flock, None))
sa_in_queue.put((False, lpath, 'blobep', 'saskey', 'container',
'blob', 'blockid', 0, 4, flock, None))
args = MagicMock()
args.pageblob = False
args.autovhd = False
args.timeout = None
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
with requests_mock.mock() as m:
m.put('mock://blobepcontainer/blob?saskey', status_code=201)
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
bcw = blobxfer.BlobChunkWorker(
exc_list, sa_in_queue, sa_out_queue, args, sbs)
try:
bcw.putblobdata(lpath, 'container', 'blob', 'blockid',
0, 4, flock, None)
except Exception:
pytest.fail('unexpected Exception raised')
m.get('mock://blobepcontainer/blob?saskey', status_code=200)
try:
bcw.getblobrange(lpath, 'container', 'blob', 0, 4, flock, None)
except Exception:
pytest.fail('unexpected Exception raised')
m.get('mock://blobepcontainer/blob?saskey', status_code=201)
bcw.run()
assert len(exc_list) > 0
exc_list = []
sa_in_queue = queue.Queue()
sa_out_queue = queue.Queue()
sa_in_queue.put((True, lpath, 'blobep', 'saskey', 'container',
'blob', 'blockid', 0, 4, flock, None))
sa_in_queue.put((False, lpath, 'blobep', 'saskey', 'container',
'blob', 'blockid', 0, 4, flock, None))
args.pageblob = True
with requests_mock.mock() as m:
m.put('mock://blobepcontainer/blob?saskey', status_code=200)
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
bcw = blobxfer.BlobChunkWorker(
exc_list, sa_in_queue, sa_out_queue, args, sbs)
with pytest.raises(IOError):
bcw.putblobdata(lpath, 'container', 'blob', 'blockid',
0, 4, flock, None)
def test_blobchunkworker_run(tmpdir):
lpath = str(tmpdir.join('test.tmp'))
with open(lpath, 'wt') as f:
f.write(str(uuid.uuid4()))
exc_list = []
sa_in_queue = queue.Queue()
sa_out_queue = queue.Queue()
flock = threading.Lock()
sa_in_queue.put((True, lpath, 'blobep', 'saskey', 'container', 'blob',
'blockid', 0, 4, flock, None))
sa_in_queue.put((False, lpath, 'blobep', 'saskey', 'container', 'blob',
'blockid', 0, 4, flock, None))
args = MagicMock()
args.pageblob = False
args.autovhd = False
args.timeout = None
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
with requests_mock.mock() as m:
m.put('mock://blobepcontainer/blob?saskey', status_code=201)
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
bcw = blobxfer.BlobChunkWorker(exc_list, sa_in_queue, sa_out_queue,
args, sbs)
try:
bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 4, flock,
None)
except Exception:
pytest.fail('unexpected Exception raised')
m.get('mock://blobepcontainer/blob?saskey', status_code=200)
try:
bcw.getblobrange(lpath, 'container', 'blob', 0, 4, flock, None)
except Exception:
pytest.fail('unexpected Exception raised')
m.get('mock://blobepcontainer/blob?saskey', status_code=201)
bcw.run()
assert len(exc_list) > 0
exc_list = []
sa_in_queue = queue.Queue()
sa_out_queue = queue.Queue()
sa_in_queue.put((True, lpath, 'blobep', 'saskey', 'container', 'blob',
'blockid', 0, 4, flock, None))
sa_in_queue.put((False, lpath, 'blobep', 'saskey', 'container', 'blob',
'blockid', 0, 4, flock, None))
args.pageblob = True
with requests_mock.mock() as m:
m.put('mock://blobepcontainer/blob?saskey', status_code=200)
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
bcw = blobxfer.BlobChunkWorker(exc_list, sa_in_queue, sa_out_queue,
args, sbs)
with pytest.raises(IOError):
bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 4, flock,
None)
def test_generate_xferspec_download_invalid(patched_azure_request):
args = MagicMock()
args.storageaccount = "blobep"
args.container = "container"
args.storageaccountkey = "saskey"
args.chunksizebytes = 5
args.timeout = None
sa_in_queue = queue.Queue()
with requests_mock.mock() as m:
m.head("mock://blobepcontainer/blob?saskey", headers={"content-length": "-1", "content-md5": "md5"})
sbs = blobxfer.SasBlobService("mock://blobep", "saskey", None)
with pytest.raises(ValueError):
blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, "tmppath", "blob", None, None, True)
def test_generate_xferspec_download(tmpdir):
lpath = str(tmpdir.join('test.tmp'))
args = MagicMock()
args.storageaccount = 'blobep'
args.container = 'container'
args.storageaccountkey = 'saskey'
args.chunksizebytes = 5
args.timeout = None
sa_in_queue = queue.Queue()
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
with requests_mock.mock() as m:
m.head('mock://blobepcontainer/blob?saskey',
headers={
'content-length': '-1',
'content-md5': 'md5'
})
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
with pytest.raises(ValueError):
blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath,
'blob', None, None, True)
m.head('mock://blobepcontainer/blob?saskey',
headers={
'content-length': '6',
'content-md5': 'md5'
})
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', None, None, True)
assert 6 == cl
assert 2 == nsops
assert 'md5' == md5
assert None != fd
fd.close()
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', None, None, False)
assert None == fd
with open(lpath, 'wt') as f:
f.write('012345')
m.head('mock://blobepcontainer/blob?saskey',
headers={
'content-length': '6',
'content-md5': '1qmpM8iq/FHlWsBmK25NSg=='
})
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', None, None, True)
assert cl is None
def test_generate_xferspec_download_invalid(patched_azure_request):
args = MagicMock()
args.storageaccount = 'blobep'
args.container = 'container'
args.storageaccountkey = 'saskey'
args.chunksizebytes = 5
args.timeout = None
sa_in_queue = queue.Queue()
with requests_mock.mock() as m:
m.head('mock://blobepcontainer/blob?saskey', headers={
'content-length': '-1', 'content-md5': 'md5'})
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
with pytest.raises(ValueError):
blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, 'tmppath', 'blob', None, None, True)
def test_generate_xferspec_download_invalid(patched_azure_request):
args = MagicMock()
args.storageaccount = 'blobep'
args.container = 'container'
args.storageaccountkey = 'saskey'
args.chunksizebytes = 5
args.timeout = None
sa_in_queue = queue.Queue()
with requests_mock.mock() as m:
m.head('mock://blobepcontainer/blob?saskey', headers={
'content-length': '-1', 'content-md5': 'md5'})
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
with pytest.raises(ValueError):
blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, 'tmppath', 'blob', None, None, True)
def test_main2(patched_parseargs, tmpdir):
lpath = str(tmpdir.join('test.tmp'))
args = MagicMock()
patched_parseargs.return_value = args
args.include = None
args.stripcomponents = 1
args.delete = False
args.rsaprivatekey = None
args.rsapublickey = None
args.numworkers = 64
args.storageaccount = 'blobep'
args.container = 'container'
args.chunksizebytes = 5
args.localresource = lpath
args.blobep = '.blobep'
args.timeout = 10
args.pageblob = False
args.autovhd = False
args.managementep = None
args.managementcert = None
args.subscriptionid = None
args.chunksizebytes = None
args.download = False
args.upload = True
args.remoteresource = None
args.saskey = None
args.storageaccountkey = 'key'
with open(lpath, 'wt') as f:
f.write(str(uuid.uuid4()))
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
with patch('azure.storage.blob.BlobService') as mock:
args.createcontainer = True
args.pageblob = False
args.autovhd = False
args.collate = None
mock.return_value = MagicMock()
mock.return_value.create_container = _mock_blobservice_create_container
blobxfer.main()
args.createcontainer = False
args.pageblob = True
args.autovhd = False
blobxfer.main()
def test_main2(patched_parseargs, tmpdir):
lpath = str(tmpdir.join('test.tmp'))
args = MagicMock()
patched_parseargs.return_value = args
args.include = None
args.stripcomponents = 1
args.delete = False
args.rsaprivatekey = None
args.rsapublickey = None
args.numworkers = 64
args.storageaccount = 'blobep'
args.container = 'container'
args.chunksizebytes = 5
args.localresource = lpath
args.blobep = '.blobep'
args.timeout = 10
args.pageblob = False
args.autovhd = False
args.managementep = None
args.managementcert = None
args.subscriptionid = None
args.chunksizebytes = None
args.download = False
args.upload = True
args.remoteresource = None
args.saskey = None
args.storageaccountkey = 'key'
with open(lpath, 'wt') as f:
f.write(str(uuid.uuid4()))
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
with patch('azure.storage.blob.BlobService') as mock:
args.createcontainer = True
args.pageblob = False
args.autovhd = False
args.collate = None
mock.return_value = MagicMock()
mock.return_value.create_container = _mock_blobservice_create_container
blobxfer.main()
args.createcontainer = False
args.pageblob = True
args.autovhd = False
blobxfer.main()
def test_main2(patched_parseargs, tmpdir):
lpath = str(tmpdir.join("test.tmp"))
args = MagicMock()
patched_parseargs.return_value = args
args.numworkers = 64
args.storageaccount = "blobep"
args.container = "container"
args.chunksizebytes = 5
args.localresource = lpath
args.blobep = ".blobep"
args.timeout = 10
args.pageblob = False
args.autovhd = False
args.managementep = None
args.managementcert = None
args.subscriptionid = None
args.chunksizebytes = None
args.download = False
args.upload = True
args.remoteresource = None
args.saskey = None
args.storageaccountkey = "key"
with open(lpath, "wt") as f:
f.write(str(uuid.uuid4()))
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount("mock", adapter)
with patch("azure.storage.blob.BlobService") as mock:
args.createcontainer = True
args.pageblob = False
args.autovhd = False
args.keeprootdir = False
args.collate = None
mock.return_value = MagicMock()
mock.return_value.create_container = _mock_blobservice_create_container
blobxfer.main()
args.createcontainer = False
args.pageblob = True
args.autovhd = False
blobxfer.main()
def test_generate_xferspec_download(tmpdir):
lpath = str(tmpdir.join("test.tmp"))
args = MagicMock()
args.storageaccount = "blobep"
args.container = "container"
args.storageaccountkey = "saskey"
args.chunksizebytes = 5
args.timeout = None
sa_in_queue = queue.Queue()
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount("mock", adapter)
with requests_mock.mock() as m:
m.head("mock://blobepcontainer/blob?saskey", headers={"content-length": "-1", "content-md5": "md5"})
sbs = blobxfer.SasBlobService("mock://blobep", "saskey", None)
with pytest.raises(ValueError):
blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath, "blob", None, None, True)
m.head("mock://blobepcontainer/blob?saskey", headers={"content-length": "6", "content-md5": "md5"})
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, "blob", None, None, True
)
assert 6 == cl
assert 2 == nsops
assert "md5" == md5
assert None != fd
fd.close()
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, "blob", None, None, False
)
assert None == fd
with open(lpath, "wt") as f:
f.write("012345")
m.head(
"mock://blobepcontainer/blob?saskey",
headers={"content-length": "6", "content-md5": "1qmpM8iq/FHlWsBmK25NSg=="},
)
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, "blob", None, None, True
)
assert cl is None
def test_generate_xferspec_download(tmpdir):
lpath = str(tmpdir.join('test.tmp'))
args = MagicMock()
args.storageaccount = 'blobep'
args.container = 'container'
args.storageaccountkey = 'saskey'
args.chunksizebytes = 5
args.timeout = None
sa_in_queue = queue.Queue()
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
with requests_mock.mock() as m:
m.head('mock://blobepcontainer/blob?saskey', headers={
'content-length': '-1', 'content-md5': 'md5'})
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
with pytest.raises(ValueError):
blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', None, None, True)
m.head('mock://blobepcontainer/blob?saskey', headers={
'content-length': '6', 'content-md5': 'md5'})
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', None, None, True)
assert 6 == cl
assert 2 == nsops
assert 'md5' == md5
assert None != fd
fd.close()
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', None, None, False)
assert None == fd
with open(lpath, 'wt') as f:
f.write('012345')
m.head('mock://blobepcontainer/blob?saskey', headers={
'content-length': '6', 'content-md5': '1qmpM8iq/FHlWsBmK25NSg=='})
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', None, None, True)
assert cl is None
def test_main1(patched_parseargs, tmpdir):
lpath = str(tmpdir.join('test.tmp'))
args = MagicMock()
args.numworkers = 0
args.localresource = ''
args.storageaccount = 'blobep'
args.container = 'container'
args.storageaccountkey = 'saskey'
args.chunksizebytes = 5
args.pageblob = False
args.autovhd = False
patched_parseargs.return_value = args
with pytest.raises(ValueError):
blobxfer.main()
args.localresource = lpath
args.blobep = ''
with pytest.raises(ValueError):
blobxfer.main()
args.blobep = 'blobep'
args.upload = True
args.download = True
with pytest.raises(ValueError):
blobxfer.main()
args.upload = None
args.download = None
with pytest.raises(ValueError):
blobxfer.main()
args.storageaccountkey = None
args.timeout = -1
args.saskey = ''
with pytest.raises(ValueError):
blobxfer.main()
args.saskey = None
args.storageaccountkey = None
args.managementcert = 'cert.spam'
args.subscriptionid = '1234'
with pytest.raises(ValueError):
blobxfer.main()
args.managementcert = 'cert.pem'
args.managementep = None
with pytest.raises(ValueError):
blobxfer.main()
args.managementep = 'mep'
args.subscriptionid = None
with pytest.raises(ValueError):
blobxfer.main()
args.subscriptionid = '1234'
args.pageblob = True
args.autovhd = True
with pytest.raises(ValueError):
blobxfer.main()
args.pageblob = False
args.autovhd = False
with patch('azure.servicemanagement.ServiceManagementService') as mock:
mock.return_value = MagicMock()
mock.return_value.get_storage_account_keys = \
_mock_get_storage_account_keys
mock.return_value.get_storage_account_properties = \
_mock_get_storage_account_properties
with pytest.raises(ValueError):
blobxfer.main()
args.managementep = None
args.managementcert = None
args.subscriptionid = None
args.remoteresource = 'blob'
args.chunksizebytes = None
with patch('azure.storage.blob.BlobService') as mock:
mock.return_value = None
with pytest.raises(ValueError):
blobxfer.main()
args.storageaccountkey = None
args.saskey = 'saskey'
args.remoteresource = None
args.download = True
with pytest.raises(ValueError):
blobxfer.main()
args.download = False
args.upload = True
args.remoteresource = None
args.storageaccountkey = ''
args.saskey = None
with pytest.raises(ValueError):
blobxfer.main()
args.storageaccountkey = None
args.saskey = 'saskey'
with open(lpath, 'wt') as f:
f.write(str(uuid.uuid4()))
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
with requests_mock.mock() as m:
m.put('https://blobep.blobep/container/blob?saskey'
'&comp=block&blockid=00000000', status_code=201)
m.put('https://blobep.blobep/container/' + lpath +
'?saskey&blockid=00000000&comp=block', status_code=201)
m.put('https://blobep.blobep/container/' + lpath +
'?saskey&comp=blocklist', status_code=201)
m.get('https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>' + lpath + '</Name>'
'<Properties><Content-Length>6</Content-Length>'
'<Content-MD5>md5</Content-MD5><BlobType>BlockBlob</BlobType>'
'</Properties></Blob></Blobs></EnumerationResults>')
args.progressbar = False
args.keeprootdir = False
args.skiponmatch = True
blobxfer.main()
args.progressbar = True
args.download = True
args.upload = False
args.remoteresource = 'blob'
args.localresource = str(tmpdir)
m.head('https://blobep.blobep/container/blob?saskey', headers={
'content-length': '6', 'content-md5': '1qmpM8iq/FHlWsBmK25NSg=='})
m.get('https://blobep.blobep/container/blob?saskey', content=b'012345')
blobxfer.main()
args.remoteresource = '.'
args.keepmismatchedmd5files = False
m.get('https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>blob</Name><Properties>'
'<Content-Length>6</Content-Length><Content-MD5>'
'</Content-MD5><BlobType>BlockBlob</BlobType></Properties>'
'</Blob></Blobs></EnumerationResults>')
m.get('https://blobep.blobep/container/?saskey')
with pytest.raises(SystemExit):
blobxfer.main()
m.get('https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>blob</Name><Properties>'
'<Content-Length>6</Content-Length><Content-MD5>md5'
'</Content-MD5><BlobType>BlockBlob</BlobType></Properties>'
'</Blob></Blobs></EnumerationResults>')
blobxfer.main()
tmplpath = str(tmpdir.join('test', 'test2', 'test3'))
print(tmplpath)
args.localresource = tmplpath
blobxfer.main()
args.localresource = str(tmpdir)
notmp_lpath = '/'.join(lpath.strip('/').split('/')[1:])
with requests_mock.mock() as m:
args.download = False
args.upload = True
args.remoteresource = None
args.skiponmatch = False
m.put('https://blobep.blobep/container/test.tmp?saskey'
'&comp=block&blockid=00000000', status_code=200)
m.put('https://blobep.blobep/container/test.tmp?saskey&comp=blocklist',
status_code=201)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=block&blockid=00000000', status_code=200)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=blocklist', status_code=201)
m.put('https://blobep.blobep/container/' + notmp_lpath +
'?saskey&comp=block&blockid=00000000', status_code=200)
m.put('https://blobep.blobep/container/' + notmp_lpath +
'?saskey&comp=blocklist', status_code=201)
with pytest.raises(SystemExit):
blobxfer.main()
args.recursive = False
m.put('https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=blocklist', status_code=201)
m.put('https://blobep.blobep/container/test.tmp.blobtmp?saskey'
'&comp=blocklist', status_code=201)
m.put('https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=block&blockid=00000000', status_code=200)
m.put('https://blobep.blobep/container/blob?saskey&comp=blocklist',
status_code=201)
with pytest.raises(SystemExit):
blobxfer.main()
args.pageblob = True
args.upload = True
args.download = False
m.put('https://blobep.blobep/container/blob.blobtmp?saskey',
status_code=201)
m.put('https://blobep.blobep/container/test.tmp?saskey',
status_code=201)
m.put('https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=properties', status_code=200)
m.put('https://blobep.blobep/container/test.tmp?saskey'
'&comp=properties', status_code=200)
m.put('https://blobep.blobep/container/blob?saskey', status_code=201)
with pytest.raises(IOError):
blobxfer.main()
m.put('https://blobep.blobep/container/blobsaskey', status_code=200)
with pytest.raises(IOError):
blobxfer.main()
m.put('https://blobep.blobep/container/' + notmp_lpath +
'?saskey&comp=blocklist', status_code=201)
m.put('https://blobep.blobep/container/blob?saskey', status_code=201)
args.pageblob = False
blobxfer.main()
args.pageblob = False
args.autovhd = True
blobxfer.main()
def test_generate_xferspec_download(tmpdir):
lpath = str(tmpdir.join('test.tmp'))
args = MagicMock()
args.rsakey = None
args.storageaccount = 'blobep'
args.container = 'container'
args.storageaccountkey = 'saskey'
args.chunksizebytes = 5
args.timeout = None
sa_in_queue = queue.PriorityQueue()
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
with requests_mock.mock() as m:
m.head('mock://blobepcontainer/blob?saskey',
headers={
'content-length': '-1',
'content-md5': 'md5'
})
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
with pytest.raises(ValueError):
blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath,
'blob', True,
[None, None, None])
assert sa_in_queue.qsize() == 0
m.head('mock://blobepcontainer/blob?saskey',
headers={
'content-length': '6',
'content-md5': 'md5'
})
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', True, [None, None, None])
assert sa_in_queue.qsize() == 2
assert 2 == nsops
assert 6 == cl
assert 2 == nsops
assert 'md5' == md5
assert fd is not None
fd.close()
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', False, [None, None, None])
assert 2 == nsops
assert fd is None
assert sa_in_queue.qsize() == 4
with open(lpath, 'wt') as f:
f.write('012345')
m.head('mock://blobepcontainer/blob?saskey',
headers={
'content-length': '6',
'content-md5': '1qmpM8iq/FHlWsBmK25NSg=='
})
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', True, [None, None, None])
assert nsops is None
assert cl is None
assert sa_in_queue.qsize() == 4
sa_in_queue = queue.PriorityQueue()
args.rsaprivatekey = _RSAKEY
args.rsapublickey = None
symkey, signkey = blobxfer.generate_aes256_keys()
args.encmode = blobxfer._ENCRYPTION_MODE_CHUNKEDBLOB
metajson = blobxfer.EncryptionMetadataJson(args,
symkey,
signkey,
iv=b'0',
encdata_signature=b'0',
preencrypted_md5=None)
encmeta = metajson.construct_metadata_json()
goodencjson = json.loads(encmeta[blobxfer._ENCRYPTION_METADATA_NAME])
goodauthjson = json.loads(
encmeta[blobxfer._ENCRYPTION_METADATA_AUTH_NAME])
metajson2 = blobxfer.EncryptionMetadataJson(args, None, None, None,
None, None)
metajson2.parse_metadata_json('blob', args.rsaprivatekey,
args.rsapublickey, encmeta)
assert metajson2.symkey == symkey
assert metajson2.signkey == signkey
assert metajson2.encmode == args.encmode
assert metajson2.chunksizebytes == args.chunksizebytes + \
blobxfer._AES256CBC_HMACSHA256_OVERHEAD_BYTES + 1
encjson = json.loads(encmeta[blobxfer._ENCRYPTION_METADATA_NAME])
encjson[blobxfer._ENCRYPTION_METADATA_LAYOUT][
blobxfer._ENCRYPTION_METADATA_CHUNKSTRUCTURE] = 'X'
headers = {
'content-length':
'64',
'content-md5':
'md5',
'x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME:
json.dumps(encjson),
'x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME:
json.dumps(goodauthjson),
}
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath,
'blob', False,
[None, None, None])
# switch to full blob mode tests
args.encmode = blobxfer._ENCRYPTION_MODE_FULLBLOB
metajson = blobxfer.EncryptionMetadataJson(args,
symkey,
signkey,
iv=b'0',
encdata_signature=b'0',
preencrypted_md5=None)
encmeta = metajson.construct_metadata_json()
goodencjson = json.loads(encmeta[blobxfer._ENCRYPTION_METADATA_NAME])
goodauthjson = json.loads(
encmeta[blobxfer._ENCRYPTION_METADATA_AUTH_NAME])
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(goodencjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \
json.dumps(goodauthjson)
encjson = copy.deepcopy(goodencjson)
encjson[blobxfer._ENCRYPTION_METADATA_AGENT][
blobxfer._ENCRYPTION_METADATA_PROTOCOL] = 'X'
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(encjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath,
'blob', False,
[None, None, None])
encjson = copy.deepcopy(goodencjson)
encjson[blobxfer._ENCRYPTION_METADATA_AGENT][
blobxfer._ENCRYPTION_METADATA_ENCRYPTION_ALGORITHM] = 'X'
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(encjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath,
'blob', False,
[None, None, None])
encjson = copy.deepcopy(goodencjson)
encjson[blobxfer._ENCRYPTION_METADATA_INTEGRITY_AUTH][
blobxfer._ENCRYPTION_METADATA_ALGORITHM] = 'X'
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(encjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath,
'blob', False,
[None, None, None])
encjson = copy.deepcopy(goodencjson)
encjson[blobxfer._ENCRYPTION_METADATA_WRAPPEDCONTENTKEY][
blobxfer._ENCRYPTION_METADATA_ALGORITHM] = 'X'
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(encjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath,
'blob', False,
[None, None, None])
authjson = copy.deepcopy(goodauthjson)
authjson.pop(blobxfer._ENCRYPTION_METADATA_AUTH_METAAUTH, None)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(goodencjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \
json.dumps(authjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath,
'blob', False,
[None, None, None])
authjson = copy.deepcopy(goodauthjson)
authjson[blobxfer._ENCRYPTION_METADATA_AUTH_METAAUTH].pop(
blobxfer._ENCRYPTION_METADATA_AUTH_ENCODING, None)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(goodencjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \
json.dumps(authjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath,
'blob', False,
[None, None, None])
authjson = copy.deepcopy(goodauthjson)
authjson[blobxfer._ENCRYPTION_METADATA_AUTH_METAAUTH][
blobxfer._ENCRYPTION_METADATA_ALGORITHM] = 'X'
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(goodencjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \
json.dumps(authjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath,
'blob', False,
[None, None, None])
authjson = copy.deepcopy(goodauthjson)
authjson[blobxfer._ENCRYPTION_METADATA_AUTH_METAAUTH][
blobxfer._ENCRYPTION_METADATA_MAC] = blobxfer.base64encode(b'X')
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(goodencjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \
json.dumps(authjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath,
'blob', False,
[None, None, None])
args.chunksizebytes = 5
metajson.chunksizebytes = args.chunksizebytes
metajson.md5 = headers['content-md5']
args.encmode = blobxfer._ENCRYPTION_MODE_FULLBLOB
encjson = copy.deepcopy(goodencjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(encjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \
json.dumps(goodauthjson)
hcl = int(headers['content-length'])
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', False,
[hcl, headers['content-md5'], metajson])
assert hcl == cl
calcops = hcl // args.chunksizebytes
hclmod = hcl % args.chunksizebytes
if hclmod > 0:
calcops += 1
assert calcops == nsops
assert headers['content-md5'] == md5
assert fd is None
assert sa_in_queue.qsize() == nsops
data = sa_in_queue.get()
assert data is not None
def test_main1(patched_sms_saprops, patched_sms_sakeys, patched_parseargs,
tmpdir):
lpath = str(tmpdir.join('test.tmp'))
args = MagicMock()
args.include = None
args.stripcomponents = 0
args.delete = False
args.rsaprivatekey = None
args.rsapublickey = None
args.rsakeypassphrase = None
args.numworkers = 0
args.localresource = ''
args.storageaccount = 'blobep'
args.container = 'container'
args.storageaccountkey = 'saskey'
args.chunksizebytes = 5
args.pageblob = False
args.autovhd = False
patched_parseargs.return_value = args
with pytest.raises(ValueError):
blobxfer.main()
args.localresource = lpath
args.blobep = ''
with pytest.raises(ValueError):
blobxfer.main()
args.blobep = 'blobep'
args.upload = True
args.download = True
with pytest.raises(ValueError):
blobxfer.main()
args.upload = None
args.download = None
with pytest.raises(ValueError):
blobxfer.main()
args.storageaccountkey = None
args.timeout = -1
args.saskey = ''
with pytest.raises(ValueError):
blobxfer.main()
args.saskey = None
args.storageaccountkey = None
args.managementcert = 'cert.spam'
args.subscriptionid = '1234'
with pytest.raises(ValueError):
blobxfer.main()
args.managementcert = 'cert.pem'
args.managementep = None
with pytest.raises(ValueError):
blobxfer.main()
args.managementep = 'mep'
args.subscriptionid = None
with pytest.raises(ValueError):
blobxfer.main()
args.subscriptionid = '1234'
args.pageblob = True
args.autovhd = True
with pytest.raises(ValueError):
blobxfer.main()
args.pageblob = False
args.autovhd = False
with patch('azure.servicemanagement.ServiceManagementService') as mock:
mock.return_value = MagicMock()
mock.return_value.get_storage_account_keys = \
_mock_get_storage_account_keys
mock.return_value.get_storage_account_properties = \
_mock_get_storage_account_properties
with pytest.raises(ValueError):
blobxfer.main()
args.managementep = None
args.managementcert = None
args.subscriptionid = None
args.remoteresource = 'blob'
args.chunksizebytes = None
with patch('azure.storage.blob.BlobService') as mock:
mock.return_value = None
with pytest.raises(ValueError):
blobxfer.main()
args.storageaccountkey = None
args.saskey = 'saskey'
args.remoteresource = None
args.download = True
with pytest.raises(ValueError):
blobxfer.main()
args.download = False
args.upload = True
args.remoteresource = None
args.storageaccountkey = ''
args.saskey = None
with pytest.raises(ValueError):
blobxfer.main()
args.collate = 'collatetmp'
with pytest.raises(ValueError):
blobxfer.main()
args.collate = None
args.storageaccountkey = None
args.saskey = ''
with pytest.raises(ValueError):
blobxfer.main()
args.saskey = None
with pytest.raises(ValueError):
blobxfer.main()
args.managementcert = '0'
args.managementep = ''
args.subscriptionid = '0'
with pytest.raises(ValueError):
blobxfer.main()
args.managementcert = 'test.pem'
with pytest.raises(ValueError):
blobxfer.main()
args.managementep = 'mep.mep'
ssk = MagicMock()
ssk.storage_service_keys = MagicMock()
ssk.storage_service_keys.primary = ''
patched_sms_sakeys.return_value = ssk
ssp = MagicMock()
ssp.storage_service_properties = MagicMock()
ssp.storage_service_properties.endpoints = ['blobep']
patched_sms_saprops.return_value = ssp
with pytest.raises(ValueError):
blobxfer.main()
ssk.storage_service_keys.primary = 'key1'
args.storageaccountkey = None
args.rsaprivatekey = ''
args.rsapublickey = ''
with pytest.raises(ValueError):
blobxfer.main()
args.rsaprivatekey = ''
args.rsapublickey = None
args.encmode = blobxfer._ENCRYPTION_MODE_FULLBLOB
with pytest.raises(IOError):
blobxfer.main()
args.rsaprivatekey = None
args.storageaccountkey = None
args.managementcert = None
args.managementep = None
args.subscriptionid = None
args.saskey = 'saskey'
with open(lpath, 'wt') as f:
f.write(str(uuid.uuid4()))
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
with requests_mock.mock() as m:
m.put(
'https://blobep.blobep/container/blob?saskey'
'&comp=block&blockid=00000000',
status_code=201)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&blockid=00000000&comp=block',
status_code=201)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=blocklist',
status_code=201)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=block&blockid=00000000',
status_code=201)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=metadata',
status_code=200)
m.get(
'https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>' + lpath + '</Name>'
'<Properties><Content-Length>6</Content-Length>'
'<Content-MD5>md5</Content-MD5><BlobType>BlockBlob</BlobType>'
'</Properties><Metadata/></Blob></Blobs></EnumerationResults>')
args.progressbar = False
args.skiponmatch = True
blobxfer.main()
args.progressbar = True
args.download = True
args.upload = False
args.remoteresource = None
with pytest.raises(ValueError):
blobxfer.main()
args.remoteresource = 'blob'
args.localresource = str(tmpdir)
m.head('https://blobep.blobep/container/blob?saskey',
headers={
'content-length': '6',
'content-md5': '1qmpM8iq/FHlWsBmK25NSg=='
})
m.get('https://blobep.blobep/container/blob?saskey', content=b'012345')
blobxfer.main()
args.pageblob = False
args.autovhd = False
args.skiponmatch = False
pemcontents = _RSAKEY.private_bytes(
encoding=cryptography.hazmat.primitives.serialization.Encoding.PEM,
format=cryptography.hazmat.primitives.serialization.PrivateFormat.
PKCS8,
encryption_algorithm=cryptography.hazmat.primitives.serialization.
NoEncryption())
pempath = str(tmpdir.join('rsa.pem'))
with open(pempath, 'wb') as f:
f.write(pemcontents)
args.rsaprivatekey = pempath
blobxfer.main()
os.remove(pempath)
args.rsaprivatekey = None
args.skiponmatch = True
args.remoteresource = '.'
args.keepmismatchedmd5files = False
m.get(
'https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>blob</Name><Properties>'
'<Content-Length>6</Content-Length><Content-MD5>'
'</Content-MD5><BlobType>BlockBlob</BlobType></Properties>'
'<Metadata/></Blob></Blobs></EnumerationResults>')
m.get('https://blobep.blobep/container/?saskey')
with pytest.raises(SystemExit):
blobxfer.main()
m.get(
'https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>blob</Name><Properties>'
'<Content-Length>6</Content-Length><Content-MD5>md5'
'</Content-MD5><BlobType>BlockBlob</BlobType></Properties>'
'<Metadata/></Blob></Blobs></EnumerationResults>')
blobxfer.main()
tmplpath = str(tmpdir.join('test', 'test2', 'test3'))
args.localresource = tmplpath
blobxfer.main()
args.localresource = str(tmpdir)
notmp_lpath = '/'.join(lpath.strip('/').split('/')[1:])
with requests_mock.mock() as m:
args.delete = True
args.download = False
args.upload = True
args.remoteresource = None
args.skiponmatch = False
m.put(
'https://blobep.blobep/container/test.tmp?saskey'
'&comp=block&blockid=00000000',
status_code=200)
m.put('https://blobep.blobep/container/test.tmp?saskey&comp=blocklist',
status_code=201)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=block&blockid=00000000',
status_code=200)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=blocklist',
status_code=201)
m.put('https://blobep.blobep/container/' + notmp_lpath +
'?saskey&comp=block&blockid=00000000',
status_code=200)
m.put('https://blobep.blobep/container/' + notmp_lpath +
'?saskey&comp=blocklist',
status_code=201)
m.get(
'https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>blob</Name><Properties>'
'<Content-Length>6</Content-Length><Content-MD5>md5'
'</Content-MD5><BlobType>BlockBlob</BlobType></Properties>'
'<Metadata/></Blob></Blobs></EnumerationResults>')
m.delete('https://blobep.blobep/container/blob?saskey',
status_code=202)
with pytest.raises(SystemExit):
blobxfer.main()
args.recursive = False
m.put(
'https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=blocklist',
status_code=201)
m.put(
'https://blobep.blobep/container/test.tmp.blobtmp?saskey'
'&comp=blocklist',
status_code=201)
m.put(
'https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=block&blockid=00000000',
status_code=200)
m.put('https://blobep.blobep/container/blob?saskey&comp=blocklist',
status_code=201)
with pytest.raises(SystemExit):
blobxfer.main()
args.stripcomponents = None
args.collate = '.'
args.pageblob = True
args.upload = True
args.download = False
m.put('https://blobep.blobep/container/blob.blobtmp?saskey',
status_code=201)
m.put('https://blobep.blobep/container/test.tmp?saskey',
status_code=201)
m.put(
'https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=properties',
status_code=200)
m.put(
'https://blobep.blobep/container/test.tmp?saskey'
'&comp=properties',
status_code=200)
m.put('https://blobep.blobep/container/blob?saskey', status_code=201)
with pytest.raises(IOError):
blobxfer.main()
args.stripcomponents = None
m.put('https://blobep.blobep/container/blobsaskey', status_code=200)
with pytest.raises(IOError):
blobxfer.main()
args.stripcomponents = None
args.pageblob = False
m.put('https://blobep.blobep/container/' + notmp_lpath +
'?saskey&comp=blocklist',
status_code=201)
m.put('https://blobep.blobep/container/blob?saskey', status_code=201)
blobxfer.main()
args.stripcomponents = None
args.autovhd = True
blobxfer.main()
args.stripcomponents = None
args.include = 'nofiles'
with pytest.raises(SystemExit):
blobxfer.main()
args.stripcomponents = None
args.include = '*'
blobxfer.main()
args.include = None
args.stripcomponents = None
args.pageblob = False
args.autovhd = False
pempath = str(tmpdir.join('rsa.pem'))
with open(pempath, 'wb') as f:
f.write(pemcontents)
args.rsaprivatekey = pempath
m.put(
'https://blobep.blobep/container/rsa.pem?saskey&comp=block'
'&blockid=00000000',
status_code=201)
m.put('https://blobep.blobep/container/rsa.pem?saskey&comp=blocklist',
status_code=201)
m.put('https://blobep.blobep/container/rsa.pem?saskey&comp=metadata',
status_code=200)
m.put('https://blobep.blobep/container/blob?saskey&comp=metadata',
status_code=200)
m.put(
'https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=metadata',
status_code=200)
m.put(
'https://blobep.blobep/container/test.tmp.blobtmp?saskey'
'&comp=metadata',
status_code=200)
m.put('https://blobep.blobep/container/test.tmp?saskey&comp=metadata',
status_code=200)
blobxfer.main()
args.stripcomponents = None
args.download = True
args.upload = False
args.rsaprivatekey = pempath
args.remoteresource = 'blob'
args.localresource = str(tmpdir)
m.head('https://blobep.blobep/container/blob?saskey',
headers={
'content-length': '6',
'content-md5': '1qmpM8iq/FHlWsBmK25NSg=='
})
m.get('https://blobep.blobep/container/blob?saskey', content=b'012345')
# TODO add encrypted data json
blobxfer.main()
def test_main1(
patched_sms_saprops, patched_sms_sakeys, patched_parseargs, tmpdir):
lpath = str(tmpdir.join('test.tmp'))
args = MagicMock()
args.include = None
args.stripcomponents = 0
args.delete = False
args.rsaprivatekey = None
args.rsapublickey = None
args.rsakeypassphrase = None
args.numworkers = 0
args.localresource = ''
args.storageaccount = 'blobep'
args.container = 'container'
args.storageaccountkey = 'saskey'
args.chunksizebytes = 5
args.pageblob = False
args.autovhd = False
patched_parseargs.return_value = args
with pytest.raises(ValueError):
blobxfer.main()
args.localresource = lpath
args.blobep = ''
with pytest.raises(ValueError):
blobxfer.main()
args.blobep = 'blobep'
args.upload = True
args.download = True
with pytest.raises(ValueError):
blobxfer.main()
args.upload = None
args.download = None
with pytest.raises(ValueError):
blobxfer.main()
args.storageaccountkey = None
args.timeout = -1
args.saskey = ''
with pytest.raises(ValueError):
blobxfer.main()
args.saskey = None
args.storageaccountkey = None
args.managementcert = 'cert.spam'
args.subscriptionid = '1234'
with pytest.raises(ValueError):
blobxfer.main()
args.managementcert = 'cert.pem'
args.managementep = None
with pytest.raises(ValueError):
blobxfer.main()
args.managementep = 'mep'
args.subscriptionid = None
with pytest.raises(ValueError):
blobxfer.main()
args.subscriptionid = '1234'
args.pageblob = True
args.autovhd = True
with pytest.raises(ValueError):
blobxfer.main()
args.pageblob = False
args.autovhd = False
with patch('azure.servicemanagement.ServiceManagementService') as mock:
mock.return_value = MagicMock()
mock.return_value.get_storage_account_keys = \
_mock_get_storage_account_keys
mock.return_value.get_storage_account_properties = \
_mock_get_storage_account_properties
with pytest.raises(ValueError):
blobxfer.main()
args.managementep = None
args.managementcert = None
args.subscriptionid = None
args.remoteresource = 'blob'
args.chunksizebytes = None
with patch('azure.storage.blob.BlobService') as mock:
mock.return_value = None
with pytest.raises(ValueError):
blobxfer.main()
args.storageaccountkey = None
args.saskey = 'saskey'
args.remoteresource = None
args.download = True
with pytest.raises(ValueError):
blobxfer.main()
args.download = False
args.upload = True
args.remoteresource = None
args.storageaccountkey = ''
args.saskey = None
with pytest.raises(ValueError):
blobxfer.main()
args.collate = 'collatetmp'
with pytest.raises(ValueError):
blobxfer.main()
args.collate = None
args.storageaccountkey = None
args.saskey = ''
with pytest.raises(ValueError):
blobxfer.main()
args.saskey = None
with pytest.raises(ValueError):
blobxfer.main()
args.managementcert = '0'
args.managementep = ''
args.subscriptionid = '0'
with pytest.raises(ValueError):
blobxfer.main()
args.managementcert = 'test.pem'
with pytest.raises(ValueError):
blobxfer.main()
args.managementep = 'mep.mep'
ssk = MagicMock()
ssk.storage_service_keys = MagicMock()
ssk.storage_service_keys.primary = ''
patched_sms_sakeys.return_value = ssk
ssp = MagicMock()
ssp.storage_service_properties = MagicMock()
ssp.storage_service_properties.endpoints = ['blobep']
patched_sms_saprops.return_value = ssp
with pytest.raises(ValueError):
blobxfer.main()
ssk.storage_service_keys.primary = 'key1'
args.storageaccountkey = None
args.rsaprivatekey = ''
args.rsapublickey = ''
with pytest.raises(ValueError):
blobxfer.main()
args.rsaprivatekey = ''
args.rsapublickey = None
args.encmode = blobxfer._ENCRYPTION_MODE_FULLBLOB
with pytest.raises(IOError):
blobxfer.main()
args.rsaprivatekey = None
args.storageaccountkey = None
args.managementcert = None
args.managementep = None
args.subscriptionid = None
args.saskey = 'saskey'
with open(lpath, 'wt') as f:
f.write(str(uuid.uuid4()))
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
with requests_mock.mock() as m:
m.put('https://blobep.blobep/container/blob?saskey'
'&comp=block&blockid=00000000', status_code=201)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&blockid=00000000&comp=block', status_code=201)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=blocklist', status_code=201)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=block&blockid=00000000', status_code=201)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=metadata', status_code=200)
m.get('https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>' + lpath + '</Name>'
'<Properties><Content-Length>6</Content-Length>'
'<Content-MD5>md5</Content-MD5><BlobType>BlockBlob</BlobType>'
'</Properties><Metadata/></Blob></Blobs></EnumerationResults>')
args.progressbar = False
args.skiponmatch = True
blobxfer.main()
args.progressbar = True
args.download = True
args.upload = False
args.remoteresource = None
with pytest.raises(ValueError):
blobxfer.main()
args.remoteresource = 'blob'
args.localresource = str(tmpdir)
m.head('https://blobep.blobep/container/blob?saskey', headers={
'content-length': '6', 'content-md5': '1qmpM8iq/FHlWsBmK25NSg=='})
m.get('https://blobep.blobep/container/blob?saskey', content=b'012345')
blobxfer.main()
args.pageblob = False
args.autovhd = False
args.skiponmatch = False
pemcontents = _RSAKEY.private_bytes(
encoding=cryptography.hazmat.primitives.serialization.
Encoding.PEM,
format=cryptography.hazmat.primitives.serialization.
PrivateFormat.PKCS8,
encryption_algorithm=cryptography.hazmat.primitives.
serialization.NoEncryption())
pempath = str(tmpdir.join('rsa.pem'))
with open(pempath, 'wb') as f:
f.write(pemcontents)
args.rsaprivatekey = pempath
blobxfer.main()
os.remove(pempath)
args.rsaprivatekey = None
args.skiponmatch = True
args.remoteresource = '.'
args.keepmismatchedmd5files = False
m.get('https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>blob</Name><Properties>'
'<Content-Length>6</Content-Length><Content-MD5>'
'</Content-MD5><BlobType>BlockBlob</BlobType></Properties>'
'<Metadata/></Blob></Blobs></EnumerationResults>')
m.get('https://blobep.blobep/container/?saskey')
with pytest.raises(SystemExit):
blobxfer.main()
m.get('https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>blob</Name><Properties>'
'<Content-Length>6</Content-Length><Content-MD5>md5'
'</Content-MD5><BlobType>BlockBlob</BlobType></Properties>'
'<Metadata/></Blob></Blobs></EnumerationResults>')
blobxfer.main()
tmplpath = str(tmpdir.join('test', 'test2', 'test3'))
args.localresource = tmplpath
blobxfer.main()
args.localresource = str(tmpdir)
notmp_lpath = '/'.join(lpath.strip('/').split('/')[1:])
with requests_mock.mock() as m:
args.delete = True
args.download = False
args.upload = True
args.remoteresource = None
args.skiponmatch = False
m.put('https://blobep.blobep/container/test.tmp?saskey'
'&comp=block&blockid=00000000', status_code=200)
m.put('https://blobep.blobep/container/test.tmp?saskey&comp=blocklist',
status_code=201)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=block&blockid=00000000', status_code=200)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=blocklist', status_code=201)
m.put('https://blobep.blobep/container/' + notmp_lpath +
'?saskey&comp=block&blockid=00000000', status_code=200)
m.put('https://blobep.blobep/container/' + notmp_lpath +
'?saskey&comp=blocklist', status_code=201)
m.get('https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>blob</Name><Properties>'
'<Content-Length>6</Content-Length><Content-MD5>md5'
'</Content-MD5><BlobType>BlockBlob</BlobType></Properties>'
'<Metadata/></Blob></Blobs></EnumerationResults>')
m.delete('https://blobep.blobep/container/blob?saskey',
status_code=202)
with pytest.raises(SystemExit):
blobxfer.main()
args.recursive = False
m.put('https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=blocklist', status_code=201)
m.put('https://blobep.blobep/container/test.tmp.blobtmp?saskey'
'&comp=blocklist', status_code=201)
m.put('https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=block&blockid=00000000', status_code=200)
m.put('https://blobep.blobep/container/blob?saskey&comp=blocklist',
status_code=201)
with pytest.raises(SystemExit):
blobxfer.main()
args.stripcomponents = None
args.collate = '.'
args.pageblob = True
args.upload = True
args.download = False
m.put('https://blobep.blobep/container/blob.blobtmp?saskey',
status_code=201)
m.put('https://blobep.blobep/container/test.tmp?saskey',
status_code=201)
m.put('https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=properties', status_code=200)
m.put('https://blobep.blobep/container/test.tmp?saskey'
'&comp=properties', status_code=200)
m.put('https://blobep.blobep/container/blob?saskey', status_code=201)
with pytest.raises(IOError):
blobxfer.main()
args.stripcomponents = None
m.put('https://blobep.blobep/container/blobsaskey', status_code=200)
with pytest.raises(IOError):
blobxfer.main()
args.stripcomponents = None
args.pageblob = False
m.put('https://blobep.blobep/container/' + notmp_lpath +
'?saskey&comp=blocklist', status_code=201)
m.put('https://blobep.blobep/container/blob?saskey', status_code=201)
blobxfer.main()
args.stripcomponents = None
args.autovhd = True
blobxfer.main()
args.stripcomponents = None
args.include = 'nofiles'
with pytest.raises(SystemExit):
blobxfer.main()
args.stripcomponents = None
args.include = '*'
blobxfer.main()
args.include = None
args.stripcomponents = None
args.pageblob = False
args.autovhd = False
pempath = str(tmpdir.join('rsa.pem'))
with open(pempath, 'wb') as f:
f.write(pemcontents)
args.rsaprivatekey = pempath
m.put('https://blobep.blobep/container/rsa.pem?saskey&comp=block'
'&blockid=00000000', status_code=201)
m.put('https://blobep.blobep/container/rsa.pem?saskey&comp=blocklist',
status_code=201)
m.put('https://blobep.blobep/container/rsa.pem?saskey&comp=metadata',
status_code=200)
m.put('https://blobep.blobep/container/blob?saskey&comp=metadata',
status_code=200)
m.put('https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=metadata', status_code=200)
m.put('https://blobep.blobep/container/test.tmp.blobtmp?saskey'
'&comp=metadata', status_code=200)
m.put('https://blobep.blobep/container/test.tmp?saskey&comp=metadata',
status_code=200)
blobxfer.main()
args.stripcomponents = None
args.download = True
args.upload = False
args.rsaprivatekey = pempath
args.remoteresource = 'blob'
args.localresource = str(tmpdir)
m.head('https://blobep.blobep/container/blob?saskey', headers={
'content-length': '6', 'content-md5': '1qmpM8iq/FHlWsBmK25NSg=='})
m.get('https://blobep.blobep/container/blob?saskey', content=b'012345')
# TODO add encrypted data json
blobxfer.main()
def test_generate_xferspec_download(tmpdir):
lpath = str(tmpdir.join('test.tmp'))
args = MagicMock()
args.rsakey = None
args.storageaccount = 'blobep'
args.container = 'container'
args.storageaccountkey = 'saskey'
args.chunksizebytes = 5
args.timeout = None
sa_in_queue = queue.PriorityQueue()
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
with requests_mock.mock() as m:
m.head('mock://blobepcontainer/blob?saskey', headers={
'content-length': '-1', 'content-md5': 'md5'})
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
with pytest.raises(ValueError):
blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', True,
[None, None, None])
assert sa_in_queue.qsize() == 0
m.head('mock://blobepcontainer/blob?saskey', headers={
'content-length': '6', 'content-md5': 'md5'})
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', True, [None, None, None])
assert sa_in_queue.qsize() == 2
assert 2 == nsops
assert 6 == cl
assert 2 == nsops
assert 'md5' == md5
assert fd is not None
fd.close()
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', False, [None, None, None])
assert 2 == nsops
assert fd is None
assert sa_in_queue.qsize() == 4
with open(lpath, 'wt') as f:
f.write('012345')
m.head('mock://blobepcontainer/blob?saskey', headers={
'content-length': '6', 'content-md5': '1qmpM8iq/FHlWsBmK25NSg=='})
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', True, [None, None, None])
assert nsops is None
assert cl is None
assert sa_in_queue.qsize() == 4
sa_in_queue = queue.PriorityQueue()
args.rsaprivatekey = _RSAKEY
args.rsapublickey = None
symkey, signkey = blobxfer.generate_aes256_keys()
args.encmode = blobxfer._ENCRYPTION_MODE_CHUNKEDBLOB
metajson = blobxfer.EncryptionMetadataJson(
args, symkey, signkey, iv=b'0', encdata_signature=b'0',
preencrypted_md5=None)
encmeta = metajson.construct_metadata_json()
goodencjson = json.loads(encmeta[blobxfer._ENCRYPTION_METADATA_NAME])
goodauthjson = json.loads(
encmeta[blobxfer._ENCRYPTION_METADATA_AUTH_NAME])
metajson2 = blobxfer.EncryptionMetadataJson(
args, None, None, None, None, None)
metajson2.parse_metadata_json(
'blob', args.rsaprivatekey, args.rsapublickey, encmeta)
assert metajson2.symkey == symkey
assert metajson2.signkey == signkey
assert metajson2.encmode == args.encmode
assert metajson2.chunksizebytes == args.chunksizebytes + \
blobxfer._AES256CBC_HMACSHA256_OVERHEAD_BYTES + 1
encjson = json.loads(encmeta[blobxfer._ENCRYPTION_METADATA_NAME])
encjson[blobxfer._ENCRYPTION_METADATA_LAYOUT][
blobxfer._ENCRYPTION_METADATA_CHUNKSTRUCTURE] = 'X'
headers = {
'content-length': '64',
'content-md5': 'md5',
'x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME:
json.dumps(encjson),
'x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME:
json.dumps(goodauthjson),
}
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', False,
[None, None, None])
# switch to full blob mode tests
args.encmode = blobxfer._ENCRYPTION_MODE_FULLBLOB
metajson = blobxfer.EncryptionMetadataJson(
args, symkey, signkey, iv=b'0', encdata_signature=b'0',
preencrypted_md5=None)
encmeta = metajson.construct_metadata_json()
goodencjson = json.loads(encmeta[blobxfer._ENCRYPTION_METADATA_NAME])
goodauthjson = json.loads(
encmeta[blobxfer._ENCRYPTION_METADATA_AUTH_NAME])
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(goodencjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \
json.dumps(goodauthjson)
encjson = copy.deepcopy(goodencjson)
encjson[blobxfer._ENCRYPTION_METADATA_AGENT][
blobxfer._ENCRYPTION_METADATA_PROTOCOL] = 'X'
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(encjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', False,
[None, None, None])
encjson = copy.deepcopy(goodencjson)
encjson[blobxfer._ENCRYPTION_METADATA_AGENT][
blobxfer._ENCRYPTION_METADATA_ENCRYPTION_ALGORITHM] = 'X'
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(encjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', False,
[None, None, None])
encjson = copy.deepcopy(goodencjson)
encjson[blobxfer._ENCRYPTION_METADATA_INTEGRITY_AUTH][
blobxfer._ENCRYPTION_METADATA_ALGORITHM] = 'X'
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(encjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', False,
[None, None, None])
encjson = copy.deepcopy(goodencjson)
encjson[blobxfer._ENCRYPTION_METADATA_WRAPPEDCONTENTKEY][
blobxfer._ENCRYPTION_METADATA_ALGORITHM] = 'X'
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(encjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', False,
[None, None, None])
authjson = copy.deepcopy(goodauthjson)
authjson.pop(blobxfer._ENCRYPTION_METADATA_AUTH_METAAUTH, None)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(goodencjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \
json.dumps(authjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', False,
[None, None, None])
authjson = copy.deepcopy(goodauthjson)
authjson[blobxfer._ENCRYPTION_METADATA_AUTH_METAAUTH].pop(
blobxfer._ENCRYPTION_METADATA_AUTH_ENCODING, None)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(goodencjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \
json.dumps(authjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', False,
[None, None, None])
authjson = copy.deepcopy(goodauthjson)
authjson[blobxfer._ENCRYPTION_METADATA_AUTH_METAAUTH][
blobxfer._ENCRYPTION_METADATA_ALGORITHM] = 'X'
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(goodencjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \
json.dumps(authjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', False,
[None, None, None])
authjson = copy.deepcopy(goodauthjson)
authjson[blobxfer._ENCRYPTION_METADATA_AUTH_METAAUTH][
blobxfer._ENCRYPTION_METADATA_MAC] = blobxfer.base64encode(b'X')
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(goodencjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \
json.dumps(authjson)
m.head('mock://blobepcontainer/blob?saskey', headers=headers)
with pytest.raises(RuntimeError):
blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', False,
[None, None, None])
args.chunksizebytes = 5
metajson.chunksizebytes = args.chunksizebytes
metajson.md5 = headers['content-md5']
args.encmode = blobxfer._ENCRYPTION_MODE_FULLBLOB
encjson = copy.deepcopy(goodencjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \
json.dumps(encjson)
headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \
json.dumps(goodauthjson)
hcl = int(headers['content-length'])
cl, nsops, md5, fd = blobxfer.generate_xferspec_download(
sbs, args, sa_in_queue, lpath, 'blob', False,
[hcl, headers['content-md5'], metajson])
assert hcl == cl
calcops = hcl // args.chunksizebytes
hclmod = hcl % args.chunksizebytes
if hclmod > 0:
calcops += 1
assert calcops == nsops
assert headers['content-md5'] == md5
assert fd is None
assert sa_in_queue.qsize() == nsops
data = sa_in_queue.get()
assert data is not None
def test_blobchunkworker_run(tmpdir):
lpath = str(tmpdir.join('test.tmp'))
with open(lpath, 'wt') as f:
f.write(str(uuid.uuid4()))
args = MagicMock()
args.rsakey = None
args.pageblob = True
args.autovhd = False
args.timeout = None
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
exc_list = []
flock = threading.Lock()
sa_in_queue = queue.PriorityQueue()
sa_out_queue = queue.Queue()
with requests_mock.mock() as m:
m.put('mock://blobepcontainer/blob?saskey', status_code=200)
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
bcw = blobxfer.BlobChunkWorker(
exc_list, sa_in_queue, sa_out_queue, args, sbs, True)
with pytest.raises(IOError):
bcw.putblobdata(lpath, 'container', 'blob', 'blockid',
0, 4, None, flock, None)
args.pageblob = False
with requests_mock.mock() as m:
m.put('mock://blobepcontainer/blob?saskey', status_code=201)
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
bcw = blobxfer.BlobChunkWorker(
exc_list, sa_in_queue, sa_out_queue, args, sbs, True)
bcw.putblobdata(lpath, 'container', 'blob', 'blockid',
0, 4, None, flock, None)
m.get('mock://blobepcontainer/blob?saskey', status_code=200)
bcw.getblobrange(lpath, 'container', 'blob', 0, 0, 4,
[None, None, None, None, None, False], flock, None)
# test zero-length putblob
bcw.putblobdata(
lpath, 'container', 'blob', 'blockid', 0, 0, None, flock, None)
bcw._pageblob = True
bcw.putblobdata(
lpath, 'container', 'blob', 'blockid', 0, 0, None, flock, None)
# test empty page
with open(lpath, 'wb') as f:
f.write(b'\0' * 4 * 1024 * 1024)
bcw.putblobdata(
lpath, 'container', 'blob', 'blockid', 0, 4 * 1024 * 1024,
None, flock, None)
with open(lpath, 'wb') as f:
f.write(b'\0' * 4 * 1024)
bcw.putblobdata(
lpath, 'container', 'blob', 'blockid', 0, 4 * 1024,
None, flock, None)
sa_in_queue.put((0, (lpath, 'container', 'blob', 'blockid', 0, 4,
[None, None, None, None], flock, None)))
with requests_mock.mock() as m:
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
bcw = blobxfer.BlobChunkWorker(
exc_list, sa_in_queue, sa_out_queue, args, sbs, False)
m.get('mock://blobepcontainer/blob?saskey', status_code=201)
bcw.run()
assert len(exc_list) > 0
def test_main1(patched_parseargs, tmpdir):
lpath = str(tmpdir.join('test.tmp'))
args = MagicMock()
args.numworkers = 0
args.localresource = ''
args.storageaccount = 'blobep'
args.container = 'container'
args.storageaccountkey = 'saskey'
args.chunksizebytes = 5
args.pageblob = False
args.autovhd = False
patched_parseargs.return_value = args
with pytest.raises(ValueError):
blobxfer.main()
args.localresource = lpath
args.blobep = ''
with pytest.raises(ValueError):
blobxfer.main()
args.blobep = 'blobep'
args.upload = True
args.download = True
with pytest.raises(ValueError):
blobxfer.main()
args.upload = None
args.download = None
with pytest.raises(ValueError):
blobxfer.main()
args.storageaccountkey = None
args.timeout = -1
args.saskey = ''
with pytest.raises(ValueError):
blobxfer.main()
args.saskey = None
args.storageaccountkey = None
args.managementcert = 'cert.spam'
args.subscriptionid = '1234'
with pytest.raises(ValueError):
blobxfer.main()
args.managementcert = 'cert.pem'
args.managementep = None
with pytest.raises(ValueError):
blobxfer.main()
args.managementep = 'mep'
args.subscriptionid = None
with pytest.raises(ValueError):
blobxfer.main()
args.subscriptionid = '1234'
args.pageblob = True
args.autovhd = True
with pytest.raises(ValueError):
blobxfer.main()
args.pageblob = False
args.autovhd = False
with patch('azure.servicemanagement.ServiceManagementService') as mock:
mock.return_value = MagicMock()
mock.return_value.get_storage_account_keys = \
_mock_get_storage_account_keys
mock.return_value.get_storage_account_properties = \
_mock_get_storage_account_properties
with pytest.raises(ValueError):
blobxfer.main()
args.managementep = None
args.managementcert = None
args.subscriptionid = None
args.remoteresource = 'blob'
args.chunksizebytes = None
with patch('azure.storage.blob.BlobService') as mock:
mock.return_value = None
with pytest.raises(ValueError):
blobxfer.main()
args.storageaccountkey = None
args.saskey = 'saskey'
args.remoteresource = None
args.download = True
with pytest.raises(ValueError):
blobxfer.main()
args.download = False
args.upload = True
args.remoteresource = None
args.storageaccountkey = ''
args.saskey = None
with pytest.raises(ValueError):
blobxfer.main()
args.storageaccountkey = None
args.saskey = 'saskey'
with open(lpath, 'wt') as f:
f.write(str(uuid.uuid4()))
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
with requests_mock.mock() as m:
m.put(
'https://blobep.blobep/container/blob?saskey'
'&comp=block&blockid=00000000',
status_code=201)
m.put('https://blobep.blobep/container/' + lpath +
'?saskey&blockid=00000000&comp=block',
status_code=201)
m.put('https://blobep.blobep/container/' + lpath +
'?saskey&comp=blocklist',
status_code=201)
m.get(
'https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>' + lpath + '</Name>'
'<Properties><Content-Length>6</Content-Length>'
'<Content-MD5>md5</Content-MD5><BlobType>BlockBlob</BlobType>'
'</Properties></Blob></Blobs></EnumerationResults>')
args.progressbar = False
args.keeprootdir = False
args.skiponmatch = True
blobxfer.main()
args.progressbar = True
args.download = True
args.upload = False
args.remoteresource = 'blob'
args.localresource = str(tmpdir)
m.head('https://blobep.blobep/container/blob?saskey',
headers={
'content-length': '6',
'content-md5': '1qmpM8iq/FHlWsBmK25NSg=='
})
m.get('https://blobep.blobep/container/blob?saskey', content=b'012345')
blobxfer.main()
args.remoteresource = '.'
args.keepmismatchedmd5files = False
m.get(
'https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>blob</Name><Properties>'
'<Content-Length>6</Content-Length><Content-MD5>'
'</Content-MD5><BlobType>BlockBlob</BlobType></Properties>'
'</Blob></Blobs></EnumerationResults>')
m.get('https://blobep.blobep/container/?saskey')
with pytest.raises(SystemExit):
blobxfer.main()
m.get(
'https://blobep.blobep/container?saskey&comp=list'
'&restype=container&maxresults=1000',
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>blob</Name><Properties>'
'<Content-Length>6</Content-Length><Content-MD5>md5'
'</Content-MD5><BlobType>BlockBlob</BlobType></Properties>'
'</Blob></Blobs></EnumerationResults>')
blobxfer.main()
tmplpath = str(tmpdir.join('test', 'test2', 'test3'))
print(tmplpath)
args.localresource = tmplpath
blobxfer.main()
args.localresource = str(tmpdir)
notmp_lpath = '/'.join(lpath.strip('/').split('/')[1:])
with requests_mock.mock() as m:
args.download = False
args.upload = True
args.remoteresource = None
args.skiponmatch = False
m.put(
'https://blobep.blobep/container/test.tmp?saskey'
'&comp=block&blockid=00000000',
status_code=200)
m.put('https://blobep.blobep/container/test.tmp?saskey&comp=blocklist',
status_code=201)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=block&blockid=00000000',
status_code=200)
m.put('https://blobep.blobep/container' + lpath +
'?saskey&comp=blocklist',
status_code=201)
m.put('https://blobep.blobep/container/' + notmp_lpath +
'?saskey&comp=block&blockid=00000000',
status_code=200)
m.put('https://blobep.blobep/container/' + notmp_lpath +
'?saskey&comp=blocklist',
status_code=201)
with pytest.raises(SystemExit):
blobxfer.main()
args.recursive = False
m.put(
'https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=blocklist',
status_code=201)
m.put(
'https://blobep.blobep/container/test.tmp.blobtmp?saskey'
'&comp=blocklist',
status_code=201)
m.put(
'https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=block&blockid=00000000',
status_code=200)
m.put('https://blobep.blobep/container/blob?saskey&comp=blocklist',
status_code=201)
with pytest.raises(SystemExit):
blobxfer.main()
args.pageblob = True
args.upload = True
args.download = False
m.put('https://blobep.blobep/container/blob.blobtmp?saskey',
status_code=201)
m.put('https://blobep.blobep/container/test.tmp?saskey',
status_code=201)
m.put(
'https://blobep.blobep/container/blob.blobtmp?saskey'
'&comp=properties',
status_code=200)
m.put(
'https://blobep.blobep/container/test.tmp?saskey'
'&comp=properties',
status_code=200)
m.put('https://blobep.blobep/container/blob?saskey', status_code=201)
with pytest.raises(IOError):
blobxfer.main()
m.put('https://blobep.blobep/container/blobsaskey', status_code=200)
with pytest.raises(IOError):
blobxfer.main()
m.put('https://blobep.blobep/container/' + notmp_lpath +
'?saskey&comp=blocklist',
status_code=201)
m.put('https://blobep.blobep/container/blob?saskey', status_code=201)
args.pageblob = False
blobxfer.main()
args.pageblob = False
args.autovhd = True
blobxfer.main()
def test_main1(patched_sms_saprops, patched_sms_sakeys, patched_parseargs, tmpdir):
lpath = str(tmpdir.join("test.tmp"))
args = MagicMock()
args.numworkers = 0
args.localresource = ""
args.storageaccount = "blobep"
args.container = "container"
args.storageaccountkey = "saskey"
args.chunksizebytes = 5
args.pageblob = False
args.autovhd = False
patched_parseargs.return_value = args
with pytest.raises(ValueError):
blobxfer.main()
args.localresource = lpath
args.blobep = ""
with pytest.raises(ValueError):
blobxfer.main()
args.blobep = "blobep"
args.upload = True
args.download = True
with pytest.raises(ValueError):
blobxfer.main()
args.upload = None
args.download = None
with pytest.raises(ValueError):
blobxfer.main()
args.storageaccountkey = None
args.timeout = -1
args.saskey = ""
with pytest.raises(ValueError):
blobxfer.main()
args.saskey = None
args.storageaccountkey = None
args.managementcert = "cert.spam"
args.subscriptionid = "1234"
with pytest.raises(ValueError):
blobxfer.main()
args.managementcert = "cert.pem"
args.managementep = None
with pytest.raises(ValueError):
blobxfer.main()
args.managementep = "mep"
args.subscriptionid = None
with pytest.raises(ValueError):
blobxfer.main()
args.subscriptionid = "1234"
args.pageblob = True
args.autovhd = True
with pytest.raises(ValueError):
blobxfer.main()
args.pageblob = False
args.autovhd = False
with patch("azure.servicemanagement.ServiceManagementService") as mock:
mock.return_value = MagicMock()
mock.return_value.get_storage_account_keys = _mock_get_storage_account_keys
mock.return_value.get_storage_account_properties = _mock_get_storage_account_properties
with pytest.raises(ValueError):
blobxfer.main()
args.managementep = None
args.managementcert = None
args.subscriptionid = None
args.remoteresource = "blob"
args.chunksizebytes = None
with patch("azure.storage.blob.BlobService") as mock:
mock.return_value = None
with pytest.raises(ValueError):
blobxfer.main()
args.storageaccountkey = None
args.saskey = "saskey"
args.remoteresource = None
args.download = True
with pytest.raises(ValueError):
blobxfer.main()
args.download = False
args.upload = True
args.remoteresource = None
args.storageaccountkey = ""
args.saskey = None
with pytest.raises(ValueError):
blobxfer.main()
args.keeprootdir = True
args.collate = "collatetmp"
with pytest.raises(ValueError):
blobxfer.main()
args.keeprootdir = False
args.collate = None
args.storageaccountkey = None
args.saskey = ""
with pytest.raises(ValueError):
blobxfer.main()
args.saskey = None
with pytest.raises(ValueError):
blobxfer.main()
args.managementcert = "0"
args.managementep = ""
args.subscriptionid = "0"
with pytest.raises(ValueError):
blobxfer.main()
args.managementcert = "test.pem"
with pytest.raises(ValueError):
blobxfer.main()
args.managementep = "mep.mep"
ssk = MagicMock()
ssk.storage_service_keys = MagicMock()
ssk.storage_service_keys.primary = ""
patched_sms_sakeys.return_value = ssk
ssp = MagicMock()
ssp.storage_service_properties = MagicMock()
ssp.storage_service_properties.endpoints = ["blobep"]
patched_sms_saprops.return_value = ssp
with pytest.raises(ValueError):
blobxfer.main()
ssk.storage_service_keys.primary = "key1"
args.storageaccountkey = None
with pytest.raises(Exception):
blobxfer.main()
args.storageaccountkey = None
args.managementcert = None
args.managementep = None
args.subscriptionid = None
args.saskey = "saskey"
with open(lpath, "wt") as f:
f.write(str(uuid.uuid4()))
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount("mock", adapter)
with requests_mock.mock() as m:
m.put("https://blobep.blobep/container/blob?saskey" "&comp=block&blockid=00000000", status_code=201)
m.put("https://blobep.blobep/container" + lpath + "?saskey&blockid=00000000&comp=block", status_code=201)
m.put("https://blobep.blobep/container" + lpath + "?saskey&comp=blocklist", status_code=201)
m.put("https://blobep.blobep/container" + lpath + "?saskey&comp=block&blockid=00000000", status_code=201)
m.get(
"https://blobep.blobep/container?saskey&comp=list" "&restype=container&maxresults=1000",
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>' + lpath + "</Name>"
"<Properties><Content-Length>6</Content-Length>"
"<Content-MD5>md5</Content-MD5><BlobType>BlockBlob</BlobType>"
"</Properties></Blob></Blobs></EnumerationResults>",
)
args.progressbar = False
args.keeprootdir = False
args.skiponmatch = True
blobxfer.main()
args.progressbar = True
args.download = True
args.upload = False
args.remoteresource = None
with pytest.raises(ValueError):
blobxfer.main()
args.remoteresource = "blob"
args.localresource = str(tmpdir)
m.head(
"https://blobep.blobep/container/blob?saskey",
headers={"content-length": "6", "content-md5": "1qmpM8iq/FHlWsBmK25NSg=="},
)
m.get("https://blobep.blobep/container/blob?saskey", content=b"012345")
blobxfer.main()
args.remoteresource = "."
args.keepmismatchedmd5files = False
m.get(
"https://blobep.blobep/container?saskey&comp=list" "&restype=container&maxresults=1000",
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>blob</Name><Properties>'
"<Content-Length>6</Content-Length><Content-MD5>"
"</Content-MD5><BlobType>BlockBlob</BlobType></Properties>"
"</Blob></Blobs></EnumerationResults>",
)
m.get("https://blobep.blobep/container/?saskey")
with pytest.raises(SystemExit):
blobxfer.main()
m.get(
"https://blobep.blobep/container?saskey&comp=list" "&restype=container&maxresults=1000",
text='<?xml version="1.0" encoding="utf-8"?>'
'<EnumerationResults ContainerName="https://blobep.blobep/'
'container"><Blobs><Blob><Name>blob</Name><Properties>'
"<Content-Length>6</Content-Length><Content-MD5>md5"
"</Content-MD5><BlobType>BlockBlob</BlobType></Properties>"
"</Blob></Blobs></EnumerationResults>",
)
blobxfer.main()
tmplpath = str(tmpdir.join("test", "test2", "test3"))
print(tmplpath)
args.localresource = tmplpath
blobxfer.main()
args.localresource = str(tmpdir)
notmp_lpath = "/".join(lpath.strip("/").split("/")[1:])
with requests_mock.mock() as m:
args.download = False
args.upload = True
args.remoteresource = None
args.skiponmatch = False
m.put("https://blobep.blobep/container/test.tmp?saskey" "&comp=block&blockid=00000000", status_code=200)
m.put("https://blobep.blobep/container/test.tmp?saskey&comp=blocklist", status_code=201)
m.put("https://blobep.blobep/container" + lpath + "?saskey&comp=block&blockid=00000000", status_code=200)
m.put("https://blobep.blobep/container" + lpath + "?saskey&comp=blocklist", status_code=201)
m.put("https://blobep.blobep/container/" + notmp_lpath + "?saskey&comp=block&blockid=00000000", status_code=200)
m.put("https://blobep.blobep/container/" + notmp_lpath + "?saskey&comp=blocklist", status_code=201)
with pytest.raises(SystemExit):
blobxfer.main()
args.recursive = False
m.put("https://blobep.blobep/container/blob.blobtmp?saskey" "&comp=blocklist", status_code=201)
m.put("https://blobep.blobep/container/test.tmp.blobtmp?saskey" "&comp=blocklist", status_code=201)
m.put("https://blobep.blobep/container/blob.blobtmp?saskey" "&comp=block&blockid=00000000", status_code=200)
m.put("https://blobep.blobep/container/blob?saskey&comp=blocklist", status_code=201)
with pytest.raises(SystemExit):
blobxfer.main()
args.pageblob = True
args.upload = True
args.download = False
m.put("https://blobep.blobep/container/blob.blobtmp?saskey", status_code=201)
m.put("https://blobep.blobep/container/test.tmp?saskey", status_code=201)
m.put("https://blobep.blobep/container/blob.blobtmp?saskey" "&comp=properties", status_code=200)
m.put("https://blobep.blobep/container/test.tmp?saskey" "&comp=properties", status_code=200)
m.put("https://blobep.blobep/container/blob?saskey", status_code=201)
with pytest.raises(IOError):
blobxfer.main()
m.put("https://blobep.blobep/container/blobsaskey", status_code=200)
with pytest.raises(IOError):
blobxfer.main()
m.put("https://blobep.blobep/container/" + notmp_lpath + "?saskey&comp=blocklist", status_code=201)
m.put("https://blobep.blobep/container/blob?saskey", status_code=201)
args.pageblob = False
blobxfer.main()
args.pageblob = False
args.autovhd = True
blobxfer.main()
def test_blobchunkworker_run(tmpdir):
lpath = str(tmpdir.join('test.tmp'))
with open(lpath, 'wt') as f:
f.write(str(uuid.uuid4()))
args = MagicMock()
args.rsakey = None
args.pageblob = True
args.autovhd = False
args.timeout = None
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('mock', adapter)
exc_list = []
flock = threading.Lock()
sa_in_queue = queue.PriorityQueue()
sa_out_queue = queue.Queue()
with requests_mock.mock() as m:
m.put('mock://blobepcontainer/blob?saskey', status_code=200)
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
bcw = blobxfer.BlobChunkWorker(exc_list, sa_in_queue, sa_out_queue,
args, sbs, True)
with pytest.raises(IOError):
bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 4, None,
flock, None)
args.pageblob = False
with requests_mock.mock() as m:
m.put('mock://blobepcontainer/blob?saskey', status_code=201)
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
bcw = blobxfer.BlobChunkWorker(exc_list, sa_in_queue, sa_out_queue,
args, sbs, True)
bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 4, None,
flock, None)
m.get('mock://blobepcontainer/blob?saskey', status_code=200)
bcw.getblobrange(lpath, 'container', 'blob', 0, 0, 4,
[None, None, None, None, None, False], flock, None)
# test zero-length putblob
bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 0, None,
flock, None)
bcw._pageblob = True
bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 0, None,
flock, None)
# test empty page
with open(lpath, 'wb') as f:
f.write(b'\0' * 4 * 1024 * 1024)
bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0,
4 * 1024 * 1024, None, flock, None)
with open(lpath, 'wb') as f:
f.write(b'\0' * 4 * 1024)
bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 4 * 1024,
None, flock, None)
sa_in_queue.put((0, (lpath, 'container', 'blob', 'blockid', 0, 4,
[None, None, None, None], flock, None)))
with requests_mock.mock() as m:
sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None)
bcw = blobxfer.BlobChunkWorker(exc_list, sa_in_queue, sa_out_queue,
args, sbs, False)
m.get('mock://blobepcontainer/blob?saskey', status_code=201)
bcw.run()
assert len(exc_list) > 0
