def test_sasblobservice_putblob(): session = requests.Session() adapter = requests_mock.Adapter() session.mount('mock', adapter) with requests_mock.mock() as m: m.put('mock://blobepcontainer/blob?saskey', status_code=201) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) try: sbs.put_blob('container', 'blob', None, 'PageBlob', 'md5', 4) except Exception: pytest.fail('unexpected Exception raised') m.put('mock://blobepcontainer/blob?saskey', content=b'', status_code=200) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) with pytest.raises(IOError): sbs.put_blob('container', 'blob', None, 'PageBlob', 'md5', 4) m.put('mock://blobepcontainer/blob?saskey', content=b'', status_code=201) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) sbs.put_blob('container', 'blob', None, 'BlockBlob', 'md5', 0)
def test_blobchunkworker_run(tmpdir): lpath = str(tmpdir.join('test.tmp')) with open(lpath, 'wt') as f: f.write(str(uuid.uuid4())) exc_list = [] sa_in_queue = queue.Queue() sa_out_queue = queue.Queue() flock = threading.Lock() sa_in_queue.put((True, lpath, 'blobep', 'saskey', 'container', 'blob', 'blockid', 0, 4, flock, None)) sa_in_queue.put((False, lpath, 'blobep', 'saskey', 'container', 'blob', 'blockid', 0, 4, flock, None)) args = MagicMock() args.pageblob = False args.autovhd = False args.timeout = None session = requests.Session() adapter = requests_mock.Adapter() session.mount('mock', adapter) with requests_mock.mock() as m: m.put('mock://blobepcontainer/blob?saskey', status_code=201) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) bcw = blobxfer.BlobChunkWorker(exc_list, sa_in_queue, sa_out_queue, args, sbs) try: bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 4, flock, None) except Exception: pytest.fail('unexpected Exception raised') m.get('mock://blobepcontainer/blob?saskey', status_code=200) try: bcw.getblobrange(lpath, 'container', 'blob', 0, 4, flock, None) except Exception: pytest.fail('unexpected Exception raised') m.get('mock://blobepcontainer/blob?saskey', status_code=201) bcw.run() assert len(exc_list) > 0 exc_list = [] sa_in_queue = queue.Queue() sa_out_queue = queue.Queue() sa_in_queue.put((True, lpath, 'blobep', 'saskey', 'container', 'blob', 'blockid', 0, 4, flock, None)) sa_in_queue.put((False, lpath, 'blobep', 'saskey', 'container', 'blob', 'blockid', 0, 4, flock, None)) args.pageblob = True with requests_mock.mock() as m: m.put('mock://blobepcontainer/blob?saskey', status_code=200) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) bcw = blobxfer.BlobChunkWorker(exc_list, sa_in_queue, sa_out_queue, args, sbs) with pytest.raises(IOError): bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 4, flock, None)
def test_sasblobservice_getblob(): session = requests.Session() adapter = requests_mock.Adapter() session.mount('mock', adapter) with requests_mock.mock() as m: m.get('mock://blobepcontainer/blob?saskey', content=b'data') sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) results = sbs.get_blob('container', 'blob', 'range') assert results == b'data' m.get('mock://blobepcontainer/blob?saskey', status_code=201) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) with pytest.raises(IOError): sbs.get_blob('container', 'blob', 'range')
def test_sasblobservice_getblobproperties(): session = requests.Session() adapter = requests_mock.Adapter() session.mount('mock', adapter) with requests_mock.mock() as m: m.head('mock://blobepcontainer/blobsaskey', headers={'hello': 'world'}) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) results = sbs.get_blob_properties('container', 'blob') assert results['hello'] == 'world' m.head('mock://blobepcontainer/blobsaskey', text='', status_code=201) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) with pytest.raises(IOError): sbs.get_blob_properties('container', 'blob')
def test_sasblobservice_listblobs(patched_parse): session = requests.Session() adapter = requests_mock.Adapter() session.mount('mock', adapter) with requests_mock.mock() as m: m.get('mock://blobepcontainersaskey', text='data') sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) results = sbs.list_blobs('container', 'marker') assert results == patched_parse.return_value m.get('mock://blobepcontainersaskey', text='', status_code=201) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) with pytest.raises(IOError): sbs.list_blobs('container', 'marker')
def test_sasblobservice_setblobproperties(): session = requests.Session() adapter = requests_mock.Adapter() session.mount('mock', adapter) with requests_mock.mock() as m: m.put('mock://blobepcontainer/blob?saskey', status_code=200) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) try: sbs.set_blob_properties('container', 'blob', 'md5') except Exception: pytest.fail('unexpected Exception raised') m.put('mock://blobepcontainer/blob?saskey', text='', status_code=201) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) with pytest.raises(IOError): sbs.set_blob_properties('container', 'blob', 'md5')
def test_sasblobservice_listblobs(): session = requests.Session() adapter = requests_mock.Adapter() session.mount('mock', adapter) content = b'<?xml version="1.0" encoding="utf-8"?><EnumerationResults ' + \ b'ServiceEndpoint="http://myaccount.blob.core.windows.net/" ' + \ b'ContainerName="mycontainer"><Prefix>string-value</Prefix>' + \ b'<Marker>string-value</Marker><MaxResults>int-value</MaxResults>' + \ b'<Delimiter>string-value</Delimiter><Blobs><Blob><Name>blob-name' + \ b'</Name><Snapshot>date-time-value</Snapshot><Properties>' + \ b'<Last-Modified>date-time-value</Last-Modified><Etag>etag</Etag>' + \ b'<Content-Length>2147483648</Content-Length><Content-Type>' + \ b'blob-content-type</Content-Type><Content-Encoding />' + \ b'<Content-Language /><Content-MD5>abc</Content-MD5>' + \ b'<Cache-Control /><x-ms-blob-sequence-number>sequence-number' + \ b'</x-ms-blob-sequence-number><BlobType>BlockBlob</BlobType>' + \ b'<LeaseStatus>locked|unlocked</LeaseStatus><LeaseState>' + \ b'available | leased | expired | breaking | broken</LeaseState>' + \ b'<LeaseDuration>infinite | fixed</LeaseDuration><CopyId>id' + \ b'</CopyId><CopyStatus>pending | success | aborted | failed' + \ b'</CopyStatus><CopySource>source url</CopySource><CopyProgress>' + \ b'bytes copied/bytes total</CopyProgress><CopyCompletionTime>' + \ b'datetime</CopyCompletionTime><CopyStatusDescription>' + \ b'error string</CopyStatusDescription></Properties><Metadata>' + \ b'<Name>value</Name></Metadata></Blob><BlobPrefix><Name>' + \ b'blob-prefix</Name></BlobPrefix></Blobs><NextMarker>nm' + \ b'</NextMarker></EnumerationResults>' with requests_mock.mock() as m: m.get('mock://blobepcontainer?saskey', content=content) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) result = sbs.list_blobs('container', 'marker', include='metadata') assert len(result) == 1 assert result[0].name == 'blob-name' assert result[0].properties.content_length == 2147483648 assert result[0].properties.content_md5 == 'abc' assert result[0].properties.blobtype == 'BlockBlob' assert result[0].metadata['Name'] == 'value' assert result.next_marker == 'nm' m.get('mock://blobepcontainer?saskey', content=b'', status_code=201) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) with pytest.raises(IOError): sbs.list_blobs('container', 'marker')
def test_sasblobservice_setblobmetadata(): session = requests.Session() adapter = requests_mock.Adapter() session.mount('mock', adapter) with requests_mock.mock() as m: m.put('mock://blobepcontainer/blob?saskey') sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) sbs.set_blob_metadata('container', 'blob', None) sbs.set_blob_metadata('container', 'blob', {'name': 'value'}) m.put('mock://blobepcontainer/blob?saskey', status_code=201) with pytest.raises(IOError): sbs.set_blob_metadata('container', 'blob', {'name': 'value'})
def test_generate_xferspec_download(tmpdir): lpath = str(tmpdir.join('test.tmp')) args = MagicMock() args.storageaccount = 'blobep' args.container = 'container' args.storageaccountkey = 'saskey' args.chunksizebytes = 5 args.timeout = None sa_in_queue = queue.Queue() session = requests.Session() adapter = requests_mock.Adapter() session.mount('mock', adapter) with requests_mock.mock() as m: m.head('mock://blobepcontainer/blob?saskey', headers={ 'content-length': '-1', 'content-md5': 'md5' }) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) with pytest.raises(ValueError): blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath, 'blob', None, None, True) m.head('mock://blobepcontainer/blob?saskey', headers={ 'content-length': '6', 'content-md5': 'md5' }) cl, nsops, md5, fd = blobxfer.generate_xferspec_download( sbs, args, sa_in_queue, lpath, 'blob', None, None, True) assert 6 == cl assert 2 == nsops assert 'md5' == md5 assert None != fd fd.close() cl, nsops, md5, fd = blobxfer.generate_xferspec_download( sbs, args, sa_in_queue, lpath, 'blob', None, None, False) assert None == fd with open(lpath, 'wt') as f: f.write('012345') m.head('mock://blobepcontainer/blob?saskey', headers={ 'content-length': '6', 'content-md5': '1qmpM8iq/FHlWsBmK25NSg==' }) cl, nsops, md5, fd = blobxfer.generate_xferspec_download( sbs, args, sa_in_queue, lpath, 'blob', None, None, True) assert cl is None
def test_generate_xferspec_download_invalid(patched_azure_request): args = MagicMock() args.storageaccount = 'blobep' args.container = 'container' args.storageaccountkey = 'saskey' args.chunksizebytes = 5 args.timeout = None sa_in_queue = queue.Queue() with requests_mock.mock() as m: m.head('mock://blobepcontainer/blob?saskey', headers={ 'content-length': '-1', 'content-md5': 'md5'}) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) with pytest.raises(ValueError): blobxfer.generate_xferspec_download( sbs, args, sa_in_queue, 'tmppath', 'blob', None, None, True)
def test_generate_xferspec_download(tmpdir): lpath = str(tmpdir.join('test.tmp')) args = MagicMock() args.rsakey = None args.storageaccount = 'blobep' args.container = 'container' args.storageaccountkey = 'saskey' args.chunksizebytes = 5 args.timeout = None sa_in_queue = queue.PriorityQueue() session = requests.Session() adapter = requests_mock.Adapter() session.mount('mock', adapter) with requests_mock.mock() as m: m.head('mock://blobepcontainer/blob?saskey', headers={ 'content-length': '-1', 'content-md5': 'md5' }) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) with pytest.raises(ValueError): blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath, 'blob', True, [None, None, None]) assert sa_in_queue.qsize() == 0 m.head('mock://blobepcontainer/blob?saskey', headers={ 'content-length': '6', 'content-md5': 'md5' }) cl, nsops, md5, fd = blobxfer.generate_xferspec_download( sbs, args, sa_in_queue, lpath, 'blob', True, [None, None, None]) assert sa_in_queue.qsize() == 2 assert 2 == nsops assert 6 == cl assert 2 == nsops assert 'md5' == md5 assert fd is not None fd.close() cl, nsops, md5, fd = blobxfer.generate_xferspec_download( sbs, args, sa_in_queue, lpath, 'blob', False, [None, None, None]) assert 2 == nsops assert fd is None assert sa_in_queue.qsize() == 4 with open(lpath, 'wt') as f: f.write('012345') m.head('mock://blobepcontainer/blob?saskey', headers={ 'content-length': '6', 'content-md5': '1qmpM8iq/FHlWsBmK25NSg==' }) cl, nsops, md5, fd = blobxfer.generate_xferspec_download( sbs, args, sa_in_queue, lpath, 'blob', True, [None, None, None]) assert nsops is None assert cl is None assert sa_in_queue.qsize() == 4 sa_in_queue = queue.PriorityQueue() args.rsaprivatekey = _RSAKEY args.rsapublickey = None symkey, signkey = blobxfer.generate_aes256_keys() args.encmode = blobxfer._ENCRYPTION_MODE_CHUNKEDBLOB metajson = blobxfer.EncryptionMetadataJson(args, symkey, signkey, iv=b'0', encdata_signature=b'0', preencrypted_md5=None) encmeta = metajson.construct_metadata_json() goodencjson = json.loads(encmeta[blobxfer._ENCRYPTION_METADATA_NAME]) goodauthjson = json.loads( encmeta[blobxfer._ENCRYPTION_METADATA_AUTH_NAME]) metajson2 = blobxfer.EncryptionMetadataJson(args, None, None, None, None, None) metajson2.parse_metadata_json('blob', args.rsaprivatekey, args.rsapublickey, encmeta) assert metajson2.symkey == symkey assert metajson2.signkey == signkey assert metajson2.encmode == args.encmode assert metajson2.chunksizebytes == args.chunksizebytes + \ blobxfer._AES256CBC_HMACSHA256_OVERHEAD_BYTES + 1 encjson = json.loads(encmeta[blobxfer._ENCRYPTION_METADATA_NAME]) encjson[blobxfer._ENCRYPTION_METADATA_LAYOUT][ blobxfer._ENCRYPTION_METADATA_CHUNKSTRUCTURE] = 'X' headers = { 'content-length': '64', 'content-md5': 'md5', 'x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME: json.dumps(encjson), 'x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME: json.dumps(goodauthjson), } m.head('mock://blobepcontainer/blob?saskey', headers=headers) with pytest.raises(RuntimeError): blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath, 'blob', False, [None, None, None]) # switch to full blob mode tests args.encmode = blobxfer._ENCRYPTION_MODE_FULLBLOB metajson = blobxfer.EncryptionMetadataJson(args, symkey, signkey, iv=b'0', encdata_signature=b'0', preencrypted_md5=None) encmeta = metajson.construct_metadata_json() goodencjson = json.loads(encmeta[blobxfer._ENCRYPTION_METADATA_NAME]) goodauthjson = json.loads( encmeta[blobxfer._ENCRYPTION_METADATA_AUTH_NAME]) headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \ json.dumps(goodencjson) headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \ json.dumps(goodauthjson) encjson = copy.deepcopy(goodencjson) encjson[blobxfer._ENCRYPTION_METADATA_AGENT][ blobxfer._ENCRYPTION_METADATA_PROTOCOL] = 'X' headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \ json.dumps(encjson) m.head('mock://blobepcontainer/blob?saskey', headers=headers) with pytest.raises(RuntimeError): blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath, 'blob', False, [None, None, None]) encjson = copy.deepcopy(goodencjson) encjson[blobxfer._ENCRYPTION_METADATA_AGENT][ blobxfer._ENCRYPTION_METADATA_ENCRYPTION_ALGORITHM] = 'X' headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \ json.dumps(encjson) m.head('mock://blobepcontainer/blob?saskey', headers=headers) with pytest.raises(RuntimeError): blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath, 'blob', False, [None, None, None]) encjson = copy.deepcopy(goodencjson) encjson[blobxfer._ENCRYPTION_METADATA_INTEGRITY_AUTH][ blobxfer._ENCRYPTION_METADATA_ALGORITHM] = 'X' headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \ json.dumps(encjson) m.head('mock://blobepcontainer/blob?saskey', headers=headers) with pytest.raises(RuntimeError): blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath, 'blob', False, [None, None, None]) encjson = copy.deepcopy(goodencjson) encjson[blobxfer._ENCRYPTION_METADATA_WRAPPEDCONTENTKEY][ blobxfer._ENCRYPTION_METADATA_ALGORITHM] = 'X' headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \ json.dumps(encjson) m.head('mock://blobepcontainer/blob?saskey', headers=headers) with pytest.raises(RuntimeError): blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath, 'blob', False, [None, None, None]) authjson = copy.deepcopy(goodauthjson) authjson.pop(blobxfer._ENCRYPTION_METADATA_AUTH_METAAUTH, None) headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \ json.dumps(goodencjson) headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \ json.dumps(authjson) m.head('mock://blobepcontainer/blob?saskey', headers=headers) with pytest.raises(RuntimeError): blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath, 'blob', False, [None, None, None]) authjson = copy.deepcopy(goodauthjson) authjson[blobxfer._ENCRYPTION_METADATA_AUTH_METAAUTH].pop( blobxfer._ENCRYPTION_METADATA_AUTH_ENCODING, None) headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \ json.dumps(goodencjson) headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \ json.dumps(authjson) m.head('mock://blobepcontainer/blob?saskey', headers=headers) with pytest.raises(RuntimeError): blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath, 'blob', False, [None, None, None]) authjson = copy.deepcopy(goodauthjson) authjson[blobxfer._ENCRYPTION_METADATA_AUTH_METAAUTH][ blobxfer._ENCRYPTION_METADATA_ALGORITHM] = 'X' headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \ json.dumps(goodencjson) headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \ json.dumps(authjson) m.head('mock://blobepcontainer/blob?saskey', headers=headers) with pytest.raises(RuntimeError): blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath, 'blob', False, [None, None, None]) authjson = copy.deepcopy(goodauthjson) authjson[blobxfer._ENCRYPTION_METADATA_AUTH_METAAUTH][ blobxfer._ENCRYPTION_METADATA_MAC] = blobxfer.base64encode(b'X') headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \ json.dumps(goodencjson) headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \ json.dumps(authjson) m.head('mock://blobepcontainer/blob?saskey', headers=headers) with pytest.raises(RuntimeError): blobxfer.generate_xferspec_download(sbs, args, sa_in_queue, lpath, 'blob', False, [None, None, None]) args.chunksizebytes = 5 metajson.chunksizebytes = args.chunksizebytes metajson.md5 = headers['content-md5'] args.encmode = blobxfer._ENCRYPTION_MODE_FULLBLOB encjson = copy.deepcopy(goodencjson) headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_NAME] = \ json.dumps(encjson) headers['x-ms-meta-' + blobxfer._ENCRYPTION_METADATA_AUTH_NAME] = \ json.dumps(goodauthjson) hcl = int(headers['content-length']) cl, nsops, md5, fd = blobxfer.generate_xferspec_download( sbs, args, sa_in_queue, lpath, 'blob', False, [hcl, headers['content-md5'], metajson]) assert hcl == cl calcops = hcl // args.chunksizebytes hclmod = hcl % args.chunksizebytes if hclmod > 0: calcops += 1 assert calcops == nsops assert headers['content-md5'] == md5 assert fd is None assert sa_in_queue.qsize() == nsops data = sa_in_queue.get() assert data is not None
def test_blobchunkworker_run(tmpdir): lpath = str(tmpdir.join('test.tmp')) with open(lpath, 'wt') as f: f.write(str(uuid.uuid4())) args = MagicMock() args.rsakey = None args.pageblob = True args.autovhd = False args.timeout = None session = requests.Session() adapter = requests_mock.Adapter() session.mount('mock', adapter) exc_list = [] flock = threading.Lock() sa_in_queue = queue.PriorityQueue() sa_out_queue = queue.Queue() with requests_mock.mock() as m: m.put('mock://blobepcontainer/blob?saskey', status_code=200) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) bcw = blobxfer.BlobChunkWorker(exc_list, sa_in_queue, sa_out_queue, args, sbs, True) with pytest.raises(IOError): bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 4, None, flock, None) args.pageblob = False with requests_mock.mock() as m: m.put('mock://blobepcontainer/blob?saskey', status_code=201) sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) bcw = blobxfer.BlobChunkWorker(exc_list, sa_in_queue, sa_out_queue, args, sbs, True) bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 4, None, flock, None) m.get('mock://blobepcontainer/blob?saskey', status_code=200) bcw.getblobrange(lpath, 'container', 'blob', 0, 0, 4, [None, None, None, None, None, False], flock, None) # test zero-length putblob bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 0, None, flock, None) bcw._pageblob = True bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 0, None, flock, None) # test empty page with open(lpath, 'wb') as f: f.write(b'\0' * 4 * 1024 * 1024) bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 4 * 1024 * 1024, None, flock, None) with open(lpath, 'wb') as f: f.write(b'\0' * 4 * 1024) bcw.putblobdata(lpath, 'container', 'blob', 'blockid', 0, 4 * 1024, None, flock, None) sa_in_queue.put((0, (lpath, 'container', 'blob', 'blockid', 0, 4, [None, None, None, None], flock, None))) with requests_mock.mock() as m: sbs = blobxfer.SasBlobService('mock://blobep', 'saskey', None) bcw = blobxfer.BlobChunkWorker(exc_list, sa_in_queue, sa_out_queue, args, sbs, False) m.get('mock://blobepcontainer/blob?saskey', status_code=201) bcw.run() assert len(exc_list) > 0