def test_get_url_private_dotted_name(botomock): def mock_api_call(self, operation_name, api_params): assert operation_name == 'ListObjectsV2' if api_params['Prefix'].endswith('xxx.sym'): # not found return {} return { 'Contents': [{ 'Key': api_params['Prefix'], }], } urls = ('https://s3.example.com/com.example.private/prefix/', ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): url = downloader.get_symbol_url('xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') assert ('/com.example.private/prefix/v0/xul.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xul.sym?') in url url = downloader.get_symbol_url('xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym') assert url is None assert len(botomock.calls) == 2
def test_multiple_urls_private_then_public(requestsmock, botomock): def mock_api_call(self, operation_name, api_params): assert operation_name == "ListObjectsV2" if api_params["Prefix"].endswith("xxx.sym"): # not found return {} # found return {"Contents": [{"Key": api_params["Prefix"]}]} requestsmock.head( "https://s3.example.com/public/prefix/v0/xul.pdb/" "44E4EC8C2F41492B9369D6B9A059577C2/xul.sym", text="", ) requestsmock.head( "https://s3.example.com/public/prefix/v0/xxx.pdb/" "44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym", text="Page Not Found", status_code=404, ) urls = ( "https://s3.example.com/private/prefix/", "https://s3.example.com/public/prefix/?access=public", ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): assert downloader.has_symbol("xul.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xul.sym") assert not downloader.has_symbol( "xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym")
def test_get_stream_public(requestsmock): requestsmock.get( 'https://s3.example.com/public/prefix/v0/xul.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xul.sym', content=b'LINE ONE\nLINE TWO\n') requestsmock.get( 'https://s3.example.com/public/prefix/v0/xxx.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym', content=b'Page Not Found', status_code=404, ) urls = ('https://s3.example.com/public/prefix/?access=public', ) downloader = SymbolDownloader(urls) stream = downloader.get_symbol_stream('xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') url = next(stream) assert url == ('https://s3.example.com/public/prefix/v0/xul.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xul.sym') lines = list(stream) assert lines == ['LINE ONE', 'LINE TWO'] stream = downloader.get_symbol_stream('xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym') with pytest.raises(SymbolNotFound): list(stream)
def test_multiple_urls_private_then_public(requestsmock, botomock): def mock_api_call(self, operation_name, api_params): assert operation_name == 'ListObjectsV2' if api_params['Prefix'].endswith('xxx.sym'): # not found return {} # found return { 'Contents': [{ 'Key': api_params['Prefix'], }], } requestsmock.head( 'https://s3.example.com/public/prefix/v0/xul.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xul.sym', text='') requestsmock.head( 'https://s3.example.com/public/prefix/v0/xxx.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym', text='Page Not Found', status_code=404, ) urls = ( 'https://s3.example.com/private/prefix/', 'https://s3.example.com/public/prefix/?access=public', ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): assert downloader.has_symbol('xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') assert not downloader.has_symbol( 'xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym')
def test_get_stream_public(requestsmock): requestsmock.get( "https://s3.example.com/public/prefix/v0/xul.pdb/" "44E4EC8C2F41492B9369D6B9A059577C2/xul.sym", content=b"LINE ONE\nLINE TWO\n", ) requestsmock.get( "https://s3.example.com/public/prefix/v0/xxx.pdb/" "44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym", content=b"Page Not Found", status_code=404, ) urls = ("https://s3.example.com/public/prefix/?access=public", ) downloader = SymbolDownloader(urls) stream = downloader.get_symbol_stream("xul.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xul.sym") url = next(stream) assert url == ("https://s3.example.com/public/prefix/v0/xul.pdb/" "44E4EC8C2F41492B9369D6B9A059577C2/xul.sym") lines = list(stream) assert lines == ["LINE ONE", "LINE TWO"] stream = downloader.get_symbol_stream("xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym") with pytest.raises(SymbolNotFound): list(stream)
def test_get_url_private(botomock): def mock_api_call(self, operation_name, api_params): assert operation_name == "ListObjectsV2" if api_params["Prefix"].endswith("xxx.sym"): # not found return {} return {"Contents": [{"Key": api_params["Prefix"]}]} urls = ("https://s3.example.com/private/prefix/", ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): url = downloader.get_symbol_url("xul.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xul.sym") # The bucket gets put in the top-domain. assert url.startswith("https://s3.example.com/") assert ("/private/prefix/v0/xul.pdb/" "44E4EC8C2F41492B9369D6B9A059577C2/xul.sym?") in url assert "Expires=" in url assert "AWSAccessKeyId=" in url assert "Signature=" in url url = downloader.get_symbol_url("xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym") assert url is None assert len(botomock.calls) == 2
def test_get_url_private(botomock): def mock_api_call(self, operation_name, api_params): assert operation_name == 'ListObjectsV2' if api_params['Prefix'].endswith('xxx.sym'): # not found return {} return { 'Contents': [{ 'Key': api_params['Prefix'], }], } urls = ('https://s3.example.com/private/prefix/', ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): url = downloader.get_symbol_url('xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') # The bucket gets put in the top-domain. assert url.startswith('https://s3.example.com/') assert ('/private/prefix/v0/xul.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xul.sym?') in url assert 'Expires=' in url assert 'AWSAccessKeyId=' in url assert 'Signature=' in url url = downloader.get_symbol_url('xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym') assert url is None assert len(botomock.calls) == 2
def test_has_public_case_insensitive_debugid(requestsmock): requestsmock.head( 'https://s3.example.com/public/prefix/v0/xul.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xul.sym', text='') urls = ('https://s3.example.com/public/prefix/?access=public', ) downloader = SymbolDownloader(urls) assert downloader.has_symbol('xul.pdb', '44e4ec8c2f41492b9369d6b9a059577c2', 'xul.sym')
def test_has_private_caching_and_invalidation(botomock): mock_calls = [] def mock_api_call(self, operation_name, api_params): assert operation_name == "ListObjectsV2" mock_calls.append(api_params["Prefix"]) return {"Contents": [{"Key": api_params["Prefix"]}]} urls = ("https://s3.example.com/private/prefix/", ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): assert downloader.has_symbol("xul.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xul.sym") assert len(mock_calls) == 1 assert downloader.has_symbol("xul.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xul.sym") # This should be cached assert len(mock_calls) == 1 # Now invalidate it downloader.invalidate_cache("xul.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xul.sym") assert downloader.has_symbol("xul.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xul.sym") assert len(mock_calls) == 2 # Invalidating unrecognized keys shouldn't break anything downloader.invalidate_cache("never", "44E4EC8C2F41492B9369D6B9A059577C2", "heardof")
def test_private_default_file_prefix(botomock, settings): """See doc string in test_public_default_file_prefix """ all_mock_calls = [] def mock_api_call(self, operation_name, api_params): if operation_name == 'ListObjectsV2': # the has_symbol() was called all_mock_calls.append(api_params['Prefix']) # pretend it doesn't exist return {} elif operation_name == 'GetObject': # someone wants a stream all_mock_calls.append(api_params['Key']) parsed_response = { 'Error': { 'Code': 'NoSuchKey', 'Message': 'Not found' }, } raise ClientError(parsed_response, operation_name) else: raise NotImplementedError(operation_name) urls = ( # Private URL with prefix and trailing / 'https://s3.example.com/priv-bucket/borje/', # No trailing / 'https://s3.example.com/also-priv-bucket/prrffxx', # No prefix 'https://s3.example.com/some-bucket', ) downloader = SymbolDownloader(urls, file_prefix='myprfx') with botomock(mock_api_call): assert not downloader.has_symbol( 'xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym') assert len(all_mock_calls) == 3 assert all_mock_calls[0].startswith('borje/myprfx/xxx.pdb') assert all_mock_calls[1].startswith('prrffxx/myprfx/xxx.pdb') assert all_mock_calls[2].startswith('myprfx/xxx.pdb') # reset the mutable recorder all_mock_calls = [] stream = downloader.get_symbol_stream( 'xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym') with pytest.raises(SymbolNotFound): next(stream) assert len(all_mock_calls) == 3 assert all_mock_calls[0].startswith('borje/myprfx/xxx.pdb') assert all_mock_calls[1].startswith('prrffxx/myprfx/xxx.pdb') assert all_mock_calls[2].startswith('myprfx/xxx.pdb')
def test_has_public_case_insensitive_debugid(requestsmock): requestsmock.head( "https://s3.example.com/public/prefix/v0/xul.pdb/" "44E4EC8C2F41492B9369D6B9A059577C2/xul.sym", text="", ) urls = ("https://s3.example.com/public/prefix/?access=public", ) downloader = SymbolDownloader(urls) assert downloader.has_symbol("xul.pdb", "44e4ec8c2f41492b9369d6b9a059577c2", "xul.sym")
def test_private_default_file_prefix(botomock, settings): """See doc string in test_public_default_file_prefix """ all_mock_calls = [] def mock_api_call(self, operation_name, api_params): if operation_name == "ListObjectsV2": # the has_symbol() was called all_mock_calls.append(api_params["Prefix"]) # pretend it doesn't exist return {} elif operation_name == "GetObject": # someone wants a stream all_mock_calls.append(api_params["Key"]) parsed_response = { "Error": { "Code": "NoSuchKey", "Message": "Not found" } } raise ClientError(parsed_response, operation_name) else: raise NotImplementedError(operation_name) urls = ( # Private URL with prefix and trailing / "https://s3.example.com/priv-bucket/borje/", # No trailing / "https://s3.example.com/also-priv-bucket/prrffxx", # No prefix "https://s3.example.com/some-bucket", ) downloader = SymbolDownloader(urls, file_prefix="myprfx") with botomock(mock_api_call): assert not downloader.has_symbol( "xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym") assert len(all_mock_calls) == 3 assert all_mock_calls[0].startswith("borje/myprfx/xxx.pdb") assert all_mock_calls[1].startswith("prrffxx/myprfx/xxx.pdb") assert all_mock_calls[2].startswith("myprfx/xxx.pdb") # reset the mutable recorder all_mock_calls = [] stream = downloader.get_symbol_stream( "xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym") with pytest.raises(SymbolNotFound): next(stream) assert len(all_mock_calls) == 3 assert all_mock_calls[0].startswith("borje/myprfx/xxx.pdb") assert all_mock_calls[1].startswith("prrffxx/myprfx/xxx.pdb") assert all_mock_calls[2].startswith("myprfx/xxx.pdb")
def reload_downloaders(urls, try_downloader=None): """Because the tecken.download.views module has a global instance of SymbolDownloader created at start-up, it's impossible to easily change the URL if you want to test clients with a different URL. This function hotfixes that instance to use a different URL(s). """ if isinstance(urls, str): urls = tuple([urls]) views.normal_downloader = SymbolDownloader(urls) if try_downloader: views.try_downloader = SymbolDownloader([try_downloader])
def test_has_private_case_insensitive_debugid(botomock): def mock_api_call(self, operation_name, api_params): assert operation_name == "ListObjectsV2" assert "44E4EC8C2F41492B9369D6B9A059577C2" in api_params["Prefix"] # found return {"Contents": [{"Key": api_params["Prefix"]}]} urls = ("https://s3.example.com/private/prefix/", ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): assert downloader.has_symbol("xul.pdb", "44e4ec8c2f41492b9369d6b9a059577c2", "xul.sym")
def test_get_stream_private_other_clienterrors(botomock): def mock_api_call(self, operation_name, api_params): assert operation_name == "GetObject" parsed_response = {"Error": {"Code": "403", "Message": "Forbidden"}} raise ClientError(parsed_response, operation_name) urls = ("https://s3.example.com/private/prefix/", ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): stream = downloader.get_symbol_stream( "xul.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xul.sym") with pytest.raises(ClientError): next(stream)
def test_has_private_bubble_other_clienterrors(botomock): def mock_api_call(self, operation_name, api_params): parsed_response = {'Error': {'Code': '403', 'Message': 'Not found'}} raise ClientError(parsed_response, operation_name) urls = ('https://s3.example.com/private/prefix/', ) downloader = SymbolDownloader(urls) # Expect this to raise a ClientError because the bucket ('private') # doesn't exist. So boto3 would normally trigger a ClientError # with a code 'Forbidden'. with botomock(mock_api_call): with pytest.raises(ClientError): downloader.has_symbol('xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym')
def test_has_private(botomock): def mock_api_call(self, operation_name, api_params): assert operation_name == "ListObjectsV2" if api_params["Prefix"].endswith("xxx.sym"): return {} return {"Contents": [{"Key": api_params["Prefix"]}]} urls = ("https://s3.example.com/private/prefix/", ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): assert downloader.has_symbol("xul.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xul.sym") assert downloader.time_took > 0.0 assert not downloader.has_symbol( "xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym") assert downloader.time_took > 0.0
def test_has_private_case_insensitive_debugid(botomock): def mock_api_call(self, operation_name, api_params): assert operation_name == 'ListObjectsV2' assert '44E4EC8C2F41492B9369D6B9A059577C2' in api_params['Prefix'] # found return { 'Contents': [{ 'Key': api_params['Prefix'], }], } urls = ('https://s3.example.com/private/prefix/', ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): assert downloader.has_symbol('xul.pdb', '44e4ec8c2f41492b9369d6b9a059577c2', 'xul.sym')
def test_get_url_private_caching_and_invalidation(botomock): mock_calls = [] def mock_api_call(self, operation_name, api_params): assert operation_name == 'ListObjectsV2' mock_calls.append(api_params['Prefix']) return { 'Contents': [{ 'Key': api_params['Prefix'], }], } urls = ('https://s3.example.com/private/prefix/', ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): assert downloader.get_symbol_url('xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') assert len(mock_calls) == 1 assert downloader.get_symbol_url('xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') # This should be cached assert len(mock_calls) == 1 # Now invalidate it downloader.invalidate_cache('xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') assert downloader.get_symbol_url('xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') assert len(mock_calls) == 2
def test_has_public(requestsmock): requestsmock.head( 'https://s3.example.com/public/prefix/v0/xul.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xul.sym', text='') requestsmock.head( 'https://s3.example.com/public/prefix/v0/xxx.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym', text='Page Not Found', status_code=404, ) urls = ('https://s3.example.com/public/prefix/?access=public', ) downloader = SymbolDownloader(urls, file_prefix='v0') assert downloader.has_symbol('xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') assert not downloader.has_symbol( 'xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym')
def test_get_stream_private_other_clienterrors(botomock): def mock_api_call(self, operation_name, api_params): assert operation_name == 'GetObject' parsed_response = { 'Error': { 'Code': '403', 'Message': 'Forbidden' }, } raise ClientError(parsed_response, operation_name) urls = ('https://s3.example.com/private/prefix/', ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): stream = downloader.get_symbol_stream( 'xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') with pytest.raises(ClientError): next(stream)
def test_has_public(requestsmock): requestsmock.head( "https://s3.example.com/public/prefix/v0/xul.pdb/" "44E4EC8C2F41492B9369D6B9A059577C2/xul.sym", text="", ) requestsmock.head( "https://s3.example.com/public/prefix/v0/xxx.pdb/" "44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym", text="Page Not Found", status_code=404, ) urls = ("https://s3.example.com/public/prefix/?access=public", ) downloader = SymbolDownloader(urls, file_prefix="v0") assert downloader.has_symbol("xul.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xul.sym") assert not downloader.has_symbol( "xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym")
def test_has_private_without_prefix(botomock): def mock_api_call(self, operation_name, api_params): assert operation_name == "ListObjectsV2" if api_params["Prefix"].endswith("xul.sym"): # found return {"Contents": [{"Key": api_params["Prefix"]}]} elif api_params["Prefix"].endswith("xxx.sym"): # not found return {} raise NotImplementedError(api_params) urls = ("https://s3.example.com/private", ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): assert downloader.has_symbol("xul.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xul.sym") assert not downloader.has_symbol( "xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym")
def test_get_stream_gzipped(botomock): def mock_api_call(self, operation_name, api_params): payload = b"line 1\n" b"line 2\n" b"line 3\n" buffer_ = BytesIO() with GzipFile(fileobj=buffer_, mode="w") as f: f.write(payload) payload_gz = buffer_.getvalue() return {"ContentEncoding": "gzip", "Body": BytesIO(payload_gz)} urls = ("https://s3.example.com/private/prefix/", ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): stream = downloader.get_symbol_stream( "xul.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xul.sym") bucket_name, key = next(stream) assert bucket_name == "private" assert key == ( "prefix/v0/xul.pdb/44E4EC8C2F41492B9369D6B9A059577C2/xul.sym") lines = list(stream) assert lines == ["line 1", "line 2", "line 3"]
def test_get_stream_gzipped(botomock): def mock_api_call(self, operation_name, api_params): payload = (b'line 1\n' b'line 2\n' b'line 3\n') buffer_ = BytesIO() with GzipFile(fileobj=buffer_, mode='w') as f: f.write(payload) payload_gz = buffer_.getvalue() return {'ContentEncoding': 'gzip', 'Body': BytesIO(payload_gz)} urls = ('https://s3.example.com/private/prefix/', ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): stream = downloader.get_symbol_stream( 'xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') bucket_name, key = next(stream) assert bucket_name == 'private' assert key == ( 'prefix/v0/xul.pdb/44E4EC8C2F41492B9369D6B9A059577C2/xul.sym') lines = list(stream) assert lines == ['line 1', 'line 2', 'line 3']
def test_has_private(botomock): def mock_api_call(self, operation_name, api_params): assert operation_name == 'ListObjectsV2' if api_params['Prefix'].endswith('xxx.sym'): return {} return { 'Contents': [{ 'Key': api_params['Prefix'], }], } urls = ('https://s3.example.com/private/prefix/', ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): assert downloader.has_symbol('xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') assert downloader.time_took > 0.0 assert not downloader.has_symbol( 'xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym') assert downloader.time_took > 0.0
def test_get_url_public(requestsmock): requestsmock.head( 'https://s3.example.com/public/prefix/v0/xul.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xul.sym', text='') requestsmock.head( 'https://s3.example.com/public/prefix/v0/xxx.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym', text='Page Not Found', status_code=404, ) urls = ('https://s3.example.com/public/prefix/?access=public', ) downloader = SymbolDownloader(urls) url = downloader.get_symbol_url('xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') assert url == ('https://s3.example.com/public/prefix/v0/xul.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xul.sym') url = downloader.get_symbol_url('xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym') assert url is None
def test_get_stream_gzipped_but_not_gzipped(botomock): def mock_api_call(self, operation_name, api_params): payload = b"line 1\n" b"line 2\n" b"line 3\n" return { "ContentEncoding": "gzip", # <-- note! "Body": BytesIO(payload), # but it's not gzipped! } urls = ("https://s3.example.com/private/prefix/", ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): stream = downloader.get_symbol_stream( "xul.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xul.sym") bucket_name, key = next(stream) assert bucket_name == "private" assert key == ( "prefix/v0/xul.pdb/44E4EC8C2F41492B9369D6B9A059577C2/xul.sym") # But when you start to stream it will realize that the file is not # actually gzipped and SymbolDownloader will automatically just skip # that file as if it doesn't exist. with pytest.raises(SymbolNotFound): next(stream)
def test_get_stream_gzipped_but_not_gzipped(botomock): def mock_api_call(self, operation_name, api_params): payload = (b'line 1\n' b'line 2\n' b'line 3\n') return { 'ContentEncoding': 'gzip', # <-- note! 'Body': BytesIO(payload) # but it's not gzipped! } urls = ('https://s3.example.com/private/prefix/', ) downloader = SymbolDownloader(urls) with botomock(mock_api_call): stream = downloader.get_symbol_stream( 'xul.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xul.sym') bucket_name, key = next(stream) assert bucket_name == 'private' assert key == ( 'prefix/v0/xul.pdb/44E4EC8C2F41492B9369D6B9A059577C2/xul.sym') # But when you start to stream it will realize that the file is not # actually gzipped and SymbolDownloader will automatically just skip # that file as if it doesn't exist. with pytest.raises(SymbolNotFound): next(stream)
def test_get_stream_public_content_encode_error(requestsmock): class BreakingStreamHTTPResponse(HTTPResponse): def stream(self, *a, **kwargs): raise ContentDecodingError('something terrible!') requestsmock.get( 'https://s3.example.com/public/prefix/v0/xul.pdb/' '44E4EC8C2F41492B9369D6B9A059577C2/xul.sym', raw=BreakingStreamHTTPResponse(status=200)) urls = ('https://s3.example.com/public/prefix/?access=public', ) downloader = SymbolDownloader(urls) stream = downloader.get_symbol_stream('xul.pdb', '44e4ec8c2f41492b9369d6b9a059577c2', 'xul.sym') # Because the URL exists (hence the 200 OK), but when you start # streaming it, it realizes it's there's something wrong with the # content encoding, it captures that and consider this symbol # not found. # I.e. unable to stream its content is as bad as the file not existing. # And because it's not found, the whole stream lookup is exhausted and # it finally raises a SymbolNotFound error. with pytest.raises(SymbolNotFound): list(stream)