def test_has_private_caching_and_invalidation(botomock):
mock_calls = []
def mock_api_call(self, operation_name, api_params):
assert operation_name == "ListObjectsV2"
mock_calls.append(api_params["Prefix"])
return {"Contents": [{"Key": api_params["Prefix"]}]}
urls = ("https://s3.example.com/private/prefix/", )
downloader = SymbolDownloader(urls)
with botomock(mock_api_call):
assert downloader.has_symbol("xul.pdb",
"44E4EC8C2F41492B9369D6B9A059577C2",
"xul.sym")
assert len(mock_calls) == 1
assert downloader.has_symbol("xul.pdb",
"44E4EC8C2F41492B9369D6B9A059577C2",
"xul.sym")
# This should be cached
assert len(mock_calls) == 1
# Now invalidate it
downloader.invalidate_cache("xul.pdb",
"44E4EC8C2F41492B9369D6B9A059577C2",
"xul.sym")
assert downloader.has_symbol("xul.pdb",
"44E4EC8C2F41492B9369D6B9A059577C2",
"xul.sym")
assert len(mock_calls) == 2
# Invalidating unrecognized keys shouldn't break anything
downloader.invalidate_cache("never",
"44E4EC8C2F41492B9369D6B9A059577C2",
"heardof")
def test_multiple_urls_private_then_public(requestsmock, botomock):
def mock_api_call(self, operation_name, api_params):
assert operation_name == "ListObjectsV2"
if api_params["Prefix"].endswith("xxx.sym"):
# not found
return {}
# found
return {"Contents": [{"Key": api_params["Prefix"]}]}
requestsmock.head(
"https://s3.example.com/public/prefix/v0/xul.pdb/"
"44E4EC8C2F41492B9369D6B9A059577C2/xul.sym",
text="",
)
requestsmock.head(
"https://s3.example.com/public/prefix/v0/xxx.pdb/"
"44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym",
text="Page Not Found",
status_code=404,
)
urls = (
"https://s3.example.com/private/prefix/",
"https://s3.example.com/public/prefix/?access=public",
)
downloader = SymbolDownloader(urls)
with botomock(mock_api_call):
assert downloader.has_symbol("xul.pdb",
"44E4EC8C2F41492B9369D6B9A059577C2",
"xul.sym")
assert not downloader.has_symbol(
"xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym")
def test_multiple_urls_private_then_public(requestsmock, botomock):
def mock_api_call(self, operation_name, api_params):
assert operation_name == 'ListObjectsV2'
if api_params['Prefix'].endswith('xxx.sym'):
# not found
return {}
# found
return {
'Contents': [{
'Key': api_params['Prefix'],
}],
}
requestsmock.head(
'https://s3.example.com/public/prefix/v0/xul.pdb/'
'44E4EC8C2F41492B9369D6B9A059577C2/xul.sym',
text='')
requestsmock.head(
'https://s3.example.com/public/prefix/v0/xxx.pdb/'
'44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym',
text='Page Not Found',
status_code=404,
)
urls = (
'https://s3.example.com/private/prefix/',
'https://s3.example.com/public/prefix/?access=public',
)
downloader = SymbolDownloader(urls)
with botomock(mock_api_call):
assert downloader.has_symbol('xul.pdb',
'44E4EC8C2F41492B9369D6B9A059577C2',
'xul.sym')
assert not downloader.has_symbol(
'xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym')
def test_has_private_bubble_other_clienterrors(botomock):
def mock_api_call(self, operation_name, api_params):
parsed_response = {'Error': {'Code': '403', 'Message': 'Not found'}}
raise ClientError(parsed_response, operation_name)
urls = ('https://s3.example.com/private/prefix/', )
downloader = SymbolDownloader(urls)
# Expect this to raise a ClientError because the bucket ('private')
# doesn't exist. So boto3 would normally trigger a ClientError
# with a code 'Forbidden'.
with botomock(mock_api_call):
with pytest.raises(ClientError):
downloader.has_symbol('xul.pdb',
'44E4EC8C2F41492B9369D6B9A059577C2',
'xul.sym')
def test_has_private(botomock):
def mock_api_call(self, operation_name, api_params):
assert operation_name == "ListObjectsV2"
if api_params["Prefix"].endswith("xxx.sym"):
return {}
return {"Contents": [{"Key": api_params["Prefix"]}]}
urls = ("https://s3.example.com/private/prefix/", )
downloader = SymbolDownloader(urls)
with botomock(mock_api_call):
assert downloader.has_symbol("xul.pdb",
"44E4EC8C2F41492B9369D6B9A059577C2",
"xul.sym")
assert downloader.time_took > 0.0
assert not downloader.has_symbol(
"xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym")
assert downloader.time_took > 0.0
def test_has_public(requestsmock):
requestsmock.head(
'https://s3.example.com/public/prefix/v0/xul.pdb/'
'44E4EC8C2F41492B9369D6B9A059577C2/xul.sym',
text='')
requestsmock.head(
'https://s3.example.com/public/prefix/v0/xxx.pdb/'
'44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym',
text='Page Not Found',
status_code=404,
)
urls = ('https://s3.example.com/public/prefix/?access=public', )
downloader = SymbolDownloader(urls, file_prefix='v0')
assert downloader.has_symbol('xul.pdb',
'44E4EC8C2F41492B9369D6B9A059577C2',
'xul.sym')
assert not downloader.has_symbol(
'xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym')
def test_has_public_case_insensitive_debugid(requestsmock):
requestsmock.head(
'https://s3.example.com/public/prefix/v0/xul.pdb/'
'44E4EC8C2F41492B9369D6B9A059577C2/xul.sym',
text='')
urls = ('https://s3.example.com/public/prefix/?access=public', )
downloader = SymbolDownloader(urls)
assert downloader.has_symbol('xul.pdb',
'44e4ec8c2f41492b9369d6b9a059577c2',
'xul.sym')
def test_has_public(requestsmock):
requestsmock.head(
"https://s3.example.com/public/prefix/v0/xul.pdb/"
"44E4EC8C2F41492B9369D6B9A059577C2/xul.sym",
text="",
)
requestsmock.head(
"https://s3.example.com/public/prefix/v0/xxx.pdb/"
"44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym",
text="Page Not Found",
status_code=404,
)
urls = ("https://s3.example.com/public/prefix/?access=public", )
downloader = SymbolDownloader(urls, file_prefix="v0")
assert downloader.has_symbol("xul.pdb",
"44E4EC8C2F41492B9369D6B9A059577C2",
"xul.sym")
assert not downloader.has_symbol(
"xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym")
def test_has_private_without_prefix(botomock):
def mock_api_call(self, operation_name, api_params):
assert operation_name == "ListObjectsV2"
if api_params["Prefix"].endswith("xul.sym"):
# found
return {"Contents": [{"Key": api_params["Prefix"]}]}
elif api_params["Prefix"].endswith("xxx.sym"):
# not found
return {}
raise NotImplementedError(api_params)
urls = ("https://s3.example.com/private", )
downloader = SymbolDownloader(urls)
with botomock(mock_api_call):
assert downloader.has_symbol("xul.pdb",
"44E4EC8C2F41492B9369D6B9A059577C2",
"xul.sym")
assert not downloader.has_symbol(
"xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym")
def test_private_default_file_prefix(botomock, settings):
"""See doc string in test_public_default_file_prefix
"""
all_mock_calls = []
def mock_api_call(self, operation_name, api_params):
if operation_name == 'ListObjectsV2':
# the has_symbol() was called
all_mock_calls.append(api_params['Prefix'])
# pretend it doesn't exist
return {}
elif operation_name == 'GetObject':
# someone wants a stream
all_mock_calls.append(api_params['Key'])
parsed_response = {
'Error': {
'Code': 'NoSuchKey',
'Message': 'Not found'
},
}
raise ClientError(parsed_response, operation_name)
else:
raise NotImplementedError(operation_name)
urls = (
# Private URL with prefix and trailing /
'https://s3.example.com/priv-bucket/borje/',
# No trailing /
'https://s3.example.com/also-priv-bucket/prrffxx',
# No prefix
'https://s3.example.com/some-bucket',
)
downloader = SymbolDownloader(urls, file_prefix='myprfx')
with botomock(mock_api_call):
assert not downloader.has_symbol(
'xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym')
assert len(all_mock_calls) == 3
assert all_mock_calls[0].startswith('borje/myprfx/xxx.pdb')
assert all_mock_calls[1].startswith('prrffxx/myprfx/xxx.pdb')
assert all_mock_calls[2].startswith('myprfx/xxx.pdb')
# reset the mutable recorder
all_mock_calls = []
stream = downloader.get_symbol_stream(
'xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym')
with pytest.raises(SymbolNotFound):
next(stream)
assert len(all_mock_calls) == 3
assert all_mock_calls[0].startswith('borje/myprfx/xxx.pdb')
assert all_mock_calls[1].startswith('prrffxx/myprfx/xxx.pdb')
assert all_mock_calls[2].startswith('myprfx/xxx.pdb')
def test_private_default_file_prefix(botomock, settings):
"""See doc string in test_public_default_file_prefix
"""
all_mock_calls = []
def mock_api_call(self, operation_name, api_params):
if operation_name == "ListObjectsV2":
# the has_symbol() was called
all_mock_calls.append(api_params["Prefix"])
# pretend it doesn't exist
return {}
elif operation_name == "GetObject":
# someone wants a stream
all_mock_calls.append(api_params["Key"])
parsed_response = {
"Error": {
"Code": "NoSuchKey",
"Message": "Not found"
}
}
raise ClientError(parsed_response, operation_name)
else:
raise NotImplementedError(operation_name)
urls = (
# Private URL with prefix and trailing /
"https://s3.example.com/priv-bucket/borje/",
# No trailing /
"https://s3.example.com/also-priv-bucket/prrffxx",
# No prefix
"https://s3.example.com/some-bucket",
)
downloader = SymbolDownloader(urls, file_prefix="myprfx")
with botomock(mock_api_call):
assert not downloader.has_symbol(
"xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym")
assert len(all_mock_calls) == 3
assert all_mock_calls[0].startswith("borje/myprfx/xxx.pdb")
assert all_mock_calls[1].startswith("prrffxx/myprfx/xxx.pdb")
assert all_mock_calls[2].startswith("myprfx/xxx.pdb")
# reset the mutable recorder
all_mock_calls = []
stream = downloader.get_symbol_stream(
"xxx.pdb", "44E4EC8C2F41492B9369D6B9A059577C2", "xxx.sym")
with pytest.raises(SymbolNotFound):
next(stream)
assert len(all_mock_calls) == 3
assert all_mock_calls[0].startswith("borje/myprfx/xxx.pdb")
assert all_mock_calls[1].startswith("prrffxx/myprfx/xxx.pdb")
assert all_mock_calls[2].startswith("myprfx/xxx.pdb")
def test_has_public_case_insensitive_debugid(requestsmock):
requestsmock.head(
"https://s3.example.com/public/prefix/v0/xul.pdb/"
"44E4EC8C2F41492B9369D6B9A059577C2/xul.sym",
text="",
)
urls = ("https://s3.example.com/public/prefix/?access=public", )
downloader = SymbolDownloader(urls)
assert downloader.has_symbol("xul.pdb",
"44e4ec8c2f41492b9369d6b9a059577c2",
"xul.sym")
def test_has_private(botomock):
def mock_api_call(self, operation_name, api_params):
assert operation_name == 'ListObjectsV2'
if api_params['Prefix'].endswith('xxx.sym'):
return {}
return {
'Contents': [{
'Key': api_params['Prefix'],
}],
}
urls = ('https://s3.example.com/private/prefix/', )
downloader = SymbolDownloader(urls)
with botomock(mock_api_call):
assert downloader.has_symbol('xul.pdb',
'44E4EC8C2F41492B9369D6B9A059577C2',
'xul.sym')
assert downloader.time_took > 0.0
assert not downloader.has_symbol(
'xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym')
assert downloader.time_took > 0.0
def test_has_private_caching_and_invalidation(botomock):
mock_calls = []
def mock_api_call(self, operation_name, api_params):
assert operation_name == 'ListObjectsV2'
mock_calls.append(api_params['Prefix'])
return {
'Contents': [{
'Key': api_params['Prefix'],
}],
}
urls = ('https://s3.example.com/private/prefix/', )
downloader = SymbolDownloader(urls)
with botomock(mock_api_call):
assert downloader.has_symbol('xul.pdb',
'44E4EC8C2F41492B9369D6B9A059577C2',
'xul.sym')
assert len(mock_calls) == 1
assert downloader.has_symbol('xul.pdb',
'44E4EC8C2F41492B9369D6B9A059577C2',
'xul.sym')
# This should be cached
assert len(mock_calls) == 1
# Now invalidate it
downloader.invalidate_cache('xul.pdb',
'44E4EC8C2F41492B9369D6B9A059577C2',
'xul.sym')
assert downloader.has_symbol('xul.pdb',
'44E4EC8C2F41492B9369D6B9A059577C2',
'xul.sym')
assert len(mock_calls) == 2
# Invalidating unrecognized keys shouldn't break anything
downloader.invalidate_cache('never',
'44E4EC8C2F41492B9369D6B9A059577C2',
'heardof')
def test_has_private_case_insensitive_debugid(botomock):
def mock_api_call(self, operation_name, api_params):
assert operation_name == "ListObjectsV2"
assert "44E4EC8C2F41492B9369D6B9A059577C2" in api_params["Prefix"]
# found
return {"Contents": [{"Key": api_params["Prefix"]}]}
urls = ("https://s3.example.com/private/prefix/", )
downloader = SymbolDownloader(urls)
with botomock(mock_api_call):
assert downloader.has_symbol("xul.pdb",
"44e4ec8c2f41492b9369d6b9a059577c2",
"xul.sym")
def test_has_private_without_prefix(botomock):
def mock_api_call(self, operation_name, api_params):
assert operation_name == 'ListObjectsV2'
if api_params['Prefix'].endswith('xul.sym'):
# found
return {
'Contents': [{
'Key': api_params['Prefix'],
}],
}
elif api_params['Prefix'].endswith('xxx.sym'):
# not found
return {}
raise NotImplementedError(api_params)
urls = ('https://s3.example.com/private', )
downloader = SymbolDownloader(urls)
with botomock(mock_api_call):
assert downloader.has_symbol('xul.pdb',
'44E4EC8C2F41492B9369D6B9A059577C2',
'xul.sym')
assert not downloader.has_symbol(
'xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym')
def test_has_private_case_insensitive_debugid(botomock):
def mock_api_call(self, operation_name, api_params):
assert operation_name == 'ListObjectsV2'
assert '44E4EC8C2F41492B9369D6B9A059577C2' in api_params['Prefix']
# found
return {
'Contents': [{
'Key': api_params['Prefix'],
}],
}
urls = ('https://s3.example.com/private/prefix/', )
downloader = SymbolDownloader(urls)
with botomock(mock_api_call):
assert downloader.has_symbol('xul.pdb',
'44e4ec8c2f41492b9369d6b9a059577c2',
'xul.sym')
def test_public_default_file_prefix(requestsmock, settings):
"""The idea with settings.SYMBOL_FILE_PREFIX is to make it easier
to specify the settings.SYMBOL_URLS. That settings.SYMBOL_FILE_PREFIX
is *always* used when uploading symbols. So it's *always* useful to
query for symbols with a prefix. However, it's an easy mistake to make
that you just focus on the bucket name to say where symbols come from.
In those cases, the code should "protect" you can make sure we actually
use the prefix.
However, we don't want to lose the flexibility to actually override
it on a *per URL* basis.
"""
# settings.SYMBOL_FILE_PREFIX = 'myprfx'
requestsmock.head(
'https://s3.example.com/public/start/myprfx/xxx.pdb/'
'44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym',
text='Page Not Found',
status_code=404,
)
requestsmock.head(
'https://s3.example.com/also-public/prrffxx/myprfx/xxx.pdb/'
'44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym',
text='Page Not Found',
status_code=404,
)
requestsmock.head(
'https://s3.example.com/special/myprfx/xxx.pdb/'
'44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym',
text='Page Not Found',
status_code=404,
)
urls = (
'https://s3.example.com/public/start/?access=public',
# No trailing / in the path part
'https://s3.example.com/also-public/prrffxx?access=public',
# No prefix!
'https://s3.example.com/special?access=public',
)
downloader = SymbolDownloader(urls, file_prefix='myprfx')
assert not downloader.has_symbol(
'xxx.pdb', '44E4EC8C2F41492B9369D6B9A059577C2', 'xxx.sym')
requestsmock.get(
'https://s3.example.com/public/start/myprfx/xxx.pdb/'
'44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym',
text='Page Not Found',
status_code=404,
)
requestsmock.get(
'https://s3.example.com/also-public/prrffxx/myprfx/xxx.pdb/'
'44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym',
text='Page Not Found',
status_code=404,
)
requestsmock.get(
'https://s3.example.com/special/myprfx/xxx.pdb/'
'44E4EC8C2F41492B9369D6B9A059577C2/xxx.sym',
text='Page Not Found',
status_code=404,
)
stream = downloader.get_symbol_stream('xxx.pdb',
'44E4EC8C2F41492B9369D6B9A059577C2',
'xxx.sym')
# Now try to stream it
with pytest.raises(SymbolNotFound):
list(stream)