def test_download_microsoft_symbol_task_dump_syms_failing(
botomock,
settings,
metricsmock,
requestsmock,
):
settings.DUMP_SYMS_PATH = FAKE_BROKEN_DUMP_SYMS
with open(PD__FILE, 'rb') as f:
content = f.read()
# just checking that the fixture file is sane
assert content.startswith(b'MSCF')
requestsmock.get(
'https://msdl.microsoft.com/download/symbols/ksproxy.pdb'
'/A7D6F1BB18CD4CB48/ksproxy.pd_',
content=content)
def mock_api_call(self, operation_name, api_params):
raise NotImplementedError((operation_name, api_params))
symbol = 'ksproxy.pdb'
debugid = 'A7D6F1BB18CD4CB48'
with botomock(mock_api_call):
with pytest.raises(DumpSymsError):
download_microsoft_symbol(symbol, debugid)
download_obj, = MicrosoftDownload.objects.all()
assert 'dump_syms extraction failed' in download_obj.error
assert 'Something horrible happened' in download_obj.error
def test_download_microsoft_symbol_task_dump_syms_failing(
botomock, settings, requestsmock):
settings.DUMP_SYMS_PATH = FAKE_BROKEN_DUMP_SYMS
with open(PD__FILE, "rb") as f:
content = f.read()
# just checking that the fixture file is sane
assert content.startswith(b"MSCF")
requestsmock.get(
"https://msdl.microsoft.com/download/symbols/ksproxy.pdb"
"/A7D6F1BB18CD4CB48/ksproxy.pd_",
content=content,
)
def mock_api_call(self, operation_name, api_params):
raise NotImplementedError((operation_name, api_params))
symbol = "ksproxy.pdb"
debugid = "A7D6F1BB18CD4CB48"
with pytest.raises(DumpSymsError), botomock(mock_api_call):
download_microsoft_symbol(symbol, debugid)
(download_obj, ) = MicrosoftDownload.objects.all()
assert "dump_syms extraction failed" in download_obj.error
assert "Something horrible happened" in download_obj.error
def test_download_microsoft_symbol_task_not_found(gcsmock, metricsmock,
requestsmock):
requestsmock.get(
"https://msdl.microsoft.com/download/symbols/ksproxy.pdb"
"/A7D6F1BB18CD4CB48/ksproxy.pd_",
content=b"Page Not Found",
status_code=404,
)
symbol = "ksproxy.pdb"
debugid = "A7D6F1BB18CD4CB48"
download_microsoft_symbol(symbol, debugid)
assert not FileUpload.objects.all().exists()
assert not MicrosoftDownload.objects.all().exists()
def test_download_microsoft_symbol_task_cabextract_failing(
gcsmock, metricsmock, requestsmock):
requestsmock.get(
"https://msdl.microsoft.com/download/symbols/ksproxy.pdb"
"/A7D6F1BB18CD4CB48/ksproxy.pd_",
content=b"MSCF but not a real binary",
)
symbol = "ksproxy.pdb"
debugid = "A7D6F1BB18CD4CB48"
download_microsoft_symbol(symbol, debugid)
assert not FileUpload.objects.all().exists()
download_obj, = MicrosoftDownload.objects.all()
assert "cabextract failed" in download_obj.error
def test_download_microsoft_symbol_task_wrong_file_header(
gcsmock, metricsmock, requestsmock):
requestsmock.get(
"https://msdl.microsoft.com/download/symbols/ksproxy.pdb"
"/A7D6F1BB18CD4CB48/ksproxy.pd_",
content=b"some other junk",
)
symbol = "ksproxy.pdb"
debugid = "A7D6F1BB18CD4CB48"
download_microsoft_symbol(symbol, debugid)
assert not FileUpload.objects.all().exists()
download_obj, = MicrosoftDownload.objects.all()
assert "did not start with 'MSCF'" in download_obj.error
def test_download_microsoft_symbol_task_skipped(gcsmock, metricsmock,
requestsmock):
with open(PD__FILE, "rb") as f:
content = f.read()
# just checking that the fixture file is sane
assert content.startswith(b"MSCF")
requestsmock.get(
"https://msdl.microsoft.com/download/symbols/ksproxy.pdb"
"/A7D6F1BB18CD4CB48/ksproxy.pd_",
content=content,
)
mock_bucket = gcsmock.MockBucket()
def mock_get_bucket(bucket_name):
assert bucket_name == "private"
return mock_bucket
gcsmock.get_bucket = mock_get_bucket
def mock_get_blob(key):
if key == "v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym":
return gcsmock.mock_blob_factory(key, size=729)
raise NotImplementedError(key)
mock_bucket.get_blob = mock_get_blob
def mocked_create_blob(key):
# Nothing should be uploaded.
raise NotImplementedError(key)
mock_bucket.blob = mocked_create_blob
symbol = "ksproxy.pdb"
debugid = "A7D6F1BB18CD4CB48"
download_microsoft_symbol(symbol, debugid)
download_obj, = MicrosoftDownload.objects.all()
assert not download_obj.error
assert download_obj.skipped
assert download_obj.completed_at
# The ultimate test is that it should NOT have created a file upload.
assert not FileUpload.objects.all().exists()
# Check that markus caught timings of the individual file processing
metricsmock.has_record(INCR, "tecken.microsoft_download_file_upload_skip",
1, None)
def test_download_microsoft_symbol_task_not_found(botomock, requestsmock):
requestsmock.get(
"https://msdl.microsoft.com/download/symbols/ksproxy.pdb"
"/A7D6F1BB18CD4CB48/ksproxy.pd_",
content=b"Page Not Found",
status_code=404,
)
def mock_api_call(self, operation_name, api_params):
raise NotImplementedError((operation_name, api_params))
symbol = "ksproxy.pdb"
debugid = "A7D6F1BB18CD4CB48"
with botomock(mock_api_call):
download_microsoft_symbol(symbol, debugid)
assert not FileUpload.objects.all().exists()
assert not MicrosoftDownload.objects.all().exists()
def test_download_microsoft_symbol_task_cabextract_failing(
botomock, requestsmock):
requestsmock.get(
"https://msdl.microsoft.com/download/symbols/ksproxy.pdb"
"/A7D6F1BB18CD4CB48/ksproxy.pd_",
content=b"MSCF but not a real binary",
)
def mock_api_call(self, operation_name, api_params):
raise NotImplementedError((operation_name, api_params))
symbol = "ksproxy.pdb"
debugid = "A7D6F1BB18CD4CB48"
with botomock(mock_api_call):
download_microsoft_symbol(symbol, debugid)
assert not FileUpload.objects.all().exists()
(download_obj, ) = MicrosoftDownload.objects.all()
assert "cabextract failed" in download_obj.error
def test_download_microsoft_symbol_task_wrong_file_header(
botomock, requestsmock):
requestsmock.get(
"https://msdl.microsoft.com/download/symbols/ksproxy.pdb"
"/A7D6F1BB18CD4CB48/ksproxy.pd_",
content=b"some other junk",
)
def mock_api_call(self, operation_name, api_params):
raise NotImplementedError((operation_name, api_params))
symbol = "ksproxy.pdb"
debugid = "A7D6F1BB18CD4CB48"
with botomock(mock_api_call):
download_microsoft_symbol(symbol, debugid)
assert not FileUpload.objects.all().exists()
(download_obj, ) = MicrosoftDownload.objects.all()
assert "did not start with 'MSCF'" in download_obj.error
def test_download_microsoft_symbol_task_skipped(
botomock,
metricsmock,
requestsmock,
):
with open(PD__FILE, 'rb') as f:
content = f.read()
# just checking that the fixture file is sane
assert content.startswith(b'MSCF')
requestsmock.get(
'https://msdl.microsoft.com/download/symbols/ksproxy.pdb'
'/A7D6F1BB18CD4CB48/ksproxy.pd_',
content=content)
def mock_api_call(self, operation_name, api_params):
# this comes from the UPLOAD_DEFAULT_URL in settings.Test
assert api_params['Bucket'] == 'private'
if (operation_name == 'HeadObject' and api_params['Key']
== ('v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym')):
return {'ContentLength': 729}
raise NotImplementedError((operation_name, api_params))
symbol = 'ksproxy.pdb'
debugid = 'A7D6F1BB18CD4CB48'
with botomock(mock_api_call):
download_microsoft_symbol(symbol, debugid)
download_obj, = MicrosoftDownload.objects.all()
assert not download_obj.error
assert download_obj.skipped
assert download_obj.completed_at
# The ultimate test is that it should NOT have created a file upload
assert not FileUpload.objects.all().exists()
# Check that markus caught timings of the individual file processing
metricsmock.has_record(INCR, 'tecken.microsoft_download_file_upload_skip',
1, None)
def test_download_microsoft_symbol_task_cabextract_failing(
botomock,
metricsmock,
requestsmock,
):
requestsmock.get(
'https://msdl.microsoft.com/download/symbols/ksproxy.pdb'
'/A7D6F1BB18CD4CB48/ksproxy.pd_',
content=b'MSCF but not a real binary',
)
def mock_api_call(self, operation_name, api_params):
raise NotImplementedError((operation_name, api_params))
symbol = 'ksproxy.pdb'
debugid = 'A7D6F1BB18CD4CB48'
with botomock(mock_api_call):
download_microsoft_symbol(symbol, debugid)
assert not FileUpload.objects.all().exists()
download_obj, = MicrosoftDownload.objects.all()
assert 'cabextract failed' in download_obj.error
def test_download_microsoft_symbol_task_skipped(botomock, metricsmock,
requestsmock):
with open(PD__FILE, "rb") as f:
content = f.read()
# just checking that the fixture file is sane
assert content.startswith(b"MSCF")
requestsmock.get(
"https://msdl.microsoft.com/download/symbols/ksproxy.pdb"
"/A7D6F1BB18CD4CB48/ksproxy.pd_",
content=content,
)
def mock_api_call(self, operation_name, api_params):
# this comes from the UPLOAD_DEFAULT_URL in settings.Test
assert api_params["Bucket"] == "private"
if operation_name == "HeadObject" and api_params["Key"] == (
"v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym"):
return {"ContentLength": 729}
raise NotImplementedError((operation_name, api_params))
symbol = "ksproxy.pdb"
debugid = "A7D6F1BB18CD4CB48"
with botomock(mock_api_call):
download_microsoft_symbol(symbol, debugid)
download_obj, = MicrosoftDownload.objects.all()
assert not download_obj.error
assert download_obj.skipped
assert download_obj.completed_at
# The ultimate test is that it should NOT have created a file upload
assert not FileUpload.objects.all().exists()
# Check that markus caught timings of the individual file processing
metricsmock.has_record(INCR, "tecken.microsoft_download_file_upload_skip",
1, None)
def test_download_microsoft_symbol_task_happy_path(botomock, metricsmock,
requestsmock):
with open(PD__FILE, "rb") as f:
content = f.read()
# just checking that the fixture file is sane
assert content.startswith(b"MSCF")
requestsmock.get(
"https://msdl.microsoft.com/download/symbols/ksproxy.pdb"
"/A7D6F1BB18CD4CB48/ksproxy.pd_",
content=content,
)
files_uploaded = []
def mock_api_call(self, operation_name, api_params):
# this comes from the UPLOAD_DEFAULT_URL in settings.Test
assert api_params["Bucket"] == "private"
if operation_name == "HeadObject" and api_params["Key"] == (
"v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym"):
# Pretend we've never heard of this
parsed_response = {
"Error": {
"Code": "404",
"Message": "Not found"
}
}
raise ClientError(parsed_response, operation_name)
if operation_name == "PutObject" and api_params["Key"] == (
"v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym"):
# Because .sym is in settings.COMPRESS_EXTENSIONS
assert api_params["ContentEncoding"] == "gzip"
# Because .sym is in settings.MIME_OVERRIDES
assert api_params["ContentType"] == "text/plain"
content = api_params["Body"].read()
assert isinstance(content, bytes)
# We know what the expected size is based on having run:
# $ cabextract ksproxy.pd_
# $ dump_syms ksproxy.pdb > ksproxy.sym
# $ ls -l ksproxy.sym
# 1721
assert len(content) == 729
original_content = gzip.decompress(content)
assert len(original_content) == 1721
# ...pretend to actually upload it.
files_uploaded.append(api_params["Key"])
return {}
raise NotImplementedError((operation_name, api_params))
symbol = "ksproxy.pdb"
debugid = "A7D6F1BB18CD4CB48"
with botomock(mock_api_call):
download_microsoft_symbol(symbol, debugid)
assert files_uploaded == ["v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym"]
# The ultimate test is that it should have created a file_upload
(file_upload, ) = FileUpload.objects.all()
assert file_upload.size == 729
assert file_upload.bucket_name == "private"
assert file_upload.key == "v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym"
assert not file_upload.update
assert not file_upload.upload
assert file_upload.compressed
assert file_upload.completed_at
assert file_upload.microsoft_download
# It should also have created a MicrosoftDownload record.
(download_obj, ) = MicrosoftDownload.objects.all()
assert download_obj.completed_at
assert download_obj.skipped is False
assert download_obj.error is None
# Check that markus caught timings of the individual file processing
records = [rec[1] for rec in metricsmock.get_records()]
assert len(records) == 10
assert records == [
"tecken.download_store_missing_symbol",
"tecken.download_cabextract",
"tecken.download_dump_syms",
"tecken.upload_file_exists",
"tecken.upload_gzip_payload",
"tecken.upload_put_object",
"tecken.upload_file_upload_upload",
"tecken.upload_file_upload",
"tecken.download_microsoft_download_file_upload_upload",
"tecken.download_upload_microsoft_symbol",
]
def test_download_microsoft_symbol_task_happy_path(
botomock,
metricsmock,
requestsmock,
):
with open(PD__FILE, 'rb') as f:
content = f.read()
# just checking that the fixture file is sane
assert content.startswith(b'MSCF')
requestsmock.get(
'https://msdl.microsoft.com/download/symbols/ksproxy.pdb'
'/A7D6F1BB18CD4CB48/ksproxy.pd_',
content=content)
def mock_api_call(self, operation_name, api_params):
# this comes from the UPLOAD_DEFAULT_URL in settings.Test
assert api_params['Bucket'] == 'private'
if (operation_name == 'HeadObject' and api_params['Key']
== ('v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym')):
# Pretend we've never heard of this
parsed_response = {
'Error': {
'Code': '404',
'Message': 'Not found'
},
}
raise ClientError(parsed_response, operation_name)
if (operation_name == 'PutObject' and api_params['Key']
== ('v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym')):
# Because .sym is in settings.COMPRESS_EXTENSIONS
assert api_params['ContentEncoding'] == 'gzip'
# Because .sym is in settings.MIME_OVERRIDES
assert api_params['ContentType'] == 'text/plain'
content = api_params['Body'].read()
assert isinstance(content, bytes)
# We know what the expected size is based on having run:
# $ cabextract ksproxy.pd_
# $ dump_syms ksproxy.pdb > ksproxy.sym
# $ ls -l ksproxy.sym
# 1721
assert len(content) == 729
original_content = gzip.decompress(content)
assert len(original_content) == 1721
# ...pretend to actually upload it.
return {
# Should there be anything here?
}
raise NotImplementedError((operation_name, api_params))
symbol = 'ksproxy.pdb'
debugid = 'A7D6F1BB18CD4CB48'
with botomock(mock_api_call):
download_microsoft_symbol(symbol, debugid)
# The ultimate test is that it should have created a file_upload
file_upload, = FileUpload.objects.all()
assert file_upload.size == 729
assert file_upload.bucket_name == 'private'
assert file_upload.key == 'v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym'
assert not file_upload.update
assert not file_upload.upload
assert file_upload.compressed
assert file_upload.completed_at
assert file_upload.microsoft_download
# It should also have created a MicrosoftDownload record.
download_obj, = MicrosoftDownload.objects.all()
assert download_obj.completed_at
assert download_obj.skipped is False
assert download_obj.error is None
# Check that markus caught timings of the individual file processing
records = metricsmock.get_records()
assert len(records) == 10
assert records[0][1] == 'tecken.download_store_missing_symbol'
assert records[1][1] == 'tecken.download_cabextract'
assert records[2][1] == 'tecken.download_dump_syms'
assert records[3][1] == 'tecken.upload_file_exists'
assert records[4][1] == 'tecken.upload_gzip_payload'
assert records[5][1] == 'tecken.upload_put_object'
assert records[6][1] == 'tecken.upload_file_upload_upload'
assert records[7][1] == 'tecken.upload_file_upload'
assert records[8][1] == (
'tecken.download_microsoft_download_file_upload_upload')
assert records[9][1] == 'tecken.download_upload_microsoft_symbol'
def test_download_microsoft_symbol_task_happy_path(gcsmock, metricsmock,
requestsmock):
with open(PD__FILE, "rb") as f:
content = f.read()
# just checking that the fixture file is sane
assert content.startswith(b"MSCF")
requestsmock.get(
"https://msdl.microsoft.com/download/symbols/ksproxy.pdb"
"/A7D6F1BB18CD4CB48/ksproxy.pd_",
content=content,
)
mock_bucket = gcsmock.MockBucket()
def mock_get_bucket(bucket_name):
assert bucket_name == "private"
return mock_bucket
gcsmock.get_bucket = mock_get_bucket
def mock_get_blob(key):
if key == "v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym":
# As if it doesn't exist
return None
raise NotImplementedError(key)
mock_bucket.get_blob = mock_get_blob
files_uploaded = []
def mocked_create_blob(key):
if key == "v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym":
blob = gcsmock.MockBlob(key)
def mock_upload_from_file(file):
files_uploaded.append(key)
content = file.read()
assert isinstance(content, bytes)
# We know what the expected size is based on having run:
# $ cabextract ksproxy.pd_
# $ dump_syms ksproxy.pdb > ksproxy.sym
# $ ls -l ksproxy.sym
# 1721
assert len(content) == 729
original_content = gzip.decompress(content)
assert len(original_content) == 1721
blob.upload_from_file = mock_upload_from_file
return blob
raise NotImplementedError(key)
mock_bucket.blob = mocked_create_blob
symbol = "ksproxy.pdb"
debugid = "A7D6F1BB18CD4CB48"
download_microsoft_symbol(symbol, debugid)
assert files_uploaded == ["v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym"]
# The ultimate test is that it should have created a file_upload
file_upload, = FileUpload.objects.all()
assert file_upload.size == 729
assert file_upload.bucket_name == "private"
assert file_upload.key == "v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym"
assert not file_upload.update
assert not file_upload.upload
assert file_upload.compressed
assert file_upload.completed_at
assert file_upload.microsoft_download
# It should also have created a MicrosoftDownload record.
download_obj, = MicrosoftDownload.objects.all()
assert download_obj.completed_at
assert download_obj.skipped is False
assert download_obj.error is None
# Check that markus caught timings of the individual file processing
records = metricsmock.get_records()
assert len(records) == 10
assert records[0][1] == "tecken.download_store_missing_symbol"
assert records[1][1] == "tecken.download_cabextract"
assert records[2][1] == "tecken.download_dump_syms"
assert records[3][1] == "tecken.upload_file_exists"
assert records[4][1] == "tecken.upload_gzip_payload"
assert records[5][1] == "tecken.upload_put_object"
assert records[6][1] == "tecken.upload_file_upload_upload"
assert records[7][1] == "tecken.upload_file_upload"
assert records[8][1] == (
"tecken.download_microsoft_download_file_upload_upload")
assert records[9][1] == "tecken.download_upload_microsoft_symbol"