def test_download_microsoft_symbol_task_dump_syms_failing( botomock, settings, metricsmock, requestsmock, ): settings.DUMP_SYMS_PATH = FAKE_BROKEN_DUMP_SYMS with open(PD__FILE, 'rb') as f: content = f.read() # just checking that the fixture file is sane assert content.startswith(b'MSCF') requestsmock.get( 'https://msdl.microsoft.com/download/symbols/ksproxy.pdb' '/A7D6F1BB18CD4CB48/ksproxy.pd_', content=content) def mock_api_call(self, operation_name, api_params): raise NotImplementedError((operation_name, api_params)) symbol = 'ksproxy.pdb' debugid = 'A7D6F1BB18CD4CB48' with botomock(mock_api_call): with pytest.raises(DumpSymsError): download_microsoft_symbol(symbol, debugid) download_obj, = MicrosoftDownload.objects.all() assert 'dump_syms extraction failed' in download_obj.error assert 'Something horrible happened' in download_obj.error
def test_download_microsoft_symbol_task_dump_syms_failing( botomock, settings, requestsmock): settings.DUMP_SYMS_PATH = FAKE_BROKEN_DUMP_SYMS with open(PD__FILE, "rb") as f: content = f.read() # just checking that the fixture file is sane assert content.startswith(b"MSCF") requestsmock.get( "https://msdl.microsoft.com/download/symbols/ksproxy.pdb" "/A7D6F1BB18CD4CB48/ksproxy.pd_", content=content, ) def mock_api_call(self, operation_name, api_params): raise NotImplementedError((operation_name, api_params)) symbol = "ksproxy.pdb" debugid = "A7D6F1BB18CD4CB48" with pytest.raises(DumpSymsError), botomock(mock_api_call): download_microsoft_symbol(symbol, debugid) (download_obj, ) = MicrosoftDownload.objects.all() assert "dump_syms extraction failed" in download_obj.error assert "Something horrible happened" in download_obj.error
def test_download_microsoft_symbol_task_not_found(gcsmock, metricsmock, requestsmock): requestsmock.get( "https://msdl.microsoft.com/download/symbols/ksproxy.pdb" "/A7D6F1BB18CD4CB48/ksproxy.pd_", content=b"Page Not Found", status_code=404, ) symbol = "ksproxy.pdb" debugid = "A7D6F1BB18CD4CB48" download_microsoft_symbol(symbol, debugid) assert not FileUpload.objects.all().exists() assert not MicrosoftDownload.objects.all().exists()
def test_download_microsoft_symbol_task_cabextract_failing( gcsmock, metricsmock, requestsmock): requestsmock.get( "https://msdl.microsoft.com/download/symbols/ksproxy.pdb" "/A7D6F1BB18CD4CB48/ksproxy.pd_", content=b"MSCF but not a real binary", ) symbol = "ksproxy.pdb" debugid = "A7D6F1BB18CD4CB48" download_microsoft_symbol(symbol, debugid) assert not FileUpload.objects.all().exists() download_obj, = MicrosoftDownload.objects.all() assert "cabextract failed" in download_obj.error
def test_download_microsoft_symbol_task_wrong_file_header( gcsmock, metricsmock, requestsmock): requestsmock.get( "https://msdl.microsoft.com/download/symbols/ksproxy.pdb" "/A7D6F1BB18CD4CB48/ksproxy.pd_", content=b"some other junk", ) symbol = "ksproxy.pdb" debugid = "A7D6F1BB18CD4CB48" download_microsoft_symbol(symbol, debugid) assert not FileUpload.objects.all().exists() download_obj, = MicrosoftDownload.objects.all() assert "did not start with 'MSCF'" in download_obj.error
def test_download_microsoft_symbol_task_skipped(gcsmock, metricsmock, requestsmock): with open(PD__FILE, "rb") as f: content = f.read() # just checking that the fixture file is sane assert content.startswith(b"MSCF") requestsmock.get( "https://msdl.microsoft.com/download/symbols/ksproxy.pdb" "/A7D6F1BB18CD4CB48/ksproxy.pd_", content=content, ) mock_bucket = gcsmock.MockBucket() def mock_get_bucket(bucket_name): assert bucket_name == "private" return mock_bucket gcsmock.get_bucket = mock_get_bucket def mock_get_blob(key): if key == "v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym": return gcsmock.mock_blob_factory(key, size=729) raise NotImplementedError(key) mock_bucket.get_blob = mock_get_blob def mocked_create_blob(key): # Nothing should be uploaded. raise NotImplementedError(key) mock_bucket.blob = mocked_create_blob symbol = "ksproxy.pdb" debugid = "A7D6F1BB18CD4CB48" download_microsoft_symbol(symbol, debugid) download_obj, = MicrosoftDownload.objects.all() assert not download_obj.error assert download_obj.skipped assert download_obj.completed_at # The ultimate test is that it should NOT have created a file upload. assert not FileUpload.objects.all().exists() # Check that markus caught timings of the individual file processing metricsmock.has_record(INCR, "tecken.microsoft_download_file_upload_skip", 1, None)
def test_download_microsoft_symbol_task_not_found(botomock, requestsmock): requestsmock.get( "https://msdl.microsoft.com/download/symbols/ksproxy.pdb" "/A7D6F1BB18CD4CB48/ksproxy.pd_", content=b"Page Not Found", status_code=404, ) def mock_api_call(self, operation_name, api_params): raise NotImplementedError((operation_name, api_params)) symbol = "ksproxy.pdb" debugid = "A7D6F1BB18CD4CB48" with botomock(mock_api_call): download_microsoft_symbol(symbol, debugid) assert not FileUpload.objects.all().exists() assert not MicrosoftDownload.objects.all().exists()
def test_download_microsoft_symbol_task_cabextract_failing( botomock, requestsmock): requestsmock.get( "https://msdl.microsoft.com/download/symbols/ksproxy.pdb" "/A7D6F1BB18CD4CB48/ksproxy.pd_", content=b"MSCF but not a real binary", ) def mock_api_call(self, operation_name, api_params): raise NotImplementedError((operation_name, api_params)) symbol = "ksproxy.pdb" debugid = "A7D6F1BB18CD4CB48" with botomock(mock_api_call): download_microsoft_symbol(symbol, debugid) assert not FileUpload.objects.all().exists() (download_obj, ) = MicrosoftDownload.objects.all() assert "cabextract failed" in download_obj.error
def test_download_microsoft_symbol_task_wrong_file_header( botomock, requestsmock): requestsmock.get( "https://msdl.microsoft.com/download/symbols/ksproxy.pdb" "/A7D6F1BB18CD4CB48/ksproxy.pd_", content=b"some other junk", ) def mock_api_call(self, operation_name, api_params): raise NotImplementedError((operation_name, api_params)) symbol = "ksproxy.pdb" debugid = "A7D6F1BB18CD4CB48" with botomock(mock_api_call): download_microsoft_symbol(symbol, debugid) assert not FileUpload.objects.all().exists() (download_obj, ) = MicrosoftDownload.objects.all() assert "did not start with 'MSCF'" in download_obj.error
def test_download_microsoft_symbol_task_skipped( botomock, metricsmock, requestsmock, ): with open(PD__FILE, 'rb') as f: content = f.read() # just checking that the fixture file is sane assert content.startswith(b'MSCF') requestsmock.get( 'https://msdl.microsoft.com/download/symbols/ksproxy.pdb' '/A7D6F1BB18CD4CB48/ksproxy.pd_', content=content) def mock_api_call(self, operation_name, api_params): # this comes from the UPLOAD_DEFAULT_URL in settings.Test assert api_params['Bucket'] == 'private' if (operation_name == 'HeadObject' and api_params['Key'] == ('v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym')): return {'ContentLength': 729} raise NotImplementedError((operation_name, api_params)) symbol = 'ksproxy.pdb' debugid = 'A7D6F1BB18CD4CB48' with botomock(mock_api_call): download_microsoft_symbol(symbol, debugid) download_obj, = MicrosoftDownload.objects.all() assert not download_obj.error assert download_obj.skipped assert download_obj.completed_at # The ultimate test is that it should NOT have created a file upload assert not FileUpload.objects.all().exists() # Check that markus caught timings of the individual file processing metricsmock.has_record(INCR, 'tecken.microsoft_download_file_upload_skip', 1, None)
def test_download_microsoft_symbol_task_cabextract_failing( botomock, metricsmock, requestsmock, ): requestsmock.get( 'https://msdl.microsoft.com/download/symbols/ksproxy.pdb' '/A7D6F1BB18CD4CB48/ksproxy.pd_', content=b'MSCF but not a real binary', ) def mock_api_call(self, operation_name, api_params): raise NotImplementedError((operation_name, api_params)) symbol = 'ksproxy.pdb' debugid = 'A7D6F1BB18CD4CB48' with botomock(mock_api_call): download_microsoft_symbol(symbol, debugid) assert not FileUpload.objects.all().exists() download_obj, = MicrosoftDownload.objects.all() assert 'cabextract failed' in download_obj.error
def test_download_microsoft_symbol_task_skipped(botomock, metricsmock, requestsmock): with open(PD__FILE, "rb") as f: content = f.read() # just checking that the fixture file is sane assert content.startswith(b"MSCF") requestsmock.get( "https://msdl.microsoft.com/download/symbols/ksproxy.pdb" "/A7D6F1BB18CD4CB48/ksproxy.pd_", content=content, ) def mock_api_call(self, operation_name, api_params): # this comes from the UPLOAD_DEFAULT_URL in settings.Test assert api_params["Bucket"] == "private" if operation_name == "HeadObject" and api_params["Key"] == ( "v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym"): return {"ContentLength": 729} raise NotImplementedError((operation_name, api_params)) symbol = "ksproxy.pdb" debugid = "A7D6F1BB18CD4CB48" with botomock(mock_api_call): download_microsoft_symbol(symbol, debugid) download_obj, = MicrosoftDownload.objects.all() assert not download_obj.error assert download_obj.skipped assert download_obj.completed_at # The ultimate test is that it should NOT have created a file upload assert not FileUpload.objects.all().exists() # Check that markus caught timings of the individual file processing metricsmock.has_record(INCR, "tecken.microsoft_download_file_upload_skip", 1, None)
def test_download_microsoft_symbol_task_happy_path(botomock, metricsmock, requestsmock): with open(PD__FILE, "rb") as f: content = f.read() # just checking that the fixture file is sane assert content.startswith(b"MSCF") requestsmock.get( "https://msdl.microsoft.com/download/symbols/ksproxy.pdb" "/A7D6F1BB18CD4CB48/ksproxy.pd_", content=content, ) files_uploaded = [] def mock_api_call(self, operation_name, api_params): # this comes from the UPLOAD_DEFAULT_URL in settings.Test assert api_params["Bucket"] == "private" if operation_name == "HeadObject" and api_params["Key"] == ( "v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym"): # Pretend we've never heard of this parsed_response = { "Error": { "Code": "404", "Message": "Not found" } } raise ClientError(parsed_response, operation_name) if operation_name == "PutObject" and api_params["Key"] == ( "v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym"): # Because .sym is in settings.COMPRESS_EXTENSIONS assert api_params["ContentEncoding"] == "gzip" # Because .sym is in settings.MIME_OVERRIDES assert api_params["ContentType"] == "text/plain" content = api_params["Body"].read() assert isinstance(content, bytes) # We know what the expected size is based on having run: # $ cabextract ksproxy.pd_ # $ dump_syms ksproxy.pdb > ksproxy.sym # $ ls -l ksproxy.sym # 1721 assert len(content) == 729 original_content = gzip.decompress(content) assert len(original_content) == 1721 # ...pretend to actually upload it. files_uploaded.append(api_params["Key"]) return {} raise NotImplementedError((operation_name, api_params)) symbol = "ksproxy.pdb" debugid = "A7D6F1BB18CD4CB48" with botomock(mock_api_call): download_microsoft_symbol(symbol, debugid) assert files_uploaded == ["v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym"] # The ultimate test is that it should have created a file_upload (file_upload, ) = FileUpload.objects.all() assert file_upload.size == 729 assert file_upload.bucket_name == "private" assert file_upload.key == "v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym" assert not file_upload.update assert not file_upload.upload assert file_upload.compressed assert file_upload.completed_at assert file_upload.microsoft_download # It should also have created a MicrosoftDownload record. (download_obj, ) = MicrosoftDownload.objects.all() assert download_obj.completed_at assert download_obj.skipped is False assert download_obj.error is None # Check that markus caught timings of the individual file processing records = [rec[1] for rec in metricsmock.get_records()] assert len(records) == 10 assert records == [ "tecken.download_store_missing_symbol", "tecken.download_cabextract", "tecken.download_dump_syms", "tecken.upload_file_exists", "tecken.upload_gzip_payload", "tecken.upload_put_object", "tecken.upload_file_upload_upload", "tecken.upload_file_upload", "tecken.download_microsoft_download_file_upload_upload", "tecken.download_upload_microsoft_symbol", ]
def test_download_microsoft_symbol_task_happy_path( botomock, metricsmock, requestsmock, ): with open(PD__FILE, 'rb') as f: content = f.read() # just checking that the fixture file is sane assert content.startswith(b'MSCF') requestsmock.get( 'https://msdl.microsoft.com/download/symbols/ksproxy.pdb' '/A7D6F1BB18CD4CB48/ksproxy.pd_', content=content) def mock_api_call(self, operation_name, api_params): # this comes from the UPLOAD_DEFAULT_URL in settings.Test assert api_params['Bucket'] == 'private' if (operation_name == 'HeadObject' and api_params['Key'] == ('v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym')): # Pretend we've never heard of this parsed_response = { 'Error': { 'Code': '404', 'Message': 'Not found' }, } raise ClientError(parsed_response, operation_name) if (operation_name == 'PutObject' and api_params['Key'] == ('v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym')): # Because .sym is in settings.COMPRESS_EXTENSIONS assert api_params['ContentEncoding'] == 'gzip' # Because .sym is in settings.MIME_OVERRIDES assert api_params['ContentType'] == 'text/plain' content = api_params['Body'].read() assert isinstance(content, bytes) # We know what the expected size is based on having run: # $ cabextract ksproxy.pd_ # $ dump_syms ksproxy.pdb > ksproxy.sym # $ ls -l ksproxy.sym # 1721 assert len(content) == 729 original_content = gzip.decompress(content) assert len(original_content) == 1721 # ...pretend to actually upload it. return { # Should there be anything here? } raise NotImplementedError((operation_name, api_params)) symbol = 'ksproxy.pdb' debugid = 'A7D6F1BB18CD4CB48' with botomock(mock_api_call): download_microsoft_symbol(symbol, debugid) # The ultimate test is that it should have created a file_upload file_upload, = FileUpload.objects.all() assert file_upload.size == 729 assert file_upload.bucket_name == 'private' assert file_upload.key == 'v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym' assert not file_upload.update assert not file_upload.upload assert file_upload.compressed assert file_upload.completed_at assert file_upload.microsoft_download # It should also have created a MicrosoftDownload record. download_obj, = MicrosoftDownload.objects.all() assert download_obj.completed_at assert download_obj.skipped is False assert download_obj.error is None # Check that markus caught timings of the individual file processing records = metricsmock.get_records() assert len(records) == 10 assert records[0][1] == 'tecken.download_store_missing_symbol' assert records[1][1] == 'tecken.download_cabextract' assert records[2][1] == 'tecken.download_dump_syms' assert records[3][1] == 'tecken.upload_file_exists' assert records[4][1] == 'tecken.upload_gzip_payload' assert records[5][1] == 'tecken.upload_put_object' assert records[6][1] == 'tecken.upload_file_upload_upload' assert records[7][1] == 'tecken.upload_file_upload' assert records[8][1] == ( 'tecken.download_microsoft_download_file_upload_upload') assert records[9][1] == 'tecken.download_upload_microsoft_symbol'
def test_download_microsoft_symbol_task_happy_path(gcsmock, metricsmock, requestsmock): with open(PD__FILE, "rb") as f: content = f.read() # just checking that the fixture file is sane assert content.startswith(b"MSCF") requestsmock.get( "https://msdl.microsoft.com/download/symbols/ksproxy.pdb" "/A7D6F1BB18CD4CB48/ksproxy.pd_", content=content, ) mock_bucket = gcsmock.MockBucket() def mock_get_bucket(bucket_name): assert bucket_name == "private" return mock_bucket gcsmock.get_bucket = mock_get_bucket def mock_get_blob(key): if key == "v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym": # As if it doesn't exist return None raise NotImplementedError(key) mock_bucket.get_blob = mock_get_blob files_uploaded = [] def mocked_create_blob(key): if key == "v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym": blob = gcsmock.MockBlob(key) def mock_upload_from_file(file): files_uploaded.append(key) content = file.read() assert isinstance(content, bytes) # We know what the expected size is based on having run: # $ cabextract ksproxy.pd_ # $ dump_syms ksproxy.pdb > ksproxy.sym # $ ls -l ksproxy.sym # 1721 assert len(content) == 729 original_content = gzip.decompress(content) assert len(original_content) == 1721 blob.upload_from_file = mock_upload_from_file return blob raise NotImplementedError(key) mock_bucket.blob = mocked_create_blob symbol = "ksproxy.pdb" debugid = "A7D6F1BB18CD4CB48" download_microsoft_symbol(symbol, debugid) assert files_uploaded == ["v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym"] # The ultimate test is that it should have created a file_upload file_upload, = FileUpload.objects.all() assert file_upload.size == 729 assert file_upload.bucket_name == "private" assert file_upload.key == "v0/ksproxy.pdb/A7D6F1BB18CD4CB48/ksproxy.sym" assert not file_upload.update assert not file_upload.upload assert file_upload.compressed assert file_upload.completed_at assert file_upload.microsoft_download # It should also have created a MicrosoftDownload record. download_obj, = MicrosoftDownload.objects.all() assert download_obj.completed_at assert download_obj.skipped is False assert download_obj.error is None # Check that markus caught timings of the individual file processing records = metricsmock.get_records() assert len(records) == 10 assert records[0][1] == "tecken.download_store_missing_symbol" assert records[1][1] == "tecken.download_cabextract" assert records[2][1] == "tecken.download_dump_syms" assert records[3][1] == "tecken.upload_file_exists" assert records[4][1] == "tecken.upload_gzip_payload" assert records[5][1] == "tecken.upload_put_object" assert records[6][1] == "tecken.upload_file_upload_upload" assert records[7][1] == "tecken.upload_file_upload" assert records[8][1] == ( "tecken.download_microsoft_download_file_upload_upload") assert records[9][1] == "tecken.download_upload_microsoft_symbol"