def test_start_happy_path(mocked_boto3, settings, valid_build, itertools_count, mocker): mocked_message = mocker.MagicMock() # See # https://gist.github.com/peterbe/f739b91c0674d36a1526ccb43b6844c3#file-stage-json # for a real life example of a SQS message. message = { "Message": json.dumps({ "Records": [ { "foot": "here" }, { "s3": { "object": { "key": "some/path/to/buildhub.json", "eTag": "e4eb6609382efd6b3bc9deec616ad5c0", }, "bucket": { "name": "buildhubses" }, } }, { "s3": { "object": { "key": "not/a/buildhub.json/file", "eTag": "77e09ba7e37836c2cf0ce59e1e8361ab", }, "bucket": { "name": "buildhubses" }, } }, ] }) } mocked_message.body = json.dumps(message) mocked_queue = mocker.MagicMock() mocked_queue.receive_messages().__iter__.return_value = [mocked_message] mocked_boto3.resource().get_queue_by_name.return_value = mocked_queue mocked_s3_client = mocker.MagicMock() mocked_boto3.client.return_value = mocked_s3_client def mocked_download_fileobj(bucket_name, key_name, f): # Sanity checks that the mocking is right assert bucket_name == "buildhubses" assert key_name == "some/path/to/buildhub.json" f.write(json.dumps(valid_build()).encode("utf-8")) mocked_s3_client.download_fileobj.side_effect = mocked_download_fileobj start(settings.SQS_QUEUE_URL) mocked_boto3.resource().get_queue_by_name.assert_called_with( QueueName="buildhub-s3-events") # It should have created 1 Build assert Build.objects.get() mocked_boto3.client.assert_called_with("s3", "ca-north-2", config=mock.ANY)
def test_records_legacy_message(mocked_boto3, settings, valid_build, itertools_count, mocker): """In production, in Buildhub2, the messages from S3 and routed through SNS. So the list of "Records" in a JSON string inside the "Message" key. However, if you have S3 events go straight to SQS, the list of "Records" is part of the main object and a top-level key. """ mocked_message = mocker.MagicMock() mocked_message.body = json.dumps({ "Records": [ { "foot": "here" }, { "s3": { "object": { "key": "some/path/to/buildhub.json", "eTag": "e4eb6609382efd6b3bc9deec616ad5c0", }, "bucket": { "name": "buildhubses" }, } }, { "s3": { "object": { "key": "not/a/buildhub.json/file", "eTag": "77e09ba7e37836c2cf0ce59e1e8361ab", }, "bucket": { "name": "buildhubses" }, } }, ] }) mocked_queue = mocker.MagicMock() mocked_queue.receive_messages().__iter__.return_value = [mocked_message] mocked_boto3.resource().get_queue_by_name.return_value = mocked_queue mocked_s3_client = mocker.MagicMock() mocked_boto3.client.return_value = mocked_s3_client def mocked_download_fileobj(bucket_name, key_name, f): # Sanity checks that the mocking is right assert bucket_name == "buildhubses" assert key_name == "some/path/to/buildhub.json" f.write(json.dumps(valid_build()).encode("utf-8")) mocked_s3_client.download_fileobj.side_effect = mocked_download_fileobj start(settings.SQS_QUEUE_URL) mocked_boto3.resource().get_queue_by_name.assert_called_with( QueueName="buildhub-s3-events") # It should have created 1 Build assert Build.objects.get()
def test_start_happy_path( mocked_boto3, settings, valid_build, itertools_count, mocker, ): mocked_message = mocker.MagicMock() mocked_message.body = json.dumps({ 'Records': [{ 'foot': 'here' }, { 's3': { 'object': { 'key': 'some/path/to/buildhub.json', 'eTag': 'e4eb6609382efd6b3bc9deec616ad5c0', }, 'bucket': { 'name': 'buildhubses', } } }, { 's3': { 'object': { 'key': 'not/a/buildhub.json/file', 'eTag': '77e09ba7e37836c2cf0ce59e1e8361ab', }, 'bucket': { 'name': 'buildhubses', } } }] }) mocked_queue = mocker.MagicMock() mocked_queue.receive_messages().__iter__.return_value = [mocked_message] mocked_boto3.resource().get_queue_by_name.return_value = mocked_queue mocked_s3_client = mocker.MagicMock() mocked_boto3.client.return_value = mocked_s3_client def mocked_download_fileobj(bucket_name, key_name, f): # Sanity checks that the mocking is right assert bucket_name == 'buildhubses' assert key_name == 'some/path/to/buildhub.json' f.write(json.dumps(valid_build()).encode('utf-8')) mocked_s3_client.download_fileobj.side_effect = mocked_download_fileobj start(settings.SQS_QUEUE_URL) mocked_boto3.resource().get_queue_by_name.assert_called_with( QueueName='buildhub-s3-events') # It should have created 1 Build assert Build.objects.get()
def test_ingest_idempotently( mocked_boto3, settings, valid_build, itertools_count, mocker ): mocked_message = mocker.MagicMock() message = { "Message": json.dumps( { "Records": [ { "s3": { "object": { "key": "some/path/to/buildhub.json", "eTag": "e4eb6609382efd6b3bc9deec616ad5c0", }, "bucket": {"name": "buildhubses"}, } } ] } ) } mocked_message.body = json.dumps(message) mocked_queue = mocker.MagicMock() mocked_queue.receive_messages().__iter__.return_value = [mocked_message] mocked_boto3.resource().get_queue_by_name.return_value = mocked_queue mocked_s3_client = mocker.MagicMock() mocked_boto3.client.return_value = mocked_s3_client build = valid_build() Build.insert(build) def mocked_download_fileobj(bucket_name, key_name, f): # Sanity checks that the mocking is right assert bucket_name == "buildhubses" assert key_name == "some/path/to/buildhub.json" f.write(json.dumps(build).encode("utf-8")) mocked_s3_client.download_fileobj.side_effect = mocked_download_fileobj start(settings.SQS_QUEUE_URL) mocked_boto3.resource().get_queue_by_name.assert_called_with( QueueName="buildhub-s3-events" ) # It should have created no new Builds assert Build.objects.all().count() == 1
def test_signed_s3_bucket(mocked_boto3, settings, valid_build, itertools_count, mocker): settings.UNSIGNED_SQS_S3_CLIENT = False mocked_message = mocker.MagicMock() message = { "Message": json.dumps({ "Records": [ { "foot": "here" }, { "s3": { "object": { "key": "firefox-99-buildhub.json", "eTag": "e4eb6609382efd6b3bc9deec616ad5c0", }, "bucket": { "name": "buildhubses" }, } }, ] }) } mocked_message.body = json.dumps(message) mocked_queue = mocker.MagicMock() mocked_queue.receive_messages().__iter__.return_value = [mocked_message] mocked_boto3.resource().get_queue_by_name.return_value = mocked_queue mocked_s3_client = mocker.MagicMock() mocked_boto3.client.return_value = mocked_s3_client def mocked_download_fileobj(bucket_name, key_name, f): # Sanity checks that the mocking is right assert bucket_name == "buildhubses" assert key_name == "firefox-99-buildhub.json" f.write(json.dumps(valid_build()).encode("utf-8")) mocked_s3_client.download_fileobj.side_effect = mocked_download_fileobj start(settings.SQS_QUEUE_URL) mocked_boto3.resource().get_queue_by_name.assert_called_with( QueueName="buildhub-s3-events") # It should have created 1 Build assert Build.objects.get() mocked_boto3.client.assert_called_with("s3", "ca-north-2")
def test_start_file_not_found( mocked_boto3, settings, valid_build, itertools_count, mocker ): mocked_message = mocker.MagicMock() message = { "Message": json.dumps( { "Records": [ { "s3": { "object": { "key": "some/path/to/buildhub.json", "eTag": "e4eb6609382efd6b3bc9deec616ad5c0", }, "bucket": {"name": "buildhubses"}, } } ] } ) } mocked_message.body = json.dumps(message) mocked_queue = mocker.MagicMock() mocked_queue.receive_messages().__iter__.return_value = [mocked_message] mocked_boto3.resource().get_queue_by_name.return_value = mocked_queue mocked_s3_client = mocker.MagicMock() mocked_boto3.client.return_value = mocked_s3_client def mocked_download_fileobj(bucket_name, key_name, f): # Sanity checks that the mocking is right assert bucket_name == "buildhubses" assert key_name == "some/path/to/buildhub.json" parsed_response = {"Error": {"Code": "404", "Message": "Not found"}} raise ClientError(parsed_response, "GetObject") mocked_s3_client.download_fileobj.side_effect = mocked_download_fileobj start(settings.SQS_QUEUE_URL) mocked_boto3.resource().get_queue_by_name.assert_called_with( QueueName="buildhub-s3-events" ) # It should have created 1 Build assert not Build.objects.all().exists()
def test_ingest_idempotently( mocked_boto3, settings, valid_build, itertools_count, mocker, ): mocked_message = mocker.MagicMock() mocked_message.body = json.dumps({ 'Records': [ { 's3': { 'object': { 'key': 'some/path/to/buildhub.json', 'eTag': 'e4eb6609382efd6b3bc9deec616ad5c0', }, 'bucket': { 'name': 'buildhubses', } } }, ] }) mocked_queue = mocker.MagicMock() mocked_queue.receive_messages().__iter__.return_value = [mocked_message] mocked_boto3.resource().get_queue_by_name.return_value = mocked_queue mocked_s3_client = mocker.MagicMock() mocked_boto3.client.return_value = mocked_s3_client build = valid_build() Build.insert(build) def mocked_download_fileobj(bucket_name, key_name, f): # Sanity checks that the mocking is right assert bucket_name == 'buildhubses' assert key_name == 'some/path/to/buildhub.json' f.write(json.dumps(build).encode('utf-8')) mocked_s3_client.download_fileobj.side_effect = mocked_download_fileobj start(settings.SQS_QUEUE_URL) mocked_boto3.resource().get_queue_by_name.assert_called_with( QueueName='buildhub-s3-events') # It should have created no new Builds assert Build.objects.all().count() == 1
def test_not_valid_buildhub_json( mocked_boto3, settings, valid_build, itertools_count, mocker, ): mocked_message = mocker.MagicMock() mocked_message.body = json.dumps({ 'Records': [ { 's3': { 'object': { 'key': 'some/path/to/buildhub.json', 'eTag': 'e4eb6609382efd6b3bc9deec616ad5c0', }, 'bucket': { 'name': 'buildhubses', } } }, ] }) mocked_queue = mocker.MagicMock() mocked_queue.receive_messages().__iter__.return_value = [mocked_message] mocked_boto3.resource().get_queue_by_name.return_value = mocked_queue mocked_s3_client = mocker.MagicMock() mocked_boto3.client.return_value = mocked_s3_client def mocked_download_fileobj(bucket_name, key_name, f): # Sanity checks that the mocking is right assert bucket_name == 'buildhubses' assert key_name == 'some/path/to/buildhub.json' build = valid_build() build['source']['junk'] = True # will make it invalid f.write(json.dumps(build).encode('utf-8')) mocked_s3_client.download_fileobj.side_effect = mocked_download_fileobj with pytest.raises(ValidationError) as exception: start(settings.SQS_QUEUE_URL) err_msg = "Additional properties are not allowed ('junk' was unexpected)" assert err_msg in str(exception.value)
def test_start_file_not_found( mocked_boto3, settings, valid_build, itertools_count, mocker, ): mocked_message = mocker.MagicMock() mocked_message.body = json.dumps({ 'Records': [ { 's3': { 'object': { 'key': 'some/path/to/buildhub.json', 'eTag': 'e4eb6609382efd6b3bc9deec616ad5c0', }, 'bucket': { 'name': 'buildhubses', } } }, ] }) mocked_queue = mocker.MagicMock() mocked_queue.receive_messages().__iter__.return_value = [mocked_message] mocked_boto3.resource().get_queue_by_name.return_value = mocked_queue mocked_s3_client = mocker.MagicMock() mocked_boto3.client.return_value = mocked_s3_client def mocked_download_fileobj(bucket_name, key_name, f): # Sanity checks that the mocking is right assert bucket_name == 'buildhubses' assert key_name == 'some/path/to/buildhub.json' parsed_response = {'Error': {'Code': '404', 'Message': 'Not found'}} raise ClientError(parsed_response, 'GetObject') mocked_s3_client.download_fileobj.side_effect = mocked_download_fileobj start(settings.SQS_QUEUE_URL) mocked_boto3.resource().get_queue_by_name.assert_called_with( QueueName='buildhub-s3-events') # It should have created 1 Build assert not Build.objects.all().exists()
def test_not_valid_buildhub_json( mocked_boto3, settings, valid_build, itertools_count, mocker ): mocked_message = mocker.MagicMock() message = { "Message": json.dumps( { "Records": [ { "s3": { "object": { "key": "some/path/to/buildhub.json", "eTag": "e4eb6609382efd6b3bc9deec616ad5c0", }, "bucket": {"name": "buildhubses"}, } } ] } ) } mocked_message.body = json.dumps(message) mocked_queue = mocker.MagicMock() mocked_queue.receive_messages().__iter__.return_value = [mocked_message] mocked_boto3.resource().get_queue_by_name.return_value = mocked_queue mocked_s3_client = mocker.MagicMock() mocked_boto3.client.return_value = mocked_s3_client def mocked_download_fileobj(bucket_name, key_name, f): # Sanity checks that the mocking is right assert bucket_name == "buildhubses" assert key_name == "some/path/to/buildhub.json" build = valid_build() build["source"]["junk"] = True # will make it invalid f.write(json.dumps(build).encode("utf-8")) mocked_s3_client.download_fileobj.side_effect = mocked_download_fileobj with pytest.raises(ValidationError) as exception: start(settings.SQS_QUEUE_URL) err_msg = "Additional properties are not allowed ('junk' was unexpected)" assert err_msg in str(exception.value)
def test_bad_client_errors( mocked_boto3, settings, valid_build, itertools_count, mocker, ): mocked_message = mocker.MagicMock() mocked_message.body = json.dumps({ 'Records': [ { 's3': { 'object': { 'key': 'some/path/to/buildhub.json', 'eTag': 'e4eb6609382efd6b3bc9deec616ad5c0', }, 'bucket': { 'name': 'buildhubses', } } }, ] }) mocked_queue = mocker.MagicMock() mocked_queue.receive_messages().__iter__.return_value = [mocked_message] mocked_boto3.resource().get_queue_by_name.return_value = mocked_queue mocked_s3_client = mocker.MagicMock() mocked_boto3.client.return_value = mocked_s3_client def mocked_download_fileobj(bucket_name, key_name, f): # Sanity checks that the mocking is right assert bucket_name == 'buildhubses' assert key_name == 'some/path/to/buildhub.json' parsed_response = {'Error': {'Code': '500', 'Message': 'Oh no!'}} raise ClientError(parsed_response, 'GetObject') mocked_s3_client.download_fileobj.side_effect = mocked_download_fileobj with pytest.raises(ClientError) as exception: start(settings.SQS_QUEUE_URL) assert "An error occurred (500)" in str(exception.value)
def test_bad_client_errors( mocked_boto3, settings, valid_build, itertools_count, mocker ): mocked_message = mocker.MagicMock() message = { "Message": json.dumps( { "Records": [ { "s3": { "object": { "key": "some/path/to/buildhub.json", "eTag": "e4eb6609382efd6b3bc9deec616ad5c0", }, "bucket": {"name": "buildhubses"}, } } ] } ) } mocked_message.body = json.dumps(message) mocked_queue = mocker.MagicMock() mocked_queue.receive_messages().__iter__.return_value = [mocked_message] mocked_boto3.resource().get_queue_by_name.return_value = mocked_queue mocked_s3_client = mocker.MagicMock() mocked_boto3.client.return_value = mocked_s3_client def mocked_download_fileobj(bucket_name, key_name, f): # Sanity checks that the mocking is right assert bucket_name == "buildhubses" assert key_name == "some/path/to/buildhub.json" parsed_response = {"Error": {"Code": "500", "Message": "Oh no!"}} raise ClientError(parsed_response, "GetObject") mocked_s3_client.download_fileobj.side_effect = mocked_download_fileobj with pytest.raises(ClientError) as exception: start(settings.SQS_QUEUE_URL) assert "An error occurred (500)" in str(exception.value)