def _api_call(self, **kwargs): operation_model = service_model.operation_model(operation_name) event_name = ( 'before-parameter-build.{endpoint_prefix}.{operation_name}') self.meta.events.emit(event_name.format( endpoint_prefix=service_model.endpoint_prefix, operation_name=operation_name), params=kwargs, model=operation_model) request_dict = self._serializer.serialize_to_request( kwargs, operation_model) self.meta.events.emit( 'before-call.{endpoint_prefix}.{operation_name}'.format( endpoint_prefix=service_model.endpoint_prefix, operation_name=operation_name), model=operation_model, params=request_dict, request_signer=self._request_signer) http, parsed_response = self._endpoint.make_request( operation_model, request_dict) self.meta.events.emit( 'after-call.{endpoint_prefix}.{operation_name}'.format( endpoint_prefix=service_model.endpoint_prefix, operation_name=operation_name), http_response=http, parsed=parsed_response, model=operation_model) if http.status_code >= 300: raise ClientError(parsed_response, operation_name) else: return parsed_response
def test_get_file_keys_from_s3_with_manifest_id(self): """Test get_file_keys_from_s3_with_manifest_id.""" files = self.report_processor.get_file_keys_from_s3_with_manifest_id( "request_id", "s3_path", "manifest_id") self.assertEqual(files, []) with patch("masu.processor.parquet.parquet_report_processor.settings", ENABLE_PARQUET_PROCESSING=True): with patch( "masu.processor.parquet.parquet_report_processor.get_s3_resource" ) as mock_s3: files = self.report_processor.get_file_keys_from_s3_with_manifest_id( "request_id", None, "manifest_id") self.assertEqual(files, []) with patch("masu.processor.parquet.parquet_report_processor.settings", ENABLE_PARQUET_PROCESSING=True): with patch( "masu.processor.parquet.parquet_report_processor.get_s3_resource" ) as mock_s3: mock_s3.side_effect = ClientError({}, "Error") files = self.report_processor.get_file_keys_from_s3_with_manifest_id( "request_id", "s3_path", "manifest_id") self.assertEqual(files, [])
def test_bucket_exists_with_non_existent_bucket(self): # connection_manager.call is called twice, and should throw the # Not Found ClientError only for the first call. self.template.s3_details = { "bucket_name": "bucket-name", "bucket_key": "bucket-key" } self.template.connection_manager.call.side_effect = [ ClientError( { "Error": { "Code": 404, "Message": "Not Found" } }, sentinel.operation ), None ] existance = self.template._bucket_exists() assert existance is False
def test_invoke_does_raise_if_service_error(runner, mock_cli_factory): deployed_resources = DeployedResources({"resources": []}) mock_cli_factory.create_config_obj.return_value = FakeConfig( deployed_resources) invoke_handler = mock.Mock(spec=LambdaInvokeHandler) invoke_handler.invoke.side_effect = ClientError( {'Error': { 'Code': 'LambdaError', 'Message': 'Error message' }}, 'Invoke') mock_cli_factory.create_lambda_invoke_handler.return_value = invoke_handler mock_reader = mock.Mock(spec=PipeReader) mock_reader.read.return_value = 'barbaz' mock_cli_factory.create_stdin_reader.return_value = mock_reader with runner.isolated_filesystem(): cli.create_new_project_skeleton('testproject') os.chdir('testproject') result = _run_cli_command(runner, cli.invoke, ['-n', 'foo'], cli_factory=mock_cli_factory) assert result.exit_code == 1 assert invoke_handler.invoke.call_args == mock.call('barbaz') assert ("Error: got 'LambdaError' exception back from Lambda\n" "Error message") in result.output
def _mock_describe_document(Name): # Using two wait flags to simulate that AWS is taking time to delete and # create documents if Name not in mock_asiaq_document_contents: if wait_flags['delete']: wait_flags['delete'] = False return {'Document': {'Name': Name, 'Status': 'Active'}} else: wait_flags['delete'] = True raise ClientError( { 'Error': { 'Code': 'Mock_code', 'Message': 'mock message' } }, 'DescribeDocument') else: if wait_flags['create']: wait_flags['create'] = False res = {'Document': {'Name': Name, 'Status': 'Creating'}} else: wait_flags['create'] = True res = {'Document': {'Name': Name, 'Status': 'Active'}} return res
def test_execute_lambda_occ_conflict( self, mock_start_transaction, mock_session, mock_transaction, mock_is_occ_conflict_exception, mock_is_retriable_exception, mock_logger_warning, mock_executor, mock_throw_if_closed, mock_no_throw_abort): ce = ClientError(MOCK_CLIENT_ERROR_MESSAGE, MOCK_MESSAGE) mock_start_transaction.return_value = mock_transaction mock_is_occ_conflict_exception.return_value = True mock_is_retriable_exception.return_value = False qldb_session = QldbSession(mock_session, MOCK_READ_AHEAD, MOCK_RETRY_LIMIT, mock_executor) mock_lambda = Mock() mock_lambda.side_effect = ce self.assertRaises(ClientError, qldb_session.execute_lambda, mock_lambda) mock_throw_if_closed.assert_called_once_with() mock_start_transaction.assert_has_calls( [call(), call(), call(), call(), call()]) mock_no_throw_abort.assert_has_calls([ call(mock_transaction), call(mock_transaction), call(mock_transaction), call(mock_transaction), call(mock_transaction) ]) mock_is_occ_conflict_exception.assert_has_calls( [call(ce), call(ce), call(ce), call(ce), call(ce)]) self.assertEqual(mock_lambda.call_count, qldb_session._retry_limit + 1) self.assertEqual(mock_logger_warning.call_count, qldb_session._retry_limit + 1) mock_transaction.commit.assert_not_called()
def test_storage_open_no_write(self): """ Test opening file in write mode and closing without writing. A file should be created as by obj.put(...). """ name = 'test_open_no_write.txt' # Set the encryption flag used for puts self.storage.encryption = True self.storage.reduced_redundancy = True self.storage.default_acl = 'public-read' file = self.storage.open(name, 'w') self.storage.bucket.Object.assert_called_with(name) obj = self.storage.bucket.Object.return_value obj.load.side_effect = ClientError( { 'Error': {}, 'ResponseMetadata': { 'HTTPStatusCode': 404 } }, 'head_bucket') # Set the name of the mock object obj.key = name # Save the internal file before closing file.close() obj.load.assert_called_once_with() obj.put.assert_called_once_with(ACL='public-read', Body=b"", ContentType='text/plain', ServerSideEncryption='AES256', StorageClass='REDUCED_REDUNDANCY')
def test_vcenter_info_exists_returns_False(get_handler): ( secret_manager, system_manager_mock, secrets_manager_mock, handler, ) = __create_secret_manager_subject(get_handler) # setup secrets_manager_mock.get_secret_value.side_effect = ClientError( {"Error": { "Message": "bla", "Code": "ResourceNotFoundException" }}, "secretsmanager:GetSecretValue", ) key = f"/rpcv/{__STAGE}/orgs/{__ORD_ID}/clusters/{__CLUSTER_ID}/vcenters/{__IP}" # when actual = secret_manager.secret_info_exists(key) # then secrets_manager_mock.get_secret_value.assert_called_with(SecretId=key) assert actual is False
def test_setup_bucket_with_resources_upload_failure(mocker, caplog, mock_generated_bucket_name, expected_bucket_name, provided_bucket_name, expected_remove_bucket): """Verify that create_bucket_with_batch_resources behaves as expected in case of upload failure.""" region = "eu-west-1" mock_artifact_dir = "artifact_dir" stack_name = "test-stack" error = "ExpiredToken" client_error = ClientError({"Error": {"Code": error}}, "upload_fileobj") mocker.patch("pcluster.utils.generate_random_name_with_prefix", side_effect=[mock_artifact_dir, mock_generated_bucket_name]) mocker.patch("pcluster.utils.create_s3_bucket") check_bucket_mock = mocker.patch("pcluster.utils.check_s3_bucket_exists") mocker.patch("pcluster.utils.upload_resources_artifacts", side_effect=client_error) cleanup_s3_mock = mocker.patch("pcluster.utils.cleanup_s3_resources") pcluster_config_mock = _mock_pcluster_config(mocker, "slurm", region, provided_bucket_name) storage_data = pcluster_config_mock.to_storage() with pytest.raises(ClientError, match=error): _setup_bucket_with_resources(pcluster_config_mock, storage_data, stack_name, {}) if provided_bucket_name: check_bucket_mock.assert_called_with(provided_bucket_name) else: check_bucket_mock.assert_not_called() # if resource upload fails we delete the bucket cleanup_s3_mock.assert_called_with(expected_bucket_name, mock_artifact_dir, expected_remove_bucket) assert_that(caplog.text).contains( "Unable to upload cluster resources to the S3 bucket")
def get_parameter(self, **kwargs): """Mocked get_parameter function that returns a value for the key from a dict Keyword Arguments: Name (str): The name of the parameter to retrieve Returns: dict: Parameter dictionary containing this parameter's value """ # Raise a botocore ClientError if the param doesn't exist if kwargs.get('Name') not in self._parameters: err = { 'Error': { 'Code': 403, 'Message': 'parameter does not exist' } } raise ClientError(err, 'get_parameter') return { 'Parameter': { 'Value': self._parameters.get(kwargs.get('Name')) } }
def test_it_provides_logs_for_failed_rollback_client_error( mock_error_handler, mock_delete, mock_load, mock_verify_integrity, message_stub): parquet_file = MagicMock() parquet_file.num_row_groups = 1 mock_load.return_value = parquet_file mock_s3 = MagicMock() mock_s3.delete_object.side_effect = ClientError({}, "DeleteObject") mock_verify_integrity.side_effect = IntegrityCheckFailedError( "Some error", mock_s3, "bucket", "test/basic.parquet", "new_version") mock_delete.return_value = pa.BufferOutputStream(), {"DeletedRows": 1} execute("https://queue/url", message_stub(), "receipt_handle") mock_verify_integrity.assert_called() assert mock_error_handler.call_args_list == [ call(ANY, ANY, "Object version integrity check failed: Some error"), call( ANY, ANY, "ClientError: An error occurred (Unknown) when calling the DeleteObject operation: Unknown. " "Version rollback caused by version integrity conflict failed", "ObjectRollbackFailed", False, ), ]
def test_dict_added_to_unprocessed_items_after_5_failures(self): base = BaseDispatcher('test_subject', 'send_lots', 'send_one', max_batch_size=3) client_error = ClientError( {"Error": { "message": "Something went wrong", "code": 0 }}, "A Test") base._individual_dispatch_method = Mock(side_effect=[ client_error, client_error, client_error, client_error, client_error ]) test_payload = {"abc": 123} base._send_individual_payload(test_payload) base._individual_dispatch_method.assert_has_calls([ call(**test_payload), call(**test_payload), call(**test_payload), call(**test_payload), call(**test_payload) ]) self.assertEqual([test_payload], base.unprocessed_items)
def test_encrypt_and_push_creds_to_s3_kms_failure(log_mock, boto_mock): """CLI - Outputs - Encrypt and push creds to s3 - kms failure""" props = { 'secret': OutputProperty(description='short description of secret needed', value='1908AGSG98A8908AG', cred_requirement=True) } err_response = { 'Error': { 'Code': 100, 'Message': 'BAAAD', 'BucketName': 'bucket' } } # Add ClientError side_effect to mock boto_mock.side_effect = ClientError(err_response, 'operation') encrypt_and_push_creds_to_s3('us-east-1', 'bucket', 'key', props, 'test_alias') log_mock.assert_called_with( 'An error occurred during credential encryption')
if _has_required_boto(): region = 'us-east-1' access_key = 'GKTADJGHEIQSXMKKRBJ08H' secret_key = 'askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs' conn_parameters = { 'region': region, 'key': access_key, 'keyid': secret_key, 'profile': {} } error_message = 'An error occurred (101) when calling the {0} operation: Test-defined error' not_found_error = ClientError( { 'Error': { 'Code': 'ResourceNotFoundException', 'Message': "Test-defined error" } }, 'msg') error_content = {'Error': {'Code': 101, 'Message': "Test-defined error"}} domain_ret = dict( DomainName='testdomain', ElasticsearchClusterConfig={}, EBSOptions={}, AccessPolicies={}, SnapshotOptions={}, AdvancedOptions={}, ElasticsearchVersion='1.5', )
def test_individual_write_throws_exceptions(self): mock_client_error = ClientError( {'Error': { 'Code': 500, 'Message': 'broken' }}, "Dynamo") kinesis_client = KinesisBatchDispatcher( stream_name='test_stream', partition_key_identifier='m_id') mock_boto3 = Mock() kinesis_client._aws_service = mock_boto3 test_payloads = [{ 'm_id': 1, 'message': 'message contents 1' }, { 'm_id': 2, 'message': 'message contents 2' }] # All records fail in first attempt failure_response = { 'FailedRecordCount': 2, 'Records': [{ 'm_id': 1, 'message': 'message contents 1', 'ErrorCode': 'badness' }, { 'm_id': 2, 'message': 'message contents 2', 'ErrorCode': 'badness' }], 'EncryptionType': 'NONE' } kinesis_client._batch_dispatch_method = Mock( side_effect=[failure_response]) kinesis_client._individual_dispatch_method = Mock(side_effect=[ mock_client_error, mock_client_error, mock_client_error, mock_client_error, mock_client_error, mock_client_error, mock_client_error, mock_client_error, mock_client_error, mock_client_error ]) for test_payload in test_payloads: kinesis_client.submit_payload(test_payload) kinesis_client.flush_payloads() kinesis_client._batch_dispatch_method.assert_called_once_with( **{ 'Records': [{ 'Data': '{"m_id": 1, "message": "message contents 1"}', 'PartitionKey': '1' }, { 'Data': '{"m_id": 2, "message": "message contents 2"}', 'PartitionKey': '2' }], 'StreamName': 'test_stream' }) kinesis_client._individual_dispatch_method.assert_has_calls([ call(Data='{"m_id": 1, "message": "message contents 1"}', PartitionKey='1', StreamName='test_stream'), call(Data='{"m_id": 1, "message": "message contents 1"}', PartitionKey='1', StreamName='test_stream'), call(Data='{"m_id": 1, "message": "message contents 1"}', PartitionKey='1', StreamName='test_stream'), call(Data='{"m_id": 1, "message": "message contents 1"}', PartitionKey='1', StreamName='test_stream'), call(Data='{"m_id": 1, "message": "message contents 1"}', PartitionKey='1', StreamName='test_stream'), call(Data='{"m_id": 2, "message": "message contents 2"}', PartitionKey='2', StreamName='test_stream'), call(Data='{"m_id": 2, "message": "message contents 2"}', PartitionKey='2', StreamName='test_stream'), call(Data='{"m_id": 2, "message": "message contents 2"}', PartitionKey='2', StreamName='test_stream'), call(Data='{"m_id": 2, "message": "message contents 2"}', PartitionKey='2', StreamName='test_stream'), call(Data='{"m_id": 2, "message": "message contents 2"}', PartitionKey='2', StreamName='test_stream') ]) self.assertEqual(test_payloads, kinesis_client.unprocessed_items)
def test_batch_write_throws_exceptions(self): kinesis_client = KinesisBatchDispatcher( stream_name='test_stream', partition_key_identifier='m_id') mock_client_error = ClientError( {'Error': { 'Code': 500, 'Message': 'broken' }}, "Dynamo") mock_boto3 = Mock() kinesis_client._aws_service = mock_boto3 test_payloads = [{ 'm_id': 1, 'message': 'message contents 1' }, { 'm_id': 2, 'message': 'message contents 2' }, { 'm_id': 3, 'message': 'message contents 3' }, { 'm_id': 4, 'message': 'message contents 4' }, { 'm_id': 5, 'message': 'message contents 5' }] kinesis_client._batch_dispatch_method = Mock(side_effect=[ mock_client_error, mock_client_error, mock_client_error, mock_client_error, mock_client_error ]) kinesis_client._individual_dispatch_method = Mock() for test_payload in test_payloads: kinesis_client.submit_payload(test_payload) kinesis_client.flush_payloads() kinesis_client._batch_dispatch_method.assert_has_calls([ call( **{ 'Records': [{ 'Data': '{"m_id": 1, "message": "message contents 1"}', 'PartitionKey': '1' }, { 'Data': '{"m_id": 2, "message": "message contents 2"}', 'PartitionKey': '2' }, { 'Data': '{"m_id": 3, "message": "message contents 3"}', 'PartitionKey': '3' }, { 'Data': '{"m_id": 4, "message": "message contents 4"}', 'PartitionKey': '4' }, { 'Data': '{"m_id": 5, "message": "message contents 5"}', 'PartitionKey': '5' }], 'StreamName': 'test_stream' }), call( **{ 'Records': [{ 'Data': '{"m_id": 1, "message": "message contents 1"}', 'PartitionKey': '1' }, { 'Data': '{"m_id": 2, "message": "message contents 2"}', 'PartitionKey': '2' }, { 'Data': '{"m_id": 3, "message": "message contents 3"}', 'PartitionKey': '3' }, { 'Data': '{"m_id": 4, "message": "message contents 4"}', 'PartitionKey': '4' }, { 'Data': '{"m_id": 5, "message": "message contents 5"}', 'PartitionKey': '5' }], 'StreamName': 'test_stream' }), call( **{ 'Records': [{ 'Data': '{"m_id": 1, "message": "message contents 1"}', 'PartitionKey': '1' }, { 'Data': '{"m_id": 2, "message": "message contents 2"}', 'PartitionKey': '2' }, { 'Data': '{"m_id": 3, "message": "message contents 3"}', 'PartitionKey': '3' }, { 'Data': '{"m_id": 4, "message": "message contents 4"}', 'PartitionKey': '4' }, { 'Data': '{"m_id": 5, "message": "message contents 5"}', 'PartitionKey': '5' }], 'StreamName': 'test_stream' }) ]) kinesis_client._individual_dispatch_method.assert_not_called() self.assertEqual(test_payloads, kinesis_client.unprocessed_items)
if _has_required_boto(): region = "us-east-1" access_key = "GKTADJGHEIQSXMKKRBJ08H" secret_key = "askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs" conn_parameters = { "region": region, "key": access_key, "keyid": secret_key, "profile": {}, } error_message = "An error occurred (101) when calling the {0} operation: Test-defined error" not_found_error = ClientError( { "Error": { "Code": "ResourceNotFoundException", "Message": "Test-defined error", } }, "msg", ) topic_rule_not_found_error = ClientError( {"Error": {"Code": "UnauthorizedException", "Message": "Test-defined error"}}, "msg", ) error_content = {"Error": {"Code": 101, "Message": "Test-defined error"}} policy_ret = dict( policyName="testpolicy", policyDocument='{"Version": "2012-10-17", "Statement": [{"Action": ["iot:Publish"], "Resource": ["*"], "Effect": "Allow"}]}', policyArn="arn:aws:iot:us-east-1:123456:policy/my_policy", policyVersionId=1, defaultVersionId=1,
def mocked_poller(account, stream): raise ClientError({"Error": { "Message": "", "Code": "AccessDenied" }}, "sts:AssumeRole")
def func(): self.count += 1 raise ClientError({"Error": {"Code": 42}}, "something")
def test_absent_with_failure(self): self.conn.delete_bucket.side_effect = ClientError(error_content, 'delete_bucket') result = salt_states['boto_s3_bucket.absent']('test', 'testbucket') self.assertFalse(result['result']) self.assertTrue('An error occurred' in result['comment'])
if not HAS_BOTO: return False elif LooseVersion(boto3.__version__) < LooseVersion(required_boto3_version): return False else: return True if _has_required_boto(): region = 'us-east-1' access_key = 'GKTADJGHEIQSXMKKRBJ08H' secret_key = 'askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs' conn_parameters = {'region': region, 'key': access_key, 'keyid': secret_key, 'profile': {}} error_message = 'An error occurred (101) when calling the {0} operation: Test-defined error' not_found_error = ClientError({ 'Error': { 'Code': '404', 'Message': "Test-defined error" } }, 'msg') error_content = { 'Error': { 'Code': 101, 'Message': "Test-defined error" } } list_ret = { 'Buckets': [{ 'Name': 'mybucket', 'CreationDate': None }], 'Owner': { 'DisplayName': 'testuser',
def writer_side_effect(*args, **kwargs): raise ClientError({'Error': {'Code': 403, 'Message': 'Access denied'}}, 'information')
def test_that_when_describing_thing_type_and_boto3_returns_error_an_error_the_describe_thing_type_method_returns_error(self): self.conn.describe_thing_type.side_effect = ClientError(error_content, 'describe_thing_type') result = boto_iot.describe_thing_type(thingTypeName='mythingtype', **conn_parameters) self.assertEqual(result.get('error', {}).get('message'), error_message.format('describe_thing_type'))
def test_entire_environment_process_together(monkeypatch): boto3_client = MagicMock() boto3_client.describe_task_definition.side_effect = ClientError({}, None) boto3_client.register_task_definition.return_value = {} boto3_client.run_task.return_value = {} monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client)) flow_runner = MagicMock() monkeypatch.setattr( "prefect.engine.get_default_flow_runner_class", MagicMock(return_value=flow_runner), ) monkeypatch.setenv("AWS_ACCESS_KEY_ID", "id") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "secret") monkeypatch.setenv("AWS_SESSION_TOKEN", "session") monkeypatch.setenv("REGION_NAME", "region") with prefect.context({"flow_run_id": "id"}), set_temporary_config( {"cloud.auth_token": "test", "logging.extra_loggers": "['test_logger']",} ): storage = Docker(registry_url="test", image_name="image", image_tag="tag") flow = Flow("name", storage=storage) environment = FargateTaskEnvironment( containerDefinitions=[ { "name": "flow-container", "image": "image", "command": [], "environment": [], "essential": True, } ], cluster="test", family="test", taskDefinition="test", ) assert environment assert environment.aws_access_key_id == "id" assert environment.aws_secret_access_key == "secret" assert environment.aws_session_token == "session" assert environment.region_name == "region" environment.setup(flow=flow) assert boto3_client.describe_task_definition.called assert boto3_client.register_task_definition.called assert boto3_client.register_task_definition.call_args[1]["family"] == "test" assert boto3_client.register_task_definition.call_args[1][ "containerDefinitions" ] == [ { "name": "flow-container", "image": "test/image:tag", "command": [ "/bin/sh", "-c", "python -c 'import prefect; prefect.environments.FargateTaskEnvironment().run_flow()'", ], "environment": [ { "name": "PREFECT__CLOUD__GRAPHQL", "value": prefect.config.cloud.graphql, }, {"name": "PREFECT__CLOUD__USE_LOCAL_SECRETS", "value": "false"}, { "name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudFlowRunner", }, { "name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudTaskRunner", }, {"name": "PREFECT__LOGGING__LOG_TO_CLOUD", "value": "true"}, { "name": "PREFECT__LOGGING__EXTRA_LOGGERS", "value": "['test_logger']", }, ], "essential": True, } ] environment.execute(flow=flow) assert boto3_client.run_task.called assert boto3_client.run_task.call_args[1]["taskDefinition"] == "test" assert boto3_client.run_task.call_args[1]["overrides"] == { "containerOverrides": [ { "name": "flow-container", "environment": [ { "name": "PREFECT__CLOUD__AUTH_TOKEN", "value": prefect.config.cloud.get("auth_token"), }, {"name": "PREFECT__CONTEXT__FLOW_RUN_ID", "value": "id"}, {"name": "PREFECT__CONTEXT__IMAGE", "value": "test/image:tag"}, ], } ] } assert boto3_client.run_task.call_args[1]["launchType"] == "FARGATE" assert boto3_client.run_task.call_args[1]["cluster"] == "test" with tempfile.TemporaryDirectory() as directory: d = Local(directory) d.add_flow(prefect.Flow("name")) gql_return = MagicMock( return_value=MagicMock( data=MagicMock( flow_run=[ GraphQLResult( { "flow": GraphQLResult( {"name": "name", "storage": d.serialize(),} ) } ) ], ) ) ) client = MagicMock() client.return_value.graphql = gql_return monkeypatch.setattr("prefect.environments.execution.base.Client", client) with set_temporary_config({"cloud.auth_token": "test"}): environment.run_flow() assert flow_runner.call_args[1]["flow"].name == "name"
return False elif LooseVersion(found_botocore_version) < LooseVersion(required_botocore_version): return False else: return True if _has_required_boto(): region = 'us-east-1' access_key = 'GKTADJGHEIQSXMKKRBJ08H' secret_key = 'askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs' conn_parameters = {'region': region, 'key': access_key, 'keyid': secret_key, 'profile': {}} error_message = 'An error occurred (101) when calling the {0} operation: Test-defined error' not_found_error = ClientError({ 'Error': { 'Code': 'ResourceNotFoundException', 'Message': "Test-defined error" } }, 'msg') topic_rule_not_found_error = ClientError({ 'Error': { 'Code': 'UnauthorizedException', 'Message': "Test-defined error" } }, 'msg') error_content = { 'Error': { 'Code': 101, 'Message': "Test-defined error" } } policy_ret = dict(policyName='testpolicy',
def test_task_client_error_one_file(self): """ Test retries for restore error for one file. """ exp_event = {} exp_event["config"] = {"glacier-bucket": "some_bucket"} exp_event["input"] = { "granules": [{ "granuleId": "MOD09GQ.A0219114.N5aUCG.006.0656338553321", "keys": [KEY1] }] } os.environ['RESTORE_RETRY_SLEEP_SECS'] = '.5' requests_db.request_id_generator = Mock(side_effect=[ REQUEST_GROUP_ID_EXP_1, REQUEST_ID1, REQUEST_ID2, REQUEST_ID3 ]) boto3.client = Mock() s3_cli = boto3.client('s3') s3_cli.head_object = Mock() s3_cli.restore_object = Mock(side_effect=[ ClientError({'Error': { 'Code': 'NoSuchBucket' }}, 'restore_object'), ClientError({'Error': { 'Code': 'NoSuchBucket' }}, 'restore_object'), ClientError({'Error': { 'Code': 'NoSuchBucket' }}, 'restore_object') ]) CumulusLogger.info = Mock() CumulusLogger.error = Mock() mock_ssm_get_parameter(1) #exp_gran = {} #exp_gran['granuleId'] = 'MOD09GQ.A0219114.N5aUCG.006.0656338553321' exp_gran = { 'granuleId': 'MOD09GQ.A0219114.N5aUCG.006.0656338553321', 'files': [{ 'key': FILE1, 'dest_bucket': PROTECTED_BUCKET, 'success': False, 'err_msg': 'An error occurred (NoSuchBucket) when calling the restore_object ' 'operation: Unknown' }] } exp_err = f"One or more files failed to be requested. {exp_gran}" try: request_files.task(exp_event, self.context) self.fail("RestoreRequestError expected") except request_files.RestoreRequestError as err: self.assertEqual(exp_err, str(err)) del os.environ['RESTORE_RETRY_SLEEP_SECS'] boto3.client.assert_called_with('ssm') s3_cli.head_object.assert_called_with(Bucket='some_bucket', Key=FILE1) restore_req_exp = { 'Days': 5, 'GlacierJobParameters': { 'Tier': 'Standard' } } s3_cli.restore_object.assert_called_with( Bucket='some_bucket', Key=FILE1, RestoreRequest=restore_req_exp)
def batch_get_item(self, **kwargs): """Mock batch_get_item method and return mimicking dynamodb response Keyword Argments: exception (bool): True raise exception. Returns: (dict): Response dictionary containing fake results. """ if self.exception: err = {'Error': {'Code': 400, 'Message': 'raising test exception'}} raise ClientError(err, 'batch_get_item') if not kwargs.get('RequestItems'): err = { 'Error': { 'Code': 403, 'Message': 'raising test exceptionParameter validation failed' } } raise ParamValidationError(report=err) # Validate query keys for _, item_value in kwargs['RequestItems'].iteritems(): if not item_value.get('Keys'): err = { 'Error': { 'Code': 400, 'Message': '[Keys] parameter is required' } } raise ParamValidationError(report=err) self._validate_keys(item_value['Keys']) response = { 'UnprocessedKeys': {}, 'Responses': { 'test_table_name': [{ 'ioc_value': { 'S': '1.1.1.2' }, 'sub_type': { 'S': 'mal_ip' } }, { 'ioc_value': { 'S': 'evil.com' }, 'sub_type': { 'S': 'c2_domain' } }] }, 'ResponseMetadata': { 'RetryAttempts': 0, 'HTTPStatusCode': 200, 'RequestId': 'ABCD1234', 'HTTPHeaders': {} } } if self.has_unprocessed_keys: response['UnprocessedKeys'] = { 'test_table_name': { 'Keys': [{ 'ioc_value': { 'S': 'foo' } }, { 'ioc_value': { 'S': 'bar' } }] } } return response
def test_task_client_error_3_times(self): """ Test three files, two successful, one errors on all retries and fails. """ exp_event = {} exp_event["config"] = {"glacier-bucket": "some_bucket"} gran = {} granule_id = "MOD09GQ.A0219114.N5aUCG.006.0656338553321" gran["granuleId"] = granule_id keys = [KEY1, KEY3, KEY4] gran["keys"] = keys exp_event["input"] = {"granules": [gran]} requests_db.request_id_generator = Mock(side_effect=[ REQUEST_GROUP_ID_EXP_1, REQUEST_ID1, REQUEST_GROUP_ID_EXP_3, REQUEST_ID2, REQUEST_ID3, REQUEST_ID4 ]) boto3.client = Mock() s3_cli = boto3.client('s3') s3_cli.head_object = Mock() s3_cli.restore_object = Mock(side_effect=[ None, ClientError({'Error': { 'Code': 'NoSuchBucket' }}, 'restore_object'), None, ClientError({'Error': { 'Code': 'NoSuchBucket' }}, 'restore_object'), ClientError({'Error': { 'Code': 'NoSuchKey' }}, 'restore_object') ]) CumulusLogger.info = Mock() CumulusLogger.error = Mock() mock_ssm_get_parameter(3) exp_gran = {} exp_gran['granuleId'] = granule_id exp_files = [] exp_file = {} exp_file['key'] = FILE1 exp_file['dest_bucket'] = PROTECTED_BUCKET exp_file['success'] = True exp_file['err_msg'] = '' exp_files.append(exp_file) exp_file = {} exp_file['key'] = FILE3 exp_file['dest_bucket'] = None exp_file['success'] = False exp_file['err_msg'] = 'An error occurred (NoSuchKey) when calling the restore_object ' \ 'operation: Unknown' exp_files.append(exp_file) exp_file = {} exp_file['key'] = FILE4 exp_file['dest_bucket'] = PUBLIC_BUCKET exp_file['success'] = True exp_file['err_msg'] = '' exp_files.append(exp_file) exp_gran['files'] = exp_files exp_err = f"One or more files failed to be requested. {exp_gran}" print_rows("begin") try: request_files.task(exp_event, self.context) self.fail("RestoreRequestError expected") except request_files.RestoreRequestError as err: self.assertEqual(exp_err, str(err)) print_rows("end")
def writer_side_effect(*args, **kwargs): raise ClientError({'Error': {}}, 'some operation')
def test_task_client_error_2_times(self): """ Test two files, first successful, second has two errors, then success. """ file1 = {"key": FILE1, "dest_bucket": "sndbx-cumulus-protected"} file2 = {"key": FILE2, "dest_bucket": "sndbx-cumulus-protected"} exp_event = {} exp_event["config"] = {"glacier-bucket": "some_bucket"} gran = {} granule_id = "MOD09GQ.A0219114.N5aUCG.006.0656338553321" gran["granuleId"] = granule_id keys = [] keys.append(file1) keys.append(file2) gran["keys"] = keys exp_event["input"] = {"granules": [gran]} requests_db.request_id_generator = Mock(side_effect=[ REQUEST_GROUP_ID_EXP_1, REQUEST_ID1, REQUEST_GROUP_ID_EXP_2, REQUEST_ID2, REQUEST_ID3 ]) boto3.client = Mock() s3_cli = boto3.client('s3') s3_cli.head_object = Mock() s3_cli.restore_object = Mock(side_effect=[ None, ClientError({'Error': { 'Code': 'NoSuchBucket' }}, 'restore_object'), ClientError({'Error': { 'Code': 'NoSuchBucket' }}, 'restore_object'), None ]) CumulusLogger.info = Mock() CumulusLogger.error = Mock() mock_ssm_get_parameter(2) exp_gran = {} exp_gran['granuleId'] = granule_id exp_files = [] exp_file = {} exp_file['key'] = FILE1 exp_file['dest_bucket'] = "sndbx-cumulus-protected" exp_file['success'] = True exp_file['err_msg'] = '' exp_files.append(exp_file) exp_file = {} exp_file['key'] = FILE2 exp_file['dest_bucket'] = "sndbx-cumulus-protected" exp_file['success'] = True exp_file['err_msg'] = '' exp_files.append(exp_file) exp_gran['files'] = exp_files print_rows("begin") result = request_files.task(exp_event, self.context) self.assertEqual(exp_gran, result) print_rows("end")