def test_emr_launch_function():
app = core.App()
stack = core.Stack(app, 'test-stack')
vpc = ec2.Vpc(stack, 'Vpc')
success_topic = sns.Topic(stack, 'SuccessTopic')
failure_topic = sns.Topic(stack, 'FailureTopic')
profile = emr_profile.EMRProfile(
stack, 'test-profile',
profile_name='test-profile',
vpc=vpc)
configuration = cluster_configuration.ClusterConfiguration(
stack, 'test-configuration', configuration_name='test-configuration')
function = emr_launch_function.EMRLaunchFunction(
stack, 'test-function',
launch_function_name='test-function',
emr_profile=profile,
cluster_configuration=configuration,
cluster_name='test-cluster',
success_topic=success_topic,
failure_topic=failure_topic,
allowed_cluster_config_overrides=configuration.override_interfaces['default'],
wait_for_cluster_start=False
)
resolved_function = stack.resolve(function.to_json())
print(default_function)
print(resolved_function)
assert default_function == resolved_function
def test_emr_secure_launch_function(self):
stack = core.Stack(core.App(), 'test-stack')
vpc = ec2.Vpc(stack, 'Vpc')
success_topic = sns.Topic(stack, 'SuccessTopic')
failure_topic = sns.Topic(stack, 'FailureTopic')
profile = emr_profile.EMRProfile(
stack, 'test-profile',
profile_name='test-profile',
vpc=vpc,)
configuration = cluster_configuration.ClusterConfiguration(
stack, 'test-configuration',
configuration_name='test-configuration',
secret_configurations={
'SecretConfiguration': secretsmanager.Secret(stack, 'Secret')
})
function = emr_launch_function.EMRLaunchFunction(
stack, 'test-function',
description='test description',
launch_function_name='test-function',
emr_profile=profile,
cluster_configuration=configuration,
cluster_name='test-cluster',
success_topic=success_topic,
failure_topic=failure_topic,
allowed_cluster_config_overrides=configuration.override_interfaces['default'],
wait_for_cluster_start=False
)
self.print_and_assert(self.default_function, function)
def test_default_configuration():
cluster_config = cluster_configuration.ClusterConfiguration(
stack, 'test-instance-group-config', configuration_name='test-cluster')
config = copy.deepcopy(default_config)
resolved_config = stack.resolve(cluster_config.to_json())
print(config)
print(resolved_config)
assert resolved_config == config
def test_get_function(self):
stack = core.Stack(core.App(), 'test-stack', env=core.Environment(account='123456789012', region='us-east-1'))
vpc = ec2.Vpc.from_lookup(stack, 'test-vpc', vpc_id='vpc-12345678')
success_topic = sns.Topic(stack, 'SuccessTopic')
failure_topic = sns.Topic(stack, 'FailureTopic')
profile = emr_profile.EMRProfile(
stack, 'test-profile',
profile_name='test-profile',
vpc=vpc)
configuration = cluster_configuration.ClusterConfiguration(
stack, 'test-configuration', configuration_name='test-configuration')
function = emr_launch_function.EMRLaunchFunction(
stack, 'test-function',
launch_function_name='test-function',
emr_profile=profile,
cluster_configuration=configuration,
cluster_name='test-cluster',
description='test description',
success_topic=success_topic,
failure_topic=failure_topic,
allowed_cluster_config_overrides=configuration.override_interfaces['default'],
wait_for_cluster_start=False
)
ssm = boto3.client('ssm')
ssm.put_parameter(
Name=f'{emr_profile.SSM_PARAMETER_PREFIX}/{profile.namespace}/{profile.profile_name}',
Value=json.dumps(profile.to_json()))
ssm.put_parameter(
Name=f'{cluster_configuration.SSM_PARAMETER_PREFIX}/'
f'{configuration.namespace}/{configuration.configuration_name}',
Value=json.dumps(configuration.to_json()))
ssm.put_parameter(
Name=f'{emr_launch_function.SSM_PARAMETER_PREFIX}/{function.namespace}/{function.launch_function_name}',
Value=json.dumps(function.to_json()))
restored_function = emr_launch_function.EMRLaunchFunction.from_stored_function(
stack, 'test-restored-function',
namespace=function.namespace,
launch_function_name=function.launch_function_name,
)
self.assertEquals(function.to_json(), restored_function.to_json())
def test_bootstrap_action_config():
bucket = s3.Bucket(stack, 'test-bucket')
bootstrap_code = emr_code.Code.from_path(path='./docs',
deployment_bucket=bucket,
deployment_prefix='prefix')
bootstrap_action = emr_code.EMRBootstrapAction(
name='Bootstrap',
path=f'{bootstrap_code.s3_path}/bootstrap_action',
args=['Arg1', 'Arg2'],
code=bootstrap_code)
cluster_config = cluster_configuration.ClusterConfiguration(
stack,
'test-bootstrap-action-config',
configuration_name='test-cluster',
bootstrap_actions=[bootstrap_action])
config = copy.deepcopy(default_config)
config['ClusterConfiguration']['BootstrapActions'] = [{
'Name': 'Bootstrap',
'ScriptBootstrapAction': {
'Path': {
'Fn::Join': [
'',
[
's3://', {
'Ref': 'testbucketE6E05ABE'
}, '/prefix/bootstrap_action'
]
]
},
'Args': ['Arg1', 'Arg2']
}
}]
config['ConfigurationArtifacts'] = [{
'Bucket': {
'Ref': 'testbucketE6E05ABE'
},
'Path': 'prefix/*'
}]
resolved_config = stack.resolve(cluster_config.to_json())
print(config)
print(resolved_config)
assert resolved_config == config
def test_disabling_glue_metastore():
cluster_config = cluster_configuration.ClusterConfiguration(
stack,
'test-disable-glue-metastore',
configuration_name='test-cluster',
use_glue_catalog=False)
config = copy.deepcopy(default_config)
config['ClusterConfiguration']['Configurations'] = [{
'Classification': 'hive-site',
'Properties': {}
}, {
'Classification': 'spark-hive-site',
'Properties': {}
}]
resolved_config = stack.resolve(cluster_config.to_json())
print(config)
print(resolved_config)
assert resolved_config == config