def test_emr_launch_function():
app = core.App()
stack = core.Stack(app, 'test-stack')
vpc = ec2.Vpc(stack, 'Vpc')
success_topic = sns.Topic(stack, 'SuccessTopic')
failure_topic = sns.Topic(stack, 'FailureTopic')
profile = emr_profile.EMRProfile(
stack, 'test-profile',
profile_name='test-profile',
vpc=vpc)
configuration = cluster_configuration.ClusterConfiguration(
stack, 'test-configuration', configuration_name='test-configuration')
function = emr_launch_function.EMRLaunchFunction(
stack, 'test-function',
launch_function_name='test-function',
emr_profile=profile,
cluster_configuration=configuration,
cluster_name='test-cluster',
success_topic=success_topic,
failure_topic=failure_topic,
allowed_cluster_config_overrides=configuration.override_interfaces['default'],
wait_for_cluster_start=False
)
resolved_function = stack.resolve(function.to_json())
print(default_function)
print(resolved_function)
assert default_function == resolved_function
def test_emr_secure_launch_function(self):
stack = core.Stack(core.App(), 'test-stack')
vpc = ec2.Vpc(stack, 'Vpc')
success_topic = sns.Topic(stack, 'SuccessTopic')
failure_topic = sns.Topic(stack, 'FailureTopic')
profile = emr_profile.EMRProfile(
stack, 'test-profile',
profile_name='test-profile',
vpc=vpc,)
configuration = cluster_configuration.ClusterConfiguration(
stack, 'test-configuration',
configuration_name='test-configuration',
secret_configurations={
'SecretConfiguration': secretsmanager.Secret(stack, 'Secret')
})
function = emr_launch_function.EMRLaunchFunction(
stack, 'test-function',
description='test description',
launch_function_name='test-function',
emr_profile=profile,
cluster_configuration=configuration,
cluster_name='test-cluster',
success_topic=success_topic,
failure_topic=failure_topic,
allowed_cluster_config_overrides=configuration.override_interfaces['default'],
wait_for_cluster_start=False
)
self.print_and_assert(self.default_function, function)
def launch_function_config(self, emr_profile, cluster_configuration,
default_fail_if_cluster_running):
return emr_launch_function.EMRLaunchFunction(
self,
self._cluster_name,
namespace=self._cluster_name,
launch_function_name="launch-fn",
emr_profile=emr_profile,
cluster_configuration=cluster_configuration,
cluster_name=self._cluster_name,
default_fail_if_cluster_running=default_fail_if_cluster_running,
allowed_cluster_config_overrides=cluster_configuration.
override_interfaces['default'],
cluster_tags=[core.Tag(key='Group', value='AWSDemo')])
def test_get_function(self):
stack = core.Stack(core.App(), 'test-stack', env=core.Environment(account='123456789012', region='us-east-1'))
vpc = ec2.Vpc.from_lookup(stack, 'test-vpc', vpc_id='vpc-12345678')
success_topic = sns.Topic(stack, 'SuccessTopic')
failure_topic = sns.Topic(stack, 'FailureTopic')
profile = emr_profile.EMRProfile(
stack, 'test-profile',
profile_name='test-profile',
vpc=vpc)
configuration = cluster_configuration.ClusterConfiguration(
stack, 'test-configuration', configuration_name='test-configuration')
function = emr_launch_function.EMRLaunchFunction(
stack, 'test-function',
launch_function_name='test-function',
emr_profile=profile,
cluster_configuration=configuration,
cluster_name='test-cluster',
description='test description',
success_topic=success_topic,
failure_topic=failure_topic,
allowed_cluster_config_overrides=configuration.override_interfaces['default'],
wait_for_cluster_start=False
)
ssm = boto3.client('ssm')
ssm.put_parameter(
Name=f'{emr_profile.SSM_PARAMETER_PREFIX}/{profile.namespace}/{profile.profile_name}',
Value=json.dumps(profile.to_json()))
ssm.put_parameter(
Name=f'{cluster_configuration.SSM_PARAMETER_PREFIX}/'
f'{configuration.namespace}/{configuration.configuration_name}',
Value=json.dumps(configuration.to_json()))
ssm.put_parameter(
Name=f'{emr_launch_function.SSM_PARAMETER_PREFIX}/{function.namespace}/{function.launch_function_name}',
Value=json.dumps(function.to_json()))
restored_function = emr_launch_function.EMRLaunchFunction.from_stored_function(
stack, 'test-restored-function',
namespace=function.namespace,
launch_function_name=function.launch_function_name,
)
self.assertEquals(function.to_json(), restored_function.to_json())
# Load our Basic Cluster Configuration created in the cluster_configurations example
cluster_config = cluster_configuration.ClusterConfiguration.from_stored_configuration(
stack, 'ClusterConfiguration', 'high-mem-instance-group-cluster')
# Create a new State Machine to launch a cluster with the Basic configuration
# Don't allow only ClusterName to be overwritten at launch time.
# Unless specifically indicated, fail to start if a cluster
# of the same name is already running.
launch_function = emr_launch_function.EMRLaunchFunction(
stack,
'EMRLaunchFunction',
launch_function_name='launch-sns-triggered-pipeline-cluster',
cluster_configuration=cluster_config,
emr_profile=sse_s3_profile,
cluster_name='sns-triggered-pipeline',
success_topic=success_topic,
failure_topic=failure_topic,
allowed_cluster_config_overrides={
'ClusterName':
cluster_config.override_interfaces['default']['ClusterName']
},
default_fail_if_cluster_running=True,
)
deployment_bucket = s3.Bucket.from_bucket_name(
stack, 'ArtifactsBucket', f'{NAMING_PREFIX}-artifacts') \
if launch_function.emr_profile.artifacts_bucket is None \
else launch_function.emr_profile.artifacts_bucket
# Prepare the scripts executed by our Steps for deployment
# This uses the Artifacts bucket defined in Cluster Configuration used by our
account=os.environ["CDK_DEFAULT_ACCOUNT"],
region=os.environ["CDK_DEFAULT_REGION"]))
# Load our SSE-KMS EMR Profile created in the emr_profiles example
sse_kms_profile = emr_profile.EMRProfile.from_stored_profile(
stack, 'EMRProfile', 'sse-kms-profile')
# Load our Basic Cluster Configuration created in the cluster_configurations example
cluster_config = cluster_configuration.ClusterConfiguration.from_stored_configuration(
stack, 'ClusterConfiguration', 'basic-instance-group-cluster')
# Create a new State Machine to launch a cluster with the Basic configuration
# Unless specifically indicated, fail to start if a cluster of the same name
# is already running. Allow any parameter in the default override_interface to
# be overwritten.
launch_function = emr_launch_function.EMRLaunchFunction(
stack,
'EMRLaunchFunction',
launch_function_name='launch-basic-cluster',
cluster_configuration=cluster_config,
emr_profile=sse_kms_profile,
cluster_name='basic-cluster',
default_fail_if_cluster_running=True,
allowed_cluster_config_overrides=cluster_config.
override_interfaces['default'],
cluster_tags=[core.Tag('Key1', 'Value1'),
core.Tag('Key2', 'Value2')],
wait_for_cluster_start=True)
app.synth()
# Load our Basic Cluster Configuration created in the cluster_configurations example
cluster_config = cluster_configuration.ClusterConfiguration.from_stored_configuration(
stack, 'ClusterConfiguration', 'high-mem-instance-group-cluster')
# Create a new State Machine to launch a cluster with the Basic configuration
# Allow the Name, Instances.InstanceGroups.1.InstanceCount, and
# Instances.InstanceGroups.1.InstanceType to be overwritten at runtime and assign
# simple names to them. Unless specifically indicated, fail to start if a cluster
# of the same name is already running.
launch_function = emr_launch_function.EMRLaunchFunction(
stack,
'EMRLaunchFunction',
launch_function_name='launch-sns-triggered-pipeline-cluster',
cluster_configuration=cluster_config,
emr_profile=sse_s3_profile,
cluster_name='sns-triggered-pipeline',
success_topic=success_topic,
failure_topic=failure_topic,
default_fail_if_cluster_running=True,
)
deployment_bucket = s3.Bucket.from_bucket_name(
stack, 'ArtifactsBucket', f'{NAMING_PREFIX}-artifacts') \
if launch_function.emr_profile.artifacts_bucket is None \
else launch_function.emr_profile.artifacts_bucket
# Prepare the scripts executed by our Steps for deployment
# This uses the Artifacts bucket defined in Cluster Configuration used by our
# Launch Function
step_code = emr_code.Code.from_path(