def test_emr_launch_function(): app = core.App() stack = core.Stack(app, 'test-stack') vpc = ec2.Vpc(stack, 'Vpc') success_topic = sns.Topic(stack, 'SuccessTopic') failure_topic = sns.Topic(stack, 'FailureTopic') profile = emr_profile.EMRProfile( stack, 'test-profile', profile_name='test-profile', vpc=vpc) configuration = cluster_configuration.ClusterConfiguration( stack, 'test-configuration', configuration_name='test-configuration') function = emr_launch_function.EMRLaunchFunction( stack, 'test-function', launch_function_name='test-function', emr_profile=profile, cluster_configuration=configuration, cluster_name='test-cluster', success_topic=success_topic, failure_topic=failure_topic, allowed_cluster_config_overrides=configuration.override_interfaces['default'], wait_for_cluster_start=False ) resolved_function = stack.resolve(function.to_json()) print(default_function) print(resolved_function) assert default_function == resolved_function
def test_emr_secure_launch_function(self): stack = core.Stack(core.App(), 'test-stack') vpc = ec2.Vpc(stack, 'Vpc') success_topic = sns.Topic(stack, 'SuccessTopic') failure_topic = sns.Topic(stack, 'FailureTopic') profile = emr_profile.EMRProfile( stack, 'test-profile', profile_name='test-profile', vpc=vpc,) configuration = cluster_configuration.ClusterConfiguration( stack, 'test-configuration', configuration_name='test-configuration', secret_configurations={ 'SecretConfiguration': secretsmanager.Secret(stack, 'Secret') }) function = emr_launch_function.EMRLaunchFunction( stack, 'test-function', description='test description', launch_function_name='test-function', emr_profile=profile, cluster_configuration=configuration, cluster_name='test-cluster', success_topic=success_topic, failure_topic=failure_topic, allowed_cluster_config_overrides=configuration.override_interfaces['default'], wait_for_cluster_start=False ) self.print_and_assert(self.default_function, function)
def test_default_configuration(): cluster_config = cluster_configuration.ClusterConfiguration( stack, 'test-instance-group-config', configuration_name='test-cluster') config = copy.deepcopy(default_config) resolved_config = stack.resolve(cluster_config.to_json()) print(config) print(resolved_config) assert resolved_config == config
def test_get_function(self): stack = core.Stack(core.App(), 'test-stack', env=core.Environment(account='123456789012', region='us-east-1')) vpc = ec2.Vpc.from_lookup(stack, 'test-vpc', vpc_id='vpc-12345678') success_topic = sns.Topic(stack, 'SuccessTopic') failure_topic = sns.Topic(stack, 'FailureTopic') profile = emr_profile.EMRProfile( stack, 'test-profile', profile_name='test-profile', vpc=vpc) configuration = cluster_configuration.ClusterConfiguration( stack, 'test-configuration', configuration_name='test-configuration') function = emr_launch_function.EMRLaunchFunction( stack, 'test-function', launch_function_name='test-function', emr_profile=profile, cluster_configuration=configuration, cluster_name='test-cluster', description='test description', success_topic=success_topic, failure_topic=failure_topic, allowed_cluster_config_overrides=configuration.override_interfaces['default'], wait_for_cluster_start=False ) ssm = boto3.client('ssm') ssm.put_parameter( Name=f'{emr_profile.SSM_PARAMETER_PREFIX}/{profile.namespace}/{profile.profile_name}', Value=json.dumps(profile.to_json())) ssm.put_parameter( Name=f'{cluster_configuration.SSM_PARAMETER_PREFIX}/' f'{configuration.namespace}/{configuration.configuration_name}', Value=json.dumps(configuration.to_json())) ssm.put_parameter( Name=f'{emr_launch_function.SSM_PARAMETER_PREFIX}/{function.namespace}/{function.launch_function_name}', Value=json.dumps(function.to_json())) restored_function = emr_launch_function.EMRLaunchFunction.from_stored_function( stack, 'test-restored-function', namespace=function.namespace, launch_function_name=function.launch_function_name, ) self.assertEquals(function.to_json(), restored_function.to_json())
def test_bootstrap_action_config(): bucket = s3.Bucket(stack, 'test-bucket') bootstrap_code = emr_code.Code.from_path(path='./docs', deployment_bucket=bucket, deployment_prefix='prefix') bootstrap_action = emr_code.EMRBootstrapAction( name='Bootstrap', path=f'{bootstrap_code.s3_path}/bootstrap_action', args=['Arg1', 'Arg2'], code=bootstrap_code) cluster_config = cluster_configuration.ClusterConfiguration( stack, 'test-bootstrap-action-config', configuration_name='test-cluster', bootstrap_actions=[bootstrap_action]) config = copy.deepcopy(default_config) config['ClusterConfiguration']['BootstrapActions'] = [{ 'Name': 'Bootstrap', 'ScriptBootstrapAction': { 'Path': { 'Fn::Join': [ '', [ 's3://', { 'Ref': 'testbucketE6E05ABE' }, '/prefix/bootstrap_action' ] ] }, 'Args': ['Arg1', 'Arg2'] } }] config['ConfigurationArtifacts'] = [{ 'Bucket': { 'Ref': 'testbucketE6E05ABE' }, 'Path': 'prefix/*' }] resolved_config = stack.resolve(cluster_config.to_json()) print(config) print(resolved_config) assert resolved_config == config
def test_disabling_glue_metastore(): cluster_config = cluster_configuration.ClusterConfiguration( stack, 'test-disable-glue-metastore', configuration_name='test-cluster', use_glue_catalog=False) config = copy.deepcopy(default_config) config['ClusterConfiguration']['Configurations'] = [{ 'Classification': 'hive-site', 'Properties': {} }, { 'Classification': 'spark-hive-site', 'Properties': {} }] resolved_config = stack.resolve(cluster_config.to_json()) print(config) print(resolved_config) assert resolved_config == config