def env_setup(emr_cluster_name, aws_region, aws_credentials, timeout_ms,
retry_ms, long_timeout_ms):
run_job_flow_args = dict(Instances={
'InstanceCount': 3,
'KeepJobFlowAliveWhenNoSteps': True,
'MasterInstanceType': 'c3.medium',
'Placement': {
'AvailabilityZone': 'test_zone'
},
'SlaveInstanceType': 'c3.xlarge',
},
JobFlowRole='EMR_EC2_DefaultRole',
LogUri='s3://mybucket/log/',
Name=emr_cluster_name,
ServiceRole='EMR_DefaultRole',
VisibleToAllUsers=True)
emr_client = Boto3Util.create_emr_client(aws_region)
emr_cluster_id = emr_client.run_job_flow(
**run_job_flow_args)['JobFlowId']
emr_cluster_client = EMRClusterClient(emr_cluster_id, aws_region,
aws_credentials, timeout_ms,
retry_ms, long_timeout_ms)
return emr_cluster_client, emr_cluster_id
def setup_method(self, method):
super(EMRSystemUnitTestBase, self).setup_method(method)
# Setup EMR mock
self.mock_emr = moto.mock_emr()
self.mock_emr.start()
self.emr_cluster_name = "test clustester for unit and integration tests"
run_job_flow_args = dict(Instances={
'InstanceCount': 3,
'KeepJobFlowAliveWhenNoSteps': True,
'MasterInstanceType': 'c3.medium',
'Placement': {
'AvailabilityZone': 'test_zone'
},
'SlaveInstanceType': 'c3.xlarge',
},
JobFlowRole='EMR_EC2_DefaultRole',
LogUri='s3://{}/log/'.format(
self.default_log_bucket),
Name=self.emr_cluster_name,
ServiceRole='EMR_DefaultRole',
VisibleToAllUsers=True)
emr_client = Boto3Util.create_emr_client(self.default_aws_region)
self.emr_cluster_id = emr_client.run_job_flow(
**run_job_flow_args)['JobFlowId']
logging.debug("Test case specific EMR cluster id is {}".format(
self.emr_cluster_id))
# Setup S3 mock
self.mock_s3 = moto.mock_s3()
self.mock_s3.start()
self.s3_resource = Boto3Util.create_s3_resource()
self.s3_resource.create_bucket(Bucket=self.default_dev_landing_bucket)
self.s3_resource.create_bucket(Bucket=self.default_dev_lake_bucket)
self.s3_resource.create_bucket(Bucket=self.default_dev_mart_cal_bucket)
self.s3_resource.create_bucket(
Bucket=self.default_dev_application_bucket)
self.s3_resource.create_bucket(Bucket=self.default_log_bucket)