def env_setup(emr_cluster_name, aws_region, aws_credentials, timeout_ms,
retry_ms, long_timeout_ms):
run_job_flow_args = dict(Instances={
'InstanceCount': 3,
'KeepJobFlowAliveWhenNoSteps': True,
'MasterInstanceType': 'c3.medium',
'Placement': {
'AvailabilityZone': 'test_zone'
},
'SlaveInstanceType': 'c3.xlarge',
},
JobFlowRole='EMR_EC2_DefaultRole',
LogUri='s3://mybucket/log/',
Name=emr_cluster_name,
ServiceRole='EMR_DefaultRole',
VisibleToAllUsers=True)
emr_client = Boto3Util.create_emr_client(aws_region)
emr_cluster_id = emr_client.run_job_flow(
**run_job_flow_args)['JobFlowId']
emr_cluster_client = EMRClusterClient(emr_cluster_id, aws_region,
aws_credentials, timeout_ms,
retry_ms, long_timeout_ms)
return emr_cluster_client, emr_cluster_id
def _create_emr_cluster_client(self, emr_cluster_id):
return EMRClusterClient(
emr_cluster_id,
self.aws_region,
self.aws_api_credentials,
self.api_action_timeout_seconds,
self.api_action_polling_interval_seconds,
self.api_long_timeout_seconds
)
def get_step_status_mocked(emr_step_id):
return "FAILED", EMRClusterClient.FailureDetails(
"test_reason", "test_message",
MockedMethod.log_file_template.format(
emr_cluster_id=MockedMethod.emr_cluster_id,
emr_step_id=emr_step_id))