def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, host='iam.amazonaws.com', debug=0, https_connection_factory=None, path='/', security_token=None, validate_certs=True, profile_name=None, mock_iam_instance_profiles=None, mock_iam_roles=None, mock_iam_role_policies=None): """Mock out connection to IAM. mock_iam_instance_profiles maps profile name to a dictionary containing: create_date -- ISO creation datetime path -- IAM path role_name -- name of single role for this instance profile, or None mock_iam_roles maps role name to a dictionary containing: assume_role_policy_document -- a JSON-then-URI-encoded policy doc create_date -- ISO creation datetime path -- IAM path mock_iam_role_policies maps policy name to a dictionary containing: policy_document -- JSON-then-URI-encoded policy doc role_name -- name of single role for this policy (always defined) We don't currently support role IDs or ARNs because our code doesn't use them. """ self.mock_iam_instance_profiles = combine_values( {}, mock_iam_instance_profiles) self.mock_iam_roles = combine_values({}, mock_iam_roles) self.mock_iam_role_policies = combine_values( {}, mock_iam_role_policies)
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, region=None, mock_s3_fs=None, mock_emr_job_flows=None, mock_emr_failures=None, mock_emr_output=None, max_days_ago=DEFAULT_MAX_DAYS_AGO, max_job_flows_returned=DEFAULT_MAX_JOB_FLOWS_RETURNED, max_simulation_steps=100): """Create a mock version of EmrConnection. Most of these args are the same as for the real EmrConnection, and are ignored. By default, jobs will run to conclusion, and if their output dir is on S3, create a single empty output file. You can manually decide that some jobs will fail, or give them different output by setting mock_emr_failures/mock_emr_output. Job flows are given IDs j-MOCKJOBFLOW0, j-MOCKJOBFLOW1, etc. Step numbers are 0-indexed. Extra args: :param mock_s3_fs: a mock S3 filesystem to point to (just a dictionary mapping bucket name to key name to bytes) :param mock_emr_job_flows: a mock set of EMR job flows to point to (just a map from job flow ID to a :py:class:`MockEmrObject` representing a job flow) :param mock_emr_failures: a map from ``(job flow ID, step_num)`` to a failure message (or ``None`` for the default message) :param mock_emr_output: a map from ``(job flow ID, step_num)`` to a list of ``str``s representing file contents to output when the job completes :type max_job_flows_returned: int :param max_job_flows_returned: the maximum number of job flows that :py:meth:`describe_jobflows` can return, to simulate a real limitation of EMR :type max_days_ago: int :param max_days_ago: the maximum amount of days that EMR will go back in time :type max_simulation_steps: int :param max_simulation_steps: the maximum number of times we can simulate the progress of EMR job flows (to protect against simulating forever) """ self.mock_s3_fs = combine_values({}, mock_s3_fs) self.mock_emr_job_flows = combine_values({}, mock_emr_job_flows) self.mock_emr_failures = combine_values({}, mock_emr_failures) self.mock_emr_output = combine_values({}, mock_emr_output) self.max_days_ago = max_days_ago self.max_job_flows_returned = max_job_flows_returned self.simulation_steps_left = max_simulation_steps if region is not None: self.endpoint = region.endpoint else: self.endpoint = 'elasticmapreduce.amazonaws.com'
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, host=None, debug=0, https_connection_factory=None, calling_format=None, path='/', provider='aws', bucket_class=None, mock_s3_fs=None): """Mock out a connection to S3. Most of these args are the same as for the real S3Connection, and are ignored. You can set up a mock filesystem to share with other objects by specifying mock_s3_fs. The mock filesystem is just a map from bucket name to key name to bytes. """ # use mock_s3_fs even if it's {} self.mock_s3_fs = combine_values({}, mock_s3_fs) self.endpoint = host or 's3.amazonaws.com'
def __init__( self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, host=None, debug=0, https_connection_factory=None, calling_format=None, path="/", provider="aws", bucket_class=None, mock_s3_fs=None, ): """Mock out a connection to S3. Most of these args are the same as for the real S3Connection, and are ignored. You can set up a mock filesystem to share with other objects by specifying mock_s3_fs. The mock filesystem is just a map from bucket name to key name to bytes. """ # use mock_s3_fs even if it's {} self.mock_s3_fs = combine_values({}, mock_s3_fs) self.endpoint = host or "s3.amazonaws.com"
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, host='iam.amazonaws.com', debug=0, https_connection_factory=None, path='/', security_token=None, validate_certs=True, profile_name=None, mock_iam_instance_profiles=None, mock_iam_roles=None, mock_iam_role_policies=None): """Mock out connection to IAM. mock_iam_instance_profiles maps profile name to a dictionary containing: create_date -- ISO creation datetime path -- IAM path role_name -- name of single role for this instance profile, or None mock_iam_roles maps role name to a dictionary containing: assume_role_policy_document -- a JSON-then-URI-encoded policy doc create_date -- ISO creation datetime path -- IAM path mock_iam_role_policies maps policy name to a dictionary containing: policy_document -- JSON-then-URI-encoded policy doc role_name -- name of single role for this policy (always defined) We don't currently support role IDs or ARNs because our code doesn't use them. """ self.mock_iam_instance_profiles = combine_values( {}, mock_iam_instance_profiles) self.mock_iam_roles = combine_values({}, mock_iam_roles) self.mock_iam_role_policies = combine_values({}, mock_iam_role_policies)
def __init__(self, region_name=None, api_version=None, use_ssl=True, verify=None, endpoint_url=None, aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None, config=None, mock_iam_instance_profiles=None, mock_iam_roles=None, mock_iam_role_attached_policies=None): """Mock out connection to IAM. mock_iam_instance_profiles maps profile name to a dict containing: create_date -- ISO creation datetime path -- IAM path role_name -- name of single role for this instance profile, or None mock_iam_roles maps role name to a dict containing: assume_role_policy_document -- a JSON-then-URI-encoded policy doc create_date -- ISO creation datetime path -- IAM path mock_iam_role_attached_policies maps role to a list of ARNs for attached (managed) policies. We don't track which managed policies exist or what their contents are. We also don't support role IDs. """ # if not passed dictionaries for these mock values, create our own self.mock_iam_instance_profiles = combine_values( {}, mock_iam_instance_profiles) self.mock_iam_roles = combine_values({}, mock_iam_roles) self.mock_iam_role_attached_policies = combine_values( {}, mock_iam_role_attached_policies) endpoint_url = endpoint_url or 'https://iam.amazonaws.com' region_name = region_name or 'aws-global' self.meta = MockClientMeta(endpoint_url=endpoint_url, region_name=region_name)
def test_falseish_values(self): # everything but None is a legit value self.assertEqual(combine_values(True, False), False) self.assertEqual(combine_values(1, 0), 0) self.assertEqual(combine_values('full', ''), '') self.assertEqual(combine_values([1, 2, 3], []), []) self.assertEqual(combine_values((1, 2, 3), ()), ()) self.assertEqual(combine_values({'a': 'b'}, {}), {}) self.assertEqual(combine_values(set([1]), set()), set())
def test_falseish_values(self): # everything but None is a legit value self.assertEqual(combine_values(True, False), False) self.assertEqual(combine_values(1, 0), 0) self.assertEqual(combine_values("full", ""), "") self.assertEqual(combine_values([1, 2, 3], []), []) self.assertEqual(combine_values((1, 2, 3), ()), ()) self.assertEqual(combine_values({"a": "b"}, {}), {}) self.assertEqual(combine_values(set([1]), set()), set())
def test_falseish_values(self): # everything but None is a legit value assert_equal(combine_values(True, False), False) assert_equal(combine_values(1, 0), 0) assert_equal(combine_values('full', ''), '') assert_equal(combine_values([1, 2, 3], []), []) assert_equal(combine_values((1, 2, 3), ()), ()) assert_equal(combine_values({'a': 'b'}, {}), {}) assert_equal(combine_values(set([1]), set()), set())
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, host='iam.amazonaws.com', debug=0, https_connection_factory=None, path='/', security_token=None, validate_certs=True, profile_name=None, mock_iam_instance_profiles=None, mock_iam_roles=None, mock_iam_role_policies=None, mock_iam_role_attached_policies=None): """Mock out connection to IAM. mock_iam_instance_profiles maps profile name to a dictionary containing: create_date -- ISO creation datetime path -- IAM path role_name -- name of single role for this instance profile, or None mock_iam_roles maps role name to a dictionary containing: assume_role_policy_document -- a JSON-then-URI-encoded policy doc create_date -- ISO creation datetime path -- IAM path mock_iam_role_policies maps policy name to a dictionary containing: policy_document -- JSON-then-URI-encoded policy doc role_name -- name of single role for this policy (always defined) mock_iam_role_attached_policies maps role to a list of ARNs for attached (managed) policies. We don't track which managed policies exist or what their contents are. We also don't support role IDs. """ self.mock_iam_instance_profiles = combine_values( {}, mock_iam_instance_profiles) self.mock_iam_roles = combine_values({}, mock_iam_roles) self.mock_iam_role_policies = combine_values( {}, mock_iam_role_policies) self.mock_iam_role_attached_policies = combine_values( {}, mock_iam_role_attached_policies)
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, region=None, mock_s3_fs=None, mock_emr_job_flows=None, mock_emr_failures=None, mock_emr_output=None, max_simulation_steps=100): """Create a mock version of EmrConnection. Most of these args are the same as for the real EmrConnection, and are ignored. By default, jobs will run to conclusion, and if their output dir is on S3, create a single empty output file. You can manually decide that some jobs will fail, or give them different output by setting mock_emr_failures/mock_emr_output. Job flows are given IDs j-MOCKJOBFLOW0, j-MOCKJOBFLOW1, etc. Step numbers are 0-indexed. Extra args: mock_s3_fs -- a mock S3 filesystem to point to (just a dictionary mapping bucket name to key name to bytes) mock_emr_job_flows -- a mock set of EMR job flows to point to (just a map from job flow ID to a MockEmrObject representing a job flow) mock_emr_failures -- a map from (job flow ID, step_num) to a failure message (or None for the default message) mock_emr_output -- a map from (job flow ID, step_num) to a list of strs representing file contents to output when the job completes max_simulation_steps -- the maximum number of times we can simulate the the progress of EMR job flows (to protect against simulating forever) """ self.mock_s3_fs = combine_values({}, mock_s3_fs) self.mock_emr_job_flows = combine_values({}, mock_emr_job_flows) self.mock_emr_failures = combine_values({}, mock_emr_failures) self.mock_emr_output = combine_values({}, mock_emr_output) self.simulation_steps_left = max_simulation_steps
def test_skips_None(self): self.assertEqual(combine_values(None, ['cat']), ['cat']) self.assertEqual(combine_values(['cat'], None), ['cat']) self.assertEqual(combine_values(None, None, ['cat'], None), ['cat'])
def test_skips_None(self): assert_equal(combine_values(None, 'one'), 'one') assert_equal(combine_values('one', None), 'one') assert_equal(combine_values(None, None, 'one', None), 'one')
def test_all_None(self): assert_equal(combine_values(None, None, None), None)
def test_picks_last_value(self): assert_equal(combine_values(1, 2, 3), 3)
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, region=None, mock_s3_fs=None, mock_emr_job_flows=None, mock_emr_failures=None, mock_emr_output=None, max_days_ago=DEFAULT_MAX_DAYS_AGO, max_job_flows_returned=DEFAULT_MAX_JOB_FLOWS_RETURNED, simulation_iterator=None, security_token=None): """Create a mock version of EmrConnection. Most of these args are the same as for the real EmrConnection, and are ignored. By default, jobs will run to conclusion, and if their output dir is on S3, create a single empty output file. You can manually decide that some jobs will fail, or give them different output by setting mock_emr_failures/mock_emr_output. Job flows are given IDs j-MOCKJOBFLOW0, j-MOCKJOBFLOW1, etc. Step numbers are 0-indexed. Extra args: :param mock_s3_fs: a mock S3 filesystem to point to (just a dictionary mapping bucket name to key name to bytes) :param mock_emr_job_flows: a mock set of EMR job flows to point to (just a map from job flow ID to a :py:class:`MockEmrObject` representing a job flow) :param mock_emr_failures: a map from ``(job flow ID, step_num)`` to a failure message (or ``None`` for the default message) :param mock_emr_output: a map from ``(job flow ID, step_num)`` to a list of ``bytes``s representing file contents to output when the job completes :type max_job_flows_returned: int :param max_job_flows_returned: the maximum number of job flows that :py:meth:`describe_jobflows` can return, to simulate a real limitation of EMR :type max_days_ago: int :param max_days_ago: the maximum amount of days that EMR will go back in time :param simulation_iterator: we call ``next()`` on this each time we simulate progress. If there is no next element, we bail out. """ # check this now; strs will cause problems later in Python 3 if mock_emr_output and any( any(not isinstance(part, bytes) for part in parts) for parts in mock_emr_output.values()): raise TypeError('mock EMR output must be bytes') self.mock_s3_fs = combine_values({}, mock_s3_fs) self.mock_emr_job_flows = combine_values({}, mock_emr_job_flows) self.mock_emr_failures = combine_values({}, mock_emr_failures) self.mock_emr_output = combine_values({}, mock_emr_output) self.max_days_ago = max_days_ago self.max_job_flows_returned = max_job_flows_returned self.simulation_iterator = simulation_iterator if region is not None: self.endpoint = region.endpoint else: self.endpoint = 'elasticmapreduce.amazonaws.com'
def test_skips_None(self): self.assertEqual(combine_values(None, 'one'), 'one') self.assertEqual(combine_values('one', None), 'one') self.assertEqual(combine_values(None, None, 'one', None), 'one')
def test_picks_last_value(self): self.assertEqual(combine_values(1, 2, 3), 3)
def test_skips_None(self): assert_equal(combine_values(None, ['cat']), ['cat']) assert_equal(combine_values(['cat'], None), ['cat']) assert_equal(combine_values(None, None, ['cat'], None), ['cat'])
def test_empty(self): self.assertEqual(combine_values(), None)
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None, endpoint_url=None, region_name=None, mock_s3_fs=None, mock_emr_clusters=None, mock_emr_failures=None, mock_emr_self_termination=None, mock_emr_output=None, max_clusters_returned=DEFAULT_MAX_CLUSTERS_RETURNED, max_steps_returned=DEFAULT_MAX_STEPS_RETURNED): """Create a mock version boto3 EMR clients. By default, jobs will run to conclusion, and if their output dir is on S3, create a single empty output file. You can manually decide that some jobs will fail, or give them different output by setting mock_emr_failures/mock_emr_output. Clusters are given IDs j-MOCKCLUSTER0, j-MOCKCLUSTER1, etc. Step numbers are 0-indexed. Extra args: :param mock_s3_fs: a mock S3 filesystem to point to (usually you just want to use an empty dictionary). :param mock_emr_clusters: map from cluster ID to an EMRObject, in the format returned by describe_cluster(), plus ``_bootstrapactions``, ``_instancegroups``, and ``_steps`` fields. :param mock_emr_failures: a set of ``(cluster ID, step_num)`` for steps that should fail. :param mock_emr_self_termination: a set of cluster IDs that should simulate master node termination once cluster is up :param mock_emr_output: a map from ``(cluster ID, step_num)`` to a list of ``str``s representing file contents to output when the job completes :type max_clusters_returned: int :param max_clusters_returned: the maximum number of clusters that :py:meth:`list_clusters` can return, to simulate a real limitation of EMR :type max_steps_returned: int :param max_steps_returned: the maximum number of clusters that :py:meth:`list_steps` can return, to simulate a real limitation of EMR :type max_days_ago: int :param max_days_ago: the maximum amount of days that EMR will go back in time """ # check this now; strs will cause problems later in Python 3 if mock_emr_output and any( any(not isinstance(part, bytes) for part in parts) for parts in mock_emr_output.values()): raise TypeError('mock EMR output must be bytes') self.mock_s3_fs = combine_values({}, mock_s3_fs) self.mock_emr_clusters = combine_values({}, mock_emr_clusters) self.mock_emr_failures = combine_values(set(), mock_emr_failures) self.mock_emr_self_termination = combine_values( set(), mock_emr_self_termination) self.mock_emr_output = combine_values({}, mock_emr_output) self.max_clusters_returned = max_clusters_returned self.max_steps_returned = max_steps_returned region_name = region_name or _DEFAULT_AWS_REGION if not endpoint_url: if region_name == _DEFAULT_AWS_REGION: # not entirely sure why boto3 1.4.4 uses a different format for # us-east-1, but there it is. according to AWS docs, the # host name is elasticmapreduce.<region>.amazonaws.com. endpoint_url = ('https://elasticmapreduce.%s.amazonaws.com' % region_name) else: endpoint_url = ('https://%s.elasticmapreduce.amazonaws.com' % region_name) self.meta = MockClientMeta(endpoint_url=endpoint_url, region_name=region_name)
def test_skips_None(self): self.assertEqual(combine_values(None, "one"), "one") self.assertEqual(combine_values("one", None), "one") self.assertEqual(combine_values(None, None, "one", None), "one")
def test_all_None(self): self.assertEqual(combine_values(None, None, None), None)
def test_empty(self): assert_equal(combine_values(), None)