def _create_mrjob_role_with_attached_policy(conn, role_document, policy_arn):
# create role
role_name = 'mrjob-' + random_identifier()
conn.create_role(role_name, json.dumps(role_document))
_attach_role_policy(conn, role_name, policy_arn)
return role_name
def _create_mrjob_role_with_attached_policy(conn, role_document, policy_arn):
# create role
role_name = 'mrjob-' + random_identifier()
conn.create_role(role_name, json.dumps(role_document))
_attach_role_policy(conn, role_name, policy_arn)
return role_name
def test_read_large_bz2_file(self):
# catch incorrect use of bz2 library (Issue #814)
input_bz2_path = os.path.join(self.tmp_dir, "input.bz2")
input_bz2 = bz2.BZ2File(input_bz2_path, "wb")
# can't just repeat same value, because we need the file to be
# compressed! 50000 lines is too few to catch the bug.
with random_seed(0):
for _ in range(100000):
input_bz2.write((random_identifier() + "\n").encode("ascii"))
input_bz2.close()
# now expect to read back the same bytes
with random_seed(0):
num_lines = 0
for line in read_file(input_bz2_path):
self.assertEqual(line, (random_identifier() + "\n").encode("ascii"))
num_lines += 1
self.assertEqual(num_lines, 100000)
def _create_mrjob_role_with_attached_policy(conn, role_document, policy_arn):
# create role
role_name = 'mrjob-' + random_identifier()
_get_response(conn, 'CreateRole', {
'AssumeRolePolicyDocument': json.dumps(role_document),
'RoleName': role_name})
_get_response(conn, 'AttachRolePolicy', {
'PolicyArn': policy_arn,
'RoleName': role_name})
return role_name
def test_read_large_bz2_file(self):
# catch incorrect use of bz2 library (Issue #814)
input_bz2_path = os.path.join(self.tmp_dir, 'input.bz2')
input_bz2 = bz2.BZ2File(input_bz2_path, 'wb')
# can't just repeat same value, because we need the file to be
# compressed! 50000 lines is too few to catch the bug.
with random_seed(0):
for _ in range(100000):
input_bz2.write((random_identifier() + '\n').encode('ascii'))
input_bz2.close()
# now expect to read back the same bytes
with random_seed(0):
num_lines = 0
for line in read_file(input_bz2_path):
self.assertEqual(line,
(random_identifier() + '\n').encode('ascii'))
num_lines += 1
self.assertEqual(num_lines, 100000)
def _create_mrjob_role_with_attached_policy(conn, role_document, policy_arn):
# create role
role_name = 'mrjob-' + random_identifier()
_get_response(
conn, 'CreateRole', {
'AssumeRolePolicyDocument': json.dumps(role_document),
'RoleName': role_name
})
_get_response(conn, 'AttachRolePolicy', {
'PolicyArn': policy_arn,
'RoleName': role_name
})
return role_name
def _create_mrjob_role_with_policies(conn, role, policies):
# create role
role_name = 'mrjob-' + random_identifier()
_get_response(conn, 'CreateRole', {
'AssumeRolePolicyDocument': json.dumps(role),
'RoleName': role_name})
for i, policy in enumerate(policies):
# each policy needs a unique name
if len(policies) == 1:
policy_name = role_name
else:
policy_name = '%s-%d' % (role_name, i)
_get_response(conn, 'PutRolePolicy', {
'PolicyDocument': json.dumps(policy),
'PolicyName': policy_name,
'RoleName': role_name})
return role_name
def _create_mrjob_role_with_policies(conn, role, policies):
# create role
role_name = 'mrjob-' + random_identifier()
_get_response(conn, 'CreateRole', {
'AssumeRolePolicyDocument': json.dumps(role),
'RoleName': role_name
})
for i, policy in enumerate(policies):
# each policy needs a unique name
if len(policies) == 1:
policy_name = role_name
else:
policy_name = '%s-%d' % (role_name, i)
_get_response(
conn, 'PutRolePolicy', {
'PolicyDocument': json.dumps(policy),
'PolicyName': policy_name,
'RoleName': role_name
})
return role_name