def test_get_partitioner_and_cleaner_exceptions(self):
Args = namedtuple("Args", ["database", "table", "profile"])
args = Args("nodb", "notable", "noprofile")
no_profile_mock = MagicMock()
no_profile_mock.side_effect = GlutilError(
error_type="ProfileNotFound", message="No such profile noprofile")
original_init = Partitioner.__init__
Partitioner.__init__ = no_profile_mock
cli = Cli()
try:
with captured_output() as (out, err):
cli.get_partitioner(args)
output = out.getvalue().strip()
output.should.equal(
"No such profile noprofile\n\tConfirm that noprofile is a locally configured aws profile."
)
self.exit_mock.assert_called_with(1)
no_access_mock = MagicMock()
no_access_mock.side_effect = GlutilError(
error_type="AccessDenied",
message="You do not have permissions to run GetTable")
Partitioner.__init__ = no_access_mock
with captured_output() as (out, err):
cli.get_partitioner(args)
output = out.getvalue().strip()
output.should.equal(
"You do not have permissions to run GetTable\n\tConfirm that noprofile has the glue:GetTable permission."
)
self.exit_mock.assert_called_with(1)
with captured_output() as (out, err):
cli.get_partitioner(Args("nodb", "notable", None))
output = out.getvalue().strip()
output.should.equal(
"You do not have permissions to run GetTable\n\tDid you mean to run this with a profile specified?"
)
self.exit_mock.assert_called_with(1)
not_found_mock = MagicMock()
not_found_mock.side_effect = GlutilError(
error_type="EntityNotFound",
message="Error, could not find table notable")
Partitioner.__init__ = not_found_mock
with captured_output() as (out, err):
cli.get_partitioner(args)
output = out.getvalue().strip()
output.should.equal(
"Error, could not find table notable\n\tConfirm notable exists, and you have the ability to access it."
)
self.exit_mock.assert_called_with(1)
finally:
# NOTE: this must stay, otherwise tests run after this will still
# have Partitioner.__init__ set to a mock
Partitioner.__init__ = original_init
def test_delete_bad_partitions_error_output(self):
self.s3.create_bucket(Bucket=self.bucket)
self.helper.make_database_and_table()
cli = Cli()
partitioner = Partitioner(self.database,
self.table,
aws_region=self.region)
partition = self.helper.create_partition_data(prefix="not-this-table")
partitioner.create_partitions([partition])
mock = MagicMock()
mock.return_value = [{
"PartitionValues": partition.values,
"ErrorDetail": {
"ErrorCode": "PartitionNotFound",
"ErrorMessage": "Partition not found"
}
}]
partitioner.delete_partitions = mock
partitioner_mock = MagicMock(return_value=partitioner)
cli.get_partitioner = partitioner_mock
expected_output = f"Found 1 partitions to delete\nDeleting the following partitions:\n\t{partition}\nOne or more errors occurred when attempting to delete partitions\nError on {partition.values}: PartitionNotFound"
out, err = self.get_cmd_output(
cli, ["delete-bad-partitions", self.database, self.table])
out.should.equal(expected_output)
self.exit_mock.assert_called_with(1)
def test_update_partitions_error_output(self):
self.s3.create_bucket(Bucket=self.bucket)
self.helper.make_database_and_table()
cli = Cli()
partitioner = Partitioner(self.database,
self.table,
aws_region=self.region)
partition = self.helper.create_partition_data()
partition.location = "s3://old-bucket/old-table/"
partitioner.create_partitions([partition])
mock = MagicMock()
mock.return_value = [{
"PartitionValues": partition.values,
"ErrorDetail": {
"ErrorCode": "PartitionNotFound",
"ErrorMessage": "Partition not found"
}
}]
partitioner.update_partition_locations = mock
partitioner_mock = MagicMock(return_value=partitioner)
cli.get_partitioner = partitioner_mock
expected_output = f"Found 1 moved partitions\n\t{partition}\nOne or more errors occurred when attempting to update partitions\nError on {partition.values}: PartitionNotFound"
out, err = self.get_cmd_output(
cli, ["update-partitions", self.database, self.table])
out.should.equal(expected_output)
self.exit_mock.assert_called_with(1)
def test_create_partitions_error_output(self):
""" Technically this should _never_ happen, but on the off chance that
batch_get_partition ever returns bad values we'll leave it in"""
self.s3.create_bucket(Bucket=self.bucket)
self.helper.make_database_and_table()
cli = Cli()
partitions = self.helper.create_many_partitions(count=10)
partitions.sort()
expected_output = f"Running Partitioner for {self.database}.{self.table}\n\tLooking for partitions in s3://{self.bucket}/{self.table}/\n\tFound 10 new partitions to create\n\t"
expected_output += ", ".join(map(str, partitions))
expected_output += f"\nOne or more errors occurred when attempting to create partitions\nError on {partitions[0].values}: AlreadyExistsException"
partitioner = Partitioner(self.database,
self.table,
aws_region=self.region)
partitioner.create_partitions([partitions[0]])
mock = MagicMock(return_value=partitions)
partitioner.partitions_to_create = mock
partitioner_mock = MagicMock(return_value=partitioner)
cli.get_partitioner = partitioner_mock
out, err = self.get_cmd_output(
cli, ["create-partitions", self.database, self.table])
out.should.equal(expected_output)
self.exit_mock.assert_called_with(1)
fresh_partitioner = Partitioner(self.database,
self.table,
aws_region=self.region)
exists = fresh_partitioner.existing_partitions()
set(exists).should.equal(set(partitions))
