def test_delete_partitions_in_groups_of_twenty_five(self):
self.s3.create_bucket(Bucket=self.bucket)
self.helper.make_database_and_table()
partitions = sorted(self.helper.create_many_partitions(count=30))
partitioner = Partitioner(self.database,
self.table,
aws_region=self.region)
partitioner.create_partitions(partitions)
mock = MagicMock(return_value=[])
partitioner.glue.batch_delete_partition = mock
existing_partitions = partitioner.existing_partitions()
partitioner.delete_partitions(existing_partitions)
first_list = [{"Values": p.values} for p in partitions[:25]]
second_list = [{"Values": p.values} for p in partitions[25:]]
calls = [
call(DatabaseName=self.database,
TableName=self.table,
PartitionsToDelete=first_list),
call(DatabaseName=self.database,
TableName=self.table,
PartitionsToDelete=second_list),
]
mock.call_count.should.equal(2)
mock.assert_has_calls(calls)
def test_delete_missing_partitions_error_output(self):
self.helper.make_database_and_table()
cli = Cli()
partitioner = Partitioner(self.database,
self.table,
aws_region=self.region)
self.s3.create_bucket(Bucket=self.bucket)
partition = self.helper.create_partition_data(save=False)
partitioner.create_partitions([partition])
mock = MagicMock()
mock.return_value = [{
"PartitionValues": partition.values,
"ErrorDetail": {
"ErrorCode": "PartitionNotFound",
"ErrorMessage": "Partition not found"
}
}]
partitioner.delete_partitions = mock
partitioner_mock = MagicMock(return_value=partitioner)
cli.get_partitioner = partitioner_mock
expected_output = f"Found 1 partitions to delete:\n\t{partition}\nOne or more errors occurred when attempting to delete partitions\nError on {partition.values}: PartitionNotFound"
out, err = self.get_cmd_output(
cli, ["delete-missing-partitions", self.database, self.table])
out.should.equal(expected_output)
self.exit_mock.assert_called_with(1)
def test_delete_partitions(self):
self.s3.create_bucket(Bucket=self.bucket)
self.helper.make_database_and_table()
self.helper.create_partition_data()
partition = self.helper.create_partition_data()
partitioner = Partitioner(self.database,
self.table,
aws_region=self.region)
partitioner.create_partitions([partition])
mock = MagicMock(return_value=[])
partitioner.glue.batch_delete_partition = mock
to_delete = partitioner.existing_partitions()
partitioner.delete_partitions(to_delete)
mock.assert_called_with(DatabaseName=self.database,
TableName=self.table,
PartitionsToDelete=[{
"Values":
to_delete[0].values
}])
def test_delete_nonexistent_partition(self):
self.s3.create_bucket(Bucket=self.bucket)
self.helper.make_database_and_table()
partitioner = Partitioner(self.database,
self.table,
aws_region=self.region)
partition = Partition(
["2019", "01", "02", "03"],
f"s3://{self.bucket}/{self.table}/2019/01/02/03/")
result = partitioner.delete_partitions([partition])
result.should.have.length_of(1)
result[0]["PartitionValues"].should.equal(["2019", "01", "02", "03"])
result[0]["ErrorDetail"]["ErrorCode"].should.equal(
"EntityNotFoundException")
