def test_delete_partitions_in_groups_of_twenty_five(self): self.s3.create_bucket(Bucket=self.bucket) self.helper.make_database_and_table() partitions = sorted(self.helper.create_many_partitions(count=30)) partitioner = Partitioner(self.database, self.table, aws_region=self.region) partitioner.create_partitions(partitions) mock = MagicMock(return_value=[]) partitioner.glue.batch_delete_partition = mock existing_partitions = partitioner.existing_partitions() partitioner.delete_partitions(existing_partitions) first_list = [{"Values": p.values} for p in partitions[:25]] second_list = [{"Values": p.values} for p in partitions[25:]] calls = [ call(DatabaseName=self.database, TableName=self.table, PartitionsToDelete=first_list), call(DatabaseName=self.database, TableName=self.table, PartitionsToDelete=second_list), ] mock.call_count.should.equal(2) mock.assert_has_calls(calls)
def test_delete_missing_partitions_error_output(self): self.helper.make_database_and_table() cli = Cli() partitioner = Partitioner(self.database, self.table, aws_region=self.region) self.s3.create_bucket(Bucket=self.bucket) partition = self.helper.create_partition_data(save=False) partitioner.create_partitions([partition]) mock = MagicMock() mock.return_value = [{ "PartitionValues": partition.values, "ErrorDetail": { "ErrorCode": "PartitionNotFound", "ErrorMessage": "Partition not found" } }] partitioner.delete_partitions = mock partitioner_mock = MagicMock(return_value=partitioner) cli.get_partitioner = partitioner_mock expected_output = f"Found 1 partitions to delete:\n\t{partition}\nOne or more errors occurred when attempting to delete partitions\nError on {partition.values}: PartitionNotFound" out, err = self.get_cmd_output( cli, ["delete-missing-partitions", self.database, self.table]) out.should.equal(expected_output) self.exit_mock.assert_called_with(1)
def test_delete_partitions(self): self.s3.create_bucket(Bucket=self.bucket) self.helper.make_database_and_table() self.helper.create_partition_data() partition = self.helper.create_partition_data() partitioner = Partitioner(self.database, self.table, aws_region=self.region) partitioner.create_partitions([partition]) mock = MagicMock(return_value=[]) partitioner.glue.batch_delete_partition = mock to_delete = partitioner.existing_partitions() partitioner.delete_partitions(to_delete) mock.assert_called_with(DatabaseName=self.database, TableName=self.table, PartitionsToDelete=[{ "Values": to_delete[0].values }])
def test_delete_nonexistent_partition(self): self.s3.create_bucket(Bucket=self.bucket) self.helper.make_database_and_table() partitioner = Partitioner(self.database, self.table, aws_region=self.region) partition = Partition( ["2019", "01", "02", "03"], f"s3://{self.bucket}/{self.table}/2019/01/02/03/") result = partitioner.delete_partitions([partition]) result.should.have.length_of(1) result[0]["PartitionValues"].should.equal(["2019", "01", "02", "03"]) result[0]["ErrorDetail"]["ErrorCode"].should.equal( "EntityNotFoundException")