def test_create_partitions_error_output(self):
""" Technically this should _never_ happen, but on the off chance that
batch_get_partition ever returns bad values we'll leave it in"""
self.s3.create_bucket(Bucket=self.bucket)
self.helper.make_database_and_table()
partitions = self.helper.create_many_partitions(count=10)
partitions.sort()
expected_output = f"Running Partitioner for {self.database}.{self.table}\n\tLooking for partitions in s3://{self.bucket}/{self.table}/\n\tFound 10 new partitions to create\n\t"
expected_output += ", ".join(map(str, partitions))
expected_output += f"\nOne or more errors occurred when attempting to create partitions\nError on {partitions[0].values}: AlreadyExistsException"
partitioner = Partitioner(self.database,
self.table,
aws_region=self.region)
partitioner.create_partitions([partitions[0]])
mock = MagicMock(return_value=partitions)
partitioner.partitions_to_create = mock
with captured_output() as (out, err):
create_found_partitions(partitioner)
output = out.getvalue().strip()
output.should.equal(expected_output)
self.exit_mock.assert_called_with(1)
fresh_partitioner = Partitioner(self.database,
self.table,
aws_region=self.region)
exists = fresh_partitioner.existing_partitions()
set(exists).should.equal(set(partitions))
def test_partitions_to_create_empty_input(self):
# this is to test the early exit
self.s3.create_bucket(Bucket=self.bucket)
self.helper.make_database_and_table()
partitioner = Partitioner(self.database,
self.table,
aws_region=self.region)
wants_to_create = partitioner.partitions_to_create([])
wants_to_create.should.have.length_of(0)
def test_partitions_to_create(self):
self.s3.create_bucket(Bucket=self.bucket)
self.helper.make_database_and_table()
already_created = self.helper.create_many_partitions(count=10,
write=True)
to_create = self.helper.create_many_partitions(count=3, write=True)
partitioner = Partitioner(self.database,
self.table,
aws_region=self.region)
partitioner.create_partitions(already_created)
found = partitioner.partitions_on_disk()
wants_to_create = partitioner.partitions_to_create(found)
set(wants_to_create).should.equal(set(to_create))