def test_create_partitions_error_output(self): """ Technically this should _never_ happen, but on the off chance that batch_get_partition ever returns bad values we'll leave it in""" self.s3.create_bucket(Bucket=self.bucket) self.helper.make_database_and_table() partitions = self.helper.create_many_partitions(count=10) partitions.sort() expected_output = f"Running Partitioner for {self.database}.{self.table}\n\tLooking for partitions in s3://{self.bucket}/{self.table}/\n\tFound 10 new partitions to create\n\t" expected_output += ", ".join(map(str, partitions)) expected_output += f"\nOne or more errors occurred when attempting to create partitions\nError on {partitions[0].values}: AlreadyExistsException" partitioner = Partitioner(self.database, self.table, aws_region=self.region) partitioner.create_partitions([partitions[0]]) mock = MagicMock(return_value=partitions) partitioner.partitions_to_create = mock with captured_output() as (out, err): create_found_partitions(partitioner) output = out.getvalue().strip() output.should.equal(expected_output) self.exit_mock.assert_called_with(1) fresh_partitioner = Partitioner(self.database, self.table, aws_region=self.region) exists = fresh_partitioner.existing_partitions() set(exists).should.equal(set(partitions))
def test_partitions_to_create_empty_input(self): # this is to test the early exit self.s3.create_bucket(Bucket=self.bucket) self.helper.make_database_and_table() partitioner = Partitioner(self.database, self.table, aws_region=self.region) wants_to_create = partitioner.partitions_to_create([]) wants_to_create.should.have.length_of(0)
def test_partitions_to_create(self): self.s3.create_bucket(Bucket=self.bucket) self.helper.make_database_and_table() already_created = self.helper.create_many_partitions(count=10, write=True) to_create = self.helper.create_many_partitions(count=3, write=True) partitioner = Partitioner(self.database, self.table, aws_region=self.region) partitioner.create_partitions(already_created) found = partitioner.partitions_on_disk() wants_to_create = partitioner.partitions_to_create(found) set(wants_to_create).should.equal(set(to_create))