def test_delete_bad_tables(self):
database_input = self.helper.create_database_input()
self.glue.create_database(**database_input)
cli = Cli()
location = "s3://bucket/root-table/"
root_table_input = self.helper.create_table_input(location=location)
self.glue.create_table(**root_table_input)
tables = []
for i in range(1, 12):
tbl_location = f"{location}{i}/"
tbl_input = self.helper.create_table_input(location=tbl_location,
random_name=True)
self.glue.create_table(**tbl_input)
tbl_input['TableInput']["DatabaseName"] = self.database
tables.append(Table(tbl_input['TableInput']))
expected_output = "Going to delete the following tables:"
tables.sort(key=lambda x: x.name)
for table in tables:
expected_output += f"\n\t{table}"
out, err = self.get_cmd_output(cli,
["delete-bad-tables", self.database])
out.should.equal(expected_output)
cleaner = DatabaseCleaner(self.database, aws_region=self.region)
found_tables = cleaner.child_tables()
found_tables.should.have.length_of(0)
def test_delete_bad_tables_error_output(self):
database_input = self.helper.create_database_input()
self.glue.create_database(**database_input)
cli = Cli()
location = "s3://bucket/root-table/"
root_table_input = self.helper.create_table_input(location=location)
self.glue.create_table(**root_table_input)
table_input = self.helper.create_table_input(location=location,
name=f"test_table-bazer")
self.glue.create_table(**table_input)
mock = MagicMock()
mock.return_value = [{
"TableName": "test_table-bazer",
"ErrorDetail": {
"ErrorCode": "EntityNotFoundException",
"ErrorMessage": "Table not found",
},
}]
cleaner = DatabaseCleaner(self.database, aws_region=self.region)
cleaner.delete_tables = mock
cleaner_mock = MagicMock(return_value=cleaner)
cli.get_database_cleaner = cleaner_mock
expected_output = f"Going to delete the following tables:\n\t<Table {self.database} / test_table-bazer : {location}>\nOne or more errors occurred when attempting to delete tables\nError on test_table-bazer: EntityNotFoundException"
out, err = self.get_cmd_output(cli,
["delete-bad-tables", self.database])
mock.assert_called()
out.should.equal(expected_output)
self.exit_mock.assert_called_with(1)
def tests_refresh_tree(self):
client = boto3.client("glue", region_name=self.region)
database_input = self.helper.create_database_input()
client.create_database(**database_input)
table1_input = self.helper.create_table_input(
name="table", location="s3://test-bucket/table/")
client.create_table(**table1_input)
table1_input["TableInput"]["DatabaseName"] = table1_input["DatabaseName"]
table1 = Table(table1_input["TableInput"])
table2_input = self.helper.create_table_input(
name="table-foobarbaz", location="s3://test-bucket/table/")
client.create_table(**table2_input)
table2_input["TableInput"]["DatabaseName"] = table2_input["DatabaseName"]
table2 = Table(table2_input["TableInput"])
cleaner = DatabaseCleaner("test_database", aws_region=self.region)
cleaner.table_trees # call to prime the pump
result = cleaner.delete_tables([table1, table2])
result.should.be.empty
cleaner.child_tables().should_not.be.empty
cleaner.refresh_trees()
cleaner.child_tables().should.be.empty
def test_delete_table_that_doesnt_exist(self):
client = boto3.client("glue", region_name=self.region)
database_input = self.helper.create_database_input()
client.create_database(**database_input)
table_input = self.helper.create_table_input(
name="table", location="s3://test-bucket/table/")
table_input["TableInput"]["DatabaseName"] = table_input["DatabaseName"]
table = Table(table_input["TableInput"])
cleaner = DatabaseCleaner("test_database", aws_region=self.region)
result = cleaner.delete_tables([table])
result.should.have.length_of(1)
result[0]["TableName"].should.equal("table")
result[0]["ErrorDetail"]["ErrorCode"].should.equal("EntityNotFoundException")
def test_child_tables_same_location(self):
client = boto3.client("glue", region_name=self.region)
database_input = self.helper.create_database_input()
client.create_database(**database_input)
table1_input = self.helper.create_table_input(
name="table", location="s3://test-bucket/table/")
client.create_table(**table1_input)
table2_input = self.helper.create_table_input(
name="table-foobarbaz", location="s3://test-bucket/table/")
client.create_table(**table2_input)
cleaner = DatabaseCleaner("test_database", aws_region=self.region)
child_tables = cleaner.child_tables()
child_tables.should.have.length_of(1)
child_tables[0].location.should.equal("s3://test-bucket/table/")
child_tables[0].name.should.equal("table-foobarbaz")
def test_child_tables_same_location_reverse_order(self):
""" This should be 90% the same as test_child_tables_same_location,
the only difference is the order in which the tables are added """
client = boto3.client("glue", region_name=self.region)
database_input = self.helper.create_database_input()
client.create_database(**database_input)
table2_input = self.helper.create_table_input(
name="table-foobarbaz", location="s3://test-bucket/table/")
client.create_table(**table2_input)
table1_input = self.helper.create_table_input(
name="table", location="s3://test-bucket/table/")
client.create_table(**table1_input)
cleaner = DatabaseCleaner("test_database", aws_region=self.region)
child_tables = cleaner.child_tables()
child_tables.should.have.length_of(1)
child_tables[0].location.should.equal("s3://test-bucket/table/")
child_tables[0].name.should.equal("table-foobarbaz")
def get_database_cleaner(self, args):
try:
return DatabaseCleaner(args.database, aws_profile=args.profile)
except GlutilError as e: # pragma: no cover
message = e.message
if e.error_type == "AccessDenied":
if args.profile:
message += f"\n\tConfirm that {args.profile} has the glue:GetTable permission."
else:
message += "\n\tDid you mean to run this with a profile specified?"
if e.error_type == "EntityNotFound":
message += f"\n\tConfirm {args.table} exists, and you have the ability to access it."
print(message)
sys.exit(1)
def test_table_trees(self):
client = boto3.client("glue", region_name=self.region)
database_input = self.helper.create_database_input()
client.create_database(**database_input)
table1_input = self.helper.create_table_input(
location="s3://test-bucket/table/")
client.create_table(**table1_input)
table2_input = self.helper.create_table_input(
name="child",
location="s3://test-bucket/table/child/")
client.create_table(**table2_input)
cleaner = DatabaseCleaner("test_database", aws_region=self.region)
trees = cleaner.table_trees
tree = trees["test-bucket"]
child_tables = tree.child_tables()
child_tables.should.have.length_of(2)
child_tables[0].location.should.equal("s3://test-bucket/table/")
child_tables[1].location.should.equal("s3://test-bucket/table/child/")
first_node = tree.children["table"]
first_node.tables.should.have.length_of(1)
first_node.tables[0].location.should.equal("s3://test-bucket/table/")
table_child_tables = first_node.child_tables()
table_child_tables.should.have.length_of(1)
table_child_tables[0].location.should.equal(
"s3://test-bucket/table/child/")
child_node = first_node.children["child"]
child_node.tables.should.have.length_of(1)
child_node.tables[0].location.should.equal(
"s3://test-bucket/table/child/")
child_child_tables = child_node.child_tables()
child_child_tables.should.be.empty