def test_tag_all_with_detailed_list(self, mocker): scanned_resources = { "eu-central-1": [ Resource( "arn:aws:sqs:eu-central-1:111111111111:someq", "someq", "queue", tags, ), Resource( "arn:aws:sqs:eu-central-1:111111111111:someq2", "someq2", "queue", tags, ), ], "global": [ Resource( "arn:aws:cloudfront::111111111111:distribution/EMS6KR7IENMDE", "EMS6KR7IENMDE", "distribution", tags, ), Resource( "arn:aws:route53::111111111111:healthcheck/f665452c-bf56-4a43-8b5d-319c3b8d0a70", "f665452c-bf56-4a43-8b5d-319c3b8d0a70", "healthcheck", tags, ), ], } some_config = Config( [Tag("Owner", "Fritz"), Tag("Project", "CRM")], default_region="eu-central-1", ) runner = CliRunner() scan_mock = mocker.patch( "taggercli.commands.tag.scan_region_and_global", return_value=scanned_resources, ) mocker.patch("taggercli.commands.tag.init_config") mocker.patch("taggercli.commands.tag.get_config", return_value=some_config) actual = runner.invoke(cli, ["tag", "all"], input="y\n n\n") assert not actual.exception assert actual.exit_code == 0 assert actual.stdout.find("Scanning completed") assert actual.stdout.find( f"Found {len(scanned_resources['global'])} global resources") assert actual.stdout.find( f"Found {len(scanned_resources['eu-central-1'])} resources in region eu-central-1" ) scan_mock.assert_called_once_with("eu-central-1")
def test_report_command(self, mocker, tmpdir): expected_file_name = "report.html" expected_output_path = str(tmpdir) + "/" + expected_file_name resources_with_tag_diffs = [ ResourceWithTagDiffs( Resource( "arn:aws:sqs:eu-central-1:111111111111:someq", "someq", "queue", tags, ), diff_for_improperly_tagged_resource, ), ResourceWithTagDiffs( Resource( "arn:aws:sqs:eu-central-1:111111111111:someq2", "someq2", "queue", [Tag("Project", "Super1")], ), diff_for_properly_tagged_resource, ), ResourceWithTagDiffs( Resource( "arn:aws:apigateway:eu-central-1::/apis/e5zcg2s231", "e5zcg2s231", "apigateway", tags, ), diff_for_properly_tagged_resource, ), ] some_config = Config([Tag("Owner", "Fritz"), Tag("Project", "CRM")], default_region="eu-west-1") mocker.patch( "taggercli.commands.report.scan_and_compare_resources", return_value=resources_with_tag_diffs, ) mocker.patch("taggercli.commands.report.init_config") mocker.patch("taggercli.commands.report.get_config", return_value=some_config) runner = CliRunner() actual = runner.invoke( cli, [ "report", "create", "--region", "eu-central-1", "--output-path", tmpdir ], ) assert not actual.exception assert actual.exit_code == 0 with open(expected_output_path, "r") as generated_report: assert generated_report.read()
def test_parse_config(self): # path relative to tox.ini is required to make file accessible in tox ( https://stackoverflow.com/questions/53309298/how-can-i-make-test-data-files-accessible-to-pytest-tests-when-run-with-tox ) config = read_config_file("test/config/test_config.ini") assert config.profile == "some-profile" assert config.account_id == "111111111111" assert config.tags == [ Tag("service", "service1"), Tag("cost-center", "marketing"), Tag("timestamp", "20200808-1030"), ]
def test_keep_case_of_tags(self): config = read_config_file( "test/config/test_config_with_mixed_cases.ini") assert config.tags == [ Tag("service", "service1"), Tag("cost-center", "Marketing"), Tag("timestamp", "20200808-1030"), Tag("Owner", "Fritz"), ]
def test_lambda_in_tag_mode_account( self, mocker, env_for_tag_mode_account, regional_resources, global_resources, tagging_result_with_failed_res, caplog, ): mocked_region_scan = mocker.patch("src.tagging_lambda.scan_region") mocked_region_scan.return_value = regional_resources mocked_global_scan = mocker.patch("src.tagging_lambda.scan_global") mocked_global_scan.return_value = global_resources mocked_perform_tagging = mocker.patch("src.tagging_lambda.perform_tagging") mocked_perform_tagging.return_value = tagging_result_with_failed_res mocked_boto_client = mocker.patch.object(boto3, "client") mocked_boto_client.return_value.assume_role.return_value = { "Credentials": { "AccessKeyId": "access_key", "SecretAccessKey": "secret_key", "SessionToken": "token1", } } mocked_list_tags_for_resource = ( mocked_boto_client.return_value.list_tags_for_resource ) mocked_list_tags_for_resource.return_value = { "Tags": [ {"Key": "Project", "Value": "CRM"}, {"Key": "Owner", "Value": "Team2"}, ] } expected_tags = [Tag("Project", "CRM"), Tag("Owner", "Team2")] lambda_handler(None, None) mocked_list_tags_for_resource.assert_called_with( ResourceId=env_for_tag_mode_account["ACCOUNT_ID"] ) mocked_region_scan.assert_called_once_with(env_for_tag_mode_account["REGION"]) mocked_global_scan.assert_called_once() mocked_perform_tagging.assert_called_once_with( regional_resources + global_resources, expected_tags ) for failed_res in tagging_result_with_failed_res.failed_arns.keys(): assert failed_res in caplog.text
def test_scan_and_compare(self, mocker, regional_resources, global_resources): mocked_region_scanner_scan = mocker.patch.object(RegionScanner, "scan") mocked_region_scanner_scan.return_value = regional_resources mocked_global_scanner_scan = mocker.patch.object(GlobalScanner, "scan") mocked_global_scanner_scan.return_value = global_resources tags = [Tag("Owner", "Hugo"), Tag("Created", "2020-08-10")] actual = scan_and_compare_resources("eu-central-1", tags) assert len(actual) == len(regional_resources) + len(global_resources) mocked_global_scanner_scan.assert_called_with(GLOBAL_RES_TYPE_NOT_TAGGABLE) mocked_region_scanner_scan.assert_called_with( REG_RES_TYPE_NOT_SUPPORTED + REG_RES_TYPE_NOT_TAGGABLE )
def create_resource(resource: Any) -> Resource: """Map a skew resource to a taggercore resource""" return Resource( arn=resource.arn, id=resource.id, resource_type=resource.resourcetype, current_tags=[Tag(key, value) for key, value in resource.tags.items()], name=resource.name, )
def parse_tags(splitted_input: str) -> List[Tag]: tags = [] for tag in splitted_input.split(","): try: tag_without_whitespace = remove_whitespace(tag) key, value = tag_without_whitespace.split("=") tags.append(Tag(key, value)) except ValueError: raise IllegalInputError( "Please provide tags in the following format: Key=Value and separate them with ," ) return tags
def fetch_tags_from_env(config: Dict[str, Any]) -> List[Tag]: tags = [] for tag in config["TAGS"].split(","): try: tag_without_whitespace = remove_whitespace(tag) key, value = tag_without_whitespace.split("=") tags.append(Tag(key, value)) except ValueError: raise ConfigurationError( "Please provide tags in the following format: Key=Value and separate them with ," ) return tags
def test_config_command(self, mocker, tmpdir): tagger_path_mock = mocker.patch( "taggercli.commands.config.TAGGER_PATH", return_value=tmpdir) mocker.patch("taggercli.commands.config.Path.mkdir") expected_file_path = str(tmpdir) + "/config.ini" tagger_path_mock.joinpath.return_value = expected_file_path runner = CliRunner() actual = runner.invoke( cli, ["config", "create"], input= "111111111111\n eu-central-1\n admin-profile\n Owner=Fritz, Project=CRM\n", ) config = get_config() assert not actual.exception assert config.account_id == "111111111111" assert config.default_region == "eu-central-1" assert config.profile == "admin-profile" assert config.tags == [Tag("Owner", "Fritz"), Tag("Project", "CRM")] assert os.path.isfile(expected_file_path)
def test_metrics_for_dashboard(self): resources_by_service = { "sqs": [ ResourceWithTagDiffs( Resource( "arn:aws:sqs:eu-central-1:111111111111:someq", "someq", "queue", tags, ), diff_for_improperly_tagged_resource, ), ResourceWithTagDiffs( Resource( "arn:aws:sqs:eu-central-1:111111111111:someq2", "someq2", "queue", [Tag("Project", "Super1")], ), diff_for_properly_tagged_resource, ), ], "apigateway": [ ResourceWithTagDiffs( Resource( "arn:aws:apigateway:eu-central-1::/apis/e5zcg2s231", "e5zcg2s231", "apigateway", tags, ), diff_for_properly_tagged_resource, ) ], } actual = metrics_for_dashboard_by_service(resources_by_service) assert actual == { "sqs": { "number_of_resources": 2, "number_of_resources_tagged_properly": 1, "ratio_tagged_properly": 0.5, }, "apigateway": { "number_of_resources": 1, "number_of_resources_tagged_properly": 1, "ratio_tagged_properly": 1, }, }
def fetch_tags_for_account(credentials: Credentials, account_id: str) -> List[Tag]: try: client = boto3.client("organizations", **credentials) tag_response = client.list_tags_for_resource(ResourceId=account_id) tags = tag_response["Tags"] account_tags = [] for tag in tags: account_tags.append(Tag(tag["Key"], tag["Value"])) except ClientError as e: logger.error( f"Failed to retrieve tags for account {account_id}, error {e.response}" ) raise e return account_tags
from jinja2 import Environment, FileSystemLoader from rich.console import Console from taggercore.model import ResourceWithTagDiffs, Tag from taggercore.usecase import scan_and_compare_resources from taggercli.commands.util import print_tags from taggercli.config import get_config, init_config console = Console() report_group = typer.Typer() TAGGERCLI_DIR = Path(find_spec("taggercli").origin).parent TEMPLATE_PATH = TAGGERCLI_DIR.joinpath("templates") TEMPLATE_FILE_NAME = "template_report.html" REPORT_FILE_NAME = "report.html" tags = [Tag("Project", "CRM"), Tag("Owner", "Alice"), Tag("Cost-center", "Sales")] @report_group.command("create") def create_report_from_cli( region: Optional[str] = typer.Option(None, help="AWS region code"), output_path: Optional[str] = typer.Option( None, help="output path for the created html report" ), ): init_config() config = get_config() account_id = config.account_id if not region: region = config.default_region tags = config.tags
def test_data_for_dashboard_template(self): creation_datetime = datetime(2020, 8, 7, 10, 5, 23, tzinfo=timezone.utc) resources_with_tag_diffs = [ ResourceWithTagDiffs( Resource( "arn:aws:sqs:eu-central-1:111111111111:someq", "someq", "queue", tags, ), diff_for_improperly_tagged_resource, ), ResourceWithTagDiffs( Resource( "arn:aws:sqs:eu-central-1:111111111111:someq2", "someq2", "queue", [Tag("Project", "Super1")], ), diff_for_properly_tagged_resource, ), ResourceWithTagDiffs( Resource( "arn:aws:apigateway:eu-central-1::/apis/e5zcg2s231", "e5zcg2s231", "apigateway", tags, ), diff_for_properly_tagged_resource, ), ] actual = prepare_data_for_dashboard_template("111111111111", creation_datetime, resources_with_tag_diffs) assert actual == { "account_id": "111111111111", "creation_datetime": "Fri Aug 7 10:05:23 2020 (UTC)", "metrics_by_service": { "sqs": { "number_of_resources": 2, "number_of_resources_tagged_properly": 1, "ratio_tagged_properly": 0.5, }, "apigateway": { "number_of_resources": 1, "number_of_resources_tagged_properly": 1, "ratio_tagged_properly": 1, }, }, "resources_by_service": { "sqs": [ ResourceWithTagDiffs( Resource( "arn:aws:sqs:eu-central-1:111111111111:someq", "someq", "queue", tags, ), diff_for_improperly_tagged_resource, ), ResourceWithTagDiffs( Resource( "arn:aws:sqs:eu-central-1:111111111111:someq2", "someq2", "queue", [Tag("Project", "Super1")], ), diff_for_properly_tagged_resource, ), ], "apigateway": [ ResourceWithTagDiffs( Resource( "arn:aws:apigateway:eu-central-1::/apis/e5zcg2s231", "e5zcg2s231", "apigateway", tags, ), diff_for_properly_tagged_resource, ) ], }, }
# under the License. # from datetime import datetime, timezone from taggercore.model import Resource, TagDiffType, Tag, TagDiff, ResourceWithTagDiffs from typer.testing import CliRunner from taggercli.commands import ( metrics_for_dashboard_by_service, prepare_data_for_dashboard_template, ) from taggercli.config import Config from taggercli.main import cli diff_for_improperly_tagged_resource = [ TagDiff(Tag("Project", "CRM"), Tag("Project", "CRM"), TagDiffType["EXISTING"]), TagDiff(Tag("Owner", "Alice"), Tag("Owner", "Bob"), TagDiffType["NEW_VALUE"]), TagDiff(Tag(None, None), Tag("Department", "Marketing"), TagDiffType["NEW"]), ] diff_for_properly_tagged_resource = [ TagDiff(Tag("Project", "CRM"), Tag("Project", "CRM"), TagDiffType["EXISTING"]), TagDiff( Tag("Created", "2020-08-10"), Tag(None, None), TagDiffType["EXISTING_NOT_IN_SCHEMA"], ),
def create_resource(resource: Any) -> Resource: """Map a skew resource to a taggercore resource""" # print(resource) # print(dir(resource)) # print(type(resource)) # print(f"name: {resource.name}") # print(f"id: {resource.id}") # print(f"arn: {resource.arn}") # print(inspect.getmro(type(resource))) try: tag_items = [Tag(key, value) for key, value in resource.tags.items()] except Exception as e: print('error encountered') print(f"arn: {resource.arn}") print(e) return None excluded_arns = [ ] excluded_roles = [ "CiscoBaselinePasswordPolicyRole", "CloudHealthIAMRole", "ContinuousSecurityBuddyLambdaRole", "CSIRT_Investigator_Role", "CustomStacksDeploymentRole", "engineer", "InfoSecAuditor", "network_api", "network_support", "OrganizationAccountAccessRole", "owner", "network_support", "SECOPS_Investigator_Role", "StreamlineIdPManagerIAMRole", ] excluded_tags = set([ Tag('ResourceOwner', 'CIE_SysEngComputeCore'), Tag('ApplicationName', 'Infosec Qualys'), Tag('ApplicationName', 'Rebox'), Tag('StreamlineAWSManaged', 'True') ]) if(resource.resourcetype == 'role' and resource.name in excluded_roles): print('EXCLUDED DUE TO MATCHING IAM ROLES') return None if(resource.arn in excluded_arns): print('EXCLUDED DUE TO MATCHING ARNS') return None if excluded_tags.intersection(tag_items): print('EXCLUDED DUE TO MATCHING TAGS') return None return Resource( arn=resource.arn, id=resource.id, resource_type=resource.resourcetype, current_tags=tag_items, name=resource.name, )
def parse_tags(tag_config: configparser.SectionProxy) -> List[Tag]: return [Tag(key, value) for key, value in tag_config.items()]
# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # from taggercore.model import Tag, Resource from typer.testing import CliRunner from taggercli.config.config import Config from taggercli.main import cli tags = [ Tag("Project", "Super1"), Tag("Owner", "Owner1"), Tag("Created", "Yesterday") ] class TestTag: def test_tag_all_with_detailed_list(self, mocker): scanned_resources = { "eu-central-1": [ Resource( "arn:aws:sqs:eu-central-1:111111111111:someq", "someq", "queue", tags, ),
def tags() -> List[Tag]: yield [ Tag("Project", "CoolProject"), Tag("Owner", "Fritz"), Tag("Created", "2020-08-01"), ]
# "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # from taggercore.model import ResourceWithTagDiffs, Tag, TagDiffType, TagDiff, Resource diff_for_improperly_tagged_resource = [ TagDiff(Tag("Project", "Super1"), Tag("Project", "Super1"), TagDiffType["EXISTING"]), TagDiff(Tag("Owner", "Owner1"), Tag("Owner", "Owner2"), TagDiffType["NEW_VALUE"]), TagDiff(Tag(None, None), Tag("Department", "Marketing"), TagDiffType["NEW"]), ] diff_for_properly_tagged_resource = [ TagDiff(Tag("Project", "Super1"), Tag("Project", "Super1"), TagDiffType["EXISTING"]), TagDiff( Tag("Created", "Yesterday"), Tag(None, None), TagDiffType["EXISTING_NOT_IN_SCHEMA"], ),