def test_tag_all_with_detailed_list(self, mocker):
scanned_resources = {
"eu-central-1": [
Resource(
"arn:aws:sqs:eu-central-1:111111111111:someq",
"someq",
"queue",
tags,
),
Resource(
"arn:aws:sqs:eu-central-1:111111111111:someq2",
"someq2",
"queue",
tags,
),
],
"global": [
Resource(
"arn:aws:cloudfront::111111111111:distribution/EMS6KR7IENMDE",
"EMS6KR7IENMDE",
"distribution",
tags,
),
Resource(
"arn:aws:route53::111111111111:healthcheck/f665452c-bf56-4a43-8b5d-319c3b8d0a70",
"f665452c-bf56-4a43-8b5d-319c3b8d0a70",
"healthcheck",
tags,
),
],
}
some_config = Config(
[Tag("Owner", "Fritz"),
Tag("Project", "CRM")],
default_region="eu-central-1",
)
runner = CliRunner()
scan_mock = mocker.patch(
"taggercli.commands.tag.scan_region_and_global",
return_value=scanned_resources,
)
mocker.patch("taggercli.commands.tag.init_config")
mocker.patch("taggercli.commands.tag.get_config",
return_value=some_config)
actual = runner.invoke(cli, ["tag", "all"], input="y\n n\n")
assert not actual.exception
assert actual.exit_code == 0
assert actual.stdout.find("Scanning completed")
assert actual.stdout.find(
f"Found {len(scanned_resources['global'])} global resources")
assert actual.stdout.find(
f"Found {len(scanned_resources['eu-central-1'])} resources in region eu-central-1"
)
scan_mock.assert_called_once_with("eu-central-1")
def test_report_command(self, mocker, tmpdir):
expected_file_name = "report.html"
expected_output_path = str(tmpdir) + "/" + expected_file_name
resources_with_tag_diffs = [
ResourceWithTagDiffs(
Resource(
"arn:aws:sqs:eu-central-1:111111111111:someq",
"someq",
"queue",
tags,
),
diff_for_improperly_tagged_resource,
),
ResourceWithTagDiffs(
Resource(
"arn:aws:sqs:eu-central-1:111111111111:someq2",
"someq2",
"queue",
[Tag("Project", "Super1")],
),
diff_for_properly_tagged_resource,
),
ResourceWithTagDiffs(
Resource(
"arn:aws:apigateway:eu-central-1::/apis/e5zcg2s231",
"e5zcg2s231",
"apigateway",
tags,
),
diff_for_properly_tagged_resource,
),
]
some_config = Config([Tag("Owner", "Fritz"),
Tag("Project", "CRM")],
default_region="eu-west-1")
mocker.patch(
"taggercli.commands.report.scan_and_compare_resources",
return_value=resources_with_tag_diffs,
)
mocker.patch("taggercli.commands.report.init_config")
mocker.patch("taggercli.commands.report.get_config",
return_value=some_config)
runner = CliRunner()
actual = runner.invoke(
cli,
[
"report", "create", "--region", "eu-central-1",
"--output-path", tmpdir
],
)
assert not actual.exception
assert actual.exit_code == 0
with open(expected_output_path, "r") as generated_report:
assert generated_report.read()
def test_parse_config(self):
# path relative to tox.ini is required to make file accessible in tox ( https://stackoverflow.com/questions/53309298/how-can-i-make-test-data-files-accessible-to-pytest-tests-when-run-with-tox )
config = read_config_file("test/config/test_config.ini")
assert config.profile == "some-profile"
assert config.account_id == "111111111111"
assert config.tags == [
Tag("service", "service1"),
Tag("cost-center", "marketing"),
Tag("timestamp", "20200808-1030"),
]
def test_keep_case_of_tags(self):
config = read_config_file(
"test/config/test_config_with_mixed_cases.ini")
assert config.tags == [
Tag("service", "service1"),
Tag("cost-center", "Marketing"),
Tag("timestamp", "20200808-1030"),
Tag("Owner", "Fritz"),
]
def test_lambda_in_tag_mode_account(
self,
mocker,
env_for_tag_mode_account,
regional_resources,
global_resources,
tagging_result_with_failed_res,
caplog,
):
mocked_region_scan = mocker.patch("src.tagging_lambda.scan_region")
mocked_region_scan.return_value = regional_resources
mocked_global_scan = mocker.patch("src.tagging_lambda.scan_global")
mocked_global_scan.return_value = global_resources
mocked_perform_tagging = mocker.patch("src.tagging_lambda.perform_tagging")
mocked_perform_tagging.return_value = tagging_result_with_failed_res
mocked_boto_client = mocker.patch.object(boto3, "client")
mocked_boto_client.return_value.assume_role.return_value = {
"Credentials": {
"AccessKeyId": "access_key",
"SecretAccessKey": "secret_key",
"SessionToken": "token1",
}
}
mocked_list_tags_for_resource = (
mocked_boto_client.return_value.list_tags_for_resource
)
mocked_list_tags_for_resource.return_value = {
"Tags": [
{"Key": "Project", "Value": "CRM"},
{"Key": "Owner", "Value": "Team2"},
]
}
expected_tags = [Tag("Project", "CRM"), Tag("Owner", "Team2")]
lambda_handler(None, None)
mocked_list_tags_for_resource.assert_called_with(
ResourceId=env_for_tag_mode_account["ACCOUNT_ID"]
)
mocked_region_scan.assert_called_once_with(env_for_tag_mode_account["REGION"])
mocked_global_scan.assert_called_once()
mocked_perform_tagging.assert_called_once_with(
regional_resources + global_resources, expected_tags
)
for failed_res in tagging_result_with_failed_res.failed_arns.keys():
assert failed_res in caplog.text
def test_scan_and_compare(self, mocker, regional_resources, global_resources):
mocked_region_scanner_scan = mocker.patch.object(RegionScanner, "scan")
mocked_region_scanner_scan.return_value = regional_resources
mocked_global_scanner_scan = mocker.patch.object(GlobalScanner, "scan")
mocked_global_scanner_scan.return_value = global_resources
tags = [Tag("Owner", "Hugo"), Tag("Created", "2020-08-10")]
actual = scan_and_compare_resources("eu-central-1", tags)
assert len(actual) == len(regional_resources) + len(global_resources)
mocked_global_scanner_scan.assert_called_with(GLOBAL_RES_TYPE_NOT_TAGGABLE)
mocked_region_scanner_scan.assert_called_with(
REG_RES_TYPE_NOT_SUPPORTED + REG_RES_TYPE_NOT_TAGGABLE
)
def create_resource(resource: Any) -> Resource:
"""Map a skew resource to a taggercore resource"""
return Resource(
arn=resource.arn,
id=resource.id,
resource_type=resource.resourcetype,
current_tags=[Tag(key, value) for key, value in resource.tags.items()],
name=resource.name,
)
def parse_tags(splitted_input: str) -> List[Tag]:
tags = []
for tag in splitted_input.split(","):
try:
tag_without_whitespace = remove_whitespace(tag)
key, value = tag_without_whitespace.split("=")
tags.append(Tag(key, value))
except ValueError:
raise IllegalInputError(
"Please provide tags in the following format: Key=Value and separate them with ,"
)
return tags
def fetch_tags_from_env(config: Dict[str, Any]) -> List[Tag]:
tags = []
for tag in config["TAGS"].split(","):
try:
tag_without_whitespace = remove_whitespace(tag)
key, value = tag_without_whitespace.split("=")
tags.append(Tag(key, value))
except ValueError:
raise ConfigurationError(
"Please provide tags in the following format: Key=Value and separate them with ,"
)
return tags
def test_config_command(self, mocker, tmpdir):
tagger_path_mock = mocker.patch(
"taggercli.commands.config.TAGGER_PATH", return_value=tmpdir)
mocker.patch("taggercli.commands.config.Path.mkdir")
expected_file_path = str(tmpdir) + "/config.ini"
tagger_path_mock.joinpath.return_value = expected_file_path
runner = CliRunner()
actual = runner.invoke(
cli,
["config", "create"],
input=
"111111111111\n eu-central-1\n admin-profile\n Owner=Fritz, Project=CRM\n",
)
config = get_config()
assert not actual.exception
assert config.account_id == "111111111111"
assert config.default_region == "eu-central-1"
assert config.profile == "admin-profile"
assert config.tags == [Tag("Owner", "Fritz"), Tag("Project", "CRM")]
assert os.path.isfile(expected_file_path)
def test_metrics_for_dashboard(self):
resources_by_service = {
"sqs": [
ResourceWithTagDiffs(
Resource(
"arn:aws:sqs:eu-central-1:111111111111:someq",
"someq",
"queue",
tags,
),
diff_for_improperly_tagged_resource,
),
ResourceWithTagDiffs(
Resource(
"arn:aws:sqs:eu-central-1:111111111111:someq2",
"someq2",
"queue",
[Tag("Project", "Super1")],
),
diff_for_properly_tagged_resource,
),
],
"apigateway": [
ResourceWithTagDiffs(
Resource(
"arn:aws:apigateway:eu-central-1::/apis/e5zcg2s231",
"e5zcg2s231",
"apigateway",
tags,
),
diff_for_properly_tagged_resource,
)
],
}
actual = metrics_for_dashboard_by_service(resources_by_service)
assert actual == {
"sqs": {
"number_of_resources": 2,
"number_of_resources_tagged_properly": 1,
"ratio_tagged_properly": 0.5,
},
"apigateway": {
"number_of_resources": 1,
"number_of_resources_tagged_properly": 1,
"ratio_tagged_properly": 1,
},
}
def fetch_tags_for_account(credentials: Credentials,
account_id: str) -> List[Tag]:
try:
client = boto3.client("organizations", **credentials)
tag_response = client.list_tags_for_resource(ResourceId=account_id)
tags = tag_response["Tags"]
account_tags = []
for tag in tags:
account_tags.append(Tag(tag["Key"], tag["Value"]))
except ClientError as e:
logger.error(
f"Failed to retrieve tags for account {account_id}, error {e.response}"
)
raise e
return account_tags
from jinja2 import Environment, FileSystemLoader
from rich.console import Console
from taggercore.model import ResourceWithTagDiffs, Tag
from taggercore.usecase import scan_and_compare_resources
from taggercli.commands.util import print_tags
from taggercli.config import get_config, init_config
console = Console()
report_group = typer.Typer()
TAGGERCLI_DIR = Path(find_spec("taggercli").origin).parent
TEMPLATE_PATH = TAGGERCLI_DIR.joinpath("templates")
TEMPLATE_FILE_NAME = "template_report.html"
REPORT_FILE_NAME = "report.html"
tags = [Tag("Project", "CRM"), Tag("Owner", "Alice"), Tag("Cost-center", "Sales")]
@report_group.command("create")
def create_report_from_cli(
region: Optional[str] = typer.Option(None, help="AWS region code"),
output_path: Optional[str] = typer.Option(
None, help="output path for the created html report"
),
):
init_config()
config = get_config()
account_id = config.account_id
if not region:
region = config.default_region
tags = config.tags
def test_data_for_dashboard_template(self):
creation_datetime = datetime(2020,
8,
7,
10,
5,
23,
tzinfo=timezone.utc)
resources_with_tag_diffs = [
ResourceWithTagDiffs(
Resource(
"arn:aws:sqs:eu-central-1:111111111111:someq",
"someq",
"queue",
tags,
),
diff_for_improperly_tagged_resource,
),
ResourceWithTagDiffs(
Resource(
"arn:aws:sqs:eu-central-1:111111111111:someq2",
"someq2",
"queue",
[Tag("Project", "Super1")],
),
diff_for_properly_tagged_resource,
),
ResourceWithTagDiffs(
Resource(
"arn:aws:apigateway:eu-central-1::/apis/e5zcg2s231",
"e5zcg2s231",
"apigateway",
tags,
),
diff_for_properly_tagged_resource,
),
]
actual = prepare_data_for_dashboard_template("111111111111",
creation_datetime,
resources_with_tag_diffs)
assert actual == {
"account_id": "111111111111",
"creation_datetime": "Fri Aug 7 10:05:23 2020 (UTC)",
"metrics_by_service": {
"sqs": {
"number_of_resources": 2,
"number_of_resources_tagged_properly": 1,
"ratio_tagged_properly": 0.5,
},
"apigateway": {
"number_of_resources": 1,
"number_of_resources_tagged_properly": 1,
"ratio_tagged_properly": 1,
},
},
"resources_by_service": {
"sqs": [
ResourceWithTagDiffs(
Resource(
"arn:aws:sqs:eu-central-1:111111111111:someq",
"someq",
"queue",
tags,
),
diff_for_improperly_tagged_resource,
),
ResourceWithTagDiffs(
Resource(
"arn:aws:sqs:eu-central-1:111111111111:someq2",
"someq2",
"queue",
[Tag("Project", "Super1")],
),
diff_for_properly_tagged_resource,
),
],
"apigateway": [
ResourceWithTagDiffs(
Resource(
"arn:aws:apigateway:eu-central-1::/apis/e5zcg2s231",
"e5zcg2s231",
"apigateway",
tags,
),
diff_for_properly_tagged_resource,
)
],
},
}
# under the License.
#
from datetime import datetime, timezone
from taggercore.model import Resource, TagDiffType, Tag, TagDiff, ResourceWithTagDiffs
from typer.testing import CliRunner
from taggercli.commands import (
metrics_for_dashboard_by_service,
prepare_data_for_dashboard_template,
)
from taggercli.config import Config
from taggercli.main import cli
diff_for_improperly_tagged_resource = [
TagDiff(Tag("Project", "CRM"), Tag("Project", "CRM"),
TagDiffType["EXISTING"]),
TagDiff(Tag("Owner", "Alice"), Tag("Owner", "Bob"),
TagDiffType["NEW_VALUE"]),
TagDiff(Tag(None, None), Tag("Department", "Marketing"),
TagDiffType["NEW"]),
]
diff_for_properly_tagged_resource = [
TagDiff(Tag("Project", "CRM"), Tag("Project", "CRM"),
TagDiffType["EXISTING"]),
TagDiff(
Tag("Created", "2020-08-10"),
Tag(None, None),
TagDiffType["EXISTING_NOT_IN_SCHEMA"],
),
def create_resource(resource: Any) -> Resource:
"""Map a skew resource to a taggercore resource"""
# print(resource)
# print(dir(resource))
# print(type(resource))
# print(f"name: {resource.name}")
# print(f"id: {resource.id}")
# print(f"arn: {resource.arn}")
# print(inspect.getmro(type(resource)))
try:
tag_items = [Tag(key, value) for key, value in resource.tags.items()]
except Exception as e:
print('error encountered')
print(f"arn: {resource.arn}")
print(e)
return None
excluded_arns = [
]
excluded_roles = [
"CiscoBaselinePasswordPolicyRole",
"CloudHealthIAMRole",
"ContinuousSecurityBuddyLambdaRole",
"CSIRT_Investigator_Role",
"CustomStacksDeploymentRole",
"engineer",
"InfoSecAuditor",
"network_api",
"network_support",
"OrganizationAccountAccessRole",
"owner",
"network_support",
"SECOPS_Investigator_Role",
"StreamlineIdPManagerIAMRole",
]
excluded_tags = set([
Tag('ResourceOwner', 'CIE_SysEngComputeCore'),
Tag('ApplicationName', 'Infosec Qualys'),
Tag('ApplicationName', 'Rebox'),
Tag('StreamlineAWSManaged', 'True')
])
if(resource.resourcetype == 'role' and resource.name in excluded_roles):
print('EXCLUDED DUE TO MATCHING IAM ROLES')
return None
if(resource.arn in excluded_arns):
print('EXCLUDED DUE TO MATCHING ARNS')
return None
if excluded_tags.intersection(tag_items):
print('EXCLUDED DUE TO MATCHING TAGS')
return None
return Resource(
arn=resource.arn,
id=resource.id,
resource_type=resource.resourcetype,
current_tags=tag_items,
name=resource.name,
)
def parse_tags(tag_config: configparser.SectionProxy) -> List[Tag]:
return [Tag(key, value) for key, value in tag_config.items()]
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
from taggercore.model import Tag, Resource
from typer.testing import CliRunner
from taggercli.config.config import Config
from taggercli.main import cli
tags = [
Tag("Project", "Super1"),
Tag("Owner", "Owner1"),
Tag("Created", "Yesterday")
]
class TestTag:
def test_tag_all_with_detailed_list(self, mocker):
scanned_resources = {
"eu-central-1": [
Resource(
"arn:aws:sqs:eu-central-1:111111111111:someq",
"someq",
"queue",
tags,
),
def tags() -> List[Tag]:
yield [
Tag("Project", "CoolProject"),
Tag("Owner", "Fritz"),
Tag("Created", "2020-08-01"),
]
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
from taggercore.model import ResourceWithTagDiffs, Tag, TagDiffType, TagDiff, Resource
diff_for_improperly_tagged_resource = [
TagDiff(Tag("Project", "Super1"), Tag("Project", "Super1"),
TagDiffType["EXISTING"]),
TagDiff(Tag("Owner", "Owner1"), Tag("Owner", "Owner2"),
TagDiffType["NEW_VALUE"]),
TagDiff(Tag(None, None), Tag("Department", "Marketing"),
TagDiffType["NEW"]),
]
diff_for_properly_tagged_resource = [
TagDiff(Tag("Project", "Super1"), Tag("Project", "Super1"),
TagDiffType["EXISTING"]),
TagDiff(
Tag("Created", "Yesterday"),
Tag(None, None),
TagDiffType["EXISTING_NOT_IN_SCHEMA"],
),