def validate(config) -> bool:
required_sections = ["tags", "classes", "accounts"]
for section in required_sections:
if section not in config:
raise ValueError(f"Section '{section}' not found in config")
if not isinstance(config["tags"], list) or len(config["tags"]) == 0:
raise ValueError("Error in 'tags' section")
if not isinstance(config["classes"], list) or len(config["classes"]) == 0:
raise ValueError("Error in 'classes' section")
if not isinstance(config["accounts"], dict) or len(config["accounts"]) == 0:
raise ValueError("Error in 'accounts' section")
default_age = config.get("default", {}).get("age")
if default_age is not None:
default_age = parse_delta(default_age)
for cloud_id, account in config["accounts"].items():
for account_id, account_data in account.items():
if "name" not in account_data:
raise ValueError(
f"Missing 'name' for account '{cloud_id}/{account_id}"
)
if "age" in account_data:
account_data["age"] = parse_delta(account_data["age"])
else:
if default_age is None:
raise ValueError(
f"Missing 'age' for account '{cloud_id}/{account_id}' and no default age defined'"
)
account_data["age"] = default_age
return True
def expired_cleanup(event: Event):
graph = event.data
now = datetime.utcnow().replace(tzinfo=timezone.utc)
with graph.lock.read_access:
for node in graph.nodes:
cloud = node.cloud(graph)
account = node.account(graph)
region = node.region(graph)
if isinstance(node, BaseResource) and isinstance(cloud, BaseCloud) and isinstance(account, BaseAccount) and isinstance(region, BaseRegion):
if 'cloudkeeper:expires' in node.tags or ('expiration' in node.tags and node.tags['expiration'] != 'never'):
try:
if 'cloudkeeper:expires' in node.tags:
expires_tag = node.tags['cloudkeeper:expires']
expires = make_valid_timestamp(datetime.fromisoformat(expires_tag))
else:
expires_tag = node.tags['expiration']
expires = make_valid_timestamp(node.ctime + parse_delta(expires_tag))
except ValueError:
log.exception((f'Found {node.resource_type} {node.dname} in cloud {cloud.name}'
f' account {account.dname} region {region.name} age {node.age} with invalid expires tag {expires_tag}'))
continue
else:
if now > expires:
log.debug((f'Found expired resource {node.resource_type} {node.dname} in cloud {cloud.name}'
f' account {account.dname} region {region.name} age {node.age} with expires tag {expires_tag}'
f' - marking for cleanup'))
node.clean = True
def __init__(self):
super().__init__()
self.name = "cleanup_aws_loadbalancers"
self.exit = threading.Event()
if ArgumentParser.args.cleanup_aws_loadbalancers:
try:
self.age = parse_delta(
ArgumentParser.args.cleanup_aws_loadbalancers_age
)
log.debug(f"AWS Loadbalancer Cleanup Plugin Age {self.age}")
add_event_listener(EventType.SHUTDOWN, self.shutdown)
add_event_listener(
EventType.CLEANUP_PLAN,
self.loadbalancer_cleanup,
blocking=True,
timeout=3600,
)
except ValueError:
log.exception(
(
f"Error while parsing AWS Loadbalancer "
f"Cleanup Age {ArgumentParser.args.cleanup_aws_loadbalancers_age}"
)
)
else:
self.exit.set()
def __init__(self):
super().__init__()
self.name = 'cleanup_volumes'
self.exit = threading.Event()
if ArgumentParser.args.cleanup_volumes:
try:
self.age = parse_delta(ArgumentParser.args.cleanup_volumes_age)
log.debug(f'Volume Cleanup Plugin Age {self.age}')
add_event_listener(EventType.SHUTDOWN, self.shutdown)
add_event_listener(EventType.CLEANUP_PLAN,
self.volumes_cleanup,
blocking=True)
except ValueError:
log.exception(
f'Error while parsing Volume Cleanup Age {ArgumentParser.args.volclean_age}'
)
else:
self.exit.set()
def cmd_match(self, items: Iterable, args: str) -> Iterable:
'''Usage: | match [not] <attribute> <operator> <value>
Matches resources whose attribute matches a value.
Valid operators are:
> greather than
< less than
= equal to
~ regex match
has value is contained in attribute
'''
attr, action, value = None, None, None
negate_match = False
if args.startswith('not '):
negate_match = True
args = args[4:]
for action in self.match_actions.keys():
if action in args:
pos = args.index(action)
if pos == 0 or pos == len(args) - 1:
raise RuntimeError(
f"Can't have {action} at the beginning or end of match"
)
attr, value = args.split(action, 1)
attr = attr.strip()
value = value.strip()
break
if not attr or not action or not value:
raise RuntimeError(f'Invalid match {args}')
for item in items:
item_attr = self.get_item_attr(item, attr)
if item_attr is None:
continue
# We convert value for every resource even though
# chances are that types for same attributes are the
# same across all resource types.
match_item_attr = item_attr
match_value = value
if isinstance(item_attr, timedelta):
match_value = parse_delta(value)
elif isinstance(item_attr, datetime):
match_value = make_valid_timestamp(
datetime.fromisoformat(value))
elif isinstance(item_attr, bool):
match_value = strtobool(value)
elif isinstance(item_attr, int):
if str(value).isnumeric():
match_value = int(value)
else:
match_item_attr, match_value = str(item_attr), str(value)
elif isinstance(item_attr, float):
if not bool(re.search('[^0-9.]', str(value))):
match_value = float(value)
else:
match_item_attr, match_value = str(item_attr), str(value)
elif isinstance(item_attr, complex):
if not bool(re.search('[^0-9.]', str(value))):
match_value = complex(value)
else:
match_item_attr, match_value = str(item_attr), str(value)
if (not negate_match and self.match_actions[action](
match_item_attr, match_value)) or (
negate_match and not self.match_actions[action]
(match_item_attr, match_value)):
yield item
def age_range(age: timedelta) -> str:
for range in age_ranges:
if age >= age_range_lookup.get(range, parse_delta(range)):
return range
return "0s"
from cloudkeeper.baseresources import BaseInstance, BaseVolume
from cloudkeeper.args import ArgumentParser
from cloudkeeper.utils import parse_delta
from cloudkeeper.event import (
Event,
EventType,
add_event_listener,
remove_event_listener,
)
log = cloudkeeper.logging.getLogger("cloudkeeper." + __name__)
age_ranges = ["30d", "7d", "1d", "12h", "8h", "4h", "2h", "1h"]
age_range_lookup = {}
for range in age_ranges:
td = parse_delta(range)
age_range_lookup[range] = td
def age_range(age: timedelta) -> str:
for range in age_ranges:
if age >= age_range_lookup.get(range, parse_delta(range)):
return range
return "0s"
class MetricsAgeRangePlugin(BasePlugin):
def __init__(self):
super().__init__()
self.name = "metrics_age_range"
self.exit = threading.Event()
def validate_tags(self, graph: Graph):
config = self.config.read_config()
pt = ParallelTagger(self.name)
with graph.lock.read_access:
for node in graph.nodes:
cloud = node.cloud(graph)
account = node.account(graph)
region = node.region(graph)
node_classes = [
cls.__name__ for cls in inspect.getmro(node.__class__)
]
node_classes.remove('ABC')
node_classes.remove('object')
if (not isinstance(node, BaseResource)
or isinstance(node, BaseCloud)
or isinstance(node, BaseAccount)
or isinstance(node, BaseRegion)
or not isinstance(cloud, BaseCloud)
or not isinstance(account, BaseAccount)
or not isinstance(region, BaseRegion)
or node.protected):
continue
if cloud.id in config and account.id in config[cloud.id]:
class_config = {}
node_class = None
for node_class in node_classes:
node_class = node_class.lower()
if node_class in config[cloud.id][account.id]:
class_config = config[cloud.id][
account.id][node_class]
break
for tag, tag_config in class_config.items():
if region.id in tag_config:
desired_value = tag_config[region.id]
elif '*' in tag_config:
desired_value = tag_config['*']
else:
log.error(
f'No tag config for node {node.dname} class {node_class} in account {account.dname} cloud {cloud.id}'
)
continue
if tag in node.tags:
current_value = node.tags[tag]
log.debug((
f'Found {node.resource_type} {node.dname} age {node.age} in cloud {cloud.name}'
f' account {account.dname} region {region.name} with tag {tag}: {current_value}'
))
if desired_value == 'never':
continue
if current_value == 'never' and desired_value != 'never':
log_msg = f'Current value {current_value} is not allowed - setting tag {tag} to desired value {desired_value}'
log.debug(log_msg)
set_tag(pt, node, tag, desired_value, log_msg,
cloud, account, region)
continue
try:
current_value_td = parse_delta(current_value)
except ValueError:
log_msg = f"Can't parse current value {current_value} - setting tag {tag} to desired value {desired_value}"
log.error(log_msg)
set_tag(pt, node, tag, desired_value, log_msg,
cloud, account, region)
continue
try:
desired_value_td = parse_delta(desired_value)
except (AssertionError, ValueError):
log.error(
"Can't parse desired value {} into timedelta - skipping tag"
)
continue
if desired_value_td < current_value_td:
log_msg = f'Current value {current_value} is larger than desired value {desired_value} - setting tag {tag}'
log.debug(log_msg)
set_tag(pt, node, tag, desired_value, log_msg,
cloud, account, region)
pt.run()