def run_loop( run_once, include_resources, exclude_resources, include_namespaces, exclude_namespaces, rules, interval, delete_notification, deployment_time_annotation: Optional[str], resource_context_hook: Optional[Callable], dry_run: bool, ): handler = shutdown.GracefulShutdown() while True: try: api = get_kube_api() clean_up( api, include_resources=frozenset(include_resources.split(",")), exclude_resources=frozenset(exclude_resources.split(",")), include_namespaces=frozenset(include_namespaces.split(",")), exclude_namespaces=frozenset(exclude_namespaces.split(",")), rules=rules, delete_notification=delete_notification, deployment_time_annotation=deployment_time_annotation, resource_context_hook=resource_context_hook, dry_run=dry_run, ) except Exception as e: logger.exception("Failed to clean up: %s", e) if run_once or handler.shutdown_now: return with handler.safe_exit(): time.sleep(interval)
def test_clean_up_default(): api_mock = MagicMock(spec=NamespacedAPIObject, name='APIMock') def get(**kwargs): if kwargs.get('url') == 'namespaces': # kube-system is skipped data = { 'items': [{ 'metadata': { 'name': 'default' } }, { 'metadata': { 'name': 'kube-system' } }] } elif kwargs['version'] == 'v1': data = {'resources': []} elif kwargs['version'] == '/apis': data = {'groups': []} else: data = {} response = MagicMock() response.json.return_value = data return response api_mock.get = get counter = clean_up(api_mock, ALL, [], ALL, ['kube-system'], [], None, dry_run=False) assert counter['resources-processed'] == 1
def test_ignore_nonlistable_api_group(): api_mock = MagicMock(spec=NamespacedAPIObject, name='APIMock') def get(**kwargs): if kwargs.get('url') == 'namespaces': data = {'items': [{'metadata': {'name': 'ns-1'}}]} elif kwargs.get('url') == 'customfoos': data = { 'items': [{ 'metadata': { 'name': 'foo-1', 'namespace': 'ns-1', 'creationTimestamp': '2019-01-17T15:14:38Z', # invalid TTL (no unit suffix) 'annotations': { 'janitor/ttl': '123' } } }] } elif kwargs['version'] == 'v1': data = {'resources': []} elif kwargs['version'] == 'srcco.de/v1': data = { 'resources': [{ 'kind': 'CustomFoo', 'name': 'customfoos', 'namespaced': True, 'verbs': ['delete'] }] } elif kwargs['version'] == 'kaput.srcco.de/v1': raise Exception('Catch me if you can!') elif kwargs['version'] == '/apis': data = { 'groups': [ { 'preferredVersion': { 'groupVersion': 'kaput.srcco.de/v1' } }, { 'preferredVersion': { 'groupVersion': 'srcco.de/v1' } }, ] } else: data = {} response = MagicMock() response.json.return_value = data return response api_mock.get = get counter = clean_up(api_mock, ALL, [], ALL, [], [], None, dry_run=False) assert counter['resources-processed'] == 2 assert counter['customfoos-with-ttl'] == 0 assert counter['customfoos-deleted'] == 0 assert not api_mock.delete.called
def test_ignore_invalid_expiry(): api_mock = MagicMock(spec=NamespacedAPIObject, name="APIMock") def get(**kwargs): if kwargs.get("url") == "namespaces": data = {"items": [{"metadata": {"name": "ns-1"}}]} elif kwargs.get("url") == "customfoos": data = { "items": [{ "metadata": { "name": "foo-1", "namespace": "ns-1", # invalid expiry "annotations": { "janitor/expires": "123" }, } }] } elif kwargs["version"] == "v1": data = {"resources": []} elif kwargs["version"] == "srcco.de/v1": data = { "resources": [{ "kind": "CustomFoo", "name": "customfoos", "namespaced": True, "verbs": ["delete"], }] } elif kwargs["version"] == "/apis": data = { "groups": [{ "preferredVersion": { "groupVersion": "srcco.de/v1" } }] } else: data = {} response = MagicMock() response.json.return_value = data return response api_mock.get = get counter = clean_up( api_mock, ALL, [], ALL, [], [], 0, deployment_time_annotation=None, dry_run=False, ) assert counter["resources-processed"] == 2 assert counter["customfoos-with-expiry"] == 0 assert counter["customfoos-deleted"] == 0 assert not api_mock.delete.called
def test_ignore_nonlistable_api_group(): api_mock = MagicMock(spec=NamespacedAPIObject, name="APIMock") def get(**kwargs): if kwargs.get("url") == "namespaces": data = {"items": [{"metadata": {"name": "ns-1"}}]} elif kwargs.get("url") == "customfoos": data = { "items": [{ "metadata": { "name": "foo-1", "namespace": "ns-1", "creationTimestamp": "2019-01-17T15:14:38Z", # invalid TTL (no unit suffix) "annotations": { "janitor/ttl": "123" }, } }] } elif kwargs["version"] == "v1": data = {"resources": []} elif kwargs["version"] == "srcco.de/v1": data = { "resources": [{ "kind": "CustomFoo", "name": "customfoos", "namespaced": True, "verbs": ["delete"], }] } elif kwargs["version"] == "kaput.srcco.de/v1": raise Exception("Catch me if you can!") elif kwargs["version"] == "/apis": data = { "groups": [ { "preferredVersion": { "groupVersion": "kaput.srcco.de/v1" } }, { "preferredVersion": { "groupVersion": "srcco.de/v1" } }, ] } else: data = {} response = MagicMock() response.json.return_value = data return response api_mock.get = get counter = clean_up(api_mock, ALL, [], ALL, [], [], None, dry_run=False) assert counter["resources-processed"] == 2 assert counter["customfoos-with-ttl"] == 0 assert counter["customfoos-deleted"] == 0 assert not api_mock.delete.called
def run_loop(run_once, include_resources, exclude_resources, include_namespaces, exclude_namespaces, rules, interval, dry_run): handler = shutdown.GracefulShutdown() while True: try: api = get_kube_api() clean_up( api, include_resources=frozenset(include_resources.split(',')), exclude_resources=frozenset(exclude_resources.split(',')), include_namespaces=frozenset(include_namespaces.split(',')), exclude_namespaces=frozenset(exclude_namespaces.split(',')), rules=rules, dry_run=dry_run) except Exception as e: logger.exception('Failed to clean up: %s', e) if run_once or handler.shutdown_now: return with handler.safe_exit(): time.sleep(interval)
def test_clean_up_only_included_namespaces(): api_mock = MagicMock(spec=NamespacedAPIObject, name="APIMock") def get(**kwargs): if kwargs.get("url") == "namespaces/foo": data = {"metadata": {"name": "foo"}} elif kwargs.get("url") == "namespaces": # kube-system is skipped data = { "items": [ { "metadata": { "name": "default" } }, { "metadata": { "name": "foo" } }, { "metadata": { "name": "kube-system" } }, ] } elif kwargs.get("url") == "namespaces/foo": # kube-system is skipped data = {"items": [{"metadata": {"name": "foo"}}]} elif kwargs["version"] == "v1": data = {"resources": []} elif kwargs["version"] == "/apis": data = {"groups": []} else: data = {} response = MagicMock() response.json.return_value = data return response api_mock.get = get counter = clean_up( api_mock, ALL, [], ["foo"], ["kube-system"], [], delete_notification=0, deployment_time_annotation=None, dry_run=False, ) assert counter["resources-processed"] == 1
def test_ignore_invalid_expiry(): api_mock = MagicMock(spec=NamespacedAPIObject, name='APIMock') def get(**kwargs): if kwargs.get('url') == 'namespaces': data = {'items': [{'metadata': {'name': 'ns-1'}}]} elif kwargs.get('url') == 'customfoos': data = { 'items': [{ 'metadata': { 'name': 'foo-1', 'namespace': 'ns-1', # invalid expiry 'annotations': { 'janitor/expires': '123' } } }] } elif kwargs['version'] == 'v1': data = {'resources': []} elif kwargs['version'] == 'srcco.de/v1': data = { 'resources': [{ 'kind': 'CustomFoo', 'name': 'customfoos', 'namespaced': True, 'verbs': ['delete'] }] } elif kwargs['version'] == '/apis': data = { 'groups': [{ 'preferredVersion': { 'groupVersion': 'srcco.de/v1' } }] } else: data = {} response = MagicMock() response.json.return_value = data return response api_mock.get = get counter = clean_up(api_mock, ALL, [], ALL, [], [], None, dry_run=False) assert counter['resources-processed'] == 2 assert counter['customfoos-with-expiry'] == 0 assert counter['customfoos-deleted'] == 0 assert not api_mock.delete.called
def test_clean_up_default(): api_mock = MagicMock(spec=NamespacedAPIObject, name="APIMock") def get(**kwargs): if kwargs.get("url") == "namespaces": # kube-system is skipped data = { "items": [ { "metadata": { "name": "default" } }, { "metadata": { "name": "kube-system" } }, ] } elif kwargs["version"] == "v1": data = {"resources": []} elif kwargs["version"] == "/apis": data = {"groups": []} else: data = {} response = MagicMock() response.json.return_value = data return response api_mock.get = get counter = clean_up(api_mock, ALL, [], ALL, ["kube-system"], [], None, dry_run=False) assert counter["resources-processed"] == 1
def test_clean_up_by_rule(): api_mock = MagicMock(name='APIMock') rule = Rule.from_entry({ 'id': 'r1', 'resources': ['customfoos'], 'jmespath': "metadata.namespace == 'ns-1'", 'ttl': '10m' }) def get(**kwargs): if kwargs.get('url') == 'namespaces': data = {'items': [{'metadata': {'name': 'ns-1'}}]} elif kwargs.get('url') == 'customfoos': data = { 'items': [{ 'metadata': { 'name': 'foo-1', 'namespace': 'ns-1', 'creationTimestamp': '2019-01-17T15:14:38Z', } }] } elif kwargs['version'] == 'v1': data = {'resources': []} elif kwargs['version'] == 'srcco.de/v1': data = { 'resources': [{ 'kind': 'CustomFoo', 'name': 'customfoos', 'namespaced': True, 'verbs': ['delete'] }] } elif kwargs['version'] == '/apis': data = { 'groups': [{ 'preferredVersion': { 'groupVersion': 'srcco.de/v1' } }] } else: data = {} response = MagicMock() response.json.return_value = data return response api_mock.get = get counter = clean_up(api_mock, ALL, [], ALL, [], [rule], None, dry_run=False) # namespace ns-1 and object foo-1 assert counter['resources-processed'] == 2 assert counter['rule-r1-matches'] == 1 assert counter['customfoos-with-ttl'] == 1 assert counter['customfoos-deleted'] == 1 api_mock.post.assert_called_once() _, kwargs = api_mock.post.call_args assert kwargs['url'] == 'events' data = json.loads(kwargs['data']) assert data['reason'] == 'TimeToLiveExpired' assert 'rule r1 matches' in data['message'] involvedObject = { 'kind': 'CustomFoo', 'name': 'foo-1', 'namespace': 'ns-1', 'apiVersion': 'srcco.de/v1', 'resourceVersion': None, 'uid': None } assert data['involvedObject'] == involvedObject # verify that the delete call happened api_mock.delete.assert_called_once_with( data='{"propagationPolicy": "Foreground"}', namespace='ns-1', url='customfoos/foo-1', version='srcco.de/v1')
def test_clean_up_custom_resource_on_expiry(): api_mock = MagicMock(name='APIMock') def get(**kwargs): if kwargs.get('url') == 'namespaces': data = {'items': [{'metadata': {'name': 'ns-1'}}]} elif kwargs.get('url') == 'customfoos': data = { 'items': [{ 'metadata': { 'name': 'foo-1', 'namespace': 'ns-1', 'annotations': { 'janitor/expires': '2001-01-17T15:14:38Z' } } }] } elif kwargs['version'] == 'v1': data = {'resources': []} elif kwargs['version'] == 'srcco.de/v1': data = { 'resources': [{ 'kind': 'CustomFoo', 'name': 'customfoos', 'namespaced': True, 'verbs': ['delete'] }] } elif kwargs['version'] == '/apis': data = { 'groups': [{ 'preferredVersion': { 'groupVersion': 'srcco.de/v1' } }] } else: data = {} response = MagicMock() response.json.return_value = data return response api_mock.get = get counter = clean_up(api_mock, ALL, [], ALL, [], [], None, dry_run=False) # namespace ns-1 and object foo-1 assert counter['resources-processed'] == 2 assert counter['customfoos-with-expiry'] == 1 assert counter['customfoos-deleted'] == 1 api_mock.post.assert_called_once() _, kwargs = api_mock.post.call_args assert kwargs['url'] == 'events' data = json.loads(kwargs['data']) assert data['reason'] == 'ExpiryTimeReached' assert 'annotation janitor/expires is set' in data['message'] involvedObject = { 'kind': 'CustomFoo', 'name': 'foo-1', 'namespace': 'ns-1', 'apiVersion': 'srcco.de/v1', 'resourceVersion': None, 'uid': None } assert data['involvedObject'] == involvedObject # verify that the delete call happened api_mock.delete.assert_called_once_with( data='{"propagationPolicy": "Foreground"}', namespace='ns-1', url='customfoos/foo-1', version='srcco.de/v1')
def test_clean_up_by_rule(): api_mock = MagicMock(name="APIMock") rule = Rule.from_entry( { "id": "r1", "resources": ["customfoos"], "jmespath": "metadata.namespace == 'ns-1'", "ttl": "10m", } ) def get(**kwargs): if kwargs.get("url") == "namespaces": data = {"items": [{"metadata": {"name": "ns-1"}}]} elif kwargs.get("url") == "customfoos": data = { "items": [ { "metadata": { "name": "foo-1", "namespace": "ns-1", "creationTimestamp": "2019-01-17T15:14:38Z", } } ] } elif kwargs["version"] == "v1": data = {"resources": []} elif kwargs["version"] == "srcco.de/v1": data = { "resources": [ { "kind": "CustomFoo", "name": "customfoos", "namespaced": True, "verbs": ["delete"], } ] } elif kwargs["version"] == "/apis": data = {"groups": [{"preferredVersion": {"groupVersion": "srcco.de/v1"}}]} else: data = {} response = MagicMock() response.json.return_value = data return response api_mock.get = get counter = clean_up( api_mock, ALL, [], ALL, [], [rule], 0, deployment_time_annotation=None, dry_run=False, ) # namespace ns-1 and object foo-1 assert counter["resources-processed"] == 2 assert counter["rule-r1-matches"] == 1 assert counter["customfoos-with-ttl"] == 1 assert counter["customfoos-deleted"] == 1 api_mock.post.assert_called_once() _, kwargs = api_mock.post.call_args assert kwargs["url"] == "events" data = json.loads(kwargs["data"]) assert data["reason"] == "TimeToLiveExpired" assert "rule r1 matches" in data["message"] involvedObject = { "kind": "CustomFoo", "name": "foo-1", "namespace": "ns-1", "apiVersion": "srcco.de/v1", "resourceVersion": None, "uid": None, } assert data["involvedObject"] == involvedObject # verify that the delete call happened api_mock.delete.assert_called_once_with( data='{"propagationPolicy": "Background"}', namespace="ns-1", url="/customfoos/foo-1", version="srcco.de/v1", )
def test_clean_up_custom_resource_on_expiry(): api_mock = MagicMock(name="APIMock") def get(**kwargs): if kwargs.get("url") == "namespaces": data = {"items": [{"metadata": {"name": "ns-1"}}]} elif kwargs.get("url") == "customfoos": data = { "items": [ { "metadata": { "name": "foo-1", "namespace": "ns-1", "annotations": {"janitor/expires": "2001-01-17T15:14:38Z"}, } } ] } elif kwargs["version"] == "v1": data = {"resources": []} elif kwargs["version"] == "srcco.de/v1": data = { "resources": [ { "kind": "CustomFoo", "name": "customfoos", "namespaced": True, "verbs": ["delete"], } ] } elif kwargs["version"] == "/apis": data = {"groups": [{"preferredVersion": {"groupVersion": "srcco.de/v1"}}]} else: data = {} response = MagicMock() response.json.return_value = data return response api_mock.get = get counter = clean_up( api_mock, ALL, [], ALL, [], [], 0, deployment_time_annotation=None, dry_run=False, ) # namespace ns-1 and object foo-1 assert counter["resources-processed"] == 2 assert counter["customfoos-with-expiry"] == 1 assert counter["customfoos-deleted"] == 1 api_mock.post.assert_called_once() _, kwargs = api_mock.post.call_args assert kwargs["url"] == "events" data = json.loads(kwargs["data"]) assert data["reason"] == "ExpiryTimeReached" assert "annotation janitor/expires is set" in data["message"] involvedObject = { "kind": "CustomFoo", "name": "foo-1", "namespace": "ns-1", "apiVersion": "srcco.de/v1", "resourceVersion": None, "uid": None, } assert data["involvedObject"] == involvedObject # verify that the delete call happened api_mock.delete.assert_called_once_with( data='{"propagationPolicy": "Background"}', namespace="ns-1", url="/customfoos/foo-1", version="srcco.de/v1", )