def clean_up_namespaces_with_cleanup_policy(self, cleanup_policy): """ Deletes all namespaces matching the given cleanup policy """ responses = [] namespaces = self.core_api.list_namespace( label_selector=labels_to_string({CLEANUP_LABEL: cleanup_policy}) ).items namespace_names = [n.metadata.name for n in namespaces] self.logger.debug( "Deleting namespaces %s with cleanup policy %s", namespace_names, cleanup_policy, ) for namespace in namespaces: resp = self.core_api.delete_namespace( namespace.metadata.name, propagation_policy="Background" ) responses.append(resp) while self.core_api.list_namespace( label_selector=labels_to_string({CLEANUP_LABEL: cleanup_policy}) ).items: self.logger.debug( "Waiting for namespaces %s to be deleted.", namespace_names ) # TODO ugly hack to prevent race conditions when deleting namespaces time.sleep(2) return responses
def delete_resource_with_cleanup_policy( self, namespaces, cleanup_policy, method, resource_name ): """ Deletes all resources which match the given cleanup policy """ responses = [] for namespace in namespaces: self.logger.debug( "Deleting %s in namespace %s with cleanup policy %s", resource_name, namespace, cleanup_policy, ) try: resp = method( namespace, label_selector=labels_to_string({CLEANUP_LABEL: cleanup_policy}), ) responses.append(resp) except kubernetes.client.rest.ApiException: self.logger.error( "An error occured trying to delete the marked resource." ) return responses
def _find_or_create_namespace_for_host(self, from_host, api): namespaces_for_host = [ ns for ns in self.current_namespaces if from_host.matches(ns) ] self.logger.debug( "Found %s namespaces for host %s: %s", len(namespaces_for_host), from_host, [ns.metadata.name for ns in namespaces_for_host], ) if namespaces_for_host: return namespaces_for_host self.logger.debug("Creating namespace for host %s", from_host) ns_labels = {CLEANUP_LABEL: CLEANUP_ALWAYS} if isinstance(from_host, GenericClusterHost): namespace_name = convert_to_resource_name( labels_to_string(from_host.namespace_labels)) for key, value in from_host.namespace_labels.items(): ns_labels[key] = value else: namespace_name = from_host.namespace self.logger.debug("Generated namespace name '%s' for host %s", namespace_name, from_host) resp = self.namespace_exists(namespace_name, api) if resp: return [resp] resp = self.create_namespace(namespace_name, api, labels=ns_labels) return [resp]
def clean_up_cluster_role_binding(self, cleanup_policy): self.logger.info("Deleting CRBs with cleanup policy " + cleanup_policy + " globally") res = self.rbac_api.delete_collection_cluster_role_binding( label_selector=labels_to_string({CLEANUP_LABEL: cleanup_policy})) self.logger.debug(res) return [res]
def resolve_namespaces(host, namespaces_per_label_strings): logger.debug(host) if isinstance(host, ClusterHost): return [host.namespace] labels = labels_to_string(host.namespace_labels) return namespaces_per_label_strings[ labels] if labels in namespaces_per_label_strings else []
def clean_up_namespaces(self, cleanup_policy): resps = [] namespaces = self.core_api.list_namespace( label_selector=labels_to_string({CLEANUP_LABEL: cleanup_policy })).items namespace_names = [n.metadata.name for n in namespaces] self.logger.info("Deleting namespacess " + str(namespace_names) + " with cleanup policy " + cleanup_policy) for namespace in namespaces: resp = self.core_api.delete_namespace( namespace.metadata.name, propagation_policy="Background") resps.append(resp) while self.core_api.list_namespace(label_selector=labels_to_string( {CLEANUP_LABEL: cleanup_policy})).items: self.logger.debug( "Waiting for namespaces {} to be deleted.".format( namespace_names)) time.sleep(2) return resps
def clean_up_cluster_role_binding_with_cleanup_policy( self, cleanup_policy): """ Deletes all cluster role bindings matching the given cleanup policy """ self.logger.info("Deleting CRBs with cleanup policy %s globally", cleanup_policy) res = self.rbac_api.delete_collection_cluster_role_binding( label_selector=labels_to_string({CLEANUP_LABEL: cleanup_policy})) self.logger.debug(res) return [res]
def resolve_namespaces(self, host, namespaces_per_label_strings): """ Returns the namespace of a given host """ self.logger.debug(host) if isinstance(host, ClusterHost): return [host.namespace] labels = labels_to_string(host.namespace_labels) return namespaces_per_label_strings[ labels] if labels in namespaces_per_label_strings else []
def _create_project_namespace_if_missing(self, api: k8s.client.CoreV1Api): namespace_labels = { ROLE_LABEL: "daemon-runner-namespace", CLEANUP_LABEL: CLEANUP_ON_REQUEST } namespace_list = api.list_namespace( label_selector=labels_to_string(namespace_labels)) if not namespace_list.items: namespace = k8s.client.V1Namespace( metadata=k8s.client.V1ObjectMeta(name=PROJECT_NAMESPACE, labels=namespace_labels)) api.create_namespace(namespace)
def delete_res_using(self, namespaces, cleanup_policy, method, res_name): resps = [] for namespace in namespaces: self.logger.info("Deleting " + res_name + "s in " + str(namespace) + " with cleanup policy " + cleanup_policy) resp = method(namespace, label_selector=labels_to_string( {CLEANUP_LABEL: cleanup_policy})) self.logger.debug(resp) resps.append(resp) return resps
def get_namespace_label_strings(namespace_labels, namespaces): """ Returns a set of all stringified namespace labels """ # list of all namespace names with labels return { labels_to_string(namespace_label): [ namespace.metadata.name for namespace in namespaces if namespace.metadata.labels is not None and namespace_label.items() <= namespace.metadata.labels.items() ] for namespace_label in namespace_labels }
def delete_collection_namespaced_serivce(namespace, label_selector=None): if label_selector is None: label_selector = labels_to_string( {CLEANUP_LABEL: cleanup_policy}) resps = [] svcs = self.core_api.list_namespaced_service( namespace, label_selector=label_selector) for svc in svcs.items: resps.append( self.core_api.delete_namespaced_service( svc.metadata.name, namespace, k8s.client.V1DeleteOptions())) return resps
def delete_collection_namespaced_service(namespace, label_selector=None): """ Wrapper Method for deleting namespaced services by label, as delete_namespaced_service does not provide this feature. """ if label_selector is None: label_selector = labels_to_string( {CLEANUP_LABEL: cleanup_policy}) responses = [] svcs = self.core_api.list_namespaced_service( namespace, label_selector=label_selector) for svc in svcs.items: responses.append( self.core_api.delete_namespaced_service( svc.metadata.name, namespace)) return responses
def _find_or_create_namespace_for_host(self, from_host, api): namespaces_for_host = [ ns for ns in self._current_namespaces if from_host.matches(ns) ] logger.debug("Found {} namespaces for host {}: {}".format( len(namespaces_for_host), from_host, [ns.metadata.name for ns in namespaces_for_host])) if namespaces_for_host: return namespaces_for_host else: logger.debug("Creating namespace for host " + str(from_host)) ns_labels = { ROLE_LABEL: "testing_namespace", CLEANUP_LABEL: CLEANUP_ALWAYS } if isinstance(from_host, GenericClusterHost): namespace_name = convert_to_resource_name( labels_to_string(from_host.namespace_labels)) for k, v in from_host.namespace_labels.items(): ns_labels[k] = v else: namespace_name = from_host.namespace logger.debug("Generated namespace name '" + str(namespace_name) + "' for host " + str(from_host)) resp = api.create_namespace( k8s.client.V1Namespace(metadata=k8s.client.V1ObjectMeta( name=namespace_name, labels=ns_labels))) if isinstance(resp, k8s.client.V1Namespace): logger.debug( "Test namespace " + resp.metadata.name + " created succesfully, adding it to namespace list") self._current_namespaces.append(resp) time.sleep(1) while not api.list_namespaced_service_account( resp.metadata.name, field_selector="metadata.name=default").items: logger.debug( "Waiting for kubernetes to create default service account for namespace " + resp.metadata.name) time.sleep(2) return [resp] else: logger.error( "Failed to create test namespace for {}! Resp: {}".format( from_host, resp)) return []
def collect_results(self, pod_selector, api: k8s.client.CoreV1Api): """ Queries pods of runner daemon set and waits for a corresponding configmap for each to be filled. Returns the merged data of all configMaps. """ daemon_pods = [] try: daemon_pods = api.list_namespaced_pod( PROJECT_NAMESPACE, label_selector=labels_to_string(pod_selector)).items self.logger.debug("Found %s daemon runner pods", len(daemon_pods)) except k8s.client.rest.ApiException as api_exception: self.logger.error(api_exception) # Todo should we just use labels ? expected_result_map_names = [ f"{d.metadata.name}-results" for d in daemon_pods ] result_config_maps = [] # retry polling results until they are all returned while len(result_config_maps) < len(daemon_pods): try: result_config_maps = [ api.read_namespaced_config_map(name=result, namespace=PROJECT_NAMESPACE) for result in expected_result_map_names ] except k8s.client.rest.ApiException as api_exception: if api_exception.reason == "Not Found": pass else: raise api_exception self.logger.debug("Map names: %s", [m.metadata.name for m in result_config_maps]) self.logger.debug("Expected names: %s", expected_result_map_names) time.sleep(2) yamls = [yaml.safe_load(c.data["results"]) for c in result_config_maps] self.logger.debug("Found following yamls in result config maps:%s", yamls) times = { c.metadata.name: yaml.safe_load(c.data["runtimes"]) for c in result_config_maps if "runtimes" in c.data } return {k: v for yam in [y.items() for y in yamls] for k, v in yam}, times
def generate_negative_cases_for_incoming_cases(self, isolated_hosts, incoming_test_cases, other_hosts, namespaces): runtimes = {} start_time = time.time() namespace_labels = [ h.namespace_labels for h in other_hosts if isinstance(h, GenericClusterHost) ] namespaces_per_label_strings = { labels_to_string(k): [ n.metadata.name for n in namespaces if n.metadata.labels is not None and k.items() <= n.metadata.labels.items() ] for k in namespace_labels } namespace_label_resolve_time = time.time() runtimes["nsLabelResolve"] = namespace_label_resolve_time - start_time labels_per_namespace = { n.metadata.name: n.metadata.labels for n in namespaces } overlaps_per_host = { host: get_overlapping_hosts(host, namespaces_per_label_strings, labels_per_namespace, isolated_hosts + other_hosts) for host in isolated_hosts } overlap_calc_time = time.time() runtimes[ "overlapCalc"] = overlap_calc_time - namespace_label_resolve_time cases = [] for host in isolated_hosts: host_string = str(host) host_start_time = time.time() runtimes[host_string] = {} # Check for hosts that can target these to construct negative cases from logger.debug(overlaps_per_host[host]) reaching_hosts_with_ports = [ (t.from_host, t.port_string) for t in incoming_test_cases if t.to_host in overlaps_per_host[host] ] logger.debug(reaching_hosts_with_ports) reaching_host_find_time = time.time() runtimes[host_string][ "findReachingHosts"] = reaching_host_find_time - host_start_time if reaching_hosts_with_ports: reaching_hosts, _ = zip(*reaching_hosts_with_ports) ports_per_host = { host: [p for h, p in reaching_hosts_with_ports if h == host] for host in reaching_hosts } match_all_host = GenericClusterHost({}, {}) if match_all_host in reaching_hosts: # All hosts are allowed to reach (on some ports or all) => results from ALLOW all if "*" in ports_per_host[match_all_host]: logger.info("Not generating negative tests for host " + str(host) + " as all connections to it are allowed") else: case = NetworkTestCase( match_all_host, host, rand_port(ports_per_host[match_all_host]), False) cases.append(case) runtimes[host_string]["matchAllCase"] = time.time( ) - reaching_host_find_time else: inverted_hosts = set([ h for l in [invert_host(host) for host in reaching_hosts] for h in l ]) hosts_on_inverted = { h: originalHost for l, originalHost in [(invert_host(host), host) for host in reaching_hosts] for h in l } host_inversion_time = time.time() runtimes[host_string][ "hostInversion"] = host_inversion_time - reaching_host_find_time overlaps_for_inverted_hosts = { h: get_overlapping_hosts(h, namespaces_per_label_strings, labels_per_namespace, reaching_hosts) for h in inverted_hosts } overlap_calc_time = time.time() runtimes[host_string][ "overlapCalc"] = overlap_calc_time - host_inversion_time logger.debug("InvertedHosts: " + str(inverted_hosts)) negative_test_targets = [ h for h in inverted_hosts if len(overlaps_for_inverted_hosts[h]) <= 1 ] logger.debug("NegativeTestTargets: " + str(negative_test_targets)) # now remove the inverted hosts that are reachable for target in negative_test_targets: ports_for_inverted_hosts_original_host = ports_per_host[ hosts_on_inverted[target]] if ports_for_inverted_hosts_original_host: cases.append( NetworkTestCase( target, host, ports_for_inverted_hosts_original_host[0], False)) else: cases.append( NetworkTestCase(target, host, "*", False)) runtimes[host_string]["casesGen"] = time.time( ) - overlap_calc_time else: # No hosts are allowed to reach host -> it should be totally isolated # => results from default deny policy cases.append(NetworkTestCase(host, host, "*", False)) runtimes["all"] = time.time() - start_time
def collect_results(self, api: k8s.client.CoreV1Api): """ Queries pods of runner daemon set and waits for a corresponding configmap for each to be filled. Returns the merged data of all configMaps. """ # Todo fix me! # api.list_node(label_selector="!node-role.kubernetes.io/master").items non_master_nodes = api.list_node().items logger.debug("Found " + str(len(non_master_nodes)) + " non master nodes") daemon_pods = [] # we re-request daemon pods until the number exactly match because pods are sometimes overprovisioned # and then immediately deleted, causing the target number of ConfigMaps to never be reached apps_api = k8s.client.AppsV1Api() while self.runner_daemon_set is None: logger.info("Waiting for runner_daemon_set to become initialized") try: self.runner_daemon_set = apps_api.read_namespaced_daemon_set( namespace=PROJECT_NAMESPACE, name=DAEMONSET_NAME) if isinstance(self.runner_daemon_set, k8s.client.V1DaemonSet): break except k8s.client.rest.ApiException as api_exception: logger.info("exception occured!") if api_exception.reason != "Not Found": raise (api_exception) time.sleep(1) while len(daemon_pods) != len(non_master_nodes): daemon_pods = api.list_namespaced_pod( PROJECT_NAMESPACE, label_selector=labels_to_string( self.runner_daemon_set.spec.selector.match_labels)).items logger.debug("Found " + str(len(daemon_pods)) + " daemon runner pods") time.sleep(2) expected_result_map_names = [ d.metadata.name + "-results" for d in daemon_pods ] result_config_maps = [] # retry polling results until they are all returned while len(result_config_maps) < len(daemon_pods): try: result_config_maps = [ api.read_namespaced_config_map(name=result, namespace=PROJECT_NAMESPACE) for result in expected_result_map_names ] except k8s.client.rest.ApiException as api_exception: if api_exception.reason == "Not Found": pass else: raise (api_exception) logger.debug("Map names: " + str([m.metadata.name for m in result_config_maps])) logger.debug("Expected names: " + str(expected_result_map_names)) time.sleep(2) yamls = [yaml.safe_load(c.data["results"]) for c in result_config_maps] logger.debug("Found following yamls in result config maps:" + str(yamls)) times = { c.metadata.name: yaml.safe_load(c.data["runtimes"]) for c in result_config_maps if "runtimes" in c.data } return {k: v for yam in [y.items() for y in yamls] for k, v in yam}, times