def run(dry_run, thread_pool_size=10, internal=None, use_jump_host=True, defer=None): namespaces = [ namespace_info for namespace_info in queries.get_namespaces() if namespace_info.get("managedRoles") and is_in_shard(f"{namespace_info['cluster']['name']}/" + f"{namespace_info['name']}") ] ri, oc_map = ob.fetch_current_state( namespaces=namespaces, thread_pool_size=thread_pool_size, integration=QONTRACT_INTEGRATION, integration_version=QONTRACT_INTEGRATION_VERSION, override_managed_types=["RoleBinding.authorization.openshift.io"], internal=internal, use_jump_host=use_jump_host, ) defer(oc_map.cleanup) fetch_desired_state(ri, oc_map) ob.realize_data(dry_run, oc_map, ri, thread_pool_size) if ri.has_error_registered(): sys.exit(1)
def run(dry_run, thread_pool_size=10, internal=None, use_jump_host=True, providers=[], cluster_name=None, namespace_name=None, init_api_resources=False, defer=None): gqlapi = gql.get_api() namespaces = [ namespace_info for namespace_info in gqlapi.query(NAMESPACES_QUERY)['namespaces'] if is_in_shard(f"{namespace_info['cluster']['name']}/" + f"{namespace_info['name']}") ] namespaces = \ filter_namespaces_by_cluster_and_namespace( namespaces, cluster_name, namespace_name ) namespaces = canonicalize_namespaces(namespaces, providers) oc_map, ri = \ fetch_data(namespaces, thread_pool_size, internal, use_jump_host, init_api_resources=init_api_resources) defer(lambda: oc_map.cleanup()) ob.realize_data(dry_run, oc_map, ri) if ri.has_error_registered(): sys.exit(1) return ri
def run(dry_run, thread_pool_size=10, internal=None, use_jump_host=True, defer=None): gqlapi = gql.get_api() namespaces = [] for namespace_info in gqlapi.query(NAMESPACES_QUERY)['namespaces']: if not namespace_info.get('networkPoliciesAllow'): continue shard_key = (f"{namespace_info['cluster']['name']}/" f"{namespace_info['name']}") if not is_in_shard(shard_key): continue namespaces.append(namespace_info) ri, oc_map = ob.fetch_current_state( namespaces=namespaces, thread_pool_size=thread_pool_size, integration=QONTRACT_INTEGRATION, integration_version=QONTRACT_INTEGRATION_VERSION, override_managed_types=['NetworkPolicy'], internal=internal, use_jump_host=use_jump_host) defer(lambda: oc_map.cleanup()) fetch_desired_state(namespaces, ri, oc_map) ob.realize_data(dry_run, oc_map, ri) if ri.has_error_registered(): sys.exit(1)
def get_desired_state(internal, use_jump_host, thread_pool_size): gqlapi = gql.get_api() all_namespaces = gqlapi.query(QUERY)['namespaces'] namespaces = [] for namespace in all_namespaces: shard_key = f'{namespace["cluster"]["name"]}/{namespace["name"]}' if is_in_shard(shard_key): namespaces.append(namespace) ri = ResourceInventory() settings = queries.get_app_interface_settings() oc_map = OC_Map(namespaces=namespaces, integration=QONTRACT_INTEGRATION, settings=settings, internal=internal, use_jump_host=use_jump_host, thread_pool_size=thread_pool_size, init_projects=True) ob.init_specs_to_fetch(ri, oc_map, namespaces=namespaces, override_managed_types=['Namespace']) desired_state = [] for cluster, namespace, _, _ in ri: if cluster not in oc_map.clusters(): continue desired_state.append({"cluster": cluster, "namespace": namespace}) return oc_map, desired_state
def test_is_in_shard_single_shard(monkeypatch): monkeypatch.setenv("SHARDS", "1") monkeypatch.setenv("SHARD_ID", "0") # SHARDS and SHARD_ID are defined as global variables. # To get tests to pick up new environment variables, # reloading is required after each call to monkeypatch.setenv() importlib.reload(sharding) assert sharding.is_in_shard(VALUE) is True
def get_gql_namespaces_in_shard() -> List[Any]: """ Get all namespaces from qontract-server and filter those which are in our shard """ all_namespaces = queries.get_namespaces() return [ns for ns in all_namespaces if not ns.get('delete') and is_in_shard(f"{ns['cluster']['name']}/{ns['name']}")]
def fetch_current_state(thread_pool_size, internal, use_jump_host): clusters = [c for c in queries.get_clusters() if is_in_shard(c['name'])] ocm_clusters = [c['name'] for c in clusters if c.get('ocm') is not None] current_state = [] settings = queries.get_app_interface_settings() oc_map = OC_Map(clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings, internal=internal, use_jump_host=use_jump_host, thread_pool_size=thread_pool_size) groups_list = create_groups_list(clusters, oc_map) results = threaded.run(get_cluster_state, groups_list, thread_pool_size, oc_map=oc_map) current_state = list(itertools.chain.from_iterable(results)) return oc_map, current_state, ocm_clusters
def get_shard_namespaces( namespaces: Iterable[Mapping[str, Any]]) \ -> Tuple[List[Dict[str, str]], bool]: # Structure holding duplicates by namespace key duplicates: Dict[str, List[Dict[str, str]]] = {} # namespace filtered list without duplicates filtered_ns: Dict[str, Dict[str, Any]] = {} err = False for ns in namespaces: key = f'{ns["cluster"]["name"]}/{ns["name"]}' if is_in_shard(key): if key not in filtered_ns: filtered_ns[key] = cast(dict, ns) else: # Duplicated NS dupe_list_by_key = duplicates.setdefault(key, []) dupe_list_by_key.append(cast(dict, ns)) for key, dupe_list in duplicates.items(): dupe_list.append(filtered_ns[key]) delete_flags = [ns["delete"] for ns in dupe_list_by_key] if len(set(delete_flags)) > 1: # If true only some definitions in list have the delete flag. # this case will generate an error err = True # Remove the namespace found from the filtered list del filtered_ns[key] logging.error(DUPLICATES_LOG_MSG.format(key=key)) else: # If all namespaces have the same delete option # The action will be performaed logging.debug(DUPLICATES_LOG_MSG.format(key=key)) return list(filtered_ns.values()), err
def setup(thread_pool_size, internal, use_jump_host, integration, integration_version, v1, v2): """Setup required resources for triggering integrations Args: thread_pool_size (int): Thread pool size to use internal (bool): Should run for internal/extrenal/all clusters use_jump_host (bool): Should use jump host to reach clusters integration (string): Name of calling integration integration_version (string): Version of calling integration v1 (bool): Should trigger for v1 SaaS files v2 (bool): Should trigger for v2 SaaS files Returns: saasherder (SaasHerder): a SaasHerder instance jenkins_map (dict): Instance names with JenkinsApi instances oc_map (OC_Map): a dictionary of OC clients per cluster settings (dict): App-interface settings error (bool): True if one happened, False otherwise """ saas_files = queries.get_saas_files(v1=v1, v2=v2) if not saas_files: logging.error("no saas files found") return None, None, None, None, True saas_files = [sf for sf in saas_files if is_in_shard(sf["name"])] # Remove saas-file targets that are disabled for saas_file in saas_files[:]: resource_templates = saas_file["resourceTemplates"] for rt in resource_templates[:]: targets = rt["targets"] for target in targets[:]: if target["disable"]: targets.remove(target) instance = queries.get_gitlab_instance() settings = queries.get_app_interface_settings() accounts = queries.get_state_aws_accounts() gl = GitLabApi(instance, settings=settings) jenkins_map = jenkins_base.get_jenkins_map() pipelines_providers = queries.get_pipelines_providers() tkn_provider_namespaces = [ pp["namespace"] for pp in pipelines_providers if pp["provider"] == "tekton" ] oc_map = OC_Map( namespaces=tkn_provider_namespaces, integration=integration, settings=settings, internal=internal, use_jump_host=use_jump_host, thread_pool_size=thread_pool_size, ) saasherder = SaasHerder( saas_files, thread_pool_size=thread_pool_size, gitlab=gl, integration=integration, integration_version=integration_version, settings=settings, jenkins_map=jenkins_map, accounts=accounts, ) return saasherder, jenkins_map, oc_map, settings, False
def fetch_desired_state(ri, oc_map): gqlapi = gql.get_api() roles = expiration.filter(gqlapi.query(ROLES_QUERY)["roles"]) users_desired_state = [] for role in roles: permissions = [{ "cluster": a["namespace"]["cluster"], "namespace": a["namespace"]["name"], "role": a["role"], } for a in role["access"] or [] if None not in [a["namespace"], a["role"]] and a["namespace"].get("managedRoles")] if not permissions: continue service_accounts = [ bot["openshift_serviceaccount"] for bot in role["bots"] if bot.get("openshift_serviceaccount") ] for permission in permissions: cluster_info = permission["cluster"] cluster = cluster_info["name"] namespace = permission["namespace"] if not is_in_shard(f"{cluster}/{namespace}"): continue if oc_map and not oc_map.get(cluster): continue user_key = ob.determine_user_key_for_access(cluster_info) for user in role["users"]: # used by openshift-users and github integrations # this is just to simplify things a bit on the their side users_desired_state.append({ "cluster": cluster, "user": user[user_key] }) if ri is None: continue oc_resource, resource_name = construct_user_oc_resource( permission["role"], user[user_key]) try: ri.add_desired( cluster, permission["namespace"], "RoleBinding.authorization.openshift.io", resource_name, oc_resource, ) except ResourceKeyExistsError: # a user may have a Role assigned to them # from multiple app-interface roles pass for sa in service_accounts: if ri is None: continue namespace, sa_name = sa.split("/") oc_resource, resource_name = construct_sa_oc_resource( permission["role"], namespace, sa_name) try: ri.add_desired( cluster, permission["namespace"], "RoleBinding.authorization.openshift.io", resource_name, oc_resource, ) except ResourceKeyExistsError: # a ServiceAccount may have a Role assigned to it # from multiple app-interface roles pass return users_desired_state
def test_is_in_shard_three_shards_fail(monkeypatch): monkeypatch.setenv("SHARDS", "3") monkeypatch.setenv("SHARD_ID", "2") importlib.reload(sharding) assert sharding.is_in_shard(VALUE) is False
def fetch_desired_state(ri, oc_map): gqlapi = gql.get_api() roles = gqlapi.query(ROLES_QUERY)['roles'] users_desired_state = [] for role in roles: permissions = [{'cluster': a['namespace']['cluster']['name'], 'namespace': a['namespace']['name'], 'role': a['role']} for a in role['access'] or [] if None not in [a['namespace'], a['role']] and a['namespace'].get('managedRoles')] if not permissions: continue users = [user['github_username'] for user in role['users']] bot_users = [bot['github_username'] for bot in role['bots'] if bot.get('github_username')] users.extend(bot_users) service_accounts = [bot['openshift_serviceaccount'] for bot in role['bots'] if bot.get('openshift_serviceaccount')] for permission in permissions: cluster = permission['cluster'] namespace = permission['namespace'] if not is_in_shard(f"{cluster}/{namespace}"): continue if oc_map and not oc_map.get(cluster): continue for user in users: # used by openshift-users and github integrations # this is just to simplify things a bit on the their side users_desired_state.append({ 'cluster': cluster, 'user': user }) if ri is None: continue oc_resource, resource_name = \ construct_user_oc_resource(permission['role'], user) try: ri.add_desired( cluster, permission['namespace'], 'RoleBinding.authorization.openshift.io', resource_name, oc_resource ) except ResourceKeyExistsError: # a user may have a Role assigned to them # from multiple app-interface roles pass for sa in service_accounts: if ri is None: continue namespace, sa_name = sa.split('/') oc_resource, resource_name = \ construct_sa_oc_resource( permission['role'], namespace, sa_name) try: ri.add_desired( permission['cluster'], permission['namespace'], 'RoleBinding.authorization.openshift.io', resource_name, oc_resource ) except ResourceKeyExistsError: # a ServiceAccount may have a Role assigned to it # from multiple app-interface roles pass return users_desired_state