class CouchbasePersistence(BasePersistence): def __init__(self, manager): host = os.environ.get("CN_COUCHBASE_URL", "localhost") user = get_couchbase_user(manager) password = get_couchbase_password(manager) self.client = CouchbaseClient(host, user, password) def get_auth_config(self): bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") req = self.client.exec_query( "SELECT jansRevision, jansConfDyn, jansConfWebKeys " f"FROM `{bucket}` " "USE KEYS 'configuration_jans-auth'", ) if not req.ok: return {} config = req.json()["results"][0] if not config: return {} config.update({"id": "configuration_jans-auth"}) return config def modify_auth_config(self, id_, rev, conf_dynamic, conf_webkeys): conf_dynamic = json.dumps(conf_dynamic) conf_webkeys = json.dumps(conf_webkeys) bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") req = self.client.exec_query( f"UPDATE `{bucket}` USE KEYS '{id_}' " f"SET jansRevision={rev}, jansConfDyn={conf_dynamic}, " f"jansConfWebKeys={conf_webkeys} " "RETURNING jansRevision" ) if not req.ok: return False return True
class CouchbasePersistence: def __init__(self, manager): host = os.environ.get("CN_COUCHBASE_URL", "localhost") user = get_couchbase_user(manager) password = get_couchbase_password(manager) self.client = CouchbaseClient(host, user, password) def get_auth_config(self): bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") req = self.client.exec_query( f"SELECT jansConfDyn FROM `{bucket}` USE KEYS 'configuration_jans-auth'" ) if not req.ok: return {} config = req.json()["results"][0] if not config: return {} return config["jansConfDyn"]
def wait_for_couchbase(manager, **kwargs): """Wait for readiness/availability of Couchbase server based on existing entry. :param manager: An instance of :class:`~jans.pycloudlib.manager._Manager`. """ host = os.environ.get("CN_COUCHBASE_URL", "localhost") user = get_couchbase_user(manager) password = get_couchbase_password(manager) persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "couchbase") ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") bucket_prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") # only default and user buckets buckets that may have initial data; # these data also affected by LDAP mapping selection; jca_client_id = manager.config.get("jca_client_id") bucket, key = bucket_prefix, f"clients_{jca_client_id}" # if `hybrid` is selected and default mapping is stored in LDAP, # the default bucket won't have data, hence we check the user bucket instead if persistence_type == "hybrid" and ldap_mapping == "default": bucket, key = f"{bucket_prefix}_user", "groups_60B7" cb_client = CouchbaseClient(host, user, password) req = cb_client.exec_query( f"SELECT objectClass FROM {bucket} USE KEYS $key", key=key, ) if not req.ok: try: data = json.loads(req.text) err = data["errors"][0]["msg"] except (ValueError, KeyError, IndexError): err = req.reason raise WaitError(err) # request is OK, but result is not found data = req.json() if not data["results"]: raise WaitError(f"Missing document {key} in bucket {bucket}")
class CouchbaseBackend(BaseBackend): def __init__(self, manager): super().__init__() self.manager = manager hostname = os.environ.get("CN_COUCHBASE_URL", "localhost") user = get_couchbase_superuser(manager) or get_couchbase_user(manager) password = "" with contextlib.suppress(FileNotFoundError): password = get_couchbase_superuser_password(manager) password = password or get_couchbase_password(manager) self.client = CouchbaseClient(hostname, user, password) self.type = "couchbase" def get_entry(self, key, filter_="", attrs=None, **kwargs): bucket = kwargs.get("bucket") req = self.client.exec_query( f"SELECT META().id, {bucket}.* FROM {bucket} USE KEYS '{key}'") if not req.ok: return try: _attrs = req.json()["results"][0] id_ = _attrs.pop("id") entry = Entry(id_, _attrs) except IndexError: entry = None return entry def modify_entry(self, key, attrs=None, **kwargs): bucket = kwargs.get("bucket") del_flag = kwargs.get("delete_attr", False) attrs = attrs or {} if del_flag: kv = ",".join(attrs.keys()) mod_kv = f"UNSET {kv}" else: kv = ",".join( ["{}={}".format(k, json.dumps(v)) for k, v in attrs.items()]) mod_kv = f"SET {kv}" query = f"UPDATE {bucket} USE KEYS '{key}' {mod_kv}" req = self.client.exec_query(query) if req.ok: resp = req.json() status = bool(resp["status"] == "success") message = resp["status"] else: status = False message = req.text or req.reason return status, message def update_people_entries(self): # add jansAdminUIRole to default admin user admin_inum = self.manager.config.get("admin_inum") id_ = id_from_dn(f"inum={admin_inum},ou=people,o=jans") bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") kwargs = {"bucket": f"{bucket}_user"} entry = self.get_entry(id_, **kwargs) if not entry: return if "jansAdminUIRole" not in entry.attrs: entry.attrs["jansAdminUIRole"] = ["api-admin"] self.modify_entry(id_, entry.attrs, **kwargs) def update_scopes_entries(self): # add jansAdminUIRole claim to profile scope id_ = id_from_dn(self.jans_admin_ui_role_id) bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") kwargs = {"bucket": bucket} entry = self.get_entry(id_, **kwargs) if not entry: return if self.jans_admin_ui_claim not in entry.attrs["jansClaim"]: entry.attrs["jansClaim"].append(self.jans_admin_ui_claim) self.modify_entry(id_, entry.attrs, **kwargs) def update_clients_entries(self): jca_client_id = self.manager.config.get("jca_client_id") id_ = id_from_dn(f"inum={jca_client_id},ou=clients,o=jans") bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") kwargs = {"bucket": bucket} entry = self.get_entry(id_, **kwargs) if not entry: return should_update = False # modify redirect UI of config-api client hostname = self.manager.config.get("hostname") if f"https://{hostname}/admin" not in entry.attrs["jansRedirectURI"]: entry.attrs["jansRedirectURI"].append(f"https://{hostname}/admin") should_update = True # add jans_stat, SCIM users.read, SCIM users.write scopes to config-api client for scope in (self.jans_scim_scopes + self.jans_stat_scopes): if scope not in entry.attrs["jansScope"]: entry.attrs["jansScope"].append(scope) should_update = True if should_update: self.modify_entry(id_, entry.attrs, **kwargs) def update_scim_scopes_entries(self): # add jansAttrs to SCIM users.read and users.write scopes ids = map(id_from_dn, self.jans_scim_scopes) bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") kwargs = {"bucket": bucket} for id_ in ids: entry = self.get_entry(id_, **kwargs) if not entry: continue if "jansAttrs" not in entry.attrs: entry.attrs["jansAttrs"] = self.jans_attrs self.modify_entry(id_, entry.attrs, **kwargs) def update_misc(self): # 1 - fix objectclass for scim and config-api where it has lowecased objectclass instead of objectClass bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") # create the index for query self.client.exec_query( f'CREATE INDEX `def_jans_fix_oc` ON `{bucket}`(`objectclass`)') # get all scopes that has objectclass instead of objectClass req = self.client.exec_query( f"SELECT META().id, {bucket}.* FROM {bucket} WHERE `objectclass` IS NOT MISSING" ) if req.ok: resp = req.json() for doc in resp["results"]: id_ = doc.pop("id") doc["objectClass"] = doc["objectclass"][-1] self.modify_entry(id_, doc, **{"bucket": bucket}) # remove the objectclass attribute so the query above wont return results self.modify_entry(id_, {"objectclass": []}, **{ "bucket": bucket, "delete_attr": True }) # drop the index self.client.exec_query(f'DROP INDEX `{bucket}`.`def_jans_fix_oc`') def update_base_entries(self): # add jansManagerGrp to base entry id_ = id_from_dn(JANS_BASE_ID) bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") kwargs = {"bucket": bucket} entry = self.get_entry(id_, **kwargs) if not entry: return if not entry.attrs.get("jansManagerGrp"): entry.attrs["jansManagerGrp"] = JANS_MANAGER_GROUP self.modify_entry(id_, entry.attrs, **kwargs)
class CouchbaseBackend: def __init__(self, manager): hostname = os.environ.get("CN_COUCHBASE_URL", "localhost") user = get_couchbase_superuser(manager) or get_couchbase_user(manager) password = "" with contextlib.suppress(FileNotFoundError): password = get_couchbase_superuser_password(manager) password = password or get_couchbase_password(manager) self.client = CouchbaseClient(hostname, user, password) self.manager = manager self.index_num_replica = 0 def create_buckets(self, bucket_mappings, bucket_type="couchbase"): sys_info = self.client.get_system_info() if not sys_info: raise RuntimeError( "Unable to get system info from Couchbase; aborting ...") ram_info = sys_info["storageTotals"]["ram"] total_mem = (ram_info['quotaTotalPerNode'] - ram_info['quotaUsedPerNode']) / (1024 * 1024) # the minimum memory is a sum of required buckets + minimum mem for `gluu` bucket min_mem = sum(value["mem_alloc"] for value in bucket_mappings.values()) + 100 logger.info( "Memory size per node for Couchbase buckets was determined as {} MB" .format(total_mem)) logger.info( "Minimum memory size per node for Couchbase buckets was determined as {} MB" .format(min_mem)) if total_mem < min_mem: logger.warning( "Available quota on couchbase node is less than {} MB".format( min_mem)) persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") # always create `jans` bucket even when `default` mapping stored in LDAP if persistence_type == "hybrid" and ldap_mapping == "default": memsize = 100 logger.info( "Creating bucket {0} with type {1} and RAM size {2}".format( "jans", bucket_type, memsize)) prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") req = self.client.add_bucket(prefix, memsize, bucket_type) if not req.ok: logger.warning("Failed to create bucket {}; reason={}".format( "jans", req.text)) req = self.client.get_buckets() if req.ok: remote_buckets = tuple(bckt["name"] for bckt in req.json()) else: remote_buckets = () for _, mapping in bucket_mappings.items(): if mapping["bucket"] in remote_buckets: continue memsize = int((mapping["mem_alloc"] / float(min_mem)) * total_mem) logger.info( "Creating bucket {0} with type {1} and RAM size {2}".format( mapping["bucket"], bucket_type, memsize)) req = self.client.add_bucket(mapping["bucket"], memsize, bucket_type) if not req.ok: logger.warning("Failed to create bucket {}; reason={}".format( mapping["bucket"], req.text)) def create_indexes(self, bucket_mappings): buckets = [mapping["bucket"] for _, mapping in bucket_mappings.items()] prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") with open("/app/static/couchbase/index.json") as f: txt = f.read().replace("!bucket_prefix!", prefix) indexes = json.loads(txt) for bucket in buckets: if bucket not in indexes: continue query_file = "/app/tmp/index_{}.n1ql".format(bucket) logger.info( "Running Couchbase index creation for {} bucket (if not exist)" .format(bucket)) with open(query_file, "w") as f: index_list = indexes.get(bucket, {}) index_names = [] for index in index_list.get("attributes", []): if '(' in ''.join(index): attr_ = index[0] index_name_ = index[0].replace('(', '_').replace( ')', '_').replace('`', '').lower() if index_name_.endswith('_'): index_name_ = index_name_[:-1] index_name = 'def_{0}_{1}'.format(bucket, index_name_) else: attr_ = ','.join(['`{}`'.format(a) for a in index]) index_name = "def_{0}_{1}".format( bucket, '_'.join(index)) f.write( 'CREATE INDEX %s ON `%s`(%s) USING GSI WITH {"defer_build":true,"num_replica": %s};\n' % (index_name, bucket, attr_, self.index_num_replica)) index_names.append(index_name) if index_names: f.write('BUILD INDEX ON `%s` (%s) USING GSI;\n' % (bucket, ', '.join(index_names))) sic = 1 for attribs, wherec in index_list.get("static", []): attrquoted = [] for a in attribs: if '(' not in a: attrquoted.append('`{}`'.format(a)) else: attrquoted.append(a) attrquoteds = ', '.join(attrquoted) f.write( 'CREATE INDEX `{0}_static_{1:02d}` ON `{0}`({2}) WHERE ({3}) WITH {{ "num_replica": {4} }}\n' .format(bucket, sic, attrquoteds, wherec, self.index_num_replica)) sic += 1 # exec query with open(query_file) as f: for line in f: query = line.strip() if not query: continue req = self.client.exec_query(query) if not req.ok: # the following code should be ignored # - 4300: index already exists error = req.json()["errors"][0] if error["code"] in (4300, ): continue logger.warning( "Failed to execute query, reason={}".format( error["msg"])) def import_ldif(self, bucket_mappings): ctx = prepare_template_ctx(self.manager) attr_processor = AttrProcessor() for _, mapping in bucket_mappings.items(): for file_ in mapping["files"]: logger.info(f"Importing {file_} file") src = f"/app/templates/{file_}" dst = f"/app/tmp/{file_}" os.makedirs(os.path.dirname(dst), exist_ok=True) render_ldif(src, dst, ctx) with open(dst, "rb") as fd: parser = LDIFParser(fd) for dn, entry in parser.parse(): if len(entry) <= 2: continue key = id_from_dn(dn) entry["dn"] = [dn] entry = transform_entry(entry, attr_processor) data = json.dumps(entry) # using INSERT will cause duplication error, but the data is left intact query = 'INSERT INTO `%s` (KEY, VALUE) VALUES ("%s", %s)' % ( mapping["bucket"], key, data) req = self.client.exec_query(query) if not req.ok: logger.warning( "Failed to execute query, reason={}".format( req.json())) def initialize(self): num_replica = int(os.environ.get("CN_COUCHBASE_INDEX_NUM_REPLICA", 0)) num_indexer_nodes = len(self.client.get_index_nodes()) if num_replica >= num_indexer_nodes: raise ValueError( f"Number of index replica ({num_replica}) must be less than available indexer nodes ({num_indexer_nodes})" ) self.index_num_replica = num_replica bucket_mappings = get_bucket_mappings(self.manager) time.sleep(5) self.create_buckets(bucket_mappings) time.sleep(5) self.create_indexes(bucket_mappings) time.sleep(5) self.import_ldif(bucket_mappings) time.sleep(5) self.create_couchbase_shib_user() def create_couchbase_shib_user(self): self.client.create_user( 'couchbaseShibUser', self.manager.secret.get("couchbase_shib_user_password"), 'Shibboleth IDP', 'query_select[*]', )