def __init__(self, manager):
hostname = os.environ.get("CN_COUCHBASE_URL", "localhost")
user = get_couchbase_superuser(manager) or get_couchbase_user(manager)
password = ""
with contextlib.suppress(FileNotFoundError):
password = get_couchbase_superuser_password(manager)
password = password or get_couchbase_password(manager)
self.client = CouchbaseClient(hostname, user, password)
self.manager = manager
self.index_num_replica = 0
def wait_for_couchbase_conn(manager, **kwargs):
"""Wait for readiness/availability of Couchbase server based on connection status.
:param manager: An instance of :class:`~jans.pycloudlib.manager._Manager`.
"""
host = os.environ.get("CN_COUCHBASE_URL", "localhost")
user = get_couchbase_user(manager)
password = get_couchbase_password(manager)
cb_client = CouchbaseClient(host, user, password)
req = cb_client.get_buckets()
if not req.ok:
raise WaitError(f"Unable to connect to host in {host} list")
def wait_for_couchbase(manager, **kwargs):
"""Wait for readiness/availability of Couchbase server based on existing entry.
:param manager: An instance of :class:`~jans.pycloudlib.manager._Manager`.
"""
host = os.environ.get("CN_COUCHBASE_URL", "localhost")
user = get_couchbase_user(manager)
password = get_couchbase_password(manager)
persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "couchbase")
ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default")
bucket_prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
# only default and user buckets buckets that may have initial data;
# these data also affected by LDAP mapping selection;
jca_client_id = manager.config.get("jca_client_id")
bucket, key = bucket_prefix, f"clients_{jca_client_id}"
# if `hybrid` is selected and default mapping is stored in LDAP,
# the default bucket won't have data, hence we check the user bucket instead
if persistence_type == "hybrid" and ldap_mapping == "default":
bucket, key = f"{bucket_prefix}_user", "groups_60B7"
cb_client = CouchbaseClient(host, user, password)
req = cb_client.exec_query(
f"SELECT objectClass FROM {bucket} USE KEYS $key",
key=key,
)
if not req.ok:
try:
data = json.loads(req.text)
err = data["errors"][0]["msg"]
except (ValueError, KeyError, IndexError):
err = req.reason
raise WaitError(err)
# request is OK, but result is not found
data = req.json()
if not data["results"]:
raise WaitError(f"Missing document {key} in bucket {bucket}")
class CouchbasePersistence(BasePersistence):
def __init__(self, manager):
host = os.environ.get("CN_COUCHBASE_URL", "localhost")
user = get_couchbase_user(manager)
password = get_couchbase_password(manager)
self.client = CouchbaseClient(host, user, password)
def get_auth_config(self):
bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
req = self.client.exec_query(
"SELECT jansRevision, jansConfDyn, jansConfWebKeys "
f"FROM `{bucket}` "
"USE KEYS 'configuration_jans-auth'",
)
if not req.ok:
return {}
config = req.json()["results"][0]
if not config:
return {}
config.update({"id": "configuration_jans-auth"})
return config
def modify_auth_config(self, id_, rev, conf_dynamic, conf_webkeys):
conf_dynamic = json.dumps(conf_dynamic)
conf_webkeys = json.dumps(conf_webkeys)
bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
req = self.client.exec_query(
f"UPDATE `{bucket}` USE KEYS '{id_}' "
f"SET jansRevision={rev}, jansConfDyn={conf_dynamic}, "
f"jansConfWebKeys={conf_webkeys} "
"RETURNING jansRevision"
)
if not req.ok:
return False
return True
class CouchbasePersistence:
def __init__(self, manager):
host = os.environ.get("CN_COUCHBASE_URL", "localhost")
user = get_couchbase_user(manager)
password = get_couchbase_password(manager)
self.client = CouchbaseClient(host, user, password)
def get_auth_config(self):
bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
req = self.client.exec_query(
f"SELECT jansConfDyn FROM `{bucket}` USE KEYS 'configuration_jans-auth'"
)
if not req.ok:
return {}
config = req.json()["results"][0]
if not config:
return {}
return config["jansConfDyn"]
def test_no_couchbase_hosts(client_prop):
from jans.pycloudlib.persistence.couchbase import CouchbaseClient
client = CouchbaseClient("", "admin", "password")
with pytest.raises(ValueError):
getattr(client, client_prop)
def __init__(self, manager):
host = os.environ.get("CN_COUCHBASE_URL", "localhost")
user = get_couchbase_user(manager)
password = get_couchbase_password(manager)
self.client = CouchbaseClient(host, user, password)
class CouchbaseBackend(BaseBackend):
def __init__(self, manager):
super().__init__()
self.manager = manager
hostname = os.environ.get("CN_COUCHBASE_URL", "localhost")
user = get_couchbase_superuser(manager) or get_couchbase_user(manager)
password = ""
with contextlib.suppress(FileNotFoundError):
password = get_couchbase_superuser_password(manager)
password = password or get_couchbase_password(manager)
self.client = CouchbaseClient(hostname, user, password)
self.type = "couchbase"
def get_entry(self, key, filter_="", attrs=None, **kwargs):
bucket = kwargs.get("bucket")
req = self.client.exec_query(
f"SELECT META().id, {bucket}.* FROM {bucket} USE KEYS '{key}'")
if not req.ok:
return
try:
_attrs = req.json()["results"][0]
id_ = _attrs.pop("id")
entry = Entry(id_, _attrs)
except IndexError:
entry = None
return entry
def modify_entry(self, key, attrs=None, **kwargs):
bucket = kwargs.get("bucket")
del_flag = kwargs.get("delete_attr", False)
attrs = attrs or {}
if del_flag:
kv = ",".join(attrs.keys())
mod_kv = f"UNSET {kv}"
else:
kv = ",".join(
["{}={}".format(k, json.dumps(v)) for k, v in attrs.items()])
mod_kv = f"SET {kv}"
query = f"UPDATE {bucket} USE KEYS '{key}' {mod_kv}"
req = self.client.exec_query(query)
if req.ok:
resp = req.json()
status = bool(resp["status"] == "success")
message = resp["status"]
else:
status = False
message = req.text or req.reason
return status, message
def update_people_entries(self):
# add jansAdminUIRole to default admin user
admin_inum = self.manager.config.get("admin_inum")
id_ = id_from_dn(f"inum={admin_inum},ou=people,o=jans")
bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
kwargs = {"bucket": f"{bucket}_user"}
entry = self.get_entry(id_, **kwargs)
if not entry:
return
if "jansAdminUIRole" not in entry.attrs:
entry.attrs["jansAdminUIRole"] = ["api-admin"]
self.modify_entry(id_, entry.attrs, **kwargs)
def update_scopes_entries(self):
# add jansAdminUIRole claim to profile scope
id_ = id_from_dn(self.jans_admin_ui_role_id)
bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
kwargs = {"bucket": bucket}
entry = self.get_entry(id_, **kwargs)
if not entry:
return
if self.jans_admin_ui_claim not in entry.attrs["jansClaim"]:
entry.attrs["jansClaim"].append(self.jans_admin_ui_claim)
self.modify_entry(id_, entry.attrs, **kwargs)
def update_clients_entries(self):
jca_client_id = self.manager.config.get("jca_client_id")
id_ = id_from_dn(f"inum={jca_client_id},ou=clients,o=jans")
bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
kwargs = {"bucket": bucket}
entry = self.get_entry(id_, **kwargs)
if not entry:
return
should_update = False
# modify redirect UI of config-api client
hostname = self.manager.config.get("hostname")
if f"https://{hostname}/admin" not in entry.attrs["jansRedirectURI"]:
entry.attrs["jansRedirectURI"].append(f"https://{hostname}/admin")
should_update = True
# add jans_stat, SCIM users.read, SCIM users.write scopes to config-api client
for scope in (self.jans_scim_scopes + self.jans_stat_scopes):
if scope not in entry.attrs["jansScope"]:
entry.attrs["jansScope"].append(scope)
should_update = True
if should_update:
self.modify_entry(id_, entry.attrs, **kwargs)
def update_scim_scopes_entries(self):
# add jansAttrs to SCIM users.read and users.write scopes
ids = map(id_from_dn, self.jans_scim_scopes)
bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
kwargs = {"bucket": bucket}
for id_ in ids:
entry = self.get_entry(id_, **kwargs)
if not entry:
continue
if "jansAttrs" not in entry.attrs:
entry.attrs["jansAttrs"] = self.jans_attrs
self.modify_entry(id_, entry.attrs, **kwargs)
def update_misc(self):
# 1 - fix objectclass for scim and config-api where it has lowecased objectclass instead of objectClass
bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
# create the index for query
self.client.exec_query(
f'CREATE INDEX `def_jans_fix_oc` ON `{bucket}`(`objectclass`)')
# get all scopes that has objectclass instead of objectClass
req = self.client.exec_query(
f"SELECT META().id, {bucket}.* FROM {bucket} WHERE `objectclass` IS NOT MISSING"
)
if req.ok:
resp = req.json()
for doc in resp["results"]:
id_ = doc.pop("id")
doc["objectClass"] = doc["objectclass"][-1]
self.modify_entry(id_, doc, **{"bucket": bucket})
# remove the objectclass attribute so the query above wont return results
self.modify_entry(id_, {"objectclass": []}, **{
"bucket": bucket,
"delete_attr": True
})
# drop the index
self.client.exec_query(f'DROP INDEX `{bucket}`.`def_jans_fix_oc`')
def update_base_entries(self):
# add jansManagerGrp to base entry
id_ = id_from_dn(JANS_BASE_ID)
bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
kwargs = {"bucket": bucket}
entry = self.get_entry(id_, **kwargs)
if not entry:
return
if not entry.attrs.get("jansManagerGrp"):
entry.attrs["jansManagerGrp"] = JANS_MANAGER_GROUP
self.modify_entry(id_, entry.attrs, **kwargs)
class CouchbaseBackend:
def __init__(self, manager):
hostname = os.environ.get("CN_COUCHBASE_URL", "localhost")
user = get_couchbase_superuser(manager) or get_couchbase_user(manager)
password = ""
with contextlib.suppress(FileNotFoundError):
password = get_couchbase_superuser_password(manager)
password = password or get_couchbase_password(manager)
self.client = CouchbaseClient(hostname, user, password)
self.manager = manager
self.index_num_replica = 0
def create_buckets(self, bucket_mappings, bucket_type="couchbase"):
sys_info = self.client.get_system_info()
if not sys_info:
raise RuntimeError(
"Unable to get system info from Couchbase; aborting ...")
ram_info = sys_info["storageTotals"]["ram"]
total_mem = (ram_info['quotaTotalPerNode'] -
ram_info['quotaUsedPerNode']) / (1024 * 1024)
# the minimum memory is a sum of required buckets + minimum mem for `gluu` bucket
min_mem = sum(value["mem_alloc"]
for value in bucket_mappings.values()) + 100
logger.info(
"Memory size per node for Couchbase buckets was determined as {} MB"
.format(total_mem))
logger.info(
"Minimum memory size per node for Couchbase buckets was determined as {} MB"
.format(min_mem))
if total_mem < min_mem:
logger.warning(
"Available quota on couchbase node is less than {} MB".format(
min_mem))
persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap")
ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default")
# always create `jans` bucket even when `default` mapping stored in LDAP
if persistence_type == "hybrid" and ldap_mapping == "default":
memsize = 100
logger.info(
"Creating bucket {0} with type {1} and RAM size {2}".format(
"jans", bucket_type, memsize))
prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
req = self.client.add_bucket(prefix, memsize, bucket_type)
if not req.ok:
logger.warning("Failed to create bucket {}; reason={}".format(
"jans", req.text))
req = self.client.get_buckets()
if req.ok:
remote_buckets = tuple(bckt["name"] for bckt in req.json())
else:
remote_buckets = ()
for _, mapping in bucket_mappings.items():
if mapping["bucket"] in remote_buckets:
continue
memsize = int((mapping["mem_alloc"] / float(min_mem)) * total_mem)
logger.info(
"Creating bucket {0} with type {1} and RAM size {2}".format(
mapping["bucket"], bucket_type, memsize))
req = self.client.add_bucket(mapping["bucket"], memsize,
bucket_type)
if not req.ok:
logger.warning("Failed to create bucket {}; reason={}".format(
mapping["bucket"], req.text))
def create_indexes(self, bucket_mappings):
buckets = [mapping["bucket"] for _, mapping in bucket_mappings.items()]
prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans")
with open("/app/static/couchbase/index.json") as f:
txt = f.read().replace("!bucket_prefix!", prefix)
indexes = json.loads(txt)
for bucket in buckets:
if bucket not in indexes:
continue
query_file = "/app/tmp/index_{}.n1ql".format(bucket)
logger.info(
"Running Couchbase index creation for {} bucket (if not exist)"
.format(bucket))
with open(query_file, "w") as f:
index_list = indexes.get(bucket, {})
index_names = []
for index in index_list.get("attributes", []):
if '(' in ''.join(index):
attr_ = index[0]
index_name_ = index[0].replace('(', '_').replace(
')', '_').replace('`', '').lower()
if index_name_.endswith('_'):
index_name_ = index_name_[:-1]
index_name = 'def_{0}_{1}'.format(bucket, index_name_)
else:
attr_ = ','.join(['`{}`'.format(a) for a in index])
index_name = "def_{0}_{1}".format(
bucket, '_'.join(index))
f.write(
'CREATE INDEX %s ON `%s`(%s) USING GSI WITH {"defer_build":true,"num_replica": %s};\n'
% (index_name, bucket, attr_, self.index_num_replica))
index_names.append(index_name)
if index_names:
f.write('BUILD INDEX ON `%s` (%s) USING GSI;\n' %
(bucket, ', '.join(index_names)))
sic = 1
for attribs, wherec in index_list.get("static", []):
attrquoted = []
for a in attribs:
if '(' not in a:
attrquoted.append('`{}`'.format(a))
else:
attrquoted.append(a)
attrquoteds = ', '.join(attrquoted)
f.write(
'CREATE INDEX `{0}_static_{1:02d}` ON `{0}`({2}) WHERE ({3}) WITH {{ "num_replica": {4} }}\n'
.format(bucket, sic, attrquoteds, wherec,
self.index_num_replica))
sic += 1
# exec query
with open(query_file) as f:
for line in f:
query = line.strip()
if not query:
continue
req = self.client.exec_query(query)
if not req.ok:
# the following code should be ignored
# - 4300: index already exists
error = req.json()["errors"][0]
if error["code"] in (4300, ):
continue
logger.warning(
"Failed to execute query, reason={}".format(
error["msg"]))
def import_ldif(self, bucket_mappings):
ctx = prepare_template_ctx(self.manager)
attr_processor = AttrProcessor()
for _, mapping in bucket_mappings.items():
for file_ in mapping["files"]:
logger.info(f"Importing {file_} file")
src = f"/app/templates/{file_}"
dst = f"/app/tmp/{file_}"
os.makedirs(os.path.dirname(dst), exist_ok=True)
render_ldif(src, dst, ctx)
with open(dst, "rb") as fd:
parser = LDIFParser(fd)
for dn, entry in parser.parse():
if len(entry) <= 2:
continue
key = id_from_dn(dn)
entry["dn"] = [dn]
entry = transform_entry(entry, attr_processor)
data = json.dumps(entry)
# using INSERT will cause duplication error, but the data is left intact
query = 'INSERT INTO `%s` (KEY, VALUE) VALUES ("%s", %s)' % (
mapping["bucket"], key, data)
req = self.client.exec_query(query)
if not req.ok:
logger.warning(
"Failed to execute query, reason={}".format(
req.json()))
def initialize(self):
num_replica = int(os.environ.get("CN_COUCHBASE_INDEX_NUM_REPLICA", 0))
num_indexer_nodes = len(self.client.get_index_nodes())
if num_replica >= num_indexer_nodes:
raise ValueError(
f"Number of index replica ({num_replica}) must be less than available indexer nodes ({num_indexer_nodes})"
)
self.index_num_replica = num_replica
bucket_mappings = get_bucket_mappings(self.manager)
time.sleep(5)
self.create_buckets(bucket_mappings)
time.sleep(5)
self.create_indexes(bucket_mappings)
time.sleep(5)
self.import_ldif(bucket_mappings)
time.sleep(5)
self.create_couchbase_shib_user()
def create_couchbase_shib_user(self):
self.client.create_user(
'couchbaseShibUser',
self.manager.secret.get("couchbase_shib_user_password"),
'Shibboleth IDP',
'query_select[*]',
)