def import_schema(self, schema_file):
if self.moddb == BackendTypes.LDAP:
base.logIt("Importing schema {}".format(schema_file))
parser = ldif_utils.myLdifParser(schema_file)
parser.parse()
for dn, entry in parser.entries:
if 'changetype' in entry:
entry.pop('changetype')
if 'add' in entry:
entry.pop('add')
for entry_type in entry:
for e in entry[entry_type]:
base.logIt("Adding to schema, type: {} value: {}".format(entry_type, e))
ldap_operation_result = self.ldap_conn.modify(dn, {entry_type: [ldap3.MODIFY_ADD, e]})
self.log_ldap_result(ldap_operation_result)
#we need re-bind after schema operations
self.ldap_conn.rebind()
def load_test_data(self):
self.logIt("Re-binding database")
self.dbUtils.bind(force=True)
self.logIt("Checking Internet conncetion")
socket.setdefaulttimeout(3)
try:
socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect(
("8.8.8.8", 443))
except:
self.logIt("Failed to connect 8.8.8.8:443.", True)
print("Test data loader needs internet connection. Giving up ...")
return
if not base.current_app.ScimInstaller.installed():
self.logIt("Scim was not installed. Installing")
Config.installScimServer = True
base.current_app.ScimInstaller.start_installation()
self.encode_test_passwords()
self.logIt("Rendering test templates")
if Config.rdbm_type == 'spanner':
Config.rdbm_password_enc = ''
Config.templateRenderingDict[
'config_oxauth_test_ldap'] = '# Not available'
Config.templateRenderingDict[
'config_oxauth_test_couchbase'] = '# Not available'
config_oxauth_test_properties = self.fomatWithDict(
'server.name=%(hostname)s\nconfig.oxauth.issuer=http://localhost:80\nconfig.oxauth.contextPath=http://localhost:80\nconfig.oxauth.salt=%(encode_salt)s\nconfig.persistence.type=%(persistence_type)s\n\n',
self.merge_dicts(Config.__dict__, Config.templateRenderingDict))
if self.getMappingType('ldap'):
template_text = self.readFile(
os.path.join(
self.template_base,
'jans-auth/server/config-oxauth-test-ldap.properties.nrnd')
)
rendered_text = self.fomatWithDict(
template_text,
self.merge_dicts(Config.__dict__,
Config.templateRenderingDict))
config_oxauth_test_properties += '#ldap\n' + rendered_text
if self.getMappingType('couchbase'):
couchbaseDict = base.current_app.CouchbaseInstaller.couchbaseDict()
template_text = self.readFile(
os.path.join(
self.template_base,
'jans-auth/server/config-oxauth-test-couchbase.properties.nrnd'
))
rendered_text = self.fomatWithDict(
template_text,
self.merge_dicts(Config.__dict__, Config.templateRenderingDict,
couchbaseDict))
config_oxauth_test_properties += '\n#couchbase\n' + rendered_text
if self.getMappingType('rdbm'):
base.current_app.RDBMInstaller.server_time_zone()
template_text = self.readFile(
os.path.join(
self.template_base,
'jans-auth/server/config-oxauth-test-sql.properties.nrnd'))
rendered_text = self.fomatWithDict(
template_text,
self.merge_dicts(Config.__dict__,
Config.templateRenderingDict))
config_oxauth_test_properties += '\n#sql\n' + rendered_text
self.logIt("Adding custom attributs and indexes")
schema2json(
os.path.join(Config.templateFolder,
'test/jans-auth/schema/102-oxauth_test.ldif'),
os.path.join(Config.outputFolder, 'test/jans-auth/schema/'))
schema2json(
os.path.join(Config.templateFolder,
'test/scim-client/schema/103-scim_test.ldif'),
os.path.join(Config.outputFolder, 'test/scim-client/schema/'),
)
oxauth_json_schema_fn = os.path.join(
Config.outputFolder,
'test/jans-auth/schema/102-oxauth_test.json')
scim_json_schema_fn = os.path.join(
Config.outputFolder,
'test/scim-client/schema/103-scim_test.json')
jans_schema_json_files = [
oxauth_json_schema_fn, scim_json_schema_fn
]
scim_schema = base.readJsonFile(scim_json_schema_fn)
may_list = []
for attribute in scim_schema['attributeTypes']:
may_list += attribute['names']
jansPerson = {
'kind': 'STRUCTURAL',
'may': may_list,
'must': ['objectclass'],
'names': ['jansPerson'],
'oid': 'jansObjClass',
'sup': ['top'],
'x_origin': 'Jans created objectclass'
}
scim_schema['objectClasses'].append(jansPerson)
with open(scim_json_schema_fn, 'w') as w:
json.dump(scim_schema, w, indent=2)
self.dbUtils.read_jans_schema(others=jans_schema_json_files)
base.current_app.RDBMInstaller.create_tables(
jans_schema_json_files)
if Config.rdbm_type != 'spanner':
self.dbUtils.rdm_automapper()
self.writeFile(
os.path.join(
Config.outputFolder,
'test/jans-auth/server/config-oxauth-test.properties'),
config_oxauth_test_properties)
self.render_templates_folder(self.template_base)
self.logIt("Loading test ldif files")
ox_auth_test_ldif = os.path.join(
Config.outputFolder, 'test/jans-auth/data/oxauth-test-data.ldif')
ox_auth_test_user_ldif = os.path.join(
Config.outputFolder,
'test/jans-auth/data/oxauth-test-data-user.ldif')
scim_test_ldif = os.path.join(
Config.outputFolder, 'test/scim-client/data/scim-test-data.ldif')
scim_test_user_ldif = os.path.join(
Config.outputFolder,
'test/scim-client/data/scim-test-data-user.ldif')
ldif_files = (ox_auth_test_ldif, scim_test_ldif,
ox_auth_test_user_ldif, scim_test_user_ldif)
self.dbUtils.import_ldif(ldif_files)
apache_user = '******' if base.clone_type == 'deb' else 'apache'
# Client keys deployment
base.download(
'https://raw.githubusercontent.com/JanssenProject/jans-auth-server/master/client/src/test/resources/jans_test_client_keys.zip',
'/var/www/html/jans_test_client_keys.zip')
self.run([
paths.cmd_unzip, '-o', '/var/www/html/jans_test_client_keys.zip',
'-d', '/var/www/html/'
])
self.run([paths.cmd_rm, '-rf', 'jans_test_client_keys.zip'])
self.run([
paths.cmd_chown, '-R', 'root:' + apache_user,
'/var/www/html/jans-auth-client'
])
oxAuthConfDynamic_changes = {
'dynamicRegistrationCustomObjectClass':
'jansClntCustomAttributes',
'dynamicRegistrationCustomAttributes': [
"jansTrustedClnt", "myCustomAttr1", "myCustomAttr2",
"jansInclClaimsInIdTkn"
],
'dynamicRegistrationExpirationTime':
86400,
'dynamicGrantTypeDefault': [
"authorization_code", "implicit", "password",
"client_credentials", "refresh_token",
"urn:ietf:params:oauth:grant-type:uma-ticket",
"urn:openid:params:grant-type:ciba",
"urn:ietf:params:oauth:grant-type:device_code"
],
'legacyIdTokenClaims':
True,
'authenticationFiltersEnabled':
True,
'clientAuthenticationFiltersEnabled':
True,
'keyRegenerationEnabled':
True,
'openidScopeBackwardCompatibility':
False,
'forceOfflineAccessScopeToEnableRefreshToken':
False,
'dynamicRegistrationPasswordGrantTypeEnabled':
True,
'cibaEnabled':
True,
'backchannelAuthenticationRequestSigningAlgValuesSupported': [
"RS256", "RS384", "RS512", "ES256", "ES384", "ES512", "PS256",
"PS384", "PS512"
],
'backchannelClientId':
'123-123-123',
'backchannelUserCodeParameterSupported':
True,
'tokenEndpointAuthSigningAlgValuesSupported': [
'HS256', 'HS384', 'HS512', 'RS256', 'RS384', 'RS512', 'ES256',
'ES384', 'ES512', 'PS256', 'PS384', 'PS512'
],
'userInfoSigningAlgValuesSupported': [
'none', 'HS256', 'HS384', 'HS512', 'RS256', 'RS384', 'RS512',
'ES256', 'ES384', 'ES512', 'PS256', 'PS384', 'PS512'
],
'consentGatheringScriptBackwardCompatibility':
False,
'claimsParameterSupported':
True,
'grantTypesSupported': [
'urn:openid:params:grant-type:ciba', 'authorization_code',
'urn:ietf:params:oauth:grant-type:uma-ticket',
'urn:ietf:params:oauth:grant-type:device_code',
'client_credentials', 'implicit', 'refresh_token', 'password'
],
'idTokenSigningAlgValuesSupported': [
'none', 'HS256', 'HS384', 'HS512', 'RS256', 'RS384', 'RS512',
'ES256', 'ES384', 'ES512', 'PS256', 'PS384', 'PS512'
],
'requestObjectSigningAlgValuesSupported': [
'none', 'HS256', 'HS384', 'HS512', 'RS256', 'RS384', 'RS512',
'ES256', 'ES384', 'ES512', 'PS256', 'PS384', 'PS512'
],
'softwareStatementValidationClaimName':
'jwks_uri',
'softwareStatementValidationType':
'jwks_uri',
'umaGrantAccessIfNoPolicies':
True,
'rejectJwtWithNoneAlg':
False,
'removeRefreshTokensForClientOnLogout':
True,
'fapiCompatibility':
False,
'forceIdTokenHintPrecense':
False,
'introspectionScriptBackwardCompatibility':
False,
'spontaneousScopeLifetime':
0,
'tokenEndpointAuthMethodsSupported': [
'client_secret_basic', 'client_secret_post',
'client_secret_jwt', 'private_key_jwt', 'tls_client_auth',
'self_signed_tls_client_auth', 'none'
],
'sessionIdRequestParameterEnabled':
True,
'skipRefreshTokenDuringRefreshing':
False,
'enabledComponents': [
'unknown', 'health_check', 'userinfo', 'clientinfo',
'id_generation', 'registration', 'introspection',
'revoke_token', 'revoke_session', 'end_session',
'status_session', 'jans_configuration', 'ciba', 'uma', 'u2f',
'device_authz', 'stat'
]
}
if Config.get('config_patch_creds'):
data = None
datajs = None
patch_url = 'https://ox.gluu.org/protected/jans-auth/jans-auth-test-config-patch.json'
req = urllib.request.Request(patch_url)
credentials = Config.get('config_patch_creds')
encoded_credentials = base64.b64encode(credentials.encode('ascii'))
req.add_header('Authorization',
'Basic %s' % encoded_credentials.decode("ascii"))
self.logIt("Retreiving auto test ciba patch from " + patch_url)
try:
resp = urllib.request.urlopen(req)
data = resp.read()
self.logIt("Auto test ciba patch retreived")
except:
self.logIt("Can't retreive auto test ciba patch", True)
if data:
try:
datajs = json.loads(data.decode())
except:
self.logIt("Can't decode json for auto test ciba patch",
True)
if datajs:
oxAuthConfDynamic_changes.update(datajs)
self.logIt(
"oxAuthConfDynamic was updated with auto test ciba patch")
custom_scripts = ('2DAF-F995', '2DAF-F996', '4BBE-C6A8', 'A51E-76DA')
self.dbUtils.set_oxAuthConfDynamic(oxAuthConfDynamic_changes)
# Enable custom scripts
for inum in custom_scripts:
self.dbUtils.enable_script(inum)
if self.dbUtils.moddb == static.BackendTypes.LDAP:
# Update LDAP schema
openDjSchemaFolder = os.path.join(Config.ldapBaseFolder,
'config/schema/')
self.copyFile(
os.path.join(Config.outputFolder,
'test/jans-auth/schema/102-oxauth_test.ldif'),
openDjSchemaFolder)
self.copyFile(
os.path.join(Config.outputFolder,
'test/scim-client/schema/103-scim_test.ldif'),
openDjSchemaFolder)
schema_fn = os.path.join(openDjSchemaFolder,
'77-customAttributes.ldif')
obcl_parser = myLdifParser(schema_fn)
obcl_parser.parse()
for i, o in enumerate(obcl_parser.entries[0][1]['objectClasses']):
objcl = ObjectClass(o)
if 'jansCustomPerson' in objcl.tokens['NAME']:
may_list = list(objcl.tokens['MAY'])
for a in ('scimCustomFirst', 'scimCustomSecond',
'scimCustomThird'):
if not a in may_list:
may_list.append(a)
objcl.tokens['MAY'] = tuple(may_list)
obcl_parser.entries[0][1]['objectClasses'][
i] = objcl.getstr()
tmp_fn = '/tmp/77-customAttributes.ldif'
with open(tmp_fn, 'wb') as w:
ldif_writer = LDIFWriter(w)
for dn, entry in obcl_parser.entries:
ldif_writer.unparse(dn, entry)
self.copyFile(tmp_fn, openDjSchemaFolder)
self.logIt("Making opndj listen all interfaces")
ldap_operation_result = self.dbUtils.ldap_conn.modify(
'cn=LDAPS Connection Handler,cn=Connection Handlers,cn=config',
{'ds-cfg-listen-address': [ldap3.MODIFY_REPLACE, '0.0.0.0']})
if not ldap_operation_result:
self.logIt("Ldap modify operation failed {}".format(
str(self.ldap_conn.result)))
self.logIt(
"Ldap modify operation failed {}".format(
str(self.ldap_conn.result)), True)
self.dbUtils.ldap_conn.unbind()
self.logIt("Re-starting opendj")
self.restart('opendj')
self.logIt("Re-binding opendj")
# try 5 times to re-bind opendj
for i in range(5):
time.sleep(5)
self.logIt("Try binding {} ...".format(i + 1))
bind_result = self.dbUtils.ldap_conn.bind()
if bind_result:
self.logIt("Binding to opendj was successful")
break
self.logIt("Re-try in 5 seconds")
else:
self.logIt("Re-binding opendj FAILED")
sys.exit("Re-binding opendj FAILED")
for atr in ('myCustomAttr1', 'myCustomAttr2'):
dn = 'ds-cfg-attribute={},cn=Index,ds-cfg-backend-id={},cn=Backends,cn=config'.format(
atr, 'userRoot')
entry = {
'objectClass': ['top', 'ds-cfg-backend-index'],
'ds-cfg-attribute': [atr],
'ds-cfg-index-type': ['equality'],
'ds-cfg-index-entry-limit': ['4000']
}
self.logIt("Creating Index {}".format(dn))
ldap_operation_result = self.dbUtils.ldap_conn.add(
dn, attributes=entry)
if not ldap_operation_result:
self.logIt("Ldap modify operation failed {}".format(
str(self.dbUtils.ldap_conn.result)))
self.logIt(
"Ldap modify operation failed {}".format(
str(self.dbUtils.ldap_conn.result)), True)
elif self.dbUtils.moddb in (static.BackendTypes.MYSQL,
static.BackendTypes.PGSQL):
pass
elif self.dbUtils.moddb == static.BackendTypes.COUCHBASE:
self.dbUtils.cbm.exec_query(
'CREATE INDEX def_{0}_myCustomAttr1 ON `{0}`(myCustomAttr1) USING GSI WITH {{"defer_build":true}}'
.format(Config.couchbase_bucket_prefix))
self.dbUtils.cbm.exec_query(
'CREATE INDEX def_{0}_myCustomAttr2 ON `{0}`(myCustomAttr2) USING GSI WITH {{"defer_build":true}}'
.format(Config.couchbase_bucket_prefix))
self.dbUtils.cbm.exec_query(
'BUILD INDEX ON `{0}` (def_{0}_myCustomAttr1, def_{0}_myCustomAttr2)'
.format(Config.couchbase_bucket_prefix))
if self.dbUtils.moddb == static.BackendTypes.LDAP:
self.dbUtils.ldap_conn.bind()
result = self.dbUtils.search(
'ou=configuration,o=jans',
search_filter='(&(jansDbAuth=*)(objectClass=jansAppConf))',
search_scope=ldap3.BASE)
oxIDPAuthentication = json.loads(result['jansDbAuth'])
oxIDPAuthentication['config']['servers'] = [
'{0}:{1}'.format(Config.hostname, Config.ldaps_port)
]
oxIDPAuthentication_js = json.dumps(oxIDPAuthentication, indent=2)
self.dbUtils.set_configuration('jansDbAuth',
oxIDPAuthentication_js)
self.create_test_client_keystore()
# Disable token binding module
if base.os_name in ('ubuntu18', 'ubuntu20'):
self.run(['a2dismod', 'mod_token_binding'])
self.restart('apache2')
self.restart('jans-auth')
if Config.installEleven:
eleven_tokens_package = os.path.join(
Config.staticFolder, 'eleven/jans-eleven-tokens.tar.gz')
target_dir = '/var/lib/softhsm/tokens/'
if not os.path.exists(target_dir):
os.makedirs(target_dir)
self.run([
paths.cmd_tar, '-zxf', eleven_tokens_package, '-C', target_dir
])
def import_ldif(self, ldif_files, bucket=None, force=None):
base.logIt("Importing ldif file(s): {} ".format(', '.join(ldif_files)))
sql_data_fn = os.path.join(Config.outputFolder, Config.rdbm_type, 'jans_data.sql')
for ldif_fn in ldif_files:
base.logIt("Importing entries from " + ldif_fn)
parser = ldif_utils.myLdifParser(ldif_fn)
parser.parse()
for dn, entry in parser.entries:
backend_location = force if force else self.get_backend_location_for_dn(dn)
if backend_location == BackendTypes.LDAP:
if 'add' in entry and 'changetype' in entry:
base.logIt("LDAP modify add dn:{} entry:{}".format(dn, dict(entry)))
change_attr = entry['add'][0]
ldap_operation_result = self.ldap_conn.modify(dn, {change_attr: [(ldap3.MODIFY_ADD, entry[change_attr])]})
self.log_ldap_result(ldap_operation_result)
elif 'replace' in entry and 'changetype' in entry:
base.logIt("LDAP modify replace dn:{} entry:{}".format(dn, dict(entry)))
change_attr = entry['replace'][0]
ldap_operation_result = self.ldap_conn.modify(dn, {change_attr: [(ldap3.MODIFY_REPLACE, [entry[change_attr][0]])]})
self.log_ldap_result(ldap_operation_result)
elif not self.dn_exists(dn):
base.logIt("Adding LDAP dn:{} entry:{}".format(dn, dict(entry)))
ldap_operation_result = self.ldap_conn.add(dn, attributes=entry)
self.log_ldap_result(ldap_operation_result)
elif backend_location in (BackendTypes.MYSQL, BackendTypes.PGSQL):
if self.Base is None:
self.rdm_automapper()
# TODO: inserting data to sub tables to be implemented for mysql and pgsql
if 'add' in entry and 'changetype' in entry:
attribute = entry['add'][0]
new_val = entry[attribute]
sqlalchObj = self.get_sqlalchObj_for_dn(dn)
if sqlalchObj:
if isinstance(sqlalchObj.__table__.columns[attribute].type, self.json_dialects_instance):
cur_val = copy.deepcopy(getattr(sqlalchObj, attribute))
for val_ in new_val:
cur_val['v'].append(val_)
setattr(sqlalchObj, attribute, cur_val)
else:
setattr(sqlalchObj, attribute, new_val[0])
self.session.commit()
else:
base.logIt("Can't find current value for repmacement of {}".replace(str(entry)), True)
continue
elif 'replace' in entry and 'changetype' in entry:
attribute = entry['replace'][0]
new_val = self.get_rdbm_val(attribute, entry[attribute])
sqlalchObj = self.get_sqlalchObj_for_dn(dn)
if sqlalchObj:
setattr(sqlalchObj, attribute, new_val)
self.session.commit()
else:
base.logIt("Can't find current value for repmacement of {}".replace(str(entry)), True)
continue
else:
vals = {}
dn_parsed = dnutils.parse_dn(dn)
rdn_name = dn_parsed[0][0]
objectClass = self.get_clean_objcet_class(entry)
if objectClass.lower() == 'organizationalunit':
continue
vals['doc_id'] = dn_parsed[0][1]
vals['dn'] = dn
vals['objectClass'] = objectClass
#entry.pop(rdn_name)
if 'objectClass' in entry:
entry.pop('objectClass')
elif 'objectclass' in entry:
entry.pop('objectclass')
table_name = objectClass
if self.dn_exists_rdbm(dn, table_name):
base.logIt("DN {} exsits in {} skipping".format(dn, Config.rdbm_type))
continue
for lkey in entry:
vals[lkey] = self.get_rdbm_val(lkey, entry[lkey])
sqlalchCls = self.Base.classes[table_name]
for col in sqlalchCls.__table__.columns:
if isinstance(col.type, self.json_dialects_instance) and not col.name in vals:
vals[col.name] = {'v': []}
sqlalchObj = sqlalchCls()
for v in vals:
setattr(sqlalchObj, v, vals[v])
base.logIt("Adding {}".format(sqlalchObj.doc_id))
self.session.add(sqlalchObj)
self.session.commit()
elif backend_location == BackendTypes.SPANNER:
if 'add' in entry and 'changetype' in entry:
table = self.get_spanner_table_for_dn(dn)
doc_id = self.get_doc_id_from_dn(dn)
change_attr = entry['add'][0]
if table:
doc_id = self.get_doc_id_from_dn(dn)
if self.in_subtable(table, change_attr):
sub_table = '{}_{}'.format(table, change_attr)
for subval in entry[change_attr]:
typed_val = self.get_rdbm_val(change_attr, subval, rdbm_type='spanner')
dict_doc_id = self.get_sha_digest(typed_val)
self.spanner.insert_data(table=sub_table, columns=['doc_id', 'dict_doc_id', change_attr], values=[[doc_id, typed_val, typed_val]])
else:
data = self.spanner.exec_sql('SELECT {} FROM {} WHERE doc_id="{}"'.format(entry['add'][0], table, doc_id))
if data.get('rows'):
cur_data = []
if 'rows' in data and data['rows'] and data['rows'][0] and data['rows'][0][0]:
cur_data = data['rows'][0][0]
for cur_val in entry[change_attr]:
typed_val = self.get_rdbm_val(change_attr, cur_val, rdbm_type='spanner')
cur_data.append(typed_val)
self.spanner.update_data(table=table, columns=['doc_id', change_attr], values=[[doc_id, cur_data]])
elif 'replace' in entry and 'changetype' in entry:
table = self.get_spanner_table_for_dn(dn)
doc_id = self.get_doc_id_from_dn(dn)
replace_attr = entry['replace'][0]
typed_val = self.get_rdbm_val(replace_attr, entry[replace_attr], rdbm_type='spanner')
if self.in_subtable(table, replace_attr):
sub_table = '{}_{}'.format(table, replace_attr)
# TODO: how to replace ?
#for subval in typed_val:
# self.spanner.update_data(table=sub_table, columns=['doc_id', replace_attr], values=[[doc_id, subval]])
else:
self.spanner.update_data(table=table, columns=['doc_id', replace_attr], values=[[doc_id, typed_val]])
else:
vals = {}
dn_parsed = dnutils.parse_dn(dn)
rdn_name = dn_parsed[0][0]
objectClass = objectClass = self.get_clean_objcet_class(entry)
if objectClass.lower() == 'organizationalunit':
continue
doc_id = dn_parsed[0][1]
vals['doc_id'] = doc_id
vals['dn'] = dn
vals['objectClass'] = objectClass
if 'objectClass' in entry:
entry.pop('objectClass')
elif 'objectclass' in entry:
entry.pop('objectclass')
table_name = objectClass
subtable_data = []
for lkey in entry:
spanner_vals = self.get_rdbm_val(lkey, entry[lkey], rdbm_type='spanner')
if not self.in_subtable(table_name, lkey):
vals[lkey] = spanner_vals
else:
sub_table = '{}_{}'.format(table_name, lkey)
sub_table_columns = ['doc_id', 'dict_doc_id', lkey]
sub_table_values = []
for subtableval in spanner_vals:
dict_doc_id = self.get_sha_digest(subtableval)
sub_table_values.append([doc_id, dict_doc_id, subtableval])
subtable_data.append((sub_table, sub_table_columns, sub_table_values))
columns = [ *vals.keys() ]
values = [ vals[lkey] for lkey in columns ]
self.spanner.insert_data(table=table_name, columns=columns, values=[values])
for sdata in subtable_data:
self.spanner.insert_data(table=sdata[0], columns=sdata[1], values=sdata[2])
elif backend_location == BackendTypes.COUCHBASE:
if len(entry) < 3:
continue
key, document = ldif_utils.get_document_from_entry(dn, entry)
cur_bucket = bucket if bucket else self.get_bucket_for_dn(dn)
base.logIt("Addnig document {} to Couchebase bucket {}".format(key, cur_bucket))
n1ql_list = []
if 'changetype' in document:
if 'replace' in document:
attribute = document['replace']
n1ql_list.append('UPDATE `%s` USE KEYS "%s" SET `%s`=%s' % (cur_bucket, key, attribute, json.dumps(document[attribute])))
elif 'add' in document:
attribute = document['add']
result = self.check_attribute_exists(key, attribute)
data = document[attribute]
if result:
if isinstance(data, list):
for d in data:
n1ql_list.append('UPDATE `%s` USE KEYS "%s" SET `%s`=ARRAY_APPEND(`%s`, %s)' % (cur_bucket, key, attribute, attribute, json.dumps(d)))
else:
n1ql_list.append('UPDATE `%s` USE KEYS "%s" SET `%s`=ARRAY_APPEND(`%s`, %s)' % (cur_bucket, key, attribute, attribute, json.dumps(data)))
else:
if attribute in attribDataTypes.listAttributes and not isinstance(data, list):
data = [data]
n1ql_list.append('UPDATE `%s` USE KEYS "%s" SET `%s`=%s' % (cur_bucket, key, attribute, json.dumps(data)))
else:
for k in document:
try:
kdata = json.loads(document[k])
if isinstance(kdata, dict):
document[k] = kdata
except:
pass
n1ql_list.append('UPSERT INTO `%s` (KEY, VALUE) VALUES ("%s", %s)' % (cur_bucket, key, json.dumps(document)))
for q in n1ql_list:
self.cbm.exec_query(q)
def load_test_data(self):
if not self.scimInstaller.installed():
self.logIt("Scim was not installed. Installing")
Config.installScimServer = True
self.scimInstaller.start_installation()
self.encode_test_passwords()
self.logIt("Rendering test templates")
Config.templateRenderingDict[
'config_oxauth_test_ldap'] = '# Not available'
Config.templateRenderingDict[
'config_oxauth_test_couchbase'] = '# Not available'
if self.getMappingType('ldap'):
template_text = self.readFile(
os.path.join(
self.template_base,
'oxauth/server/config-oxauth-test-ldap.properties.nrnd'))
rendered_text = self.fomatWithDict(
template_text,
self.merge_dicts(Config.__dict__,
Config.templateRenderingDict))
Config.templateRenderingDict[
'config_oxauth_test_ldap'] = rendered_text
if self.getMappingType('couchbase'):
template_text = self.readFile(
os.path.join(
self.template_base,
'oxauth/server/config-oxauth-test-couchbase.properties.nrnd'
))
rendered_text = self.fomatWithDict(
template_text,
self.merge_dicts(Config.__dict__,
Config.templateRenderingDict))
Config.templateRenderingDict[
'config_oxauth_test_couchbase'] = rendered_text
self.render_templates_folder(self.template_base)
self.logIt("Loading test ldif files")
if not self.passportInstaller.installed():
self.passportInstaller.generate_configuration()
ox_auth_test_ldif = os.path.join(
Config.outputFolder, 'test/oxauth/data/oxauth-test-data.ldif')
ox_auth_test_user_ldif = os.path.join(
Config.outputFolder, 'test/oxauth/data/oxauth-test-data-user.ldif')
scim_test_ldif = os.path.join(
Config.outputFolder, 'test/scim-client/data/scim-test-data.ldif')
scim_test_user_ldif = os.path.join(
Config.outputFolder,
'test/scim-client/data/scim-test-data-user.ldif')
ldif_files = (ox_auth_test_ldif, scim_test_ldif,
ox_auth_test_user_ldif, scim_test_user_ldif)
self.dbUtils.import_ldif(ldif_files)
apache_user = '******' if base.clone_type == 'deb' else 'apache'
# Client keys deployment
base.download(
'https://raw.githubusercontent.com/GluuFederation/oxAuth/master/Client/src/test/resources/oxauth_test_client_keys.zip',
'/var/www/html/oxauth_test_client_keys.zip')
self.run([
paths.cmd_unzip, '-o', '/var/www/html/oxauth_test_client_keys.zip',
'-d', '/var/www/html/'
])
self.run([paths.cmd_rm, '-rf', 'oxauth_test_client_keys.zip'])
self.run([
paths.cmd_chown, '-R', 'root:' + apache_user,
'/var/www/html/oxauth-client'
])
oxAuthConfDynamic_changes = {
'dynamicRegistrationCustomObjectClass':
'oxAuthClientCustomAttributes',
'dynamicRegistrationCustomAttributes': [
"oxAuthTrustedClient", "myCustomAttr1", "myCustomAttr2",
"oxIncludeClaimsInIdToken"
],
'dynamicRegistrationExpirationTime':
86400,
'dynamicGrantTypeDefault': [
"authorization_code", "implicit", "password",
"client_credentials", "refresh_token",
"urn:ietf:params:oauth:grant-type:uma-ticket"
],
'legacyIdTokenClaims':
True,
'authenticationFiltersEnabled':
True,
'clientAuthenticationFiltersEnabled':
True,
'keyRegenerationEnabled':
True,
'openidScopeBackwardCompatibility':
False,
}
custom_scripts = ('2DAF-F995', '2DAF-F996', '4BBE-C6A8')
self.dbUtils.set_oxAuthConfDynamic(oxAuthConfDynamic_changes)
# Enable custom scripts
for inum in custom_scripts:
self.dbUtils.enable_script(inum)
if self.dbUtils.moddb == static.BackendTypes.LDAP:
# Update LDAP schema
openDjSchemaFolder = os.path.join(Config.ldapBaseFolder,
'config/schema/')
self.copyFile(
os.path.join(Config.outputFolder,
'test/oxauth/schema/102-oxauth_test.ldif'),
openDjSchemaFolder)
self.copyFile(
os.path.join(Config.outputFolder,
'test/scim-client/schema/103-scim_test.ldif'),
openDjSchemaFolder)
schema_fn = os.path.join(openDjSchemaFolder,
'77-customAttributes.ldif')
obcl_parser = myLdifParser(schema_fn)
obcl_parser.parse()
for i, o in enumerate(obcl_parser.entries[0][1]['objectClasses']):
objcl = ObjectClass(o)
if 'gluuCustomPerson' in objcl.tokens['NAME']:
may_list = list(objcl.tokens['MAY'])
for a in ('scimCustomFirst', 'scimCustomSecond',
'scimCustomThird'):
if not a in may_list:
may_list.append(a)
objcl.tokens['MAY'] = tuple(may_list)
obcl_parser.entries[0][1]['objectClasses'][
i] = objcl.getstr()
tmp_fn = '/tmp/77-customAttributes.ldif'
with open(tmp_fn, 'wb') as w:
ldif_writer = LDIFWriter(w)
for dn, entry in obcl_parser.entries:
ldif_writer.unparse(dn, entry)
self.copyFile(tmp_fn, openDjSchemaFolder)
cwd = os.path.join(Config.ldapBaseFolder, 'bin')
dsconfigCmd = (
'{} --trustAll --no-prompt --hostname {} --port {} '
'--bindDN "{}" --bindPasswordFile /home/ldap/.pw set-connection-handler-prop '
'--handler-name "LDAPS Connection Handler" --set listen-address:0.0.0.0'
).format(os.path.join(Config.ldapBaseFolder,
'bin/dsconfig'), Config.ldap_hostname,
Config.ldap_admin_port, Config.ldap_binddn)
self.run(['/bin/su', 'ldap', '-c', dsconfigCmd], cwd=cwd)
self.dbUtils.ldap_conn.unbind()
self.restart('opendj')
#wait 10 seconds to start opendj
time.sleep(10)
for atr in ('myCustomAttr1', 'myCustomAttr2'):
cmd = (
'create-backend-index --backend-name userRoot --type generic '
'--index-name {} --set index-type:equality --set index-entry-limit:4000 '
'--hostName {} --port {} --bindDN "{}" -j /home/ldap/.pw '
'--trustAll --noPropertiesFile --no-prompt').format(
atr, Config.ldap_hostname, Config.ldap_admin_port,
Config.ldap_binddn)
dsconfigCmd = '{1} {2}'.format(Config.ldapBaseFolder,
os.path.join(cwd, 'dsconfig'),
cmd)
self.run(['/bin/su', 'ldap', '-c', dsconfigCmd], cwd=cwd)
else:
self.dbUtils.cbm.exec_query(
'CREATE INDEX def_gluu_myCustomAttr1 ON `gluu`(myCustomAttr1) USING GSI WITH {"defer_build":true}'
)
self.dbUtils.cbm.exec_query(
'CREATE INDEX def_gluu_myCustomAttr2 ON `gluu`(myCustomAttr2) USING GSI WITH {"defer_build":true}'
)
self.dbUtils.cbm.exec_query(
'BUILD INDEX ON `gluu` (def_gluu_myCustomAttr1, def_gluu_myCustomAttr2)'
)
self.dbUtils.ldap_conn.bind()
result = self.dbUtils.search('ou=configuration,o=gluu',
search_filter='(oxIDPAuthentication=*)',
search_scope=ldap3.BASE)
oxIDPAuthentication = json.loads(result['oxIDPAuthentication'])
oxIDPAuthentication['config']['servers'] = [
'{0}:{1}'.format(Config.hostname, Config.ldaps_port)
]
oxIDPAuthentication_js = json.dumps(oxIDPAuthentication, indent=2)
self.dbUtils.set_configuration('oxIDPAuthentication',
oxIDPAuthentication_js)
self.create_test_client_keystore()
# Disable token binding module
if base.os_name in ('ubuntu18', 'ubuntu20'):
self.run(['a2dismod', 'mod_token_binding'])
self.restart('apache2')
self.restart('oxauth')
def load_test_data(self):
if not self.scimInstaller.installed():
self.logIt("Scim was not installed. Installing")
Config.installScimServer = True
self.scimInstaller.start_installation()
self.encode_test_passwords()
self.logIt("Rendering test templates")
Config.templateRenderingDict[
'config_oxauth_test_ldap'] = '# Not available'
Config.templateRenderingDict[
'config_oxauth_test_couchbase'] = '# Not available'
config_oxauth_test_properties = self.fomatWithDict(
'server.name=%(hostname)s\nconfig.oxauth.issuer=http://localhost:80\nconfig.oxauth.contextPath=http://localhost:80\nconfig.oxauth.salt=%(encode_salt)s\nconfig.persistence.type=%(persistence_type)s\n\n',
self.merge_dicts(Config.__dict__, Config.templateRenderingDict))
if self.getMappingType('ldap'):
template_text = self.readFile(
os.path.join(
self.template_base,
'oxauth/server/config-oxauth-test-ldap.properties.nrnd'))
rendered_text = self.fomatWithDict(
template_text,
self.merge_dicts(Config.__dict__,
Config.templateRenderingDict))
config_oxauth_test_properties += '#ldap\n' + rendered_text
if self.getMappingType('couchbase'):
template_text = self.readFile(
os.path.join(
self.template_base,
'oxauth/server/config-oxauth-test-couchbase.properties.nrnd'
))
rendered_text = self.fomatWithDict(
template_text,
self.merge_dicts(Config.__dict__,
Config.templateRenderingDict))
config_oxauth_test_properties += '\n#couchbase\n' + rendered_text
self.writeFile(
os.path.join(Config.outputFolder,
'test/oxauth/server/config-oxauth-test.properties'),
config_oxauth_test_properties)
self.render_templates_folder(self.template_base)
self.logIt("Loading test ldif files")
ox_auth_test_ldif = os.path.join(
Config.outputFolder, 'test/oxauth/data/oxauth-test-data.ldif')
ox_auth_test_user_ldif = os.path.join(
Config.outputFolder, 'test/oxauth/data/oxauth-test-data-user.ldif')
scim_test_ldif = os.path.join(
Config.outputFolder, 'test/scim-client/data/scim-test-data.ldif')
scim_test_user_ldif = os.path.join(
Config.outputFolder,
'test/scim-client/data/scim-test-data-user.ldif')
ldif_files = (ox_auth_test_ldif, scim_test_ldif,
ox_auth_test_user_ldif, scim_test_user_ldif)
self.dbUtils.import_ldif(ldif_files)
apache_user = '******' if base.clone_type == 'deb' else 'apache'
# Client keys deployment
base.download(
'https://raw.githubusercontent.com/JanssenProject/jans-auth-server/master/client/src/test/resources/oxauth_test_client_keys.zip',
'/var/www/html/oxauth_test_client_keys.zip')
self.run([
paths.cmd_unzip, '-o', '/var/www/html/oxauth_test_client_keys.zip',
'-d', '/var/www/html/'
])
self.run([paths.cmd_rm, '-rf', 'oxauth_test_client_keys.zip'])
self.run([
paths.cmd_chown, '-R', 'root:' + apache_user,
'/var/www/html/oxauth-client'
])
oxAuthConfDynamic_changes = {
'dynamicRegistrationCustomObjectClass':
'oxAuthClientCustomAttributes',
'dynamicRegistrationCustomAttributes': [
"oxAuthTrustedClient", "myCustomAttr1", "myCustomAttr2",
"oxIncludeClaimsInIdToken"
],
'dynamicRegistrationExpirationTime':
86400,
'dynamicGrantTypeDefault': [
"authorization_code", "implicit", "password",
"client_credentials", "refresh_token",
"urn:ietf:params:oauth:grant-type:uma-ticket"
],
'legacyIdTokenClaims':
True,
'authenticationFiltersEnabled':
True,
'clientAuthenticationFiltersEnabled':
True,
'keyRegenerationEnabled':
True,
'openidScopeBackwardCompatibility':
False,
}
custom_scripts = ('2DAF-F995', '2DAF-F996', '4BBE-C6A8')
self.dbUtils.set_oxAuthConfDynamic(oxAuthConfDynamic_changes)
# Enable custom scripts
for inum in custom_scripts:
self.dbUtils.enable_script(inum)
if self.dbUtils.moddb == static.BackendTypes.LDAP:
# Update LDAP schema
openDjSchemaFolder = os.path.join(Config.ldapBaseFolder,
'config/schema/')
self.copyFile(
os.path.join(Config.outputFolder,
'test/oxauth/schema/102-oxauth_test.ldif'),
openDjSchemaFolder)
self.copyFile(
os.path.join(Config.outputFolder,
'test/scim-client/schema/103-scim_test.ldif'),
openDjSchemaFolder)
schema_fn = os.path.join(openDjSchemaFolder,
'77-customAttributes.ldif')
obcl_parser = myLdifParser(schema_fn)
obcl_parser.parse()
for i, o in enumerate(obcl_parser.entries[0][1]['objectClasses']):
objcl = ObjectClass(o)
if 'gluuCustomPerson' in objcl.tokens['NAME']:
may_list = list(objcl.tokens['MAY'])
for a in ('scimCustomFirst', 'scimCustomSecond',
'scimCustomThird'):
if not a in may_list:
may_list.append(a)
objcl.tokens['MAY'] = tuple(may_list)
obcl_parser.entries[0][1]['objectClasses'][
i] = objcl.getstr()
tmp_fn = '/tmp/77-customAttributes.ldif'
with open(tmp_fn, 'wb') as w:
ldif_writer = LDIFWriter(w)
for dn, entry in obcl_parser.entries:
ldif_writer.unparse(dn, entry)
self.copyFile(tmp_fn, openDjSchemaFolder)
self.logIt("Making opndj listen all interfaces")
ldap_operation_result = self.dbUtils.ldap_conn.modify(
'cn=LDAPS Connection Handler,cn=Connection Handlers,cn=config',
{'ds-cfg-listen-address': [ldap3.MODIFY_REPLACE, '0.0.0.0']})
if not ldap_operation_result:
self.logIt("Ldap modify operation failed {}".format(
str(self.ldap_conn.result)))
self.logIt(
"Ldap modify operation failed {}".format(
str(self.ldap_conn.result)), True)
self.dbUtils.ldap_conn.unbind()
self.logIt("Re-starting opendj")
self.restart('opendj')
self.logIt("Re-binding opendj")
# try 5 times to re-bind opendj
for i in range(5):
time.sleep(5)
self.logIt("Try binding {} ...".format(i + 1))
bind_result = self.dbUtils.ldap_conn.bind()
if bind_result:
self.logIt("Binding to opendj was successful")
break
self.logIt("Re-try in 5 seconds")
else:
self.logIt("Re-binding opendj FAILED")
sys.exit("Re-binding opendj FAILED")
for atr in ('myCustomAttr1', 'myCustomAttr2'):
dn = 'ds-cfg-attribute={},cn=Index,ds-cfg-backend-id={},cn=Backends,cn=config'.format(
atr, 'userRoot')
entry = {
'objectClass': ['top', 'ds-cfg-backend-index'],
'ds-cfg-attribute': [atr],
'ds-cfg-index-type': ['equality'],
'ds-cfg-index-entry-limit': ['4000']
}
self.logIt("Creating Index {}".format(dn))
ldap_operation_result = self.dbUtils.ldap_conn.add(
dn, attributes=entry)
if not ldap_operation_result:
self.logIt("Ldap modify operation failed {}".format(
str(self.dbUtils.ldap_conn.result)))
self.logIt(
"Ldap modify operation failed {}".format(
str(self.dbUtils.ldap_conn.result)), True)
else:
self.dbUtils.cbm.exec_query(
'CREATE INDEX def_gluu_myCustomAttr1 ON `gluu`(myCustomAttr1) USING GSI WITH {"defer_build":true}'
)
self.dbUtils.cbm.exec_query(
'CREATE INDEX def_gluu_myCustomAttr2 ON `gluu`(myCustomAttr2) USING GSI WITH {"defer_build":true}'
)
self.dbUtils.cbm.exec_query(
'BUILD INDEX ON `gluu` (def_gluu_myCustomAttr1, def_gluu_myCustomAttr2)'
)
self.dbUtils.ldap_conn.bind()
result = self.dbUtils.search('ou=configuration,o=jans',
search_filter='(jansIDPAuthn=*)',
search_scope=ldap3.BASE)
oxIDPAuthentication = json.loads(result['jansIDPAuthn'])
oxIDPAuthentication['config']['servers'] = [
'{0}:{1}'.format(Config.hostname, Config.ldaps_port)
]
oxIDPAuthentication_js = json.dumps(oxIDPAuthentication, indent=2)
self.dbUtils.set_configuration('jansIDPAuthn', oxIDPAuthentication_js)
self.create_test_client_keystore()
# Disable token binding module
if base.os_name in ('ubuntu18', 'ubuntu20'):
self.run(['a2dismod', 'mod_token_binding'])
self.restart('apache2')
self.restart('jans-auth')
def import_ldif(self, ldif_files, bucket=None, force=None):
for ldif_fn in ldif_files:
parser = ldif_utils.myLdifParser(ldif_fn)
parser.parse()
for dn, entry in parser.entries:
backend_location = force if force else self.get_backend_location_for_dn(
dn)
if backend_location == BackendTypes.LDAP:
if not self.dn_exists(dn):
base.logIt("Adding LDAP dn:{} entry:{}".format(
dn, dict(entry)))
self.ldap_conn.add(dn, attributes=entry)
elif backend_location == BackendTypes.COUCHBASE:
if len(entry) < 3:
continue
key, document = ldif_utils.get_document_from_entry(
dn, entry)
cur_bucket = bucket if bucket else self.get_bucket_for_dn(
dn)
base.logIt(
"Addnig document {} to Couchebase bucket {}".format(
key, cur_bucket))
n1ql_list = []
if 'changetype' in document:
if 'replace' in document:
attribute = document['replace']
n1ql_list.append(
'UPDATE `%s` USE KEYS "%s" SET `%s`=%s' %
(cur_bucket, key, attribute,
json.dumps(document[attribute])))
elif 'add' in document:
attribute = document['add']
result = self.check_attribute_exists(
key, attribute)
data = document[attribute]
if result:
if isinstance(data, list):
for d in data:
n1ql_list.append(
'UPDATE `%s` USE KEYS "%s" SET `%s`=ARRAY_APPEND(`%s`, %s)'
% (cur_bucket, key, attribute,
attribute, json.dumps(d)))
else:
n1ql_list.append(
'UPDATE `%s` USE KEYS "%s" SET `%s`=ARRAY_APPEND(`%s`, %s)'
% (cur_bucket, key, attribute,
attribute, json.dumps(data)))
else:
if attribute in attribDataTypes.listAttributes and not isinstance(
data, list):
data = [data]
n1ql_list.append(
'UPDATE `%s` USE KEYS "%s" SET `%s`=%s' %
(cur_bucket, key, attribute,
json.dumps(data)))
else:
n1ql_list.append(
'UPSERT INTO `%s` (KEY, VALUE) VALUES ("%s", %s)' %
(cur_bucket, key, json.dumps(document)))
for q in n1ql_list:
self.cbm.exec_query(q)
def import_ldif(self, ldif_files, bucket=None, force=None):
base.logIt("Importing ldif file(s): {} ".format(', '.join(ldif_files)))
sql_data_fn = os.path.join(Config.outputFolder, Config.rdbm_type, 'jans_data.sql')
for ldif_fn in ldif_files:
base.logIt("Importing entries from " + ldif_fn)
parser = ldif_utils.myLdifParser(ldif_fn)
parser.parse()
for dn, entry in parser.entries:
backend_location = force if force else self.get_backend_location_for_dn(dn)
if backend_location == BackendTypes.LDAP:
if 'add' in entry and 'changetype' in entry:
base.logIt("LDAP modify add dn:{} entry:{}".format(dn, dict(entry)))
change_attr = entry['add'][0]
ldap_operation_result = self.ldap_conn.modify(dn, {change_attr: [(ldap3.MODIFY_ADD, entry[change_attr])]})
self.log_ldap_result(ldap_operation_result)
elif 'replace' in entry and 'changetype' in entry:
base.logIt("LDAP modify replace dn:{} entry:{}".format(dn, dict(entry)))
change_attr = entry['replace'][0]
ldap_operation_result = self.ldap_conn.modify(dn, {change_attr: [(ldap3.MODIFY_REPLACE, [entry[change_attr][0]])]})
self.log_ldap_result(ldap_operation_result)
elif not self.dn_exists(dn):
base.logIt("Adding LDAP dn:{} entry:{}".format(dn, dict(entry)))
ldap_operation_result = self.ldap_conn.add(dn, attributes=entry)
self.log_ldap_result(ldap_operation_result)
elif backend_location in (BackendTypes.MYSQL, BackendTypes.PGSQL):
if self.Base is None:
self.rdm_automapper()
if 'add' in entry and 'changetype' in entry:
attribute = entry['add'][0]
new_val = entry[attribute]
sqlalchObj = self.get_sqlalchObj_for_dn(dn)
if sqlalchObj:
if isinstance(sqlalchObj.__table__.columns[attribute].type, self.json_dialects_instance):
cur_val = copy.deepcopy(getattr(sqlalchObj, attribute))
for val_ in new_val:
cur_val['v'].append(val_)
setattr(sqlalchObj, attribute, cur_val)
else:
setattr(sqlalchObj, attribute, new_val[0])
self.session.commit()
else:
base.logIt("Can't find current value for repmacement of {}".replace(str(entry)), True)
continue
elif 'replace' in entry and 'changetype' in entry:
attribute = entry['replace'][0]
new_val = self.get_rdbm_val(attribute, entry[attribute])
sqlalchObj = self.get_sqlalchObj_for_dn(dn)
if sqlalchObj:
setattr(sqlalchObj, attribute, new_val)
self.session.commit()
else:
base.logIt("Can't find current value for repmacement of {}".replace(str(entry)), True)
continue
else:
vals = {}
dn_parsed = dnutils.parse_dn(dn)
rdn_name = dn_parsed[0][0]
objectClass = entry.get('objectClass') or entry.get('objectclass')
if objectClass:
if 'top' in objectClass:
objectClass.remove('top')
if len(objectClass) == 1 and objectClass[0].lower() == 'organizationalunit':
continue
objectClass = objectClass[-1]
vals['doc_id'] = dn_parsed[0][1]
vals['dn'] = dn
vals['objectClass'] = objectClass
#entry.pop(rdn_name)
if 'objectClass' in entry:
entry.pop('objectClass')
elif 'objectclass' in entry:
entry.pop('objectclass')
table_name = objectClass
if self.dn_exists_rdbm(dn, table_name):
base.logIt("DN {} exsits in {} skipping".format(dn, Config.rdbm_type))
continue
for lkey in entry:
vals[lkey] = self.get_rdbm_val(lkey, entry[lkey])
sqlalchCls = self.Base.classes[table_name]
for col in sqlalchCls.__table__.columns:
if isinstance(col.type, self.json_dialects_instance) and not col.name in vals:
vals[col.name] = {'v': []}
sqlalchObj = sqlalchCls()
for v in vals:
setattr(sqlalchObj, v, vals[v])
base.logIt("Adding {}".format(sqlalchObj.doc_id))
self.session.add(sqlalchObj)
self.session.commit()
elif backend_location == BackendTypes.COUCHBASE:
if len(entry) < 3:
continue
key, document = ldif_utils.get_document_from_entry(dn, entry)
cur_bucket = bucket if bucket else self.get_bucket_for_dn(dn)
base.logIt("Addnig document {} to Couchebase bucket {}".format(key, cur_bucket))
n1ql_list = []
if 'changetype' in document:
if 'replace' in document:
attribute = document['replace']
n1ql_list.append('UPDATE `%s` USE KEYS "%s" SET `%s`=%s' % (cur_bucket, key, attribute, json.dumps(document[attribute])))
elif 'add' in document:
attribute = document['add']
result = self.check_attribute_exists(key, attribute)
data = document[attribute]
if result:
if isinstance(data, list):
for d in data:
n1ql_list.append('UPDATE `%s` USE KEYS "%s" SET `%s`=ARRAY_APPEND(`%s`, %s)' % (cur_bucket, key, attribute, attribute, json.dumps(d)))
else:
n1ql_list.append('UPDATE `%s` USE KEYS "%s" SET `%s`=ARRAY_APPEND(`%s`, %s)' % (cur_bucket, key, attribute, attribute, json.dumps(data)))
else:
if attribute in attribDataTypes.listAttributes and not isinstance(data, list):
data = [data]
n1ql_list.append('UPDATE `%s` USE KEYS "%s" SET `%s`=%s' % (cur_bucket, key, attribute, json.dumps(data)))
else:
for k in document:
try:
kdata = json.loads(document[k])
if isinstance(kdata, dict):
document[k] = kdata
except:
pass
n1ql_list.append('UPSERT INTO `%s` (KEY, VALUE) VALUES ("%s", %s)' % (cur_bucket, key, json.dumps(document)))
for q in n1ql_list:
self.cbm.exec_query(q)
