def import_users(self):
"""Imports users from all the configured LDAP plugins into OGDS.
"""
session = create_session()
# Set all SQL users inactive first - the ones still contained in the LDAP
# will be set active again below (in the same transaction).
for user in session.query(User):
user.active = 0
for plugin in self._ldap_plugins():
ldap_userfolder = plugin._getLDAPUserFolder()
uid_attr = self._get_uid_attr(ldap_userfolder)
ldap_util = ILDAPSearch(ldap_userfolder)
ldap_users = ldap_util.get_users()
for ldap_user in ldap_users:
dn, info = ldap_user
# Ignore users without an UID in LDAP
if not uid_attr in info:
continue
userid = info[uid_attr]
# Skip users with uid longer than SQL 'userid' column
# FIXME: Increase size of SQL column to 64
if len(userid) > 30:
continue
if not self.user_exists(userid):
# Create the new user
user = User(userid)
session.add(user)
else:
# Get the existing user
user = session.query(User).filter_by(userid=userid).first()
# Iterate over all SQL columns and update their values
columns = User.__table__.columns
for col in columns:
if col.name == 'userid':
# We already set the userid when creating the user
# object, and it may not be called the same in LDAP as
# in our SQL model
continue
value = info.get(col.name)
# We can't store sequences in SQL columns. So if we do get a multi-valued field
# to be stored directly in OGDS, we treat it as a multi-line string and join it.
if isinstance(value, list) or isinstance(value, tuple):
value = ' '.join([str(v) for v in value])
setattr(user, col.name, value)
# Set the user active
user.active = 1
logger.info("Imported user '%s'..." % userid)
session.flush()
def import_users(self):
"""Imports users from all the configured LDAP plugins into OGDS.
"""
session = create_session()
# Set all SQL users inactive first - the ones still contained in the LDAP
# will be set active again below (in the same transaction).
for user in session.query(User):
user.active = 0
for plugin in self._ldap_plugins():
ldap_userfolder = plugin._getLDAPUserFolder()
uid_attr = self._get_uid_attr(ldap_userfolder)
ldap_util = ILDAPSearch(ldap_userfolder)
ldap_users = ldap_util.get_users()
for ldap_user in ldap_users:
dn, info = ldap_user
# Ignore users without an UID in LDAP
if not uid_attr in info:
continue
userid = info[uid_attr]
# Skip users with uid longer than SQL 'userid' column
# FIXME: Increase size of SQL column to 64
if len(userid) > 30:
continue
if not self.user_exists(userid):
# Create the new user
user = User(userid)
session.add(user)
else:
# Get the existing user
user = session.query(User).filter_by(userid=userid).first()
# Iterate over all SQL columns and update their values
columns = User.__table__.columns
for col in columns:
if col.name == 'userid':
# We already set the userid when creating the user
# object, and it may not be called the same in LDAP as
# in our SQL model
continue
value = info.get(col.name)
# We can't store sequences in SQL columns. So if we do get a multi-valued field
# to be stored directly in OGDS, we treat it as a multi-line string and join it.
if isinstance(value, list) or isinstance(value, tuple):
value = ' '.join([str(v) for v in value])
setattr(user, col.name, value)
# Set the user active
user.active = 1
logger.info("Imported user '%s'..." % userid)
session.flush()
def import_groups(self):
"""Imports groups from all the configured LDAP plugins into OGDS.
"""
session = create_session()
# Set all SQL groups inactive first - the ones still contained in the
# LDAP will be set active again below (in the same transaction).
#
# Also set their `users` attribute to an empty collection in order
# to clear out memberships from the `groups_users` association table
# before importing them, so that memberships from groups that have
# been deleted in LDAP get removed from OGDS.
for group in session.query(Group):
group.active = False
group.users = []
for plugin in self._ldap_plugins():
ldap_userfolder = plugin._getLDAPUserFolder()
ldap_util = ILDAPSearch(ldap_userfolder)
logger.info(u'Groups base: %s' % ldap_userfolder.groups_base)
logger.info(u'Group filter: %r' % ldap_util.get_group_filter())
ldap_groups = ldap_util.get_groups()
for ldap_group in ldap_groups:
dn, info = ldap_group
# Group name is in the 'cn' attribute, which may be
# mapped to 'fullname'
if 'cn' in info:
groupid = info['cn']
if isinstance(groupid, list):
groupid = groupid[0]
else:
groupid = info['fullname']
groupid = groupid.decode('utf-8')
info['groupid'] = groupid
# Skip groups with groupid longer than SQL 'groupid' column
if len(groupid) > GROUP_ID_LENGTH:
logger.warn(u"Skipping group '{}' - "
u"groupid too long!".format(groupid))
continue
if not self.group_exists(groupid):
# Create the new group
group = Group(groupid)
session.add(group)
else:
# Get the existing group
try:
group = self.get_sql_group(groupid)
except MultipleResultsFound:
# Duplicate group with slightly different spelling
# (casing, whitespace, ...) that may not be considered
# different by the SQL backend's unique constraint.
# We therefore enforce uniqueness ourselves.
logger.warn(
u"Skipping duplicate group '{}'!".format(groupid))
continue
# Iterate over all SQL columns and update their values
columns = Group.__table__.columns
for col in columns:
setattr(group, col.name,
self._convert_value(info.get(col.name)))
# Sync group title
title_attribute = self.get_group_title_ldap_attribute()
if title_attribute and info.get(title_attribute):
setattr(group, 'title',
self._convert_value(info.get(title_attribute)))
contained_users = []
group_members = ldap_util.get_group_members(info)
logger.info(u"Importing group '{}'...".format(groupid))
for user_dn in group_members:
ldap_user = ldap_util.entry_by_dn(user_dn)
if ldap_user is None:
logger.warn(USER_NOT_FOUND_LDAP.format(user_dn))
continue
user_dn, user_info = ldap_user
if isinstance(user_dn, str):
user_dn = user_dn.decode('utf-8')
if not ldap_util.is_ad:
if 'userid' not in user_info:
logger.warn(NO_UID_MSG.format(user_dn))
continue
userid = user_info['userid']
else:
# Active Directory
uid_found = False
for uid_key in AD_UID_KEYS:
if uid_key in user_info:
userid = user_info[uid_key]
uid_found = True
break
if not uid_found:
# No suitable UID found, skip this user
logger.warn(
NO_UID_AD_MSG.format(user_dn, AD_UID_KEYS))
continue
if isinstance(userid, list):
userid = userid[0]
if isinstance(userid, str):
userid = userid.decode('utf-8')
try:
user = self.get_sql_user(userid)
except NoResultFound:
logger.warn(USER_NOT_FOUND_SQL.format(userid))
continue
except MultipleResultsFound:
# Duplicate user - skip (see above).
logger.warn(
u" Skipping duplicate user '{}'!".format(userid))
continue
contained_users.append(user)
logger.info(
u"Importing user '{}' into group '{}'...".format(
userid, groupid))
group.users = contained_users
group.active = True
session.flush()
logger.info(u"Done.")
def import_users(self):
"""Imports users from all the configured LDAP plugins into OGDS.
"""
session = create_session()
# Set all SQL users inactive first - the ones still contained in the
# LDAP will be set active again below (in the same transaction).
for user in session.query(User):
user.active = False
for plugin in self._ldap_plugins():
ldap_userfolder = plugin._getLDAPUserFolder()
uid_attr = self._get_uid_attr(ldap_userfolder)
ldap_util = ILDAPSearch(ldap_userfolder)
logger.info(u'Users base: %s' % ldap_userfolder.users_base)
logger.info(u'User filter: %s' % ldap_util.get_user_filter())
ldap_users = ldap_util.get_users()
for ldap_user in ldap_users:
dn, info = ldap_user
# Ignore users without an UID in LDAP
if uid_attr not in info:
continue
userid = info[uid_attr]
userid = userid.decode('utf-8')
# Skip users with uid longer than SQL 'userid' column
if len(userid) > USER_ID_LENGTH:
logger.warn(u"Skipping user '{}' - "
u"userid too long!".format(userid))
continue
if not self.user_exists(userid):
# Create the new user
user = User(userid)
session.add(user)
else:
# Get the existing user
try:
user = self.get_sql_user(userid)
except MultipleResultsFound:
# Duplicate user with slightly different spelling
# (casing, whitespace, ...) that may not be considered
# different by the SQL backend's unique constraint.
# We therefore enforce uniqueness ourselves.
logger.warn(
u"Skipping duplicate user '{}'!".format(userid))
continue
# Iterate over all SQL columns and update their values
columns = User.__table__.columns
for col in columns:
if col.name == 'userid':
# We already set the userid when creating the user
# object, and it may not be called the same in LDAP as
# in our SQL model
continue
value = info.get(col.name)
# We can't store sequences in SQL columns. So if we do get
# a multi-valued field to be stored directly in OGDS, we
# treat it as a multi-line string and join it.
if isinstance(value, list) or isinstance(value, tuple):
value = ' '.join([str(v) for v in value])
if isinstance(value, str):
value = value.decode('utf-8')
# Truncate purely descriptive user fields if necessary
if isinstance(col.type, String):
if value and len(value) > col.type.length:
logger.warn(u"Truncating value %r for column %r "
u"(user: %r)" %
(value, col.name, userid))
value = value[:col.type.length]
setattr(user, col.name, value)
# Set the user active
user.active = True
logger.info(u"Imported user '{}'".format(userid))
session.flush()
def import_groups(self):
"""Imports groups from all the configured LDAP plugins into OGDS.
"""
session = create_session()
for plugin in self._ldap_plugins():
ldap_userfolder = plugin._getLDAPUserFolder()
ldap_util = ILDAPSearch(ldap_userfolder)
ldap_groups = ldap_util.get_groups()
for ldap_group in ldap_groups:
dn, info = ldap_group
# Group name is in the 'cn' attribute, which may be mapped to 'fullname'
if 'cn' in info:
groupid = info['cn']
if isinstance(groupid, list):
groupid = groupid[0]
else:
groupid = info['fullname']
groupid = groupid.decode('utf-8')
info['groupid'] = groupid
if not self.group_exists(groupid):
# Create the new group
group = Group(groupid)
session.add(group)
else:
# Get the existing group
group = session.query(Group).filter_by(groupid=groupid).first()
# Iterate over all SQL columns and update their values
columns = Group.__table__.columns
for col in columns:
value = info.get(col.name)
# We can't store sequences in SQL columns. So if we do get
# a multi-valued field to be stored directly in OGDS, we
# treat it as a multi-line string and join it.
if isinstance(value, list) or isinstance(value, tuple):
value = ' '.join([str(v) for v in value])
setattr(group, col.name, value)
contained_users = []
group_members = ldap_util.get_group_members(info)
for user_dn in group_members:
try:
ldap_user = ldap_util.entry_by_dn(user_dn)
user_dn, user_info = ldap_user
if not ldap_util.is_ad:
if not 'userid' in user_info:
logger.warn(NO_UID_MSG % user_dn)
continue
userid = user_info['userid']
else:
# Active Directory
uid_found = False
for uid_key in AD_UID_KEYS:
if uid_key in user_info:
userid = user_info[uid_key]
uid_found = True
break
if not uid_found:
# No suitable UID found, skip this user
logger.warn(NO_UID_AD_MSG % (AD_UID_KEYS,
user_dn))
continue
if isinstance(userid, list):
userid = userid[0]
user = self.get_sql_user(userid)
if user is None:
logger.warn(USER_NOT_FOUND_SQL % userid)
continue
contained_users.append(user)
logger.info("Importing user '%s'..." % userid)
except NO_SUCH_OBJECT:
logger.warn(USER_NOT_FOUND_LDAP % user_dn)
group.users = contained_users
session.flush()
def import_groups(self):
"""Imports groups from all the configured LDAP plugins into OGDS.
"""
session = create_session()
for plugin in self._ldap_plugins():
ldap_userfolder = plugin._getLDAPUserFolder()
ldap_util = ILDAPSearch(ldap_userfolder)
ldap_groups = ldap_util.get_groups()
for ldap_group in ldap_groups:
dn, info = ldap_group
# Group name is in the 'cn' attribute, which may be mapped to 'fullname'
if 'cn' in info:
groupid = info['cn']
if isinstance(groupid, list):
groupid = groupid[0]
else:
groupid = info['fullname']
groupid = groupid.decode('utf-8')
info['groupid'] = groupid
if not self.group_exists(groupid):
# Create the new group
group = Group(groupid)
session.add(group)
else:
# Get the existing group
group = session.query(Group).filter_by(
groupid=groupid).first()
# Iterate over all SQL columns and update their values
columns = Group.__table__.columns
for col in columns:
value = info.get(col.name)
# We can't store sequences in SQL columns. So if we do get
# a multi-valued field to be stored directly in OGDS, we
# treat it as a multi-line string and join it.
if isinstance(value, list) or isinstance(value, tuple):
value = ' '.join([str(v) for v in value])
setattr(group, col.name, value)
contained_users = []
group_members = ldap_util.get_group_members(info)
for user_dn in group_members:
try:
ldap_user = ldap_util.entry_by_dn(user_dn)
user_dn, user_info = ldap_user
if not ldap_util.is_ad:
if not 'userid' in user_info:
logger.warn(NO_UID_MSG % user_dn)
continue
userid = user_info['userid']
else:
# Active Directory
uid_found = False
for uid_key in AD_UID_KEYS:
if uid_key in user_info:
userid = user_info[uid_key]
uid_found = True
break
if not uid_found:
# No suitable UID found, skip this user
logger.warn(NO_UID_AD_MSG %
(AD_UID_KEYS, user_dn))
continue
if isinstance(userid, list):
userid = userid[0]
user = self.get_sql_user(userid)
if user is None:
logger.warn(USER_NOT_FOUND_SQL % userid)
continue
contained_users.append(user)
logger.info("Importing user '%s'..." % userid)
except NO_SUCH_OBJECT:
logger.warn(USER_NOT_FOUND_LDAP % user_dn)
group.users = contained_users
session.flush()
def import_groups(self):
"""Imports groups from all the configured LDAP plugins into OGDS.
"""
session = create_session()
# Set all SQL groups inactive first - the ones still contained in the
# LDAP will be set active again below (in the same transaction).
#
# Also set their `users` attribute to an empty collection in order
# to clear out memberships from the `groups_users` association table
# before importing them, so that memberships from groups that have
# been deleted in LDAP get removed from OGDS.
for group in session.query(Group):
group.active = False
group.users = []
for plugin in self._ldap_plugins():
ldap_userfolder = plugin._getLDAPUserFolder()
ldap_util = ILDAPSearch(ldap_userfolder)
logger.info(u"Groups base: %s" % ldap_userfolder.groups_base)
logger.info(u"Group filter: %r" % ldap_util.get_group_filter())
ldap_groups = ldap_util.get_groups()
for ldap_group in ldap_groups:
dn, info = ldap_group
# Group name is in the 'cn' attribute, which may be
# mapped to 'fullname'
if "cn" in info:
groupid = info["cn"]
if isinstance(groupid, list):
groupid = groupid[0]
else:
groupid = info["fullname"]
groupid = groupid.decode("utf-8")
info["groupid"] = groupid
# Skip groups with groupid longer than SQL 'groupid' column
if len(groupid) > GROUP_ID_LENGTH:
logger.warn(u"Skipping group '{}' - " u"groupid too long!".format(groupid))
continue
if not self.group_exists(groupid):
# Create the new group
group = Group(groupid)
session.add(group)
else:
# Get the existing group
try:
group = self.get_sql_group(groupid)
except MultipleResultsFound:
# Duplicate group with slightly different spelling
# (casing, whitespace, ...) that may not be considered
# different by the SQL backend's unique constraint.
# We therefore enforce uniqueness ourselves.
logger.warn(u"Skipping duplicate group '{}'!".format(groupid))
continue
# Iterate over all SQL columns and update their values
columns = Group.__table__.columns
for col in columns:
value = info.get(col.name)
# We can't store sequences in SQL columns. So if we do get
# a multi-valued field to be stored directly in OGDS, we
# treat it as a multi-line string and join it.
if isinstance(value, list) or isinstance(value, tuple):
value = " ".join([str(v) for v in value])
if isinstance(value, str):
value = value.decode("utf-8")
setattr(group, col.name, value)
contained_users = []
group_members = ldap_util.get_group_members(info)
logger.info(u"Importing group '{}'...".format(groupid))
for user_dn in group_members:
try:
ldap_user = ldap_util.entry_by_dn(user_dn)
user_dn, user_info = ldap_user
if isinstance(user_dn, str):
user_dn = user_dn.decode("utf-8")
if not ldap_util.is_ad:
if "userid" not in user_info:
logger.warn(NO_UID_MSG.format(user_dn))
continue
userid = user_info["userid"]
else:
# Active Directory
uid_found = False
for uid_key in AD_UID_KEYS:
if uid_key in user_info:
userid = user_info[uid_key]
uid_found = True
break
if not uid_found:
# No suitable UID found, skip this user
logger.warn(NO_UID_AD_MSG.format(user_dn, AD_UID_KEYS))
continue
if isinstance(userid, list):
userid = userid[0]
if isinstance(userid, str):
userid = userid.decode("utf-8")
try:
user = self.get_sql_user(userid)
except NoResultFound:
logger.warn(USER_NOT_FOUND_SQL.format(userid))
continue
except MultipleResultsFound:
# Duplicate user - skip (see above).
logger.warn(u" Skipping duplicate user '{}'!".format(userid))
continue
contained_users.append(user)
logger.info(u"Importing user '{}' into group '{}'...".format(userid, groupid))
except NO_SUCH_OBJECT:
logger.warn(USER_NOT_FOUND_LDAP.format(user_dn))
group.users = contained_users
group.active = True
session.flush()
logger.info(u"Done.")
def import_users(self):
"""Imports users from all the configured LDAP plugins into OGDS.
"""
session = create_session()
# Set all SQL users inactive first - the ones still contained in the
# LDAP will be set active again below (in the same transaction).
for user in session.query(User):
user.active = False
for plugin in self._ldap_plugins():
ldap_userfolder = plugin._getLDAPUserFolder()
uid_attr = self._get_uid_attr(ldap_userfolder)
ldap_util = ILDAPSearch(ldap_userfolder)
logger.info(u"Users base: %s" % ldap_userfolder.users_base)
logger.info(u"User filter: %s" % ldap_util.get_user_filter())
ldap_users = ldap_util.get_users()
for ldap_user in ldap_users:
dn, info = ldap_user
# Ignore users without an UID in LDAP
if uid_attr not in info:
continue
userid = info[uid_attr]
userid = userid.decode("utf-8")
# Skip users with uid longer than SQL 'userid' column
if len(userid) > USER_ID_LENGTH:
logger.warn(u"Skipping user '{}' - " u"userid too long!".format(userid))
continue
if not self.user_exists(userid):
# Create the new user
user = User(userid)
session.add(user)
else:
# Get the existing user
try:
user = self.get_sql_user(userid)
except MultipleResultsFound:
# Duplicate user with slightly different spelling
# (casing, whitespace, ...) that may not be considered
# different by the SQL backend's unique constraint.
# We therefore enforce uniqueness ourselves.
logger.warn(u"Skipping duplicate user '{}'!".format(userid))
continue
# Iterate over all SQL columns and update their values
columns = User.__table__.columns
for col in columns:
if col.name == "userid":
# We already set the userid when creating the user
# object, and it may not be called the same in LDAP as
# in our SQL model
continue
value = info.get(col.name)
# We can't store sequences in SQL columns. So if we do get
# a multi-valued field to be stored directly in OGDS, we
# treat it as a multi-line string and join it.
if isinstance(value, list) or isinstance(value, tuple):
value = " ".join([str(v) for v in value])
if isinstance(value, str):
value = value.decode("utf-8")
setattr(user, col.name, value)
# Set the user active
user.active = True
logger.info(u"Imported user '{}'".format(userid))
session.flush()