def get_all_users(self, username, password):
base_dn = 'OU=上海二三四五网络科技有限公司,DC=trump,DC=local'
self.l.simple_bind_s(username + "@trump.local", password)
PAGE_SIZE = 500 # 设置每页返回的条数
pg_ctrl = SimplePagedResultsControl(True, size=PAGE_SIZE, cookie="")
userdata = []
retdata = []
while True:
msgid = self.l.search_ext(base_dn,
ldap.SCOPE_SUBTREE,
"(&(objectClass=person))",
None,
serverctrls=[pg_ctrl])
_a, res_data, _b, srv_ctrls = self.l.result3(msgid)
userdata.extend(res_data)
cookie = srv_ctrls[0].cookie
if cookie:
pg_ctrl.cookie = cookie
else:
break
print('totalnum:', len(userdata))
for u in userdata:
retdict = {}
for k, v in u[1].items():
retdict[k] = v[0]
retdata.append(retdict)
return retdata
def get_ldap_lists(l, search_strings, parent_list=None):
results = set()
known_ldap_resp_ctrls = {
SimplePagedResultsControl.controlType: SimplePagedResultsControl,
}
req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='')
if parent_list:
filterstr = search_strings['get_all_sub_lists_filter'] % escape_filter_chars(parent_list)
else:
filterstr = search_strings['get_all_lists_filter']
while True:
msgid = l.search_ext(search_strings['list_search_string'],
ldap.SCOPE_SUBTREE,
serverctrls=[req_ctrl],
attrlist=(search_strings['list_cn_field'], search_strings['list_name_field']),
filterstr=filterstr)
rtype, rdata, rmsgid, serverctrls = l.result3(msgid, resp_ctrl_classes=known_ldap_resp_ctrls)
results |= {(data[search_strings['list_cn_field']][0], data[search_strings['list_name_field']][0]) for (dn, data) in rdata}
pctrls = [c for c in serverctrls
if c.controlType == SimplePagedResultsControl.controlType]
cookie = pctrls[0].cookie
if not cookie:
break
req_ctrl.cookie = cookie
return results
def paged_search_iter(self, filter, attrs=None):
# Test for version 2.3 API
try:
lc = SimplePagedResultsControl \
(ldap.LDAP_CONTROL_PAGE_OID, True, (self.page_size, ''))
api_version = 3
except AttributeError:
# New version 2.4 API
lc = SimplePagedResultsControl \
(True, size = self.page_size, cookie = '')
sc = \
{ SimplePagedResultsControl.controlType :
SimplePagedResultsControl
}
api_version = 4
res = self.ldcon.search_ext \
( self.cfg.LDAP_BASE_DN
, ldap.SCOPE_SUBTREE
, filter
, attrlist = attrs
, serverctrls = [lc]
)
while True:
params = (res, )
if api_version == 3:
rtype, rdata, rmsgid, serverctrls = self.ldcon.result3(res)
pctrls = \
[c for c in serverctrls
if c.controlType == ldap.LDAP_CONTROL_PAGE_OID
]
else:
rtype, rdata, rmsgid, serverctrls = self.ldcon.result3 \
(res, resp_ctrl_classes = sc)
pctrls = \
[c for c in serverctrls
if c.controlType == SimplePagedResultsControl.controlType
]
for r in rdata:
if not r[0]:
continue
r = LDAP_Search_Result(r)
yield r
if pctrls:
if api_version == 3:
x, cookie = pctrls[0].controlValue
lc.controlValue = (self.page_size, cookie)
else:
cookie = pctrls[0].cookie
lc.cookie = cookie
if not cookie:
break
res = self.ldcon.search_ext \
( self.cfg.LDAP_BASE_DN
, ldap.SCOPE_SUBTREE
, filter
, attrs
, serverctrls = [lc]
)
else:
break
def paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)',
attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None,
timeout=-1, sizelimit=0):
req_ctrl = SimplePagedResultsControl(
True, size=self.page_size, cookie='',
)
result_pages = 0
all_results = []
msgid = self.search_ext(
base, scope, filterstr, attrlist=attrlist,
serverctrls=(serverctrls or [])+[req_ctrl],
)
while True:
rtype, rdata, rmsgid, rctrls = self.result3(msgid)
all_results.extend(rdata)
result_pages += 1
pctrls = [c for c in rctrls if c.controlType == sprc_ct]
if not pctrls:
continue
if not pctrls[0].cookie:
break
req_ctrl.cookie = pctrls[0].cookie
msgid = self.search_ext(
base, scope, filterstr, attrlist=attrlist,
serverctrls=(serverctrls or [])+[req_ctrl]
)
return all_results
def paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)',
attrlist=None, attrsonly=0, serverctrls=None,
clientctrls=None, timeout=-1, sizelimit=0):
"""
Behaves exactly like LDAPObject.search_ext_s() but internally uses the
simple paged results control to retrieve search results in chunks.
"""
req_ctrl = SimplePagedResultsControl(True, size=self.page_size,
cookie='')
# Send first search request
msgid = self.search_ext(base, ldap.SCOPE_SUBTREE, filterstr,
attrlist=attrlist,
serverctrls=(serverctrls or []) + [req_ctrl])
results = []
while True:
rtype, rdata, rmsgid, rctrls = self.result3(msgid)
results.extend(rdata)
# Extract the simple paged results response control
pctrls = [c for c in rctrls if c.controlType ==
SimplePagedResultsControl.controlType]
if pctrls:
if pctrls[0].cookie:
# Copy cookie from response control to request control
req_ctrl.cookie = pctrls[0].cookie
msgid = self.search_ext(base, ldap.SCOPE_SUBTREE,
filterstr, attrlist=attrlist,
serverctrls=(serverctrls or []) +
[req_ctrl])
else:
break
return results
def _paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)', attrlist=None, attrsonly=0,
serverctrls=None, clientctrls=None, timeout=-1, sizelimit=0, page_size=10):
"""
Behaves similarly to LDAPObject.search_ext_s() but internally uses the
simple paged results control to retrieve search results in chunks.
Taken from the python-ldap paged_search_ext_s.py demo, showing how to use
the paged results control: https://bitbucket.org/jaraco/python-ldap/
"""
request_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='')
results = []
while True:
msgid = self.conn.search_ext(base, scope, filterstr=filterstr, attrlist=attrlist, attrsonly=attrsonly,
serverctrls=(serverctrls or []) + [request_ctrl], clientctrls=clientctrls,
timeout=timeout, sizelimit=sizelimit)
result_type, result_data, result_msgid, result_ctrls = self.conn.result3(msgid)
results.extend(result_data)
# Extract the simple paged results response control
paged_ctrls = [c for c in result_ctrls if c.controlType == SimplePagedResultsControl.controlType]
if paged_ctrls and paged_ctrls[0].cookie:
# Copy cookie from response control to request control
request_ctrl.cookie = paged_ctrls[0].cookie
else:
break
return results
def search(base_dn, search_filter, attributes):
"""Iterative LDAP search using page control.
:param base_dn: str -- The base DN from which to start the search.
:param search_filter: str -- Representation of the filter to apply
in the search.
:param attributes: list -- Attributes to be retrieved for each
entry. If ``None``, all attributes will be
retrieved.
:returns: A generator which yields one search result at a time as a
tuple containing a `dn` as ``str`` and `attributes` as
``dict``.
"""
connection, settings = current_ldap
page_ctrl = SimplePagedResultsControl(True, size=settings['page_size'], cookie='')
while True:
msg_id = connection.search_ext(base_dn, SCOPE_SUBTREE, filterstr=search_filter, attrlist=attributes,
serverctrls=[page_ctrl], timeout=settings['timeout'])
try:
_, r_data, __, server_ctrls = connection.result3(msg_id, timeout=settings['timeout'])
except NO_SUCH_OBJECT:
break
for dn, entry in r_data:
if dn:
yield dn, entry
page_ctrl.cookie = get_page_cookie(server_ctrls)
if not page_ctrl.cookie:
# End of results
break
def get_ldap_list_membership(l, search_strings, list_name):
results = set()
known_ldap_resp_ctrls = {
SimplePagedResultsControl.controlType: SimplePagedResultsControl,
}
req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='')
while True:
msgid = l.search_ext(search_strings['user_search_string'],
ldap.SCOPE_SUBTREE,
serverctrls=[req_ctrl],
attrlist=[search_strings['user_account_name_field']],
filterstr=search_strings['user_membership_filter'] % escape_filter_chars(list_name)
)
rtype, rdata, rmsgid, serverctrls = l.result3(msgid, resp_ctrl_classes=known_ldap_resp_ctrls)
results |= {data[1][search_strings['user_account_name_field']][0] for data in rdata}
pctrls = [c for c in serverctrls
if c.controlType == SimplePagedResultsControl.controlType]
cookie = pctrls[0].cookie
if not cookie:
break
req_ctrl.cookie = cookie
return results
def _paged_search(self, connection, base, scope, filterstr, attrlist):
page_size = self.settings_dict.get('PAGE_SIZE', 1000)
results = []
pg_ctrl = SimplePagedResultsControl(True, page_size, "")
scope = ldap.SCOPE_SUBTREE
pages = 0
while True:
pages += 1
msgid = connection.search_ext(
base,
scope=scope,
filterstr=filterstr,
attrlist=attrlist,
serverctrls=[pg_ctrl]
)
rtype, rdata, rmsgid, serverctrls = connection.result3(msgid)
results.extend(rdata)
cookie = serverctrls[0].cookie
if cookie:
pg_ctrl.cookie = cookie
search = connection.search_ext(
base,
scope=scope,
filterstr=filterstr,
attrlist=attrlist,
serverctrls=[pg_ctrl]
)
else:
break
return results
def query_users_paged(self, options):
"""
Connects to LDAP server and attempts to query and return all users
by iterating through each page result. Requires LDAP v3.
"""
page_size = options.get('page_size', 500)
criticality = options.get('criticality', True)
cookie = options.get('cookie', '')
full_record = options.get('full_record', False)
fields = self.ldap_query_fields
if full_record:
fields = None
# search the server for users
first_pass = True
pg_ctrl = SimplePagedResultsControl(criticality, page_size, cookie)
LOG.debug("self.ldap_query_fields = %r", self.ldap_query_fields)
while first_pass or pg_ctrl.cookie:
first_pass = False
msgid = self.ldap_connection.search_ext(
self.settings['base_dn'], ldap.SCOPE_SUBTREE, self.settings['filter'],
fields,
serverctrls=[pg_ctrl]
)
result_type, ldap_users, msgid, serverctrls = self.ldap_connection.result3(msgid)
pg_ctrl.cookie = serverctrls[0].cookie
for user in ldap_users:
if user[0] and user[1]:
yield _clean_record(user[1])
# disconnect and return results
self.ldap_connection.unbind_s()
def pagedsearch(self, search_flt, searchreq_attrlist):
req_ctrl = SimplePagedResultsControl(True,size=self.page_size,cookie='')
known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType:SimplePagedResultsControl,}
# Send search request
msgid = self.lcon.search_ext( self.base, ldap.SCOPE_SUBTREE, search_flt, attrlist=searchreq_attrlist, serverctrls=[req_ctrl])
pages = 0
i = 0
while True:
pages += 1
rtype, rdata, rmsgid, serverctrls = self.lcon.result3(msgid,resp_ctrl_classes=known_ldap_resp_ctrls)
yield rdata
pctrls = [c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType]
if pctrls:
if pctrls[0].cookie:
# Copy cookie from response control to request control
req_ctrl.cookie = pctrls[0].cookie
msgid = self.lcon.search_ext(self.base,ldap.SCOPE_SUBTREE,search_flt,attrlist=searchreq_attrlist,serverctrls=[req_ctrl])
else:
break
else:
print "Warning: Server ignores RFC 2696 control."
break
def search(base_dn, search_filter, attributes):
"""Iterative LDAP search using page control.
:param base_dn: str -- The base DN from which to start the search.
:param search_filter: str -- Representation of the filter to apply
in the search.
:param attributes: list -- Attributes to be retrieved for each
entry. If ``None``, all attributes will be
retrieved.
:returns: A generator which yields one search result at a time as a
tuple containing a `dn` as ``str`` and `attributes` as
``dict``.
"""
connection, settings = current_ldap
page_ctrl = SimplePagedResultsControl(True, size=settings['page_size'], cookie='')
search_filter = search_filter.encode('utf-8')
while True:
msg_id = connection.search_ext(base_dn, SCOPE_SUBTREE, filterstr=search_filter, attrlist=attributes,
serverctrls=[page_ctrl], timeout=settings['timeout'])
try:
_, r_data, __, server_ctrls = connection.result3(msg_id, timeout=settings['timeout'])
except NO_SUCH_OBJECT:
break
for dn, entry in r_data:
if dn:
yield dn, entry
page_ctrl.cookie = get_page_cookie(server_ctrls)
if not page_ctrl.cookie:
# End of results
break
def query(self, base, scope, params, fixed_rdn=None):
ocs = ["(objectClass=%s)" % o.strip() for o in params['objectClasses'].split(",")]
fltr = "(&" + "".join(ocs) + (ldap.filter.filter_format("(%s)", [fixed_rdn]) if fixed_rdn else "") + ")"
result = []
with self.lh.get_handle() as l, self.lock:
self.log.debug("LDAP Search using filter '{filter}'".format(filter=fltr))
lc = SimplePagedResultsControl(criticality=True, size=self.page_size, cookie='')
ix = 0
while True:
msgid = l.search_ext(base, ldap.SCOPE_ONELEVEL, fltr, attrlist=[self.uuid_entry], serverctrls=[lc])
res = l.result3(msgid=msgid)
result_type, results, unused_msgid, serverctrls = res
for resultdata in results:
ix += 1
if resultdata[0] is not None:
result.append(resultdata)
cookie = None
for serverctrl in serverctrls:
if serverctrl.controlType == ldap.controls.libldap.SimplePagedResultsControl.controlType:
cookie = serverctrl.cookie
if cookie:
lc.cookie = cookie
break
if not cookie:
break
return [x for x in dict(result).keys()]
def get_paged_users(self, filters, attributes):
pages = 0
result = []
page_control = SimplePagedResultsControl(True, size=self.page_size, cookie='')
res = self._conn.search_ext(
self.domain_dn,
ldap.SCOPE_SUBTREE,
filters,
attributes,
serverctrls=[page_control]
)
while True:
pages += 1
self.log.debug("Page {} done".format(pages))
rtype, rdata, rmsgid, serverctrls = self._conn.result3(res)
result.extend(rdata)
controls = [ctrl for ctrl in serverctrls if ctrl.controlType == SimplePagedResultsControl.controlType]
if not controls:
self.log.error('The server ignores RFC 2696 control')
break
if not controls[0].cookie:
break
page_control.cookie = controls[0].cookie
res = self._conn.search_ext(
self.domain_dn,
ldap.SCOPE_SUBTREE,
filters,
attributes,
serverctrls=[page_control]
)
return result
def results_for_dn(directory: ldap.ldapobject, base_dn: str,
filter: str) -> List[User]:
"""Returns a list of User objects found in the directory object for filter
:param directory: A ldap.LDAPObject that has already been bound to a
directory.
:param base_dn: The base of the directory tree to run the search filter
against.
:param filter: The LDAP search filter to run on base_dn using directory.
"""
req_ctrl = SimplePagedResultsControl(True, size=5000, cookie="")
known_ldap_resp_ctrls = {
SimplePagedResultsControl.controlType: SimplePagedResultsControl,
}
# Send search request
msgid = directory.search_ext(base_dn,
ldap.SCOPE_SUBTREE,
filterstr=LDAP_FILTER,
serverctrls=[req_ctrl])
results = []
while True:
__, result_data, __, serverctrls = directory.result3(
msgid, resp_ctrl_classes=known_ldap_resp_ctrls)
results.extend([
User(
ldap_entry["sAMAccountName"][0].decode(),
ldap_entry["mail"][0].decode(),
ldap_entry["sn"][0].decode(),
ldap_entry["givenName"][0].decode(),
) for __, ldap_entry in result_data
])
page_controls = [
control for control in serverctrls
if control.controlType == SimplePagedResultsControl.controlType
]
if page_controls:
if page_controls[0].cookie:
# Copy cookie from response control to request control
req_ctrl.cookie = page_controls[0].cookie
msgid = directory.search_ext(
base_dn,
ldap.SCOPE_SUBTREE,
filterstr=LDAP_FILTER,
serverctrls=[req_ctrl],
)
else:
break
else:
eprint("Warning: Server ignores RFC 2696 control.")
break
return results
def test_search_multiple_paging(topology_st, create_user):
"""Verify that after performing multiple simple paged searches
on a single connection without a complition, it wouldn't fail.
:id: 628b29a6-2d47-4116-a88d-00b87405ef7f
:customerscenario: True
:setup: Standalone instance, test user for binding,
varying number of users for the search base
:steps:
1. Bind as test user
2. Initiate the search with a simple paged control
3. Acquire the returned cookie only one time
4. Perform steps 2 and 3 three times in a row
:expectedresults:
1. Bind should be successful
2. Search should be successfully initiated
3. Cookie should be successfully acquired
4. No error happens
"""
users_num = 100
page_size = 30
users_list = add_users(topology_st, users_num, DEFAULT_SUFFIX)
search_flt = r'(uid=test*)'
searchreq_attrlist = ['dn', 'sn']
try:
log.info('Set user bind')
conn = create_user.bind(TEST_USER_PWD)
log.info('Create simple paged results control instance')
req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='')
controls = [req_ctrl]
for ii in range(3):
log.info('Iteration %d' % ii)
msgid = conn.search_ext(DEFAULT_SUFFIX,
ldap.SCOPE_SUBTREE,
search_flt,
searchreq_attrlist,
serverctrls=controls)
rtype, rdata, rmsgid, rctrls = conn.result3(msgid)
pctrls = [
c for c in rctrls
if c.controlType == SimplePagedResultsControl.controlType
]
# Copy cookie from response control to request control
req_ctrl.cookie = pctrls[0].cookie
msgid = conn.search_ext(DEFAULT_SUFFIX,
ldap.SCOPE_SUBTREE,
search_flt,
searchreq_attrlist,
serverctrls=controls)
finally:
del_users(users_list)
def get_ldap_lists(l, search_strings, parent_list=None):
results = set()
known_ldap_resp_ctrls = {
SimplePagedResultsControl.controlType: SimplePagedResultsControl,
}
req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='')
if parent_list:
filterstr = search_strings['get_all_sub_lists_filter'] % escape_filter_chars(parent_list)
else:
filterstr = search_strings['get_all_lists_filter']
while True:
msgid = l.search_ext(search_strings['list_search_string'],
ldap.SCOPE_SUBTREE,
serverctrls=[req_ctrl],
attrlist=(search_strings['list_cn_field'], search_strings['list_name_field']),
filterstr=filterstr)
retries = 0
rdata = None
while retries < 5:
try:
rtype, rdata, rmsgid, serverctrls = l.result3(msgid, timeout=ldap_timeout, resp_ctrl_classes=known_ldap_resp_ctrls)
except Exception:
logger.exception('Error getting ldap list memberships')
retries += 1
else:
break
# ldap sometimees returns good but does not return rdata
if rdata is None:
break
for (dn, data) in rdata:
cn_field = data[search_strings['list_cn_field']][0]
name_field = data[search_strings['list_name_field']][0]
if isinstance(cn_field, bytes):
cn_field = cn_field.decode('utf-8')
if isinstance(name_field, bytes):
name_field = name_field.decode('utf-8')
results |= {(cn_field, name_field)}
pctrls = [c for c in serverctrls
if c.controlType == SimplePagedResultsControl.controlType]
if not pctrls:
# Paging not supported
logger.debug('LDAP pagination not supported')
break
cookie = pctrls[0].cookie
if not cookie:
break
req_ctrl.cookie = cookie
return results
def _search(self,
basedn='',
scope=pyldap.SCOPE_SUBTREE,
filter='',
timeout=-1,
sizelimit=0):
if not self._handle:
self._open()
result = []
serverctrls = None
clientctrls = None
paged = SimplePagedResultsControl(criticality=False,
size=self.pagesize,
cookie='')
paged_ctrls = {
SimplePagedResultsControl.controlType: SimplePagedResultsControl
}
page = 0
while True:
serverctrls = [paged]
id = self._handle.search_ext(basedn,
scope,
filterstr=filter,
attrlist=None,
attrsonly=0,
serverctrls=serverctrls,
clientctrls=clientctrls,
timeout=timeout,
sizelimit=sizelimit)
(rtype, rdata, rmsgid,
serverctrls) = self._handle.result3(id,
resp_ctrl_classes=paged_ctrls)
result.extend(rdata)
paged.size = 0
paged.cookie = cookie = None
for sc in serverctrls:
if sc.controlType == SimplePagedResultsControl.controlType:
cookie = sc.cookie
if cookie:
paged.cookie = cookie
paged.size = self.pagesize
break
if not cookie:
break
page += 1
return result
def test_search_invalid_cookie(topology_st, create_user, invalid_cookie):
"""Verify that using invalid cookie while performing
search with the simple paged results control throws
a TypeError exception
:id: 107be12d-4fe4-47fe-ae86-f3e340a56f42
:customerscenario: True
:parametrized: yes
:setup: Standalone instance, test user for binding,
varying number of users for the search base
:steps:
1. Bind as test user
2. Initiate the search with a simple paged control
3. Put an invalid cookie (-1, 1000) to the control
4. Continue the search
:expectedresults:
1. Bind should be successful
2. Search should be successfully initiated
3. Cookie should be added
4. It should throw a TypeError exception
"""
users_num = 100
page_size = 50
users_list = add_users(topology_st, users_num, DEFAULT_SUFFIX)
search_flt = r'(uid=test*)'
searchreq_attrlist = ['dn', 'sn']
try:
log.info('Set user bind')
conn = create_user.bind(TEST_USER_PWD)
log.info('Create simple paged results control instance')
req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='')
controls = [req_ctrl]
msgid = conn.search_ext(DEFAULT_SUFFIX,
ldap.SCOPE_SUBTREE,
search_flt,
searchreq_attrlist,
serverctrls=controls)
rtype, rdata, rmsgid, rctrls = conn.result3(msgid)
log.info(
'Put an invalid cookie (%d) to the control. TypeError is expected'
% invalid_cookie)
req_ctrl.cookie = invalid_cookie
with pytest.raises(TypeError):
msgid = conn.search_ext(DEFAULT_SUFFIX,
ldap.SCOPE_SUBTREE,
search_flt,
searchreq_attrlist,
serverctrls=controls)
finally:
del_users(users_list)
def handle(self, *args, **options):
filter_string = options.get('filter', DEFAULT_FILTER)
ldap_backend = LDAPBackend()
ldap_user = _LDAPUser(backend=ldap_backend, username='')
ldap_connection = ldap_user.connection
# Initialize the LDAP controls for paging.
# Note that we pass '' for the cookie because on first iteration, it starts out empty.
ldap_controls = SimplePagedResultsControl(True,
size=PAGE_SIZE,
cookie='')
while True:
# Send search request
# If you leave out the ATTRLIST it'll return all attributes which you have permissions to access.
# You may want to adjust the scope level as well
# (perhaps "ldap.SCOPE_SUBTREE", but it can reduce performance if you don't need it).
message_id = ldap_connection.search_ext(
base=settings.AUTH_LDAP_USER_SEARCH.base_dn,
scope=ldap.SCOPE_SUBTREE,
filterstr=filter_string,
serverctrls=[ldap_controls])
# Pull the results from the search request
rtype, rdata, rmsgid, server_controls = ldap_connection.result3(
message_id)
rdata = settings.AUTH_LDAP_USER_SEARCH._process_results(rdata)
# Each "rdata" is a tuple of the form (dn, attrs), where dn is a string containing the
# DN (distinguished name) of the entry, and attrs is a dictionary containing the attributes associated
# with the entry. The keys of attrs are strings, and the associated values are lists of strings.
for distinguished_name, attributes in rdata:
username = "".join(attributes['uid'])
print(f"{username} \t| {distinguished_name}")
print(attributes)
# Look through the returned controls and find the page controls.
# This will also have our returned cookie which we need to make the next search request.
page_controls = [
control for control in server_controls
if control.controlType == SimplePagedResultsControl.controlType
]
if not page_controls:
print('Warning: Server ignores RFC 2696 control.')
break
# Ok, we did find the page control, yank the cookie from it and insert it into the control for our
# next search. If however there is no cookie, we are done!
cookie = page_controls[0].cookie
ldap_controls.cookie = cookie
if not cookie:
break
def paged_search_ext_s(self, conn, base, scope, filter, attrs, attrsonly=0,serverctrls=None,clientctrls=None,timeout=-1,sizelimit=0):
""" Helper function that implements a paged LDAP search for
the Search method below.
Args:
query: LDAP filter to apply to the search
sizelimit: max # of users to return.
attrlist: list of attributes to return. If null, all attributes
are returned
Returns:
A list of users as returned by the LDAP search
"""
ldap_page_size = 100
search_flt = r'(objectClass=*)'
searchreq_attrlist = ['cn', 'entryDN',
'entryUUID', 'mail', 'objectClass']
req_ctrl = SimplePagedResultsControl(
True, ldap_page_size, cookie='')
logging.debug('Paged search on %s for %s' % (base, filter))
# Send first search request
msgid = conn.search_ext(
base,
ldap.SCOPE_SUBTREE,
search_flt,
attrlist=searchreq_attrlist,
serverctrls=(serverctrls or []) + [req_ctrl]
)
result_pages = 0
all_results = []
while True:
rtype, rdata, rmsgid, rctrls = conn.result3(msgid)
all_results.extend(rdata)
result_pages += 1
# Extract the simple paged results response control
pctrls = [
c
for c in rctrls
if c.controlType == SimplePagedResultsControl.controlType
]
if pctrls:
if pctrls[0].cookie:
# Copy cookie from response control to request control
req_ctrl.cookie = pctrls[0].cookie
msgid = conn.search_ext(
base,
ldap.SCOPE_SUBTREE,
search_flt,
attrlist=searchreq_attrlist,
serverctrls=(serverctrls or []) + [req_ctrl]
)
else:
break
return all_results
def set_cookie(
lc_object: SimplePagedResultsControl,
pctrls: List[SimplePagedResultsControl],
) -> bool:
"""
Push latest cookie back into the page control.
"""
cookie = pctrls[0].cookie
lc_object.cookie = cookie
return bool(cookie)
def paged_search_ext_s(self,
base,
scope,
filterstr='(objectClass=*)',
attrlist=None,
attrsonly=0,
serverctrls=None,
clientctrls=None,
timeout=-1,
sizelimit=0):
"""
Behaves exactly like LDAPObject.search_ext_s() but internally uses the
simple paged results control to retrieve search results in chunks.
This is non-sense for really large results sets which you would like
to process one-by-one
"""
req_ctrl = SimplePagedResultsControl(True,
size=self.page_size,
cookie='')
# Send first search request
msgid = self.search_ext(base,
ldap.SCOPE_SUBTREE,
search_flt,
attrlist=searchreq_attrlist,
serverctrls=(serverctrls or []) +
[req_ctrl])
result_pages = 0
all_results = []
while True:
rtype, rdata, rmsgid, rctrls = self.result3(msgid)
all_results.extend(rdata)
result_pages += 1
# Extract the simple paged results response control
pctrls = [
c for c in rctrls
if c.controlType == SimplePagedResultsControl.controlType
]
if pctrls:
if pctrls[0].cookie:
# Copy cookie from response control to request control
req_ctrl.cookie = pctrls[0].cookie
msgid = self.search_ext(
base,
ldap.SCOPE_SUBTREE,
search_flt,
attrlist=searchreq_attrlist,
serverctrls=(serverctrls or []) + [req_ctrl])
else:
break
return result_pages, all_results
def search_users(req_page):
con = get_ldap_con()
sc = SimplePagedResultsControl(criticality=True, size=PAGESIZE, cookie='')
search_results = None
last_page = False
i = 0
while True:
if i >= req_page:
break
msgid = con.search_ext(base=BASEDN,
scope=ldap.SCOPE_SUBTREE,
filterstr=SEARCHFILTER.decode('utf8'),
attrlist=USERS_ATTR_LIST,
serverctrls=[sc])
# msgid is an integer representing LDAP operation id
_, search_results, _, result_sc = con.result3(msgid)
# result_data: list of tuples in the form of (dn, attrs),
# dn is a string, ie: 'dn=alvaro,dc=example,dc=com',
# attrs is a string key based dictionary.
# result_sc: servercontrols list.
pctrls = [
c for c in result_sc
if c.controlType == SimplePagedResultsControl.controlType
]
if not pctrls:
last_page = True
i += 1
break
cookie = pctrls[0].cookie
if not cookie:
last_page = True
i += 1
break
sc.cookie = cookie
i += 1
# Data mangle
# We don't need Full DN or registers with empty DN
search_results = purge_results(search_results)
ret_json = get_response_template()
ret_json['page'] = i
ret_json['quant'] = len(search_results)
ret_json['last'] = last_page
ret_json['data'] = search_results
return ret_json
def paginated_query(connect,
attributes,
base,
scope,
search_flt,
result_class=None):
# how many users to search for in each page,
# this depends on the server maximum setting
# (default highest value is 1000)
page_size = 500
req_ctrl = SimplePagedResultsControl(criticality=True,
size=page_size,
cookie="")
msgid = connect.search_ext(
base=base,
scope=scope,
filterstr=search_flt,
attrlist=attributes,
serverctrls=[req_ctrl],
)
total_results = []
pages = 0
while True: # loop over all of the pages using the same cookie,
# otherwise the search will fail
pages += 1
rtype, rdata, rmsgid, serverctrls = connect.result3(msgid)
for user in rdata:
if not result_class:
total_results.append(user)
else:
total_results.append(result_class(user))
pctrls = [
c for c in serverctrls
if c.controlType == SimplePagedResultsControl.controlType
]
if pctrls:
if pctrls[
0].cookie: # Copy cookie from response control to request control
req_ctrl.cookie = pctrls[0].cookie
msgid = connect.search_ext(
base=base,
scope=scope,
filterstr=search_flt,
attrlist=attributes,
serverctrls=[req_ctrl],
)
else:
break
else:
break
return total_results
def _paged_search(self, base_dn, scope, search_filter, attrs):
conn = self.connect()
# Get paged results to prevent exceeding server size limit
page_size = 1000
if PYTHON_LDAP_24:
lc = SimplePagedResultsControl(size=page_size, cookie='')
else:
lc = SimplePagedResultsControl(LDAP_CONTROL_PAGED_RESULTS,
True,
(page_size, ''),)
is_last_page = False
results = []
while not is_last_page:
msgid = conn.search_ext(base_dn,
scope,
search_filter,
attrs,
serverctrls=[lc])
rtype, rdata, rmsgid, serverctrls = conn.result3(msgid)
pctrls = [c for c in serverctrls
if c.controlType == LDAP_CONTROL_PAGED_RESULTS]
results.extend(rdata)
if pctrls:
if PYTHON_LDAP_24:
cookie = pctrls[0].cookie
if cookie:
lc.cookie = cookie
else:
is_last_page = True
else:
cookie = pctrls[0].controlValue[1]
if cookie:
# lc.controlValue seems to have been mutable at some
# point, now it's a tuple.
cv = list(lc.controlValue)
cv[1] = cookie
lc.controlValue = tuple(cv)
else:
is_last_page = True
else:
is_last_page = True
if results:
# If the search returned an empty result, page controls
# aren't included - so don't produce a bogus warning
logger.warn(
"Server ignores paged results control (RFC 2696).")
return results
def _search(self, basedn='', scope=ldap.SCOPE_SUBTREE, filter='', timeout=-1, sizelimit=0):
if not self._handle:
self._open()
result = []
serverctrls = None
clientctrls = None
paged = SimplePagedResultsControl(
criticality=False,
size=self.pagesize,
cookie=''
)
paged_ctrls = {SimplePagedResultsControl.controlType: SimplePagedResultsControl}
page = 0
while True:
serverctrls = [paged]
id = self._handle.search_ext(
basedn,
scope,
filterstr=filter,
attrlist=None,
attrsonly=0,
serverctrls=serverctrls,
clientctrls=clientctrls,
timeout=timeout,
sizelimit=sizelimit
)
(rtype, rdata, rmsgid, serverctrls) = self._handle.result3(
id, resp_ctrl_classes=paged_ctrls
)
result.extend(rdata)
paged.size = 0
paged.cookie = cookie = None
for sc in serverctrls:
if sc.controlType == SimplePagedResultsControl.controlType:
cookie = sc.cookie
if cookie:
paged.cookie = cookie
paged.size = self.pagesize
break
if not cookie:
break
page += 1
return result
def get_ldap_list_membership(l, search_strings, list_name):
results = set()
known_ldap_resp_ctrls = {
SimplePagedResultsControl.controlType: SimplePagedResultsControl,
}
req_ctrl = SimplePagedResultsControl(True,
size=ldap_pagination_size,
cookie='')
while True:
msgid = l.search_ext(
search_strings['user_search_string'],
ldap.SCOPE_SUBTREE,
serverctrls=[req_ctrl],
attrlist=[search_strings['user_mail_field']],
filterstr=search_strings['user_membership_filter'] %
escape_filter_chars(list_name))
retries = 0
rdata = None
while retries < 5:
try:
rtype, rdata, rmsgid, serverctrls = l.result3(
msgid,
timeout=ldap_timeout,
resp_ctrl_classes=known_ldap_resp_ctrls)
except Exception:
logger.exception('Error getting ldap list memberships')
retries += 1
else:
break
# ldap sometimees returns good but does not return rdata
if rdata is None:
break
for data in rdata:
member = data[1].get(search_strings['user_mail_field'], [None])[0]
if isinstance(member, bytes):
member = member.decode('utf-8')
results |= {member}
pctrls = [
c for c in serverctrls
if c.controlType == SimplePagedResultsControl.controlType
]
if not pctrls:
# Paging not supported
logger.debug('LDAP pagination not supported')
break
cookie = pctrls[0].cookie
if not cookie:
break
req_ctrl.cookie = cookie
return results
def get_ldap_lists(l, search_strings, parent_list=None):
results = set()
known_ldap_resp_ctrls = {
SimplePagedResultsControl.controlType: SimplePagedResultsControl,
}
req_ctrl = SimplePagedResultsControl(True,
size=ldap_pagination_size,
cookie='')
if parent_list:
filterstr = search_strings[
'get_all_sub_lists_filter'] % escape_filter_chars(parent_list)
else:
filterstr = search_strings['get_all_lists_filter']
while True:
msgid = l.search_ext(search_strings['list_search_string'],
ldap.SCOPE_SUBTREE,
serverctrls=[req_ctrl],
attrlist=(search_strings['list_cn_field'],
search_strings['list_name_field']),
filterstr=filterstr)
rtype, rdata, rmsgid, serverctrls = l.result3(
msgid,
timeout=ldap_timeout,
resp_ctrl_classes=known_ldap_resp_ctrls)
for (dn, data) in rdata:
cn_field = data[search_strings['list_cn_field']][0]
name_field = data[search_strings['list_name_field']][0]
if isinstance(cn_field, bytes):
cn_field = cn_field.decode('utf-8')
if isinstance(name_field, bytes):
name_field = name_field.decode('utf-8')
results |= {(cn_field, name_field)}
pctrls = [
c for c in serverctrls
if c.controlType == SimplePagedResultsControl.controlType
]
if not pctrls:
# Paging not supported
break
cookie = pctrls[0].cookie
if not cookie:
break
req_ctrl.cookie = cookie
return results
def _paged_search_ext_s(self,
base,
scope,
filterstr='(objectClass=*)',
attrlist=None,
attrsonly=0,
serverctrls=None,
clientctrls=None,
timeout=-1,
sizelimit=0,
page_size=10):
"""
Behaves similarly to LDAPObject.search_ext_s() but internally uses the
simple paged results control to retrieve search results in chunks.
Taken from the python-ldap paged_search_ext_s.py demo, showing how to use
the paged results control: https://bitbucket.org/jaraco/python-ldap/
"""
request_ctrl = SimplePagedResultsControl(True,
size=page_size,
cookie='')
results = []
while True:
msgid = self.conn.search_ext(base,
scope,
filterstr=filterstr,
attrlist=attrlist,
attrsonly=attrsonly,
serverctrls=(serverctrls or []) +
[request_ctrl],
clientctrls=clientctrls,
timeout=timeout,
sizelimit=sizelimit)
result_type, result_data, result_msgid, result_ctrls = self.conn.result3(
msgid)
results.extend(result_data)
# Extract the simple paged results response control
paged_ctrls = [
c for c in result_ctrls
if c.controlType == SimplePagedResultsControl.controlType
]
if paged_ctrls and paged_ctrls[0].cookie:
# Copy cookie from response control to request control
request_ctrl.cookie = paged_ctrls[0].cookie
else:
break
return results
def _paged_search(self, basedn, searchfilter, attrlist=None, pagesize=100, sizelimit=0):
"""
Performs a pages search against the LDAP server. Code lifted from:
https://gist.github.com/mattfahrner/c228ead9c516fc322d3a
"""
# Initialize the LDAP controls for paging. Note that we pass ''
# for the cookie because on first iteration, it starts out empty.
controls = SimplePagedResultsControl(True, size=pagesize, cookie='')
# Do searches until we run out of pages to get from the LDAP server.
results = []
while True:
# Send search request.
msgid = self.connection.search_ext(
basedn,
ldap.SCOPE_SUBTREE,
searchfilter,
attrlist,
serverctrls=[controls],
sizelimit=sizelimit
)
rtype, rdata, rmsgid, serverctrls = self.connection.result3(msgid)
# Each "rdata" is a tuple of the form (dn, attrs), where dn is
# a string containing the DN (distinguished name) of the entry,
# and attrs is a dictionary containing the attributes associated
# with the entry. The keys of attrs are strings, and the associated
# values are lists of strings.
for dn, attrs in rdata:
# AD returns an rdata at the end that is a reference that we want to ignore
if type(attrs) == dict:
results.append((dn, attrs))
# Get cookie for the next request.
paged_controls = self._get_pctrls(serverctrls)
if not paged_controls:
self.logger.warning(
f'paged_search.rfc2696_control_ignored searchfilter={searchfilter}'
f'attrlist={",".join(attrlist)} pagesize={pagesize} sizelimit={sizelimit}'
)
break
# Push cookie back into the main controls.
controls.cookie = paged_controls[0].cookie
# If there is no cookie, we're done!
if not paged_controls[0].cookie:
break
return results
def paged_search(self, search_dn, filter, scope, results_processor, attributes=None):
if not attributes:
attributes = ['*']
page_control = SimplePagedResultsControl(True, self.PAGE_SIZE, '')
serverctrls = [page_control]
msgid = self.connection.search_ext(
search_dn,
scope,
filter,
attrlist=attributes,
serverctrls=serverctrls
)
page = 0
records = 0
while True:
page += 1
try:
result_type, results, result_msg, serverctrls = self.connection.result3(msgid=msgid, timeout=self.LDAP_TIMEOUT)
records += len(results)
results_processor(results)
pagectrls = [
c
for c in serverctrls
if c.controlType == SimplePagedResultsControl.controlType
]
if pagectrls:
if pagectrls[0].cookie:
page_control.cookie = pagectrls[0].cookie
msgid = self.connection.search_ext(
search_dn,
scope,
filter,
attrlist=attributes,
serverctrls=[page_control]
)
else:
break
else:
break
except ldap.LDAPError, e:
print e
def test_lookup_with_pagination(udm):
"""Test serverctrls of ldap server"""
pytest.skip('FIXME??? Bug #49638: git:63ba30a2040')
from ldap.controls import SimplePagedResultsControl
from ldap.controls.sss import SSSRequestControl
name = uts.random_username()
dns = [
udm.create_user(username=name + str(i),
wait_for_replication=False,
check_for_drs_replication=False,
wait_for=False)[0] for i in range(1, 8)
]
print(('Created users:', dns))
univention.admin.modules.update()
lo = univention.uldap.getMachineConnection()
res = {}
page_size = 2
pctrl = SimplePagedResultsControl(True, size=page_size, cookie='')
sctrl = SSSRequestControl(ordering_rules=['uid:caseIgnoreOrderingMatch'])
users = univention.admin.modules.get('users/user')
ctrls = [sctrl, pctrl]
entries = []
while True:
entries.append([
x.dn for x in users.lookup(None,
lo,
'username=%s*' % (name, ),
serverctrls=ctrls,
response=res)
])
print(('Found', entries[-1]))
for control in res['ctrls']:
if control.controlType == SimplePagedResultsControl.controlType:
pctrl.cookie = control.cookie
if not pctrl.cookie:
break
assert len(entries[-1]) == page_size
found = []
for entry in entries:
found.extend(entry)
assert sorted(found) == sorted(dns)
def paged_search_ext_s(self,base,scope,filterstr='(objectClass=*)',attrlist=None,attrsonly=0,serverctrls=None,clientctrls=None,timeout=-1,sizelimit=0,criticality=True,page_size=1000):
"""
Behaves exactly like LDAPObject.search_ext_s() but internally uses the
simple paged results control to retrieve search results in chunks.
This is non-sense for really large results sets which you would like
to process one-by-one
"""
req_ctrl = SimplePagedResultsControl(criticality,size=page_size,cookie='')
# Send first search request
msgid = self.search_ext(
base,
scope,
filterstr,
attrlist=attrlist,
serverctrls=(serverctrls or [])+[req_ctrl]
)
result_pages = 0
all_results = []
while True:
rtype, rdata, rmsgid, rctrls = self.result3(msgid)
all_results.extend(rdata)
result_pages += 1
# Extract the simple paged results response control
pctrls = [
c
for c in rctrls
if c.controlType == SimplePagedResultsControl.controlType
]
if pctrls:
if pctrls[0].cookie:
# Copy cookie from response control to request control
req_ctrl.cookie = pctrls[0].cookie
msgid = self.search_ext(
base,
scope,
filterstr,
attrlist=attrlist,
serverctrls=(serverctrls or [])+[req_ctrl]
)
else:
break
return result_pages,all_results
def _search(self, search_flt, attrs, limit=None, path_prefix=None):
"""
Query this academic calendar for records matching the search filter.
Only looks one level deep in the tree to improve performance, so make
sure you pass the dn that you want to look directly inside of as
the `path` parameter.
Returns a list of records.
Each record is a dictionary containing the requested attributes.
search_flt -- LDAP-style search filter
attrs -- attributes to be retrieved
limit -- max number of records to return
path -- extra LDAP dn: is prepended to this object's basedn
ie the LDAP directory to look from relative to the root
"""
if limit is None:
limit = float('inf')
if path_prefix is None:
path = self._basedn
else:
path = '{}{}'.format(path_prefix, self._basedn)
page_control = SimplePagedResultsControl(
criticality=True, size=RemoteLDAPDatabase.PAGE_SIZE, cookie='')
results = []
pages_retrieved = 0
first_pass = True
while pages_retrieved*RemoteLDAPDatabase.PAGE_SIZE < limit \
and page_control.cookie \
or first_pass:
first_pass = False
try:
msgid = self._client.search_ext(path,
ldap.SCOPE_ONELEVEL,
search_flt,
attrlist=attrs,
serverctrls=[page_control])
except:
raise
pages_retrieved += 1
_, result_data, msgid, serverctrls = self._client.result3(msgid)
page_control.cookie = serverctrls[0].cookie
results += extract_results_from_ldap_data(result_data)
return results
def search(self, base_dn=None, scope=ldap.SCOPE_SUBTREE,
filter='(objectClass=*)', attrs=[]):
conn = self.connect()
if conn is None:
return []
if base_dn is None:
base_dn = self.base_dn
# Get paged results to prevent exceeding server size limit
page_size = 1000
if PYTHON_LDAP_24:
lc = SimplePagedResultsControl(size=page_size, cookie='')
else:
lc = SimplePagedResultsControl(LDAP_CONTROL_PAGED_RESULTS,
True,
(page_size, ''),)
is_last_page = False
results = []
while not is_last_page:
msgid = conn.search_ext(base_dn,
scope,
filter,
attrs,
serverctrls=[lc])
rtype, rdata, rmsgid, serverctrls = conn.result3(msgid)
results.extend(rdata)
pctrls = [c for c in serverctrls
if c.controlType == LDAP_CONTROL_PAGED_RESULTS]
if pctrls:
if PYTHON_LDAP_24:
cookie = pctrls[0].cookie
if cookie:
lc.cookie = cookie
else:
is_last_page = True
else:
cookie = pctrls[0].controlValue[1]
if cookie:
lc.controlValue[1] = cookie
else:
is_last_page = True
else:
is_last_page = True
logger.warn("Server ignores paged results control (RFC 2696).")
return results
def _search(self, search_flt, attrs, limit=None, path_prefix=None):
"""
Query this academic calendar for records matching the search filter.
Only looks one level deep in the tree to improve performance, so make
sure you pass the dn that you want to look directly inside of as
the `path` parameter.
Returns a list of records.
Each record is a dictionary containing the requested attributes.
search_flt -- LDAP-style search filter
attrs -- attributes to be retrieved
limit -- max number of records to return
path -- extra LDAP dn: is prepended to this object's basedn
ie the LDAP directory to look from relative to the root
"""
if limit is None:
limit = float('inf')
if path_prefix is None:
path = self._basedn
else:
path = '{}{}'.format(path_prefix, self._basedn)
page_control = SimplePagedResultsControl(
criticality=True, size=RemoteLDAPDatabase.PAGE_SIZE, cookie=''
)
results = []
pages_retrieved = 0
first_pass = True
while pages_retrieved*RemoteLDAPDatabase.PAGE_SIZE < limit \
and page_control.cookie \
or first_pass:
first_pass = False
try:
msgid = self._client.search_ext(path, ldap.SCOPE_ONELEVEL,
search_flt, attrlist=attrs,
serverctrls=[page_control])
except:
raise
pages_retrieved += 1
_, result_data, msgid, serverctrls = self._client.result3(msgid)
page_control.cookie = serverctrls[0].cookie
results += extract_results_from_ldap_data(result_data)
return results
def query(self, base, scope, params, fixed_rdn=None):
ocs = [
"(objectClass=%s)" % o.strip()
for o in params['objectClasses'].split(",")
]
fltr = "(&" + "".join(ocs) + (ldap.filter.filter_format(
"(%s)", [fixed_rdn]) if fixed_rdn else "") + ")"
result = []
with self.lh.get_handle() as l, self.lock:
self.log.debug(
"LDAP Search using filter '{filter}'".format(filter=fltr))
lc = SimplePagedResultsControl(criticality=True,
size=self.page_size,
cookie='')
ix = 0
while True:
msgid = l.search_ext(base,
ldap.SCOPE_ONELEVEL,
fltr,
attrlist=[self.uuid_entry],
serverctrls=[lc])
res = l.result3(msgid=msgid)
result_type, results, unused_msgid, serverctrls = res
for resultdata in results:
ix += 1
if resultdata[0] is not None:
result.append(resultdata)
cookie = None
for serverctrl in serverctrls:
if serverctrl.controlType == ldap.controls.libldap.SimplePagedResultsControl.controlType:
cookie = serverctrl.cookie
if cookie:
lc.cookie = cookie
break
if not cookie:
break
return [x for x in dict(result).keys()]
def get_ldap_list_membership(l, search_strings, list_name):
results = set()
known_ldap_resp_ctrls = {
SimplePagedResultsControl.controlType: SimplePagedResultsControl,
}
req_ctrl = SimplePagedResultsControl(True,
size=ldap_pagination_size,
cookie='')
while True:
msgid = l.search_ext(
search_strings['user_search_string'],
ldap.SCOPE_SUBTREE,
serverctrls=[req_ctrl],
attrlist=[search_strings['user_mail_field']],
filterstr=search_strings['user_membership_filter'] %
escape_filter_chars(list_name))
rtype, rdata, rmsgid, serverctrls = l.result3(
msgid,
timeout=ldap_timeout,
resp_ctrl_classes=known_ldap_resp_ctrls)
for data in rdata:
member = data[1].get(search_strings['user_mail_field'], [None])[0]
if isinstance(member, bytes):
member = member.decode('utf-8')
results |= {member}
pctrls = [
c for c in serverctrls
if c.controlType == SimplePagedResultsControl.controlType
]
if not pctrls:
# Paging not supported
break
cookie = pctrls[0].cookie
if not cookie:
break
req_ctrl.cookie = cookie
return results
def _make_paged_query(
conn, search_base, search_scope, ad_query, attr_list, page_size
):
"""
Makes paged query to LDAP.
Default max page size for LDAP is 1000.
"""
result = []
page_result_control = SimplePagedResultsControl(
size=page_size,
cookie=''
)
msgid = conn.search_ext(
search_base,
search_scope,
ad_query,
attr_list,
serverctrls=[page_result_control],
)
while True:
r_type, r_data, r_msgid, serverctrls = conn.result3(msgid)
result.extend(r_data)
if serverctrls:
if serverctrls[0].cookie:
page_result_control.size = page_size
page_result_control.cookie = serverctrls[0].cookie
msgid = conn.search_ext(
search_base,
search_scope,
ad_query,
attr_list,
serverctrls=[page_result_control],
)
else:
break
return result
def getSambaUsers(filter, attrs, base_dn):
print("getSambaUsers")
global ldap_connection
ldapUsers = []
page_control = SimplePagedResultsControl(True, size=1000, cookie='')
response = ldap_connection.search_ext(base_dn,
ldap.SCOPE_SUBTREE,
filter,
attrs,
serverctrls=[page_control])
try:
pages = 0
while True:
pages += 1
rtype, rdata, rmsgid, serverctrls = ldap_connection.result3(
response)
ldapUsers.extend(rdata)
controls = [
control for control in serverctrls
if control.controlType == SimplePagedResultsControl.controlType
]
if not controls:
print('The server ignores RFC 2696 control')
break
if not controls[0].cookie:
break
page_control.cookie = controls[0].cookie
response = ldap_connection.search_ext(base_dn,
ldap.SCOPE_SUBTREE,
filter,
attrs,
serverctrls=[page_control])
except:
print()
# dönen dictionary array'indeki verileri for dönerek daha basit bir hale getirir. Key olarak kullanılar dn'leri siler
results = [entry for dn, entry in ldapUsers if isinstance(entry, dict)]
return results
def _run_ldap_query(self, query):
with LDAPConnectionManager() as conn:
lc = SimplePagedResultsControl(
size=LDAP_RESULTS_PAGE_SIZE,
cookie=''
)
msgid = conn.search_ext(
settings.AUTH_LDAP_USER_SEARCH_BASE,
ldap.SCOPE_SUBTREE,
query,
serverctrls=[lc],
)
page_num = 0
while True:
page_num += 1
r_type, r_data, r_msgid, serverctrls = conn.result3(msgid)
logger.info('Pack of {} users loaded (page {})'.format(
LDAP_RESULTS_PAGE_SIZE, page_num,
))
for item in r_data:
yield item
if serverctrls:
if serverctrls[0].cookie:
lc.size = LDAP_RESULTS_PAGE_SIZE
lc.cookie = serverctrls[0].cookie
msgid = conn.search_ext(
settings.AUTH_LDAP_USER_SEARCH_BASE,
ldap.SCOPE_SUBTREE,
query,
serverctrls=[lc],
)
else:
break
else:
logger.error(
'LDAP::_run_ldap_query\tQuery: Server ignores RFC 2696 '
'control'
)
sys.exit(1)
def ldap_paged_async_search(base, scope, filt, columns):
ldap_log(' PAGED ASYNC SEARCH')
page_size = config.ldap_connection.get('page_size', 100)
if ldap_compat:
lc = SimplePagedResultsControl(size = page_size, cookie = '')
else:
lc = SimplePagedResultsControl(
LDAP_CONTROL_PAGED_RESULTS, True, (page_size, '')
)
results = []
while True:
# issue the ldap search command (async)
msgid = ldap_connection.search_ext(base, scope, filt, columns, serverctrls = [lc])
unused_code, response, unused_msgid, serverctrls = ldap_connection.result3(
msgid = msgid, timeout = config.ldap_connection.get('response_timeout', 5)
)
for result in response:
results.append(result)
# Mark current position in pagination control for next loop
cookie = None
for serverctrl in serverctrls:
if serverctrl.controlType == LDAP_CONTROL_PAGED_RESULTS:
if ldap_compat:
cookie = serverctrl.cookie
if cookie:
lc.cookie = cookie
else:
cookie = serverctrl.controlValue[1]
if cookie:
lc.controlValue = (page_size, cookie)
break
if not cookie:
break
return results
def _run_ldap_query(self, query):
self.conn = get_ldap()
lc = SimplePagedResultsControl(size=LDAP_RESULTS_PAGE_SIZE, cookie='')
msgid = self.conn.search_ext(
settings.AUTH_LDAP_USER_SEARCH_BASE,
ldap.SCOPE_SUBTREE,
query,
serverctrls=[lc],
)
page_num = 0
while True:
page_num += 1
r_type, r_data, r_msgid, serverctrls = self.conn.result3(msgid)
print "Pack of %s users loaded (page %s)" % (
LDAP_RESULTS_PAGE_SIZE,
page_num,
)
for item in r_data:
yield item
if serverctrls:
if serverctrls[0].cookie:
lc.size = LDAP_RESULTS_PAGE_SIZE
lc.cookie = serverctrls[0].cookie
msgid = self.conn.search_ext(
settings.AUTH_LDAP_USER_SEARCH_BASE,
ldap.SCOPE_SUBTREE,
query,
serverctrls=[lc],
)
else:
break
else:
logger.error(
'LDAP::_run_ldap_query\tQuery: %s\t'
'Server ignores RFC 2696 control',
)
sys.exit(-1)
self._disconnect()
def doSearch(self, searchFilter=None, retrieveAttributes=None):
"""
Searches for records in an ldap server, Research is filtered and the result is a selection of individual
fields of the returned records.
:param searchFilter: It is the filter to search the LDAP server.
The filter must be written in the LDAP filter format.
If the filter is not specified, the default filter is used (defined in the
instance of the class).
If the filter contains (@DefaultFilter) as a filter, the filter passed is added
to the default filter.
:param retrieveAttributes: This is the list of the attributes returned by the search for each record.
The list of attributes is made up of a string with the list of attributes
separated by commas.
If attributes are not specified, the default attributes (defined in the instance
of the class are returned).
If the list of attributes including @DefaultAttribute attribute, passed
attributes are added to the default attributes.
:return: Return a list of dictionaries, a dictionary for each record. The value attached
to each key in turn is a list of strings.
"""
if searchFilter:
if '(@DefaultFilter)' in searchFilter:
searchFilter = '(' + self.searchFilter[1:-1] + searchFilter[1:-1].replace('(@DefaultFilter)', '') + ')'
else:
searchFilter = self.searchFilter
if retrieveAttributes:
retrieveAttributes = [str(attribute).strip() for attribute in retrieveAttributes.split(',')]
if '@DefaultAttribute' in retrieveAttributes:
retrieveAttributes = list(set(self.userAttr + retrieveAttributes))
retrieveAttributes.remove('@DefaultAttribute')
else:
retrieveAttributes = self.userAttr
page_size = 50
ldap.set_option(ldap.OPT_REFERRALS, 0)
req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='')
ldap_user = self.doLogin(user=self.searchUser, password=self.searchPassword, mode='Search')
if ldap_user:
try:
ldap_result_id = self.ldapClient.search_ext(self.baseDN, ldap.SCOPE_SUBTREE, searchFilter,
attrlist=retrieveAttributes,
serverctrls=[req_ctrl])
result_set = []
result_pages = 0
while True:
rtype, rdata, rmsgid, rctrls = self.ldapClient.result3(ldap_result_id)
result_set.extend(rdata)
result_pages += 1
pctrls = [c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType]
if pctrls:
if pctrls[0].cookie:
req_ctrl.cookie = pctrls[0].cookie
ldap_result_id = self.ldapClient.search_ext(self.baseDN, ldap.SCOPE_SUBTREE, searchFilter,
attrlist=retrieveAttributes,
serverctrls=[req_ctrl])
else:
break
except ldap.LDAPError, e:
print e
return None
finally:
def test_ticket48808(topology, test_user):
log.info('Run multiple paging controls on a single connection')
users_num = 100
page_size = 30
users_list = add_users(topology, users_num)
search_flt = r'(uid=test*)'
searchreq_attrlist = ['dn', 'sn']
log.info('Set user bind')
topology.standalone.simple_bind_s(TEST_USER_DN, TEST_USER_PWD)
log.info('Create simple paged results control instance')
req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='')
controls = [req_ctrl]
for ii in xrange(3):
log.info('Iteration %d' % ii)
msgid = topology.standalone.search_ext(DEFAULT_SUFFIX,
ldap.SCOPE_SUBTREE,
search_flt,
searchreq_attrlist,
serverctrls=controls)
rtype, rdata, rmsgid, rctrls = topology.standalone.result3(msgid)
pctrls = [
c
for c in rctrls
if c.controlType == SimplePagedResultsControl.controlType
]
req_ctrl.cookie = pctrls[0].cookie
msgid = topology.standalone.search_ext(DEFAULT_SUFFIX,
ldap.SCOPE_SUBTREE,
search_flt,
searchreq_attrlist,
serverctrls=controls)
log.info('Set Directory Manager bind back')
topology.standalone.simple_bind_s(DN_DM, PASSWORD)
del_users(topology, users_list)
log.info('Abandon the search')
users_num = 10
page_size = 0
users_list = add_users(topology, users_num)
search_flt = r'(uid=test*)'
searchreq_attrlist = ['dn', 'sn']
log.info('Set user bind')
topology.standalone.simple_bind_s(TEST_USER_DN, TEST_USER_PWD)
log.info('Create simple paged results control instance')
req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='')
controls = [req_ctrl]
msgid = topology.standalone.search_ext(DEFAULT_SUFFIX,
ldap.SCOPE_SUBTREE,
search_flt,
searchreq_attrlist,
serverctrls=controls)
rtype, rdata, rmsgid, rctrls = topology.standalone.result3(msgid)
pctrls = [
c
for c in rctrls
if c.controlType == SimplePagedResultsControl.controlType
]
assert not pctrls[0].cookie
log.info('Set Directory Manager bind back')
topology.standalone.simple_bind_s(DN_DM, PASSWORD)
del_users(topology, users_list)
log.info("Search should fail with 'nsPagedSizeLimit = 5'"
"and 'nsslapd-pagedsizelimit = 15' with 10 users")
conf_attr = '15'
user_attr = '5'
expected_rs = ldap.SIZELIMIT_EXCEEDED
users_num = 10
page_size = 10
users_list = add_users(topology, users_num)
search_flt = r'(uid=test*)'
searchreq_attrlist = ['dn', 'sn']
conf_attr_bck = change_conf_attr(topology, DN_CONFIG,
'nsslapd-pagedsizelimit', conf_attr)
user_attr_bck = change_conf_attr(topology, TEST_USER_DN,
'nsPagedSizeLimit', user_attr)
log.info('Set user bind')
topology.standalone.simple_bind_s(TEST_USER_DN, TEST_USER_PWD)
log.info('Create simple paged results control instance')
req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='')
controls = [req_ctrl]
log.info('Expect to fail with SIZELIMIT_EXCEEDED')
with pytest.raises(expected_rs):
all_results = paged_search(topology, controls,
search_flt, searchreq_attrlist)
log.info('Set Directory Manager bind back')
topology.standalone.simple_bind_s(DN_DM, PASSWORD)
del_users(topology, users_list)
change_conf_attr(topology, DN_CONFIG,
'nsslapd-pagedsizelimit', conf_attr_bck)
change_conf_attr(topology, TEST_USER_DN,
'nsPagedSizeLimit', user_attr_bck)
log.info("Search should pass with 'nsPagedSizeLimit = 15'"
"and 'nsslapd-pagedsizelimit = 5' with 10 users")
conf_attr = '5'
user_attr = '15'
users_num = 10
page_size = 10
users_list = add_users(topology, users_num)
search_flt = r'(uid=test*)'
searchreq_attrlist = ['dn', 'sn']
conf_attr_bck = change_conf_attr(topology, DN_CONFIG,
'nsslapd-pagedsizelimit', conf_attr)
user_attr_bck = change_conf_attr(topology, TEST_USER_DN,
'nsPagedSizeLimit', user_attr)
log.info('Set user bind')
topology.standalone.simple_bind_s(TEST_USER_DN, TEST_USER_PWD)
log.info('Create simple paged results control instance')
req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='')
controls = [req_ctrl]
log.info('Search should PASS')
all_results = paged_search(topology, controls,
search_flt, searchreq_attrlist)
log.info('%d results' % len(all_results))
assert len(all_results) == len(users_list)
log.info('Set Directory Manager bind back')
topology.standalone.simple_bind_s(DN_DM, PASSWORD)
del_users(topology, users_list)
change_conf_attr(topology, DN_CONFIG,
'nsslapd-pagedsizelimit', conf_attr_bck)
change_conf_attr(topology, TEST_USER_DN,
'nsPagedSizeLimit', user_attr_bck)
retrieveAttributes = ['dNSHostName', 'distinguishedName']
# searchFilter. Options can include CN=, etc
searchFilter = "objectClass=computer"
pg_ctrl = SimplePagedResultsControl(CRITICALITY, PAGE_SIZE, COOKIE)
while first_pass or pg_ctrl.cookie:
first_pass = False
try:
msgid = l.search_ext(baseDN, searchScope, searchFilter, retrieveAttributes, serverctrls=[pg_ctrl])
except ldap.LDAPError, e:
print e
result_type, data, msgid, serverctrls = l.result3(msgid)
# print "data is: ", data
# print "serverctrls value is: ", serverctrls
# pg_ctrl.cookie = serverctrls[0].cookie
if len(serverctrls) > 0: pg_ctrl.cookie = serverctrls[0].cookie
results += [i[0] for i in data]
# convert results to a string as re.findall will not accept a List
results_str = str(results)
# Active Directory
# The regular expression to pull only the distinguishedName attribute from the LDAP query is:
AD_reg = "'CN=([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9]),"
# open our csv file that was dumped from our LDAP computer objects lookup
# read the contents of the file
# use a findall function to parse through file searching for content based on the regex
output = re.findall(AD_reg, results_str ,flags=0)
# print the results
#open file and export results
file = open('ldap_out.csv','w')
#Loop all computer hostnames and deploy Agent MSI
print ("Getting page %d" % (pages))
rtype, rdata, rmsgid, serverctrls = l.result3(msgid,resp_ctrl_classes=known_ldap_resp_ctrls)
print ('%d results' % len(rdata))
print ('serverctrls=',pprint.pprint(serverctrls))
# pprint.pprint(rdata)
pctrls = [
c
for c in serverctrls
if c.controlType == SimplePagedResultsControl.controlType
]
if pctrls:
print ('pctrls[0].size',repr(pctrls[0].size))
print ('pctrls[0].cookie',repr(pctrls[0].cookie))
if pctrls[0].cookie:
# Copy cookie from response control to request control
req_ctrl.cookie = pctrls[0].cookie
msgid = l.search_ext(
base,
ldap.SCOPE_SUBTREE,
search_flt,
attrlist=searchreq_attrlist,
serverctrls=[req_ctrl]
)
else:
break
else:
print ("Warning: Server ignores RFC 2696 control.")
break
l.unbind_s()
def discover(self, args):
import omero
ldap = self.__import_ldap__()
c = self.ctx.conn(args)
iconfig = c.sf.getConfigService()
iadmin = c.sf.getAdminService()
iquery = c.sf.getQueryService()
LDAP_PROPERTIES = """
omero.ldap.urls
omero.ldap.username
omero.ldap.password
omero.ldap.base
omero.ldap.user_filter
omero.ldap.user_mapping
omero.ldap.group_filter
omero.ldap.group_mapping
omero.ldap.new_user_group
""".split()
cfg = dict()
for key in LDAP_PROPERTIES:
cfg[key.split(".")[-1]] = iconfig.getConfigValue(key)
urls = args.urls and args.urls or cfg["urls"]
basedn = args.base and args.base or cfg["base"]
for url in urls.split(","):
self.ctx.err("Connecting to %s..." % url)
ld = ldap.initialize(url)
ld.simple_bind_s(cfg['username'], cfg['password'])
user_filter = cfg["user_filter"]
user_mapping = cfg["user_mapping"]
user_mapping = user_mapping.split(",")
omeName_mapping = None
for um in user_mapping:
parts = um.split("=")
if parts[0] == "omeName":
omeName_mapping = parts[1]
from ldap.controls import SimplePagedResultsControl
cookie = ''
# This is the limit for Active Directory, 1000. However
# the LDAP connection has a sizeLimit that overrides
# this value if the page_size exceeds it so it is safe
# to enter pretty much anything here when using paged results.
page_size = 1000
results = []
first = True
page_control = SimplePagedResultsControl(False, page_size, cookie)
while first or page_control.cookie:
first = False
try:
msgid = ld.search_ext(
basedn, ldap.SCOPE_SUBTREE,
user_filter, serverctrls=[page_control]
)
except:
self.ctx.die(1, "Failed to execute LDAP search")
result_type, results, msgid, serverctrls = ld.result3(msgid)
if serverctrls:
page_control.cookie = serverctrls[0].cookie
user_names = set()
user_dns = {}
for dn, entry in results:
omeName = entry[omeName_mapping]
if isinstance(omeName, (list, tuple)):
if len(omeName) == 1:
omeName = omeName[0]
user_names.add(omeName)
user_dns[omeName] = dn
else:
self.ctx.err("Failed to unwrap omeName: %s" %
omeName)
continue
if not user_names:
continue # Early exit!
from omero.rtypes import rlist
from omero.rtypes import rstring
from omero.rtypes import unwrap
params = omero.sys.ParametersI()
params.add("names", rlist([rstring(x) for x in user_names]))
id_names = unwrap(iquery.projection(
"select id, omeName from Experimenter "
"where omeName in (:names)", params))
for eid, omeName in id_names:
try:
olddn = iadmin.lookupLdapAuthExperimenter(eid)
dn = user_dns[omeName]
except omero.ApiUsageException:
continue # Unknown user
if olddn:
if olddn.lower() != dn.lower():
self.ctx.err("Found different DN for %s: %s"
% (omeName, olddn))
else:
self.ctx.dbg("DN already set for %s: %s"
% (omeName, olddn))
else:
if args.commands:
self.ctx.out("%s ldap setdn %s %s"
% (sys.argv[0], omeName, dn))
else:
self.ctx.out("Experimenter:%s\tomeName=%s\t%s"
% (eid, omeName, dn))
def paged_search_ext_s(self,base,scope,filterstr='(objectClass=*)',attrlist=None,attrsonly=0,serverctrls=None,clientctrls=None,timeout=-1,sizelimit=0):
"""
Behaves exactly like LDAPObject.search_ext_s() but internally uses the
simple paged results control to retrieve search results in chunks.
This is non-sense for really large results sets which you would like
to process one-by-one
"""
while True: # loop for reconnecting if necessary
req_ctrl = SimplePagedResultsControl(True,size=self.page_size,cookie='')
try:
# Send first search request
msgid = self.search_ext(
base,
scope,
filterstr=filterstr,
attrlist=attrlist,
attrsonly=attrsonly,
serverctrls=(serverctrls or [])+[req_ctrl],
clientctrls=clientctrls,
timeout=timeout,
sizelimit=sizelimit
)
result_pages = 0
all_results = []
while True:
rtype, rdata, rmsgid, rctrls = self.result3(msgid)
all_results.extend(rdata)
result_pages += 1
# Extract the simple paged results response control
pctrls = [
c
for c in rctrls
if c.controlType == SimplePagedResultsControl.controlType
]
if pctrls:
if pctrls[0].cookie:
# Copy cookie from response control to request control
req_ctrl.cookie = pctrls[0].cookie
msgid = self.search_ext(
base,
scope,
filterstr=filterstr,
attrlist=attrlist,
attrsonly=attrsonly,
serverctrls=(serverctrls or [])+[req_ctrl],
clientctrls=clientctrls,
timeout=timeout,
sizelimit=sizelimit
)
else:
break # no more pages available
except ldap.SERVER_DOWN,e:
try:
self.reconnect(self._uri)
except AttributeError:
raise e
else:
return result_pages,all_results
def paged_search_ext_s(self,base,scope,filterstr='(objectClass=*)',attrlist=None,attrsonly=0,serverctrls=None,clientctrls=None,timeout=-1,sizelimit=0, page_size=1000):
"""
Behaves exactly like LDAPObject.search_ext_s() but internally uses the
simple paged results control to retrieve search results in chunks.
Taken on 2013-07-10 from :
https://bitbucket.org/jaraco/python-ldap/src/f208b6338a28/Demo/paged_search_ext_s.py
Converted to a generator
Added range support based on :
http://stackoverflow.com/a/1711862/591064
"""
req_ctrl = SimplePagedResultsControl(True,size=page_size,cookie='')
# Send first search request
msgid = self.search_ext(
base,
scope,
filterstr=filterstr,
attrlist=attrlist,
attrsonly=attrsonly,
serverctrls=(serverctrls or [])+[req_ctrl],
clientctrls=clientctrls,
timeout=timeout,
sizelimit=sizelimit
)
while True:
rtype, rdata, rmsgid, rctrls = self.result3(msgid)
#singleresult is a list of pyldap tuple in the form [(dn, dict),(dn,dict)]
for singleresult in rdata:
#Range support idea found on StackOverflow
#http://stackoverflow.com/a/1711862/591064#
dn = singleresult[0]
for attrname in singleresult[1]:
if ';range=' in attrname:
#
# parse range attr
#
actual_attrname, range_stmt = attrname.split(';')
bound_lower, bound_upper = [
int(x) for x in range_stmt.split('=')[1].split('-')
]
step = bound_upper - bound_lower + 1
while True:
attr_next = '%s;range=%d-%d' % (
actual_attrname, bound_lower, bound_upper
)
dn, attrs = self.search_s(
dn, ldap.SCOPE_BASE, attrlist = [attr_next])[0]
assert len(attrs) == 1
ret_attrname = attrs.keys()[0]
singleresult[1][actual_attrname].extend(attrs[ret_attrname])
if ret_attrname.endswith('-*'):
break
bound_lower = bound_upper + 1
bound_upper += step
yield singleresult
# Extract the simple paged results response control
pctrls = [
c
for c in rctrls
if c.controlType == SimplePagedResultsControl.controlType
]
if pctrls:
if pctrls[0].cookie:
# Copy cookie from response control to request control
req_ctrl.cookie = pctrls[0].cookie
msgid = self.search_ext(
base,
scope,
filterstr=filterstr,
attrlist=attrlist,
attrsonly=attrsonly,
serverctrls=(serverctrls or [])+[req_ctrl],
clientctrls=clientctrls,
timeout=timeout,
sizelimit=sizelimit
)
else:
break
