def search(base_dn, search_filter, attributes): """Iterative LDAP search using page control. :param base_dn: str -- The base DN from which to start the search. :param search_filter: str -- Representation of the filter to apply in the search. :param attributes: list -- Attributes to be retrieved for each entry. If ``None``, all attributes will be retrieved. :returns: A generator which yields one search result at a time as a tuple containing a `dn` as ``str`` and `attributes` as ``dict``. """ connection, settings = current_ldap page_ctrl = SimplePagedResultsControl(True, size=settings['page_size'], cookie='') while True: msg_id = connection.search_ext(base_dn, SCOPE_SUBTREE, filterstr=search_filter, attrlist=attributes, serverctrls=[page_ctrl], timeout=settings['timeout']) try: _, r_data, __, server_ctrls = connection.result3(msg_id, timeout=settings['timeout']) except NO_SUCH_OBJECT: break for dn, entry in r_data: if dn: yield dn, entry page_ctrl.cookie = get_page_cookie(server_ctrls) if not page_ctrl.cookie: # End of results break
def ldap_search_paged(l, basedn, scope, filter, attributes, timeout, page_size): # FIXME: untested from ldap.controls import SimplePagedResultsControl lc = SimplePagedResultsControl( ldap.LDAP_CONTROL_PAGE_OID, True, (page_size,'') ) # Send search request result_id = l.search_ext(basedn, scope, filter, attributes, serverctrls=[lc]) pages = 0 while True: pages += 1 log.debug('Getting page %d', pages) result_type, result_data, result_msgid, serverctrls = l.result3(result_id) log.debug('%d results', len(result_data)) if not result_data: break pctrls = [c for c in serverctrls if c.controlType == ldap.LDAP_CONTROL_PAGE_OID] if pctrls: est, cookie = pctrls[0].controlValue if cookie: lc.controlValue = (page_size, cookie) result_id = l.search_ext(basedn, scope, filter, attributes, serverctrls=[lc]) else: break else: log.warn('Server ignores RFC 2696 control.') break
def paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)', attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None, timeout=-1, sizelimit=0): req_ctrl = SimplePagedResultsControl( True, size=self.page_size, cookie='', ) result_pages = 0 all_results = [] msgid = self.search_ext( base, scope, filterstr, attrlist=attrlist, serverctrls=(serverctrls or [])+[req_ctrl], ) while True: rtype, rdata, rmsgid, rctrls = self.result3(msgid) all_results.extend(rdata) result_pages += 1 pctrls = [c for c in rctrls if c.controlType == sprc_ct] if not pctrls: continue if not pctrls[0].cookie: break req_ctrl.cookie = pctrls[0].cookie msgid = self.search_ext( base, scope, filterstr, attrlist=attrlist, serverctrls=(serverctrls or [])+[req_ctrl] ) return all_results
def paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)', attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None, timeout=-1, sizelimit=0): """ Behaves exactly like LDAPObject.search_ext_s() but internally uses the simple paged results control to retrieve search results in chunks. """ req_ctrl = SimplePagedResultsControl(True, size=self.page_size, cookie='') # Send first search request msgid = self.search_ext(base, ldap.SCOPE_SUBTREE, filterstr, attrlist=attrlist, serverctrls=(serverctrls or []) + [req_ctrl]) results = [] while True: rtype, rdata, rmsgid, rctrls = self.result3(msgid) results.extend(rdata) # Extract the simple paged results response control pctrls = [c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType] if pctrls: if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = self.search_ext(base, ldap.SCOPE_SUBTREE, filterstr, attrlist=attrlist, serverctrls=(serverctrls or []) + [req_ctrl]) else: break return results
def _paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)', attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None, timeout=-1, sizelimit=0, page_size=10): """ Behaves similarly to LDAPObject.search_ext_s() but internally uses the simple paged results control to retrieve search results in chunks. Taken from the python-ldap paged_search_ext_s.py demo, showing how to use the paged results control: https://bitbucket.org/jaraco/python-ldap/ """ request_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='') results = [] while True: msgid = self.conn.search_ext(base, scope, filterstr=filterstr, attrlist=attrlist, attrsonly=attrsonly, serverctrls=(serverctrls or []) + [request_ctrl], clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit) result_type, result_data, result_msgid, result_ctrls = self.conn.result3(msgid) results.extend(result_data) # Extract the simple paged results response control paged_ctrls = [c for c in result_ctrls if c.controlType == SimplePagedResultsControl.controlType] if paged_ctrls and paged_ctrls[0].cookie: # Copy cookie from response control to request control request_ctrl.cookie = paged_ctrls[0].cookie else: break return results
def query(self, base, scope, params, fixed_rdn=None): ocs = ["(objectClass=%s)" % o.strip() for o in params['objectClasses'].split(",")] fltr = "(&" + "".join(ocs) + (ldap.filter.filter_format("(%s)", [fixed_rdn]) if fixed_rdn else "") + ")" result = [] with self.lh.get_handle() as l, self.lock: self.log.debug("LDAP Search using filter '{filter}'".format(filter=fltr)) lc = SimplePagedResultsControl(criticality=True, size=self.page_size, cookie='') ix = 0 while True: msgid = l.search_ext(base, ldap.SCOPE_ONELEVEL, fltr, attrlist=[self.uuid_entry], serverctrls=[lc]) res = l.result3(msgid=msgid) result_type, results, unused_msgid, serverctrls = res for resultdata in results: ix += 1 if resultdata[0] is not None: result.append(resultdata) cookie = None for serverctrl in serverctrls: if serverctrl.controlType == ldap.controls.libldap.SimplePagedResultsControl.controlType: cookie = serverctrl.cookie if cookie: lc.cookie = cookie break if not cookie: break return [x for x in dict(result).keys()]
def paged_search_iter(self, filter, attrs=None): # Test for version 2.3 API try: lc = SimplePagedResultsControl \ (ldap.LDAP_CONTROL_PAGE_OID, True, (self.page_size, '')) api_version = 3 except AttributeError: # New version 2.4 API lc = SimplePagedResultsControl \ (True, size = self.page_size, cookie = '') sc = \ { SimplePagedResultsControl.controlType : SimplePagedResultsControl } api_version = 4 res = self.ldcon.search_ext \ ( self.cfg.LDAP_BASE_DN , ldap.SCOPE_SUBTREE , filter , attrlist = attrs , serverctrls = [lc] ) while True: params = (res, ) if api_version == 3: rtype, rdata, rmsgid, serverctrls = self.ldcon.result3(res) pctrls = \ [c for c in serverctrls if c.controlType == ldap.LDAP_CONTROL_PAGE_OID ] else: rtype, rdata, rmsgid, serverctrls = self.ldcon.result3 \ (res, resp_ctrl_classes = sc) pctrls = \ [c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType ] for r in rdata: if not r[0]: continue r = LDAP_Search_Result(r) yield r if pctrls: if api_version == 3: x, cookie = pctrls[0].controlValue lc.controlValue = (self.page_size, cookie) else: cookie = pctrls[0].cookie lc.cookie = cookie if not cookie: break res = self.ldcon.search_ext \ ( self.cfg.LDAP_BASE_DN , ldap.SCOPE_SUBTREE , filter , attrs , serverctrls = [lc] ) else: break
def pagedsearch(self, search_flt, searchreq_attrlist): req_ctrl = SimplePagedResultsControl(True,size=self.page_size,cookie='') known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType:SimplePagedResultsControl,} # Send search request msgid = self.lcon.search_ext( self.base, ldap.SCOPE_SUBTREE, search_flt, attrlist=searchreq_attrlist, serverctrls=[req_ctrl]) pages = 0 i = 0 while True: pages += 1 rtype, rdata, rmsgid, serverctrls = self.lcon.result3(msgid,resp_ctrl_classes=known_ldap_resp_ctrls) yield rdata pctrls = [c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType] if pctrls: if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = self.lcon.search_ext(self.base,ldap.SCOPE_SUBTREE,search_flt,attrlist=searchreq_attrlist,serverctrls=[req_ctrl]) else: break else: print "Warning: Server ignores RFC 2696 control." break
def get_all_users(self, username, password): base_dn = 'OU=上海二三四五网络科技有限公司,DC=trump,DC=local' self.l.simple_bind_s(username + "@trump.local", password) PAGE_SIZE = 500 # 设置每页返回的条数 pg_ctrl = SimplePagedResultsControl(True, size=PAGE_SIZE, cookie="") userdata = [] retdata = [] while True: msgid = self.l.search_ext(base_dn, ldap.SCOPE_SUBTREE, "(&(objectClass=person))", None, serverctrls=[pg_ctrl]) _a, res_data, _b, srv_ctrls = self.l.result3(msgid) userdata.extend(res_data) cookie = srv_ctrls[0].cookie if cookie: pg_ctrl.cookie = cookie else: break print('totalnum:', len(userdata)) for u in userdata: retdict = {} for k, v in u[1].items(): retdict[k] = v[0] retdata.append(retdict) return retdata
def get_ldap_lists(l, search_strings, parent_list=None): results = set() known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl, } req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='') if parent_list: filterstr = search_strings['get_all_sub_lists_filter'] % escape_filter_chars(parent_list) else: filterstr = search_strings['get_all_lists_filter'] while True: msgid = l.search_ext(search_strings['list_search_string'], ldap.SCOPE_SUBTREE, serverctrls=[req_ctrl], attrlist=(search_strings['list_cn_field'], search_strings['list_name_field']), filterstr=filterstr) rtype, rdata, rmsgid, serverctrls = l.result3(msgid, resp_ctrl_classes=known_ldap_resp_ctrls) results |= {(data[search_strings['list_cn_field']][0], data[search_strings['list_name_field']][0]) for (dn, data) in rdata} pctrls = [c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType] cookie = pctrls[0].cookie if not cookie: break req_ctrl.cookie = cookie return results
def get_ldap_list_membership(l, search_strings, list_name): results = set() known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl, } req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='') while True: msgid = l.search_ext(search_strings['user_search_string'], ldap.SCOPE_SUBTREE, serverctrls=[req_ctrl], attrlist=[search_strings['user_account_name_field']], filterstr=search_strings['user_membership_filter'] % escape_filter_chars(list_name) ) rtype, rdata, rmsgid, serverctrls = l.result3(msgid, resp_ctrl_classes=known_ldap_resp_ctrls) results |= {data[1][search_strings['user_account_name_field']][0] for data in rdata} pctrls = [c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType] cookie = pctrls[0].cookie if not cookie: break req_ctrl.cookie = cookie return results
def get_paged_users(self, filters, attributes): pages = 0 result = [] page_control = SimplePagedResultsControl(True, size=self.page_size, cookie='') res = self._conn.search_ext( self.domain_dn, ldap.SCOPE_SUBTREE, filters, attributes, serverctrls=[page_control] ) while True: pages += 1 self.log.debug("Page {} done".format(pages)) rtype, rdata, rmsgid, serverctrls = self._conn.result3(res) result.extend(rdata) controls = [ctrl for ctrl in serverctrls if ctrl.controlType == SimplePagedResultsControl.controlType] if not controls: self.log.error('The server ignores RFC 2696 control') break if not controls[0].cookie: break page_control.cookie = controls[0].cookie res = self._conn.search_ext( self.domain_dn, ldap.SCOPE_SUBTREE, filters, attributes, serverctrls=[page_control] ) return result
def _paged_search(self, connection, base, scope, filterstr, attrlist): page_size = self.settings_dict.get('PAGE_SIZE', 1000) results = [] pg_ctrl = SimplePagedResultsControl(True, page_size, "") scope = ldap.SCOPE_SUBTREE pages = 0 while True: pages += 1 msgid = connection.search_ext( base, scope=scope, filterstr=filterstr, attrlist=attrlist, serverctrls=[pg_ctrl] ) rtype, rdata, rmsgid, serverctrls = connection.result3(msgid) results.extend(rdata) cookie = serverctrls[0].cookie if cookie: pg_ctrl.cookie = cookie search = connection.search_ext( base, scope=scope, filterstr=filterstr, attrlist=attrlist, serverctrls=[pg_ctrl] ) else: break return results
def search(base_dn, search_filter, attributes): """Iterative LDAP search using page control. :param base_dn: str -- The base DN from which to start the search. :param search_filter: str -- Representation of the filter to apply in the search. :param attributes: list -- Attributes to be retrieved for each entry. If ``None``, all attributes will be retrieved. :returns: A generator which yields one search result at a time as a tuple containing a `dn` as ``str`` and `attributes` as ``dict``. """ connection, settings = current_ldap page_ctrl = SimplePagedResultsControl(True, size=settings['page_size'], cookie='') search_filter = search_filter.encode('utf-8') while True: msg_id = connection.search_ext(base_dn, SCOPE_SUBTREE, filterstr=search_filter, attrlist=attributes, serverctrls=[page_ctrl], timeout=settings['timeout']) try: _, r_data, __, server_ctrls = connection.result3(msg_id, timeout=settings['timeout']) except NO_SUCH_OBJECT: break for dn, entry in r_data: if dn: yield dn, entry page_ctrl.cookie = get_page_cookie(server_ctrls) if not page_ctrl.cookie: # End of results break
def _PagedAsyncSearch(ldap_conn, sizelimit, base_dn, scope, filterstr="(objectClass=*)", attrlist=None): """ Helper function that implements a paged LDAP search for the Search method below. Args: ldap_conn: our OMLdapConnection object sizelimit: max # of users to return. filterstr: LDAP filter to apply to the search attrlist: list of attributes to return. If null, all attributes are returned Returns: A list of users as returned by the LDAP search """ # Time to autodetect our library's API, because python-ldap's API intoduced # breaking changes between versions 2.3 and 2.4. use_old_paging_api = False if hasattr(ldap, "LDAP_CONTROL_PAGE_OID"): use_old_paging_api = True paged_results_control = SimplePagedResultsControl( controlType=ldap.LDAP_CONTROL_PAGE_OID, criticality=True, controlValue=(_PAGE_SIZE, "") ) page_ctrl_oid = ldap.LDAP_CONTROL_PAGE_OID else: paged_results_control = SimplePagedResultsControl(criticality=True, size=_PAGE_SIZE, cookie="") page_ctrl_oid = ldap.controls.SimplePagedResultsControl.controlType logging.debug("Paged search on %s for %s", base_dn, filterstr) users = [] ix = 0 while True: if _PAGE_SIZE == 0: serverctrls = [] else: serverctrls = [paged_results_control] msgid = ldap_conn.conn.search_ext(base_dn, scope, filterstr, attrlist=attrlist, serverctrls=serverctrls) res = ldap_conn.conn.result3(msgid=msgid) unused_code, results, unused_msgid, serverctrls = res for result in results: ix += 1 users.append(result) if sizelimit and ix >= sizelimit: break if sizelimit and ix >= sizelimit: break cookie = None for serverctrl in serverctrls: if serverctrl.controlType == page_ctrl_oid: if use_old_paging_api: unused_est, cookie = serverctrl.controlValue if cookie: paged_results_control.controlValue = (_PAGE_SIZE, cookie) else: cookie = paged_results_control.cookie = serverctrl.cookie break if not cookie: break return users
def query_users_paged(self, options): """ Connects to LDAP server and attempts to query and return all users by iterating through each page result. Requires LDAP v3. """ page_size = options.get('page_size', 500) criticality = options.get('criticality', True) cookie = options.get('cookie', '') full_record = options.get('full_record', False) fields = self.ldap_query_fields if full_record: fields = None # search the server for users first_pass = True pg_ctrl = SimplePagedResultsControl(criticality, page_size, cookie) LOG.debug("self.ldap_query_fields = %r", self.ldap_query_fields) while first_pass or pg_ctrl.cookie: first_pass = False msgid = self.ldap_connection.search_ext( self.settings['base_dn'], ldap.SCOPE_SUBTREE, self.settings['filter'], fields, serverctrls=[pg_ctrl] ) result_type, ldap_users, msgid, serverctrls = self.ldap_connection.result3(msgid) pg_ctrl.cookie = serverctrls[0].cookie for user in ldap_users: if user[0] and user[1]: yield _clean_record(user[1]) # disconnect and return results self.ldap_connection.unbind_s()
def results_for_dn(directory: ldap.ldapobject, base_dn: str, filter: str) -> List[User]: """Returns a list of User objects found in the directory object for filter :param directory: A ldap.LDAPObject that has already been bound to a directory. :param base_dn: The base of the directory tree to run the search filter against. :param filter: The LDAP search filter to run on base_dn using directory. """ req_ctrl = SimplePagedResultsControl(True, size=5000, cookie="") known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl, } # Send search request msgid = directory.search_ext(base_dn, ldap.SCOPE_SUBTREE, filterstr=LDAP_FILTER, serverctrls=[req_ctrl]) results = [] while True: __, result_data, __, serverctrls = directory.result3( msgid, resp_ctrl_classes=known_ldap_resp_ctrls) results.extend([ User( ldap_entry["sAMAccountName"][0].decode(), ldap_entry["mail"][0].decode(), ldap_entry["sn"][0].decode(), ldap_entry["givenName"][0].decode(), ) for __, ldap_entry in result_data ]) page_controls = [ control for control in serverctrls if control.controlType == SimplePagedResultsControl.controlType ] if page_controls: if page_controls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = page_controls[0].cookie msgid = directory.search_ext( base_dn, ldap.SCOPE_SUBTREE, filterstr=LDAP_FILTER, serverctrls=[req_ctrl], ) else: break else: eprint("Warning: Server ignores RFC 2696 control.") break return results
def create_controls(pagesize, LDAP_API_CHECK): """Create an LDAP control with a page size of "pagesize".""" # Initialize the LDAP controls for paging. Note that we pass '' # for the cookie because on first iteration, it starts out empty. if LDAP_API_CHECK: return SimplePagedResultsControl(True, size=pagesize, cookie='') return SimplePagedResultsControl(ldap.LDAP_CONTROL_PAGE_OID, True, (pagesize, ''))
def create_controls(self, pagesize): # Initialize the LDAP controls for paging. Note that we pass '' # for the cookie because on first iteration, it starts out empty. if LDAP24API: return SimplePagedResultsControl(True, size=pagesize, cookie='') else: return SimplePagedResultsControl(ldap.LDAP_CONTROL_PAGE_OID, True, (pagesize, ''))
def create_ldif_from_master(lo, ldif_file, base, page_size): """ create ldif file from everything from lo """ logging.info('Fetching LDIF ...') if ldif_file == '-': output = sys.stdout else: if os.path.isfile(ldif_file): os.unlink(ldif_file) output = gzip.open(ldif_file, 'wb') if hasattr(ldap, 'LDAP_CONTROL_PAGE_OID'): # python-ldap <= 2.3 logging.debug('Using old python-ldap 2.3 API') api24 = False lc = SimplePagedResultsControl(controlType=ldap.LDAP_CONTROL_PAGE_OID, criticality=True, controlValue=(page_size, '')) page_ctrl_oid = ldap.LDAP_CONTROL_PAGE_OID else: # python-ldap >= 2.4 logging.debug('Using new python-ldap 2.4 API') api24 = True lc = SimplePagedResultsControl(criticality=True, size=page_size, cookie='') page_ctrl_oid = lc.controlType while True: msgid = lo.lo.search_ext(base, ldap.SCOPE_SUBTREE, '(objectclass=*)', ['+', '*'], serverctrls=[lc]) rtype, rdata, rmsgid, serverctrls = lo.lo.result3(msgid) for dn, data in rdata: logging.debug('Processing %s ...', dn) for attr in replication.EXCLUDE_ATTRIBUTES: data.pop(attr, None) output.write(ldif.CreateLDIF(dn, data, cols=10000)) pctrls = [c for c in serverctrls if c.controlType == page_ctrl_oid] if pctrls: if api24: cookie = lc.cookie = pctrls[0].cookie else: _est, cookie = pctrls[0].controlValue lc.controlValue = (page_size, cookie) if not cookie: break else: logging.warning( "Server ignores RFC 2696 Simple Paged Results Control.") break output.close()
def create_controls(pagesize): """ Create an LDAP control with a page size of "pagesize". """ if LDAP24API: return SimplePagedResultsControl(True, size=pagesize, cookie='') else: return SimplePagedResultsControl(ldap.LDAP_CONTROL_PAGE_OID, True, (pagesize, ''))
def _search(self, basedn='', scope=pyldap.SCOPE_SUBTREE, filter='', timeout=-1, sizelimit=0): if not self._handle: self._open() result = [] serverctrls = None clientctrls = None paged = SimplePagedResultsControl(criticality=False, size=self.pagesize, cookie='') paged_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl } page = 0 while True: serverctrls = [paged] id = self._handle.search_ext(basedn, scope, filterstr=filter, attrlist=None, attrsonly=0, serverctrls=serverctrls, clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit) (rtype, rdata, rmsgid, serverctrls) = self._handle.result3(id, resp_ctrl_classes=paged_ctrls) result.extend(rdata) paged.size = 0 paged.cookie = cookie = None for sc in serverctrls: if sc.controlType == SimplePagedResultsControl.controlType: cookie = sc.cookie if cookie: paged.cookie = cookie paged.size = self.pagesize break if not cookie: break page += 1 return result
def get_ldap_lists(l, search_strings, parent_list=None): results = set() known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl, } req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='') if parent_list: filterstr = search_strings['get_all_sub_lists_filter'] % escape_filter_chars(parent_list) else: filterstr = search_strings['get_all_lists_filter'] while True: msgid = l.search_ext(search_strings['list_search_string'], ldap.SCOPE_SUBTREE, serverctrls=[req_ctrl], attrlist=(search_strings['list_cn_field'], search_strings['list_name_field']), filterstr=filterstr) retries = 0 rdata = None while retries < 5: try: rtype, rdata, rmsgid, serverctrls = l.result3(msgid, timeout=ldap_timeout, resp_ctrl_classes=known_ldap_resp_ctrls) except Exception: logger.exception('Error getting ldap list memberships') retries += 1 else: break # ldap sometimees returns good but does not return rdata if rdata is None: break for (dn, data) in rdata: cn_field = data[search_strings['list_cn_field']][0] name_field = data[search_strings['list_name_field']][0] if isinstance(cn_field, bytes): cn_field = cn_field.decode('utf-8') if isinstance(name_field, bytes): name_field = name_field.decode('utf-8') results |= {(cn_field, name_field)} pctrls = [c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType] if not pctrls: # Paging not supported logger.debug('LDAP pagination not supported') break cookie = pctrls[0].cookie if not cookie: break req_ctrl.cookie = cookie return results
def test_search_multiple_paging(topology_st, create_user): """Verify that after performing multiple simple paged searches on a single connection without a complition, it wouldn't fail. :id: 628b29a6-2d47-4116-a88d-00b87405ef7f :customerscenario: True :setup: Standalone instance, test user for binding, varying number of users for the search base :steps: 1. Bind as test user 2. Initiate the search with a simple paged control 3. Acquire the returned cookie only one time 4. Perform steps 2 and 3 three times in a row :expectedresults: 1. Bind should be successful 2. Search should be successfully initiated 3. Cookie should be successfully acquired 4. No error happens """ users_num = 100 page_size = 30 users_list = add_users(topology_st, users_num, DEFAULT_SUFFIX) search_flt = r'(uid=test*)' searchreq_attrlist = ['dn', 'sn'] try: log.info('Set user bind') conn = create_user.bind(TEST_USER_PWD) log.info('Create simple paged results control instance') req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='') controls = [req_ctrl] for ii in range(3): log.info('Iteration %d' % ii) msgid = conn.search_ext(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, search_flt, searchreq_attrlist, serverctrls=controls) rtype, rdata, rmsgid, rctrls = conn.result3(msgid) pctrls = [ c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType ] # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = conn.search_ext(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, search_flt, searchreq_attrlist, serverctrls=controls) finally: del_users(users_list)
def create_controls(pagesize): """Create an LDAP control with a page size of "pagesize".""" # Initialize the LDAP controls for paging. Note that we pass '' # for the cookie because on first iteration, it starts out empty. LDAP24API = StrictVersion(ldap.__version__) >= StrictVersion('2.4') if LDAP24API: return SimplePagedResultsControl(True, size=pagesize, cookie='') else: return SimplePagedResultsControl(ldap.LDAP_CONTROL_PAGE_OID, True, (pagesize, ''))
def test_search_invalid_cookie(topology_st, create_user, invalid_cookie): """Verify that using invalid cookie while performing search with the simple paged results control throws a TypeError exception :id: 107be12d-4fe4-47fe-ae86-f3e340a56f42 :customerscenario: True :parametrized: yes :setup: Standalone instance, test user for binding, varying number of users for the search base :steps: 1. Bind as test user 2. Initiate the search with a simple paged control 3. Put an invalid cookie (-1, 1000) to the control 4. Continue the search :expectedresults: 1. Bind should be successful 2. Search should be successfully initiated 3. Cookie should be added 4. It should throw a TypeError exception """ users_num = 100 page_size = 50 users_list = add_users(topology_st, users_num, DEFAULT_SUFFIX) search_flt = r'(uid=test*)' searchreq_attrlist = ['dn', 'sn'] try: log.info('Set user bind') conn = create_user.bind(TEST_USER_PWD) log.info('Create simple paged results control instance') req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='') controls = [req_ctrl] msgid = conn.search_ext(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, search_flt, searchreq_attrlist, serverctrls=controls) rtype, rdata, rmsgid, rctrls = conn.result3(msgid) log.info( 'Put an invalid cookie (%d) to the control. TypeError is expected' % invalid_cookie) req_ctrl.cookie = invalid_cookie with pytest.raises(TypeError): msgid = conn.search_ext(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, search_flt, searchreq_attrlist, serverctrls=controls) finally: del_users(users_list)
def handle(self, *args, **options): filter_string = options.get('filter', DEFAULT_FILTER) ldap_backend = LDAPBackend() ldap_user = _LDAPUser(backend=ldap_backend, username='') ldap_connection = ldap_user.connection # Initialize the LDAP controls for paging. # Note that we pass '' for the cookie because on first iteration, it starts out empty. ldap_controls = SimplePagedResultsControl(True, size=PAGE_SIZE, cookie='') while True: # Send search request # If you leave out the ATTRLIST it'll return all attributes which you have permissions to access. # You may want to adjust the scope level as well # (perhaps "ldap.SCOPE_SUBTREE", but it can reduce performance if you don't need it). message_id = ldap_connection.search_ext( base=settings.AUTH_LDAP_USER_SEARCH.base_dn, scope=ldap.SCOPE_SUBTREE, filterstr=filter_string, serverctrls=[ldap_controls]) # Pull the results from the search request rtype, rdata, rmsgid, server_controls = ldap_connection.result3( message_id) rdata = settings.AUTH_LDAP_USER_SEARCH._process_results(rdata) # Each "rdata" is a tuple of the form (dn, attrs), where dn is a string containing the # DN (distinguished name) of the entry, and attrs is a dictionary containing the attributes associated # with the entry. The keys of attrs are strings, and the associated values are lists of strings. for distinguished_name, attributes in rdata: username = "".join(attributes['uid']) print(f"{username} \t| {distinguished_name}") print(attributes) # Look through the returned controls and find the page controls. # This will also have our returned cookie which we need to make the next search request. page_controls = [ control for control in server_controls if control.controlType == SimplePagedResultsControl.controlType ] if not page_controls: print('Warning: Server ignores RFC 2696 control.') break # Ok, we did find the page control, yank the cookie from it and insert it into the control for our # next search. If however there is no cookie, we are done! cookie = page_controls[0].cookie ldap_controls.cookie = cookie if not cookie: break
def paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)', attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None, timeout=30, sizelimit=0): use_old_paging_api = False if hasattr(ldap, 'LDAP_CONTROL_PAGE_OID'): use_old_paging_api = True lc = SimplePagedResultsControl(ldap.LDAP_CONTROL_PAGE_OID, True, (self.page_size, '')) page_ctrl_oid = ldap.LDAP_CONTROL_PAGE_OID else: lc = ldap.controls.libldap.SimplePagedResultsControl( size=self.page_size, cookie='') page_ctrl_oid = ldap.controls.SimplePagedResultsControl.controlType msgid = self.search_ext(base, scope, filterstr, attrlist=attrlist, serverctrls=[lc]) pages = 0 all_results = [] while True: pages += 1 rtype, rdata, rmsgid, serverctrls = self.result3(msgid) all_results.extend(rdata) pctrls = [c for c in serverctrls if c.controlType == page_ctrl_oid] if pctrls: if use_old_paging_api: est, cookie = pctrls[0].controlValue lc.controlValue = (self.page_size, cookie) else: cookie = lc.cookie = pctrls[0].cookie if cookie: msgid = self.search_ext(base, ldap.SCOPE_SUBTREE, filterstr, attrlist=attrlist, serverctrls=[lc]) else: break else: raise ldap.LDAPError return all_results
def paged_search_ext_s(self, conn, base, scope, filter, attrs, attrsonly=0,serverctrls=None,clientctrls=None,timeout=-1,sizelimit=0): """ Helper function that implements a paged LDAP search for the Search method below. Args: query: LDAP filter to apply to the search sizelimit: max # of users to return. attrlist: list of attributes to return. If null, all attributes are returned Returns: A list of users as returned by the LDAP search """ ldap_page_size = 100 search_flt = r'(objectClass=*)' searchreq_attrlist = ['cn', 'entryDN', 'entryUUID', 'mail', 'objectClass'] req_ctrl = SimplePagedResultsControl( True, ldap_page_size, cookie='') logging.debug('Paged search on %s for %s' % (base, filter)) # Send first search request msgid = conn.search_ext( base, ldap.SCOPE_SUBTREE, search_flt, attrlist=searchreq_attrlist, serverctrls=(serverctrls or []) + [req_ctrl] ) result_pages = 0 all_results = [] while True: rtype, rdata, rmsgid, rctrls = conn.result3(msgid) all_results.extend(rdata) result_pages += 1 # Extract the simple paged results response control pctrls = [ c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType ] if pctrls: if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = conn.search_ext( base, ldap.SCOPE_SUBTREE, search_flt, attrlist=searchreq_attrlist, serverctrls=(serverctrls or []) + [req_ctrl] ) else: break return all_results
def paginated_query(connect, attributes, base, scope, search_flt, result_class=None): # how many users to search for in each page, # this depends on the server maximum setting # (default highest value is 1000) page_size = 500 req_ctrl = SimplePagedResultsControl(criticality=True, size=page_size, cookie="") msgid = connect.search_ext( base=base, scope=scope, filterstr=search_flt, attrlist=attributes, serverctrls=[req_ctrl], ) total_results = [] pages = 0 while True: # loop over all of the pages using the same cookie, # otherwise the search will fail pages += 1 rtype, rdata, rmsgid, serverctrls = connect.result3(msgid) for user in rdata: if not result_class: total_results.append(user) else: total_results.append(result_class(user)) pctrls = [ c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType ] if pctrls: if pctrls[ 0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = connect.search_ext( base=base, scope=scope, filterstr=search_flt, attrlist=attributes, serverctrls=[req_ctrl], ) else: break else: break return total_results
def search_users(req_page): con = get_ldap_con() sc = SimplePagedResultsControl(criticality=True, size=PAGESIZE, cookie='') search_results = None last_page = False i = 0 while True: if i >= req_page: break msgid = con.search_ext(base=BASEDN, scope=ldap.SCOPE_SUBTREE, filterstr=SEARCHFILTER.decode('utf8'), attrlist=USERS_ATTR_LIST, serverctrls=[sc]) # msgid is an integer representing LDAP operation id _, search_results, _, result_sc = con.result3(msgid) # result_data: list of tuples in the form of (dn, attrs), # dn is a string, ie: 'dn=alvaro,dc=example,dc=com', # attrs is a string key based dictionary. # result_sc: servercontrols list. pctrls = [ c for c in result_sc if c.controlType == SimplePagedResultsControl.controlType ] if not pctrls: last_page = True i += 1 break cookie = pctrls[0].cookie if not cookie: last_page = True i += 1 break sc.cookie = cookie i += 1 # Data mangle # We don't need Full DN or registers with empty DN search_results = purge_results(search_results) ret_json = get_response_template() ret_json['page'] = i ret_json['quant'] = len(search_results) ret_json['last'] = last_page ret_json['data'] = search_results return ret_json
def paged_search(self, base, sfilter, attrlist, scope='subtree', page_size=1000): if scope == 'one': _scope = ldap.SCOPE_ONELEVEL else: _scope = ldap.SCOPE_SUBTREE lc = SimplePagedResultsControl( ldap.LDAP_CONTROL_PAGE_OID,True,(page_size,'') ) # Send search request msgid = self.l.search_ext( base, _scope, sfilter, attrlist=attrlist, serverctrls=[lc] ) results = [] pages = 0 while True: pages += 1 #print "Getting page %d" % (pages,) rtype, rdata, rmsgid, serverctrls = self.l.result3(msgid) #print '%d results' % len(rdata) for dn,data in rdata: _r = data _r['dn'] = dn results.append(_r) #results += [i[0] for i in rdata] #pprint.pprint(rdata[0]) pctrls = [ c for c in serverctrls if c.controlType == ldap.LDAP_CONTROL_PAGE_OID ] if pctrls: est, cookie = pctrls[0].controlValue if cookie: lc.controlValue = (page_size, cookie) msgid = l.search_ext( base, _scope, sfilter, attrlist=attrlist, serverctrls=[lc] ) else: break else: print "Warning: Server ignores RFC 2696 control." break return results
def paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)', attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None, timeout=-1, sizelimit=0): """ Behaves exactly like LDAPObject.search_ext_s() but internally uses the simple paged results control to retrieve search results in chunks. This is non-sense for really large results sets which you would like to process one-by-one """ req_ctrl = SimplePagedResultsControl(True, size=self.page_size, cookie='') # Send first search request msgid = self.search_ext(base, ldap.SCOPE_SUBTREE, search_flt, attrlist=searchreq_attrlist, serverctrls=(serverctrls or []) + [req_ctrl]) result_pages = 0 all_results = [] while True: rtype, rdata, rmsgid, rctrls = self.result3(msgid) all_results.extend(rdata) result_pages += 1 # Extract the simple paged results response control pctrls = [ c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType ] if pctrls: if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = self.search_ext( base, ldap.SCOPE_SUBTREE, search_flt, attrlist=searchreq_attrlist, serverctrls=(serverctrls or []) + [req_ctrl]) else: break return result_pages, all_results
def _paged_search(self, base_dn, scope, search_filter, attrs): conn = self.connect() # Get paged results to prevent exceeding server size limit page_size = 1000 if PYTHON_LDAP_24: lc = SimplePagedResultsControl(size=page_size, cookie='') else: lc = SimplePagedResultsControl(LDAP_CONTROL_PAGED_RESULTS, True, (page_size, ''),) is_last_page = False results = [] while not is_last_page: msgid = conn.search_ext(base_dn, scope, search_filter, attrs, serverctrls=[lc]) rtype, rdata, rmsgid, serverctrls = conn.result3(msgid) pctrls = [c for c in serverctrls if c.controlType == LDAP_CONTROL_PAGED_RESULTS] results.extend(rdata) if pctrls: if PYTHON_LDAP_24: cookie = pctrls[0].cookie if cookie: lc.cookie = cookie else: is_last_page = True else: cookie = pctrls[0].controlValue[1] if cookie: # lc.controlValue seems to have been mutable at some # point, now it's a tuple. cv = list(lc.controlValue) cv[1] = cookie lc.controlValue = tuple(cv) else: is_last_page = True else: is_last_page = True if results: # If the search returned an empty result, page controls # aren't included - so don't produce a bogus warning logger.warn( "Server ignores paged results control (RFC 2696).") return results
def get_ldap_list_membership(l, search_strings, list_name): results = set() known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl, } req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='') while True: msgid = l.search_ext( search_strings['user_search_string'], ldap.SCOPE_SUBTREE, serverctrls=[req_ctrl], attrlist=[search_strings['user_mail_field']], filterstr=search_strings['user_membership_filter'] % escape_filter_chars(list_name)) retries = 0 rdata = None while retries < 5: try: rtype, rdata, rmsgid, serverctrls = l.result3( msgid, timeout=ldap_timeout, resp_ctrl_classes=known_ldap_resp_ctrls) except Exception: logger.exception('Error getting ldap list memberships') retries += 1 else: break # ldap sometimees returns good but does not return rdata if rdata is None: break for data in rdata: member = data[1].get(search_strings['user_mail_field'], [None])[0] if isinstance(member, bytes): member = member.decode('utf-8') results |= {member} pctrls = [ c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType ] if not pctrls: # Paging not supported logger.debug('LDAP pagination not supported') break cookie = pctrls[0].cookie if not cookie: break req_ctrl.cookie = cookie return results
def paged_search(self, filterstr='(objectClass=*)', attrlist=None, attrsonly=0, timeout=-1): """Searches Active Directory for a given search string. Uses RFC 2696 paged results control to search AD. This comes into play when the search yields more then ldap.SIZE_LIMIT results. This search uses the LDAP_BASE value read from configuration. If there is need for searching higher up in the AD tree then the configuration should be changed. Args: filterstr (str): The search term to look for in AD. attrlist (list): List of attributes to get from AD. Defaults to all. attrsonly (bool): Only gets attributes, not values. timeout (int): Time the search waits for results before giving up. Returns: list: List with all results. """ #Simple paged results control to keep track of the search status req_ctrl = SimplePagedResultsControl(ldap.LDAP_CONTROL_PAGE_OID, True, (self.LDAP_PAGE_SIZE, '')) #Send first search request msgid = self.ldap.search_ext(self.LDAP_BASE, ldap.SCOPE_SUBTREE, filterstr=filterstr, attrlist=attrlist, serverctrls=[req_ctrl], timeout=timeout) all_results = [] while True: rtype, rdata, rmsgid, rctrls = self.ldap.result3(msgid) all_results.extend(rdata) #Extract the simple paged results response control pctrls = [c for c in rctrls if c.controlType == ldap.LDAP_CONTROL_PAGE_OID] if pctrls: est, cookie = pctrls[0].controlValue if cookie: #Copy cookie from response control to request control req_ctrl.controlValue = (self.LDAP_PAGE_SIZE, cookie) #Continue the search with updated request control msgid = self.ldap.search_ext(self.LDAP_BASE, ldap.SCOPE_SUBTREE, filterstr=filterstr, attrlist=attrlist, serverctrls=[req_ctrl], timeout=timeout) else: break else: print("Warning: Server ignores RFC 2696 control.") break return all_results
def _search(self, basedn='', scope=ldap.SCOPE_SUBTREE, filter='', timeout=-1, sizelimit=0): if not self._handle: self._open() result = [] serverctrls = None clientctrls = None paged = SimplePagedResultsControl( criticality=False, size=self.pagesize, cookie='' ) paged_ctrls = {SimplePagedResultsControl.controlType: SimplePagedResultsControl} page = 0 while True: serverctrls = [paged] id = self._handle.search_ext( basedn, scope, filterstr=filter, attrlist=None, attrsonly=0, serverctrls=serverctrls, clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit ) (rtype, rdata, rmsgid, serverctrls) = self._handle.result3( id, resp_ctrl_classes=paged_ctrls ) result.extend(rdata) paged.size = 0 paged.cookie = cookie = None for sc in serverctrls: if sc.controlType == SimplePagedResultsControl.controlType: cookie = sc.cookie if cookie: paged.cookie = cookie paged.size = self.pagesize break if not cookie: break page += 1 return result
def _PagedAsyncSearch(ldap_conn, sizelimit, base_dn, scope, filterstr='(objectClass=*)', attrlist=None): """ Helper function that implements a paged LDAP search for the Search method below. Args: ldap_conn: our OMLdapConnection object sizelimit: max # of users to return. filterstr: LDAP filter to apply to the search attrlist: list of attributes to return. If null, all attributes are returned Returns: A list of users as returned by the LDAP search """ paged_results_control = SimplePagedResultsControl( ldap.LDAP_CONTROL_PAGE_OID, True, (_PAGE_SIZE, '')) logging.debug('Paged search on %s for %s', base_dn, filterstr) users = [] ix = 0 while True: if _PAGE_SIZE == 0: serverctrls = [] else: serverctrls = [paged_results_control] msgid = ldap_conn.conn.search_ext(base_dn, scope, filterstr, attrlist=attrlist, serverctrls=serverctrls) res = ldap_conn.conn.result3(msgid=msgid) unused_code, results, unused_msgid, serverctrls = res for result in results: ix += 1 users.append(result) if sizelimit and ix >= sizelimit: break if sizelimit and ix >= sizelimit: break cookie = None for serverctrl in serverctrls: if serverctrl.controlType == ldap.LDAP_CONTROL_PAGE_OID: unused_est, cookie = serverctrl.controlValue if cookie: paged_results_control.controlValue = (_PAGE_SIZE, cookie) break if not cookie: break return users
def get_ldap_lists(l, search_strings, parent_list=None): results = set() known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl, } req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='') if parent_list: filterstr = search_strings[ 'get_all_sub_lists_filter'] % escape_filter_chars(parent_list) else: filterstr = search_strings['get_all_lists_filter'] while True: msgid = l.search_ext(search_strings['list_search_string'], ldap.SCOPE_SUBTREE, serverctrls=[req_ctrl], attrlist=(search_strings['list_cn_field'], search_strings['list_name_field']), filterstr=filterstr) rtype, rdata, rmsgid, serverctrls = l.result3( msgid, timeout=ldap_timeout, resp_ctrl_classes=known_ldap_resp_ctrls) for (dn, data) in rdata: cn_field = data[search_strings['list_cn_field']][0] name_field = data[search_strings['list_name_field']][0] if isinstance(cn_field, bytes): cn_field = cn_field.decode('utf-8') if isinstance(name_field, bytes): name_field = name_field.decode('utf-8') results |= {(cn_field, name_field)} pctrls = [ c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType ] if not pctrls: # Paging not supported break cookie = pctrls[0].cookie if not cookie: break req_ctrl.cookie = cookie return results
def ldap_search(username): auth_info = AuthMode.get_or_none(AuthMode.mode == "LDAP") if not auth_info: return auth_info = model_to_dict(auth_info) config = json.loads(auth_info.get('config')) host = config.get('host') port = config.get('port') Server = "ldap://{}:{}".format(host, port) baseDN = config.get('basedn') ldapuser = config.get('account') or "" ldappass = config.get('password') or "" dn = "" try: conn = ldap.initialize(Server) retrieveAttributes = [ 'name', 'sAMAccountName', 'searchFiltername', 'mail' ] conn.protocol_version = ldap.VERSION3 conn.simple_bind_s(ldapuser, ldappass) searchScope = ldap.SCOPE_SUBTREE searchFiltername = config.get("loginname_property") or "sAMAccountName" searchFilter = '{}=*{}*'.format(searchFiltername, username) pg_ctrl = SimplePagedResultsControl(True, size=20, cookie="") ldap_result = conn.search_ext_s(baseDN, searchScope, searchFilter, retrieveAttributes, serverctrls=[pg_ctrl]) r1 = [] pg_ctrl = SimplePagedResultsControl(True, size=20, cookie="") searchFilter = 'name=*{}*'.format(username) r1 = conn.search_ext_s(baseDN, searchScope, searchFilter, retrieveAttributes, serverctrls=[pg_ctrl]) return True, list(ldap_result) + list(r1) except Exception as e: return False, str(e) return True, []
def _paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)', attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None, timeout=-1, sizelimit=0, page_size=10): """ Behaves similarly to LDAPObject.search_ext_s() but internally uses the simple paged results control to retrieve search results in chunks. Taken from the python-ldap paged_search_ext_s.py demo, showing how to use the paged results control: https://bitbucket.org/jaraco/python-ldap/ """ request_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='') results = [] while True: msgid = self.conn.search_ext(base, scope, filterstr=filterstr, attrlist=attrlist, attrsonly=attrsonly, serverctrls=(serverctrls or []) + [request_ctrl], clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit) result_type, result_data, result_msgid, result_ctrls = self.conn.result3( msgid) results.extend(result_data) # Extract the simple paged results response control paged_ctrls = [ c for c in result_ctrls if c.controlType == SimplePagedResultsControl.controlType ] if paged_ctrls and paged_ctrls[0].cookie: # Copy cookie from response control to request control request_ctrl.cookie = paged_ctrls[0].cookie else: break return results
def _paged_search(self, basedn, searchfilter, attrlist=None, pagesize=100, sizelimit=0): """ Performs a pages search against the LDAP server. Code lifted from: https://gist.github.com/mattfahrner/c228ead9c516fc322d3a """ # Initialize the LDAP controls for paging. Note that we pass '' # for the cookie because on first iteration, it starts out empty. controls = SimplePagedResultsControl(True, size=pagesize, cookie='') # Do searches until we run out of pages to get from the LDAP server. results = [] while True: # Send search request. msgid = self.connection.search_ext( basedn, ldap.SCOPE_SUBTREE, searchfilter, attrlist, serverctrls=[controls], sizelimit=sizelimit ) rtype, rdata, rmsgid, serverctrls = self.connection.result3(msgid) # Each "rdata" is a tuple of the form (dn, attrs), where dn is # a string containing the DN (distinguished name) of the entry, # and attrs is a dictionary containing the attributes associated # with the entry. The keys of attrs are strings, and the associated # values are lists of strings. for dn, attrs in rdata: # AD returns an rdata at the end that is a reference that we want to ignore if type(attrs) == dict: results.append((dn, attrs)) # Get cookie for the next request. paged_controls = self._get_pctrls(serverctrls) if not paged_controls: self.logger.warning( f'paged_search.rfc2696_control_ignored searchfilter={searchfilter}' f'attrlist={",".join(attrlist)} pagesize={pagesize} sizelimit={sizelimit}' ) break # Push cookie back into the main controls. controls.cookie = paged_controls[0].cookie # If there is no cookie, we're done! if not paged_controls[0].cookie: break return results
def paged_search(self, search_dn, filter, scope, results_processor, attributes=None): if not attributes: attributes = ['*'] page_control = SimplePagedResultsControl(True, self.PAGE_SIZE, '') serverctrls = [page_control] msgid = self.connection.search_ext( search_dn, scope, filter, attrlist=attributes, serverctrls=serverctrls ) page = 0 records = 0 while True: page += 1 try: result_type, results, result_msg, serverctrls = self.connection.result3(msgid=msgid, timeout=self.LDAP_TIMEOUT) records += len(results) results_processor(results) pagectrls = [ c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType ] if pagectrls: if pagectrls[0].cookie: page_control.cookie = pagectrls[0].cookie msgid = self.connection.search_ext( search_dn, scope, filter, attrlist=attributes, serverctrls=[page_control] ) else: break else: break except ldap.LDAPError, e: print e
def _PagedAsyncSearch(self, query, sizelimit, attrlist=None): """ Helper function that implements a paged LDAP search for the Search method below. Args: query: LDAP filter to apply to the search sizelimit: max # of users to return. attrlist: list of attributes to return. If null, all attributes are returned Returns: A list of users as returned by the LDAP search """ if not self.IsUsingLdapLibThatSupportsPaging(): logging.error('Your version of python-ldap is too old to support ' 'paged LDAP queries. Aborting search.') return None paged_results_control = SimplePagedResultsControl( ldap.LDAP_CONTROL_PAGE_OID, True, (self.ldap_page_size, '')) logging.debug('Paged search on %s for %s' % (self.ldap_base_dn, query)) users = [] ix = 0 while True: if self.ldap_page_size == 0: serverctrls = [] else: serverctrls = [paged_results_control] msgid = self.conn.search_ext(self.ldap_base_dn, ldap.SCOPE_SUBTREE, query, attrlist=attrlist, serverctrls=serverctrls) res = self.conn.result3(msgid=msgid, timeout=self.ldap_timeout) unused_code, results, unused_msgid, serverctrls = res for result in results: ix += 1 users.append(result) if sizelimit and ix >= sizelimit: break if sizelimit and ix >= sizelimit: break cookie = None for serverctrl in serverctrls: if serverctrl.controlType == ldap.LDAP_CONTROL_PAGE_OID: unused_est, cookie = serverctrl.controlValue if cookie: paged_results_control.controlValue = (self.ldap_page_size, cookie) break if not cookie: break return users
def getEnabledAccountList (ldapHandle, fhandle): # define variables for query page_size = 1000 filter = "(&(sAMAccountName=*)(userAccountControl=" + str(ENA_NUM) + "))" attributeList = ['sAMAccountName', 'mail'] lc = SimplePagedResultsControl (ldap.LDAP_CONTROL_PAGE_OID, True, (page_size, "")) msgid = ldapHandle.search_ext ( BASE, SCOPE, filter, attributeList, serverctrls = [lc] ) pages = 0 while True: pages += 1 print "Getting page %d" % pages rtype, rdata, rmsgid, serverctrls = ldapHandle.result3 (msgid) print "%d results" % len (rdata) for pi in rdata: if (len (pi) == 2) and (type (pi[1]) is dict) and (len(pi[1]) == 2): #print pi #print pi[1] accName = pi[1]['sAMAccountName'][0] mail = pi[1]['mail'][0] if (accName + SUFFIX).lower() != mail.lower(): #fhandle.write (accName + "," + mail + "\n") print (accName + "," + mail + "\n") pctrls = [ c for c in serverctrls if c.controlType == ldap.LDAP_CONTROL_PAGE_OID ] if pctrls: est, cookie = pctrls[0].controlValue if cookie: lc.controlValue = (page_size, cookie) msgid = ldapHandle.search_ext (BASE, SCOPE, filter, attributeList, serverctrls=[lc]) else: break else: print "Warning" break
def paged_search_ext_s(self,base,scope,filterstr='(objectClass=*)',attrlist=None,attrsonly=0,serverctrls=None,clientctrls=None,timeout=-1,sizelimit=0,criticality=True,page_size=1000): """ Behaves exactly like LDAPObject.search_ext_s() but internally uses the simple paged results control to retrieve search results in chunks. This is non-sense for really large results sets which you would like to process one-by-one """ req_ctrl = SimplePagedResultsControl(criticality,size=page_size,cookie='') # Send first search request msgid = self.search_ext( base, scope, filterstr, attrlist=attrlist, serverctrls=(serverctrls or [])+[req_ctrl] ) result_pages = 0 all_results = [] while True: rtype, rdata, rmsgid, rctrls = self.result3(msgid) all_results.extend(rdata) result_pages += 1 # Extract the simple paged results response control pctrls = [ c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType ] if pctrls: if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = self.search_ext( base, scope, filterstr, attrlist=attrlist, serverctrls=(serverctrls or [])+[req_ctrl] ) else: break return result_pages,all_results
def search(self, base_dn=None, scope=ldap.SCOPE_SUBTREE, filter='(objectClass=*)', attrs=[]): conn = self.connect() if conn is None: return [] if base_dn is None: base_dn = self.base_dn # Get paged results to prevent exceeding server size limit page_size = 1000 if PYTHON_LDAP_24: lc = SimplePagedResultsControl(size=page_size, cookie='') else: lc = SimplePagedResultsControl(LDAP_CONTROL_PAGED_RESULTS, True, (page_size, ''),) is_last_page = False results = [] while not is_last_page: msgid = conn.search_ext(base_dn, scope, filter, attrs, serverctrls=[lc]) rtype, rdata, rmsgid, serverctrls = conn.result3(msgid) results.extend(rdata) pctrls = [c for c in serverctrls if c.controlType == LDAP_CONTROL_PAGED_RESULTS] if pctrls: if PYTHON_LDAP_24: cookie = pctrls[0].cookie if cookie: lc.cookie = cookie else: is_last_page = True else: cookie = pctrls[0].controlValue[1] if cookie: lc.controlValue[1] = cookie else: is_last_page = True else: is_last_page = True logger.warn("Server ignores paged results control (RFC 2696).") return results
def _search(self, search_flt, attrs, limit=None, path_prefix=None): """ Query this academic calendar for records matching the search filter. Only looks one level deep in the tree to improve performance, so make sure you pass the dn that you want to look directly inside of as the `path` parameter. Returns a list of records. Each record is a dictionary containing the requested attributes. search_flt -- LDAP-style search filter attrs -- attributes to be retrieved limit -- max number of records to return path -- extra LDAP dn: is prepended to this object's basedn ie the LDAP directory to look from relative to the root """ if limit is None: limit = float('inf') if path_prefix is None: path = self._basedn else: path = '{}{}'.format(path_prefix, self._basedn) page_control = SimplePagedResultsControl( criticality=True, size=RemoteLDAPDatabase.PAGE_SIZE, cookie='' ) results = [] pages_retrieved = 0 first_pass = True while pages_retrieved*RemoteLDAPDatabase.PAGE_SIZE < limit \ and page_control.cookie \ or first_pass: first_pass = False try: msgid = self._client.search_ext(path, ldap.SCOPE_ONELEVEL, search_flt, attrlist=attrs, serverctrls=[page_control]) except: raise pages_retrieved += 1 _, result_data, msgid, serverctrls = self._client.result3(msgid) page_control.cookie = serverctrls[0].cookie results += extract_results_from_ldap_data(result_data) return results
def ldap_paged_async_search(base, scope, filt, columns): ldap_log(' PAGED ASYNC SEARCH') page_size = config.ldap_connection.get('page_size', 100) if ldap_compat: lc = SimplePagedResultsControl(size = page_size, cookie = '') else: lc = SimplePagedResultsControl( LDAP_CONTROL_PAGED_RESULTS, True, (page_size, '') ) results = [] while True: # issue the ldap search command (async) msgid = ldap_connection.search_ext(base, scope, filt, columns, serverctrls = [lc]) unused_code, response, unused_msgid, serverctrls = ldap_connection.result3( msgid = msgid, timeout = config.ldap_connection.get('response_timeout', 5) ) for result in response: results.append(result) # Mark current position in pagination control for next loop cookie = None for serverctrl in serverctrls: if serverctrl.controlType == LDAP_CONTROL_PAGED_RESULTS: if ldap_compat: cookie = serverctrl.cookie if cookie: lc.cookie = cookie else: cookie = serverctrl.controlValue[1] if cookie: lc.controlValue = (page_size, cookie) break if not cookie: break return results
def _run_ldap_query(self, query): self.conn = get_ldap() lc = SimplePagedResultsControl(size=LDAP_RESULTS_PAGE_SIZE, cookie='') msgid = self.conn.search_ext( settings.AUTH_LDAP_USER_SEARCH_BASE, ldap.SCOPE_SUBTREE, query, serverctrls=[lc], ) page_num = 0 while True: page_num += 1 r_type, r_data, r_msgid, serverctrls = self.conn.result3(msgid) print "Pack of %s users loaded (page %s)" % ( LDAP_RESULTS_PAGE_SIZE, page_num, ) for item in r_data: yield item if serverctrls: if serverctrls[0].cookie: lc.size = LDAP_RESULTS_PAGE_SIZE lc.cookie = serverctrls[0].cookie msgid = self.conn.search_ext( settings.AUTH_LDAP_USER_SEARCH_BASE, ldap.SCOPE_SUBTREE, query, serverctrls=[lc], ) else: break else: logger.error( 'LDAP::_run_ldap_query\tQuery: %s\t' 'Server ignores RFC 2696 control', ) sys.exit(-1) self._disconnect()
def list_machines(self, name=None, attrs=None): if attrs is None: attrs = COMPUTER_ATTRS oc_filter = '(objectclass=computer)' desc = self.machine_description(name) page_size = 64 lc = SimplePagedResultsControl( ldap.LDAP_CONTROL_PAGE_OID, True, (page_size,'')) conn = self.open() msgid = conn.search_ext( desc, ldap.SCOPE_SUBTREE, oc_filter, attrs, serverctrls=[lc]) rval = {} while True: # (rtype, rdata, rmsgid, serverctrls) _, rdata, _, serverctrls = conn.result3(msgid) # Synthesize the machines into a simple form. for machine in rdata: if machine[0]: for cn in machine[1]['cn']: rval[cn.lower()] = machine[1] # Read the next page of the result. pctrls = [ c for c in serverctrls if c.controlType == ldap.LDAP_CONTROL_PAGE_OID ] if pctrls: # (est, cookie) _, cookie = pctrls[0].controlValue if cookie: lc.controlValue = (page_size, cookie) msgid = conn.search_ext( desc, ldap.SCOPE_SUBTREE, oc_filter, attrs, serverctrls=[lc]) else: break return rval
def paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)', attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None, timeout=30, sizelimit=0): use_old_paging_api = False if hasattr(ldap, 'LDAP_CONTROL_PAGE_OID'): use_old_paging_api = True lc = SimplePagedResultsControl( ldap.LDAP_CONTROL_PAGE_OID,True,(self.page_size,'') ) page_ctrl_oid = ldap.LDAP_CONTROL_PAGE_OID else: lc = ldap.controls.libldap.SimplePagedResultsControl(size=self.page_size,cookie='') page_ctrl_oid = ldap.controls.SimplePagedResultsControl.controlType msgid = self.search_ext(base, scope, filterstr, attrlist=attrlist, serverctrls=[lc]) pages = 0 all_results = [] while True: pages += 1 rtype, rdata, rmsgid, serverctrls = self.result3(msgid) all_results.extend(rdata) pctrls = [ c for c in serverctrls if c.controlType == page_ctrl_oid ] if pctrls: if use_old_paging_api: est, cookie = pctrls[0].controlValue lc.controlValue = (self.page_size, cookie) else: cookie = lc.cookie = pctrls[0].cookie if cookie: msgid = self.search_ext(base, ldap.SCOPE_SUBTREE, filterstr, attrlist=attrlist, serverctrls=[lc]) else: break else: raise ldap.LDAPError return all_results
def paged_search_ext_s(self,base,scope,filterstr='(objectClass=*)',attrlist=None,attrsonly=0,serverctrls=None,clientctrls=None,timeout=-1,sizelimit=0): """ Behaves exactly like LDAPObject.search_ext_s() but internally uses the simple paged results control to retrieve search results in chunks. This is non-sense for really large results sets which you would like to process one-by-one """ while True: # loop for reconnecting if necessary req_ctrl = SimplePagedResultsControl(True,size=self.page_size,cookie='') try: # Send first search request msgid = self.search_ext( base, scope, filterstr=filterstr, attrlist=attrlist, attrsonly=attrsonly, serverctrls=(serverctrls or [])+[req_ctrl], clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit ) result_pages = 0 all_results = [] while True: rtype, rdata, rmsgid, rctrls = self.result3(msgid) all_results.extend(rdata) result_pages += 1 # Extract the simple paged results response control pctrls = [ c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType ] if pctrls: if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = self.search_ext( base, scope, filterstr=filterstr, attrlist=attrlist, attrsonly=attrsonly, serverctrls=(serverctrls or [])+[req_ctrl], clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit ) else: break # no more pages available except ldap.SERVER_DOWN,e: try: self.reconnect(self._uri) except AttributeError: raise e else: return result_pages,all_results
url = "ldap://localhost:1390/" base = "dc=stroeder,dc=de" search_flt = r'(objectClass=*)' page_size = 10 import ldap from ldap.controls import SimplePagedResultsControl ldap.set_option(ldap.OPT_REFERRALS, 0) l = ldap.initialize(url) l.protocol_version = 3 l.simple_bind_s("", "") lc = SimplePagedResultsControl( ldap.LDAP_CONTROL_PAGE_OID,True,(page_size,'') ) # Send search request msgid = l.search_ext( base, ldap.SCOPE_SUBTREE, search_flt, serverctrls=[lc] ) pages = 0 while True: pages += 1 print("Getting page %d" % (pages,)) rtype, rdata, rmsgid, serverctrls = l.result3(msgid) print('%d results' % len(rdata))
base = "dc=stroeder,dc=de" search_flt = r'(objectClass=*)' page_size = 10 import ldap,pprint from ldap.controls import SimplePagedResultsControl searchreq_attrlist=['cn','entryDN','entryUUID','mail','objectClass'] #ldap.set_option(ldap.OPT_DEBUG_LEVEL,255) ldap.set_option(ldap.OPT_REFERRALS, 0) l = ldap.initialize(url,trace_level=1) l.protocol_version = 3 l.simple_bind_s("", "") req_ctrl = SimplePagedResultsControl(True,size=page_size,cookie='') known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType:SimplePagedResultsControl, } # Send search request msgid = l.search_ext( base, ldap.SCOPE_SUBTREE, search_flt, attrlist=searchreq_attrlist, serverctrls=[req_ctrl] ) pages = 0
def discover(self, args): import omero ldap = self.__import_ldap__() c = self.ctx.conn(args) iconfig = c.sf.getConfigService() iadmin = c.sf.getAdminService() iquery = c.sf.getQueryService() LDAP_PROPERTIES = """ omero.ldap.urls omero.ldap.username omero.ldap.password omero.ldap.base omero.ldap.user_filter omero.ldap.user_mapping omero.ldap.group_filter omero.ldap.group_mapping omero.ldap.new_user_group """.split() cfg = dict() for key in LDAP_PROPERTIES: cfg[key.split(".")[-1]] = iconfig.getConfigValue(key) urls = args.urls and args.urls or cfg["urls"] basedn = args.base and args.base or cfg["base"] for url in urls.split(","): self.ctx.err("Connecting to %s..." % url) ld = ldap.initialize(url) ld.simple_bind_s(cfg['username'], cfg['password']) user_filter = cfg["user_filter"] user_mapping = cfg["user_mapping"] user_mapping = user_mapping.split(",") omeName_mapping = None for um in user_mapping: parts = um.split("=") if parts[0] == "omeName": omeName_mapping = parts[1] from ldap.controls import SimplePagedResultsControl cookie = '' # This is the limit for Active Directory, 1000. However # the LDAP connection has a sizeLimit that overrides # this value if the page_size exceeds it so it is safe # to enter pretty much anything here when using paged results. page_size = 1000 results = [] first = True page_control = SimplePagedResultsControl(False, page_size, cookie) while first or page_control.cookie: first = False try: msgid = ld.search_ext( basedn, ldap.SCOPE_SUBTREE, user_filter, serverctrls=[page_control] ) except: self.ctx.die(1, "Failed to execute LDAP search") result_type, results, msgid, serverctrls = ld.result3(msgid) if serverctrls: page_control.cookie = serverctrls[0].cookie user_names = set() user_dns = {} for dn, entry in results: omeName = entry[omeName_mapping] if isinstance(omeName, (list, tuple)): if len(omeName) == 1: omeName = omeName[0] user_names.add(omeName) user_dns[omeName] = dn else: self.ctx.err("Failed to unwrap omeName: %s" % omeName) continue if not user_names: continue # Early exit! from omero.rtypes import rlist from omero.rtypes import rstring from omero.rtypes import unwrap params = omero.sys.ParametersI() params.add("names", rlist([rstring(x) for x in user_names])) id_names = unwrap(iquery.projection( "select id, omeName from Experimenter " "where omeName in (:names)", params)) for eid, omeName in id_names: try: olddn = iadmin.lookupLdapAuthExperimenter(eid) dn = user_dns[omeName] except omero.ApiUsageException: continue # Unknown user if olddn: if olddn.lower() != dn.lower(): self.ctx.err("Found different DN for %s: %s" % (omeName, olddn)) else: self.ctx.dbg("DN already set for %s: %s" % (omeName, olddn)) else: if args.commands: self.ctx.out("%s ldap setdn %s %s" % (sys.argv[0], omeName, dn)) else: self.ctx.out("Experimenter:%s\tomeName=%s\t%s" % (eid, omeName, dn))
import time from ldap.controls import SimplePagedResultsControl import ldap.modlist as modlist def pagedLDAPSearch(base, filter, attributes, timeout=0, pageSize=100, server='xldap.cern.ch', who="", cred="", scope=ldap.SCOPE_SUBTREE): returnValue = [] ldapServer = 'ldap://' + server try: ldapConnection = ldap.initialize(ldapServer) except ldap.LDAPError, err: sys.stderr.write('Error connecting to server ' + ldapServer + '\nReason - ' + str(err)) return returnValue pagedResultsControl = SimplePagedResultsControl(ldap.LDAP_CONTROL_PAGE_OID, True, (pageSize, '')) results = [] pageCounter = 0 while True: #print 'LoopCounter----%d' % pageCounter serverControls = [pagedResultsControl] try: messageId = ldapConnection.search_ext(base, scope, filter, attributes, serverctrls = serverControls) except ldap.LDAPError, err: sys.stderr.write('Error performing user paged search: ' + str(err) + '\n') return returnValue try: