def get_all_users(self, username, password): base_dn = 'OU=上海二三四五网络科技有限公司,DC=trump,DC=local' self.l.simple_bind_s(username + "@trump.local", password) PAGE_SIZE = 500 # 设置每页返回的条数 pg_ctrl = SimplePagedResultsControl(True, size=PAGE_SIZE, cookie="") userdata = [] retdata = [] while True: msgid = self.l.search_ext(base_dn, ldap.SCOPE_SUBTREE, "(&(objectClass=person))", None, serverctrls=[pg_ctrl]) _a, res_data, _b, srv_ctrls = self.l.result3(msgid) userdata.extend(res_data) cookie = srv_ctrls[0].cookie if cookie: pg_ctrl.cookie = cookie else: break print('totalnum:', len(userdata)) for u in userdata: retdict = {} for k, v in u[1].items(): retdict[k] = v[0] retdata.append(retdict) return retdata
def get_ldap_lists(l, search_strings, parent_list=None): results = set() known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl, } req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='') if parent_list: filterstr = search_strings['get_all_sub_lists_filter'] % escape_filter_chars(parent_list) else: filterstr = search_strings['get_all_lists_filter'] while True: msgid = l.search_ext(search_strings['list_search_string'], ldap.SCOPE_SUBTREE, serverctrls=[req_ctrl], attrlist=(search_strings['list_cn_field'], search_strings['list_name_field']), filterstr=filterstr) rtype, rdata, rmsgid, serverctrls = l.result3(msgid, resp_ctrl_classes=known_ldap_resp_ctrls) results |= {(data[search_strings['list_cn_field']][0], data[search_strings['list_name_field']][0]) for (dn, data) in rdata} pctrls = [c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType] cookie = pctrls[0].cookie if not cookie: break req_ctrl.cookie = cookie return results
def paged_search_iter(self, filter, attrs=None): # Test for version 2.3 API try: lc = SimplePagedResultsControl \ (ldap.LDAP_CONTROL_PAGE_OID, True, (self.page_size, '')) api_version = 3 except AttributeError: # New version 2.4 API lc = SimplePagedResultsControl \ (True, size = self.page_size, cookie = '') sc = \ { SimplePagedResultsControl.controlType : SimplePagedResultsControl } api_version = 4 res = self.ldcon.search_ext \ ( self.cfg.LDAP_BASE_DN , ldap.SCOPE_SUBTREE , filter , attrlist = attrs , serverctrls = [lc] ) while True: params = (res, ) if api_version == 3: rtype, rdata, rmsgid, serverctrls = self.ldcon.result3(res) pctrls = \ [c for c in serverctrls if c.controlType == ldap.LDAP_CONTROL_PAGE_OID ] else: rtype, rdata, rmsgid, serverctrls = self.ldcon.result3 \ (res, resp_ctrl_classes = sc) pctrls = \ [c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType ] for r in rdata: if not r[0]: continue r = LDAP_Search_Result(r) yield r if pctrls: if api_version == 3: x, cookie = pctrls[0].controlValue lc.controlValue = (self.page_size, cookie) else: cookie = pctrls[0].cookie lc.cookie = cookie if not cookie: break res = self.ldcon.search_ext \ ( self.cfg.LDAP_BASE_DN , ldap.SCOPE_SUBTREE , filter , attrs , serverctrls = [lc] ) else: break
def paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)', attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None, timeout=-1, sizelimit=0): req_ctrl = SimplePagedResultsControl( True, size=self.page_size, cookie='', ) result_pages = 0 all_results = [] msgid = self.search_ext( base, scope, filterstr, attrlist=attrlist, serverctrls=(serverctrls or [])+[req_ctrl], ) while True: rtype, rdata, rmsgid, rctrls = self.result3(msgid) all_results.extend(rdata) result_pages += 1 pctrls = [c for c in rctrls if c.controlType == sprc_ct] if not pctrls: continue if not pctrls[0].cookie: break req_ctrl.cookie = pctrls[0].cookie msgid = self.search_ext( base, scope, filterstr, attrlist=attrlist, serverctrls=(serverctrls or [])+[req_ctrl] ) return all_results
def paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)', attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None, timeout=-1, sizelimit=0): """ Behaves exactly like LDAPObject.search_ext_s() but internally uses the simple paged results control to retrieve search results in chunks. """ req_ctrl = SimplePagedResultsControl(True, size=self.page_size, cookie='') # Send first search request msgid = self.search_ext(base, ldap.SCOPE_SUBTREE, filterstr, attrlist=attrlist, serverctrls=(serverctrls or []) + [req_ctrl]) results = [] while True: rtype, rdata, rmsgid, rctrls = self.result3(msgid) results.extend(rdata) # Extract the simple paged results response control pctrls = [c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType] if pctrls: if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = self.search_ext(base, ldap.SCOPE_SUBTREE, filterstr, attrlist=attrlist, serverctrls=(serverctrls or []) + [req_ctrl]) else: break return results
def _paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)', attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None, timeout=-1, sizelimit=0, page_size=10): """ Behaves similarly to LDAPObject.search_ext_s() but internally uses the simple paged results control to retrieve search results in chunks. Taken from the python-ldap paged_search_ext_s.py demo, showing how to use the paged results control: https://bitbucket.org/jaraco/python-ldap/ """ request_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='') results = [] while True: msgid = self.conn.search_ext(base, scope, filterstr=filterstr, attrlist=attrlist, attrsonly=attrsonly, serverctrls=(serverctrls or []) + [request_ctrl], clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit) result_type, result_data, result_msgid, result_ctrls = self.conn.result3(msgid) results.extend(result_data) # Extract the simple paged results response control paged_ctrls = [c for c in result_ctrls if c.controlType == SimplePagedResultsControl.controlType] if paged_ctrls and paged_ctrls[0].cookie: # Copy cookie from response control to request control request_ctrl.cookie = paged_ctrls[0].cookie else: break return results
def search(base_dn, search_filter, attributes): """Iterative LDAP search using page control. :param base_dn: str -- The base DN from which to start the search. :param search_filter: str -- Representation of the filter to apply in the search. :param attributes: list -- Attributes to be retrieved for each entry. If ``None``, all attributes will be retrieved. :returns: A generator which yields one search result at a time as a tuple containing a `dn` as ``str`` and `attributes` as ``dict``. """ connection, settings = current_ldap page_ctrl = SimplePagedResultsControl(True, size=settings['page_size'], cookie='') while True: msg_id = connection.search_ext(base_dn, SCOPE_SUBTREE, filterstr=search_filter, attrlist=attributes, serverctrls=[page_ctrl], timeout=settings['timeout']) try: _, r_data, __, server_ctrls = connection.result3(msg_id, timeout=settings['timeout']) except NO_SUCH_OBJECT: break for dn, entry in r_data: if dn: yield dn, entry page_ctrl.cookie = get_page_cookie(server_ctrls) if not page_ctrl.cookie: # End of results break
def get_ldap_list_membership(l, search_strings, list_name): results = set() known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl, } req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='') while True: msgid = l.search_ext(search_strings['user_search_string'], ldap.SCOPE_SUBTREE, serverctrls=[req_ctrl], attrlist=[search_strings['user_account_name_field']], filterstr=search_strings['user_membership_filter'] % escape_filter_chars(list_name) ) rtype, rdata, rmsgid, serverctrls = l.result3(msgid, resp_ctrl_classes=known_ldap_resp_ctrls) results |= {data[1][search_strings['user_account_name_field']][0] for data in rdata} pctrls = [c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType] cookie = pctrls[0].cookie if not cookie: break req_ctrl.cookie = cookie return results
def _paged_search(self, connection, base, scope, filterstr, attrlist): page_size = self.settings_dict.get('PAGE_SIZE', 1000) results = [] pg_ctrl = SimplePagedResultsControl(True, page_size, "") scope = ldap.SCOPE_SUBTREE pages = 0 while True: pages += 1 msgid = connection.search_ext( base, scope=scope, filterstr=filterstr, attrlist=attrlist, serverctrls=[pg_ctrl] ) rtype, rdata, rmsgid, serverctrls = connection.result3(msgid) results.extend(rdata) cookie = serverctrls[0].cookie if cookie: pg_ctrl.cookie = cookie search = connection.search_ext( base, scope=scope, filterstr=filterstr, attrlist=attrlist, serverctrls=[pg_ctrl] ) else: break return results
def query_users_paged(self, options): """ Connects to LDAP server and attempts to query and return all users by iterating through each page result. Requires LDAP v3. """ page_size = options.get('page_size', 500) criticality = options.get('criticality', True) cookie = options.get('cookie', '') full_record = options.get('full_record', False) fields = self.ldap_query_fields if full_record: fields = None # search the server for users first_pass = True pg_ctrl = SimplePagedResultsControl(criticality, page_size, cookie) LOG.debug("self.ldap_query_fields = %r", self.ldap_query_fields) while first_pass or pg_ctrl.cookie: first_pass = False msgid = self.ldap_connection.search_ext( self.settings['base_dn'], ldap.SCOPE_SUBTREE, self.settings['filter'], fields, serverctrls=[pg_ctrl] ) result_type, ldap_users, msgid, serverctrls = self.ldap_connection.result3(msgid) pg_ctrl.cookie = serverctrls[0].cookie for user in ldap_users: if user[0] and user[1]: yield _clean_record(user[1]) # disconnect and return results self.ldap_connection.unbind_s()
def pagedsearch(self, search_flt, searchreq_attrlist): req_ctrl = SimplePagedResultsControl(True,size=self.page_size,cookie='') known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType:SimplePagedResultsControl,} # Send search request msgid = self.lcon.search_ext( self.base, ldap.SCOPE_SUBTREE, search_flt, attrlist=searchreq_attrlist, serverctrls=[req_ctrl]) pages = 0 i = 0 while True: pages += 1 rtype, rdata, rmsgid, serverctrls = self.lcon.result3(msgid,resp_ctrl_classes=known_ldap_resp_ctrls) yield rdata pctrls = [c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType] if pctrls: if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = self.lcon.search_ext(self.base,ldap.SCOPE_SUBTREE,search_flt,attrlist=searchreq_attrlist,serverctrls=[req_ctrl]) else: break else: print "Warning: Server ignores RFC 2696 control." break
def search(base_dn, search_filter, attributes): """Iterative LDAP search using page control. :param base_dn: str -- The base DN from which to start the search. :param search_filter: str -- Representation of the filter to apply in the search. :param attributes: list -- Attributes to be retrieved for each entry. If ``None``, all attributes will be retrieved. :returns: A generator which yields one search result at a time as a tuple containing a `dn` as ``str`` and `attributes` as ``dict``. """ connection, settings = current_ldap page_ctrl = SimplePagedResultsControl(True, size=settings['page_size'], cookie='') search_filter = search_filter.encode('utf-8') while True: msg_id = connection.search_ext(base_dn, SCOPE_SUBTREE, filterstr=search_filter, attrlist=attributes, serverctrls=[page_ctrl], timeout=settings['timeout']) try: _, r_data, __, server_ctrls = connection.result3(msg_id, timeout=settings['timeout']) except NO_SUCH_OBJECT: break for dn, entry in r_data: if dn: yield dn, entry page_ctrl.cookie = get_page_cookie(server_ctrls) if not page_ctrl.cookie: # End of results break
def query(self, base, scope, params, fixed_rdn=None): ocs = ["(objectClass=%s)" % o.strip() for o in params['objectClasses'].split(",")] fltr = "(&" + "".join(ocs) + (ldap.filter.filter_format("(%s)", [fixed_rdn]) if fixed_rdn else "") + ")" result = [] with self.lh.get_handle() as l, self.lock: self.log.debug("LDAP Search using filter '{filter}'".format(filter=fltr)) lc = SimplePagedResultsControl(criticality=True, size=self.page_size, cookie='') ix = 0 while True: msgid = l.search_ext(base, ldap.SCOPE_ONELEVEL, fltr, attrlist=[self.uuid_entry], serverctrls=[lc]) res = l.result3(msgid=msgid) result_type, results, unused_msgid, serverctrls = res for resultdata in results: ix += 1 if resultdata[0] is not None: result.append(resultdata) cookie = None for serverctrl in serverctrls: if serverctrl.controlType == ldap.controls.libldap.SimplePagedResultsControl.controlType: cookie = serverctrl.cookie if cookie: lc.cookie = cookie break if not cookie: break return [x for x in dict(result).keys()]
def get_paged_users(self, filters, attributes): pages = 0 result = [] page_control = SimplePagedResultsControl(True, size=self.page_size, cookie='') res = self._conn.search_ext( self.domain_dn, ldap.SCOPE_SUBTREE, filters, attributes, serverctrls=[page_control] ) while True: pages += 1 self.log.debug("Page {} done".format(pages)) rtype, rdata, rmsgid, serverctrls = self._conn.result3(res) result.extend(rdata) controls = [ctrl for ctrl in serverctrls if ctrl.controlType == SimplePagedResultsControl.controlType] if not controls: self.log.error('The server ignores RFC 2696 control') break if not controls[0].cookie: break page_control.cookie = controls[0].cookie res = self._conn.search_ext( self.domain_dn, ldap.SCOPE_SUBTREE, filters, attributes, serverctrls=[page_control] ) return result
def results_for_dn(directory: ldap.ldapobject, base_dn: str, filter: str) -> List[User]: """Returns a list of User objects found in the directory object for filter :param directory: A ldap.LDAPObject that has already been bound to a directory. :param base_dn: The base of the directory tree to run the search filter against. :param filter: The LDAP search filter to run on base_dn using directory. """ req_ctrl = SimplePagedResultsControl(True, size=5000, cookie="") known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl, } # Send search request msgid = directory.search_ext(base_dn, ldap.SCOPE_SUBTREE, filterstr=LDAP_FILTER, serverctrls=[req_ctrl]) results = [] while True: __, result_data, __, serverctrls = directory.result3( msgid, resp_ctrl_classes=known_ldap_resp_ctrls) results.extend([ User( ldap_entry["sAMAccountName"][0].decode(), ldap_entry["mail"][0].decode(), ldap_entry["sn"][0].decode(), ldap_entry["givenName"][0].decode(), ) for __, ldap_entry in result_data ]) page_controls = [ control for control in serverctrls if control.controlType == SimplePagedResultsControl.controlType ] if page_controls: if page_controls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = page_controls[0].cookie msgid = directory.search_ext( base_dn, ldap.SCOPE_SUBTREE, filterstr=LDAP_FILTER, serverctrls=[req_ctrl], ) else: break else: eprint("Warning: Server ignores RFC 2696 control.") break return results
def test_search_multiple_paging(topology_st, create_user): """Verify that after performing multiple simple paged searches on a single connection without a complition, it wouldn't fail. :id: 628b29a6-2d47-4116-a88d-00b87405ef7f :customerscenario: True :setup: Standalone instance, test user for binding, varying number of users for the search base :steps: 1. Bind as test user 2. Initiate the search with a simple paged control 3. Acquire the returned cookie only one time 4. Perform steps 2 and 3 three times in a row :expectedresults: 1. Bind should be successful 2. Search should be successfully initiated 3. Cookie should be successfully acquired 4. No error happens """ users_num = 100 page_size = 30 users_list = add_users(topology_st, users_num, DEFAULT_SUFFIX) search_flt = r'(uid=test*)' searchreq_attrlist = ['dn', 'sn'] try: log.info('Set user bind') conn = create_user.bind(TEST_USER_PWD) log.info('Create simple paged results control instance') req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='') controls = [req_ctrl] for ii in range(3): log.info('Iteration %d' % ii) msgid = conn.search_ext(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, search_flt, searchreq_attrlist, serverctrls=controls) rtype, rdata, rmsgid, rctrls = conn.result3(msgid) pctrls = [ c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType ] # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = conn.search_ext(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, search_flt, searchreq_attrlist, serverctrls=controls) finally: del_users(users_list)
def get_ldap_lists(l, search_strings, parent_list=None): results = set() known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl, } req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='') if parent_list: filterstr = search_strings['get_all_sub_lists_filter'] % escape_filter_chars(parent_list) else: filterstr = search_strings['get_all_lists_filter'] while True: msgid = l.search_ext(search_strings['list_search_string'], ldap.SCOPE_SUBTREE, serverctrls=[req_ctrl], attrlist=(search_strings['list_cn_field'], search_strings['list_name_field']), filterstr=filterstr) retries = 0 rdata = None while retries < 5: try: rtype, rdata, rmsgid, serverctrls = l.result3(msgid, timeout=ldap_timeout, resp_ctrl_classes=known_ldap_resp_ctrls) except Exception: logger.exception('Error getting ldap list memberships') retries += 1 else: break # ldap sometimees returns good but does not return rdata if rdata is None: break for (dn, data) in rdata: cn_field = data[search_strings['list_cn_field']][0] name_field = data[search_strings['list_name_field']][0] if isinstance(cn_field, bytes): cn_field = cn_field.decode('utf-8') if isinstance(name_field, bytes): name_field = name_field.decode('utf-8') results |= {(cn_field, name_field)} pctrls = [c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType] if not pctrls: # Paging not supported logger.debug('LDAP pagination not supported') break cookie = pctrls[0].cookie if not cookie: break req_ctrl.cookie = cookie return results
def _search(self, basedn='', scope=pyldap.SCOPE_SUBTREE, filter='', timeout=-1, sizelimit=0): if not self._handle: self._open() result = [] serverctrls = None clientctrls = None paged = SimplePagedResultsControl(criticality=False, size=self.pagesize, cookie='') paged_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl } page = 0 while True: serverctrls = [paged] id = self._handle.search_ext(basedn, scope, filterstr=filter, attrlist=None, attrsonly=0, serverctrls=serverctrls, clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit) (rtype, rdata, rmsgid, serverctrls) = self._handle.result3(id, resp_ctrl_classes=paged_ctrls) result.extend(rdata) paged.size = 0 paged.cookie = cookie = None for sc in serverctrls: if sc.controlType == SimplePagedResultsControl.controlType: cookie = sc.cookie if cookie: paged.cookie = cookie paged.size = self.pagesize break if not cookie: break page += 1 return result
def test_search_invalid_cookie(topology_st, create_user, invalid_cookie): """Verify that using invalid cookie while performing search with the simple paged results control throws a TypeError exception :id: 107be12d-4fe4-47fe-ae86-f3e340a56f42 :customerscenario: True :parametrized: yes :setup: Standalone instance, test user for binding, varying number of users for the search base :steps: 1. Bind as test user 2. Initiate the search with a simple paged control 3. Put an invalid cookie (-1, 1000) to the control 4. Continue the search :expectedresults: 1. Bind should be successful 2. Search should be successfully initiated 3. Cookie should be added 4. It should throw a TypeError exception """ users_num = 100 page_size = 50 users_list = add_users(topology_st, users_num, DEFAULT_SUFFIX) search_flt = r'(uid=test*)' searchreq_attrlist = ['dn', 'sn'] try: log.info('Set user bind') conn = create_user.bind(TEST_USER_PWD) log.info('Create simple paged results control instance') req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='') controls = [req_ctrl] msgid = conn.search_ext(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, search_flt, searchreq_attrlist, serverctrls=controls) rtype, rdata, rmsgid, rctrls = conn.result3(msgid) log.info( 'Put an invalid cookie (%d) to the control. TypeError is expected' % invalid_cookie) req_ctrl.cookie = invalid_cookie with pytest.raises(TypeError): msgid = conn.search_ext(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, search_flt, searchreq_attrlist, serverctrls=controls) finally: del_users(users_list)
def handle(self, *args, **options): filter_string = options.get('filter', DEFAULT_FILTER) ldap_backend = LDAPBackend() ldap_user = _LDAPUser(backend=ldap_backend, username='') ldap_connection = ldap_user.connection # Initialize the LDAP controls for paging. # Note that we pass '' for the cookie because on first iteration, it starts out empty. ldap_controls = SimplePagedResultsControl(True, size=PAGE_SIZE, cookie='') while True: # Send search request # If you leave out the ATTRLIST it'll return all attributes which you have permissions to access. # You may want to adjust the scope level as well # (perhaps "ldap.SCOPE_SUBTREE", but it can reduce performance if you don't need it). message_id = ldap_connection.search_ext( base=settings.AUTH_LDAP_USER_SEARCH.base_dn, scope=ldap.SCOPE_SUBTREE, filterstr=filter_string, serverctrls=[ldap_controls]) # Pull the results from the search request rtype, rdata, rmsgid, server_controls = ldap_connection.result3( message_id) rdata = settings.AUTH_LDAP_USER_SEARCH._process_results(rdata) # Each "rdata" is a tuple of the form (dn, attrs), where dn is a string containing the # DN (distinguished name) of the entry, and attrs is a dictionary containing the attributes associated # with the entry. The keys of attrs are strings, and the associated values are lists of strings. for distinguished_name, attributes in rdata: username = "".join(attributes['uid']) print(f"{username} \t| {distinguished_name}") print(attributes) # Look through the returned controls and find the page controls. # This will also have our returned cookie which we need to make the next search request. page_controls = [ control for control in server_controls if control.controlType == SimplePagedResultsControl.controlType ] if not page_controls: print('Warning: Server ignores RFC 2696 control.') break # Ok, we did find the page control, yank the cookie from it and insert it into the control for our # next search. If however there is no cookie, we are done! cookie = page_controls[0].cookie ldap_controls.cookie = cookie if not cookie: break
def paged_search_ext_s(self, conn, base, scope, filter, attrs, attrsonly=0,serverctrls=None,clientctrls=None,timeout=-1,sizelimit=0): """ Helper function that implements a paged LDAP search for the Search method below. Args: query: LDAP filter to apply to the search sizelimit: max # of users to return. attrlist: list of attributes to return. If null, all attributes are returned Returns: A list of users as returned by the LDAP search """ ldap_page_size = 100 search_flt = r'(objectClass=*)' searchreq_attrlist = ['cn', 'entryDN', 'entryUUID', 'mail', 'objectClass'] req_ctrl = SimplePagedResultsControl( True, ldap_page_size, cookie='') logging.debug('Paged search on %s for %s' % (base, filter)) # Send first search request msgid = conn.search_ext( base, ldap.SCOPE_SUBTREE, search_flt, attrlist=searchreq_attrlist, serverctrls=(serverctrls or []) + [req_ctrl] ) result_pages = 0 all_results = [] while True: rtype, rdata, rmsgid, rctrls = conn.result3(msgid) all_results.extend(rdata) result_pages += 1 # Extract the simple paged results response control pctrls = [ c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType ] if pctrls: if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = conn.search_ext( base, ldap.SCOPE_SUBTREE, search_flt, attrlist=searchreq_attrlist, serverctrls=(serverctrls or []) + [req_ctrl] ) else: break return all_results
def set_cookie( lc_object: SimplePagedResultsControl, pctrls: List[SimplePagedResultsControl], ) -> bool: """ Push latest cookie back into the page control. """ cookie = pctrls[0].cookie lc_object.cookie = cookie return bool(cookie)
def paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)', attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None, timeout=-1, sizelimit=0): """ Behaves exactly like LDAPObject.search_ext_s() but internally uses the simple paged results control to retrieve search results in chunks. This is non-sense for really large results sets which you would like to process one-by-one """ req_ctrl = SimplePagedResultsControl(True, size=self.page_size, cookie='') # Send first search request msgid = self.search_ext(base, ldap.SCOPE_SUBTREE, search_flt, attrlist=searchreq_attrlist, serverctrls=(serverctrls or []) + [req_ctrl]) result_pages = 0 all_results = [] while True: rtype, rdata, rmsgid, rctrls = self.result3(msgid) all_results.extend(rdata) result_pages += 1 # Extract the simple paged results response control pctrls = [ c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType ] if pctrls: if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = self.search_ext( base, ldap.SCOPE_SUBTREE, search_flt, attrlist=searchreq_attrlist, serverctrls=(serverctrls or []) + [req_ctrl]) else: break return result_pages, all_results
def search_users(req_page): con = get_ldap_con() sc = SimplePagedResultsControl(criticality=True, size=PAGESIZE, cookie='') search_results = None last_page = False i = 0 while True: if i >= req_page: break msgid = con.search_ext(base=BASEDN, scope=ldap.SCOPE_SUBTREE, filterstr=SEARCHFILTER.decode('utf8'), attrlist=USERS_ATTR_LIST, serverctrls=[sc]) # msgid is an integer representing LDAP operation id _, search_results, _, result_sc = con.result3(msgid) # result_data: list of tuples in the form of (dn, attrs), # dn is a string, ie: 'dn=alvaro,dc=example,dc=com', # attrs is a string key based dictionary. # result_sc: servercontrols list. pctrls = [ c for c in result_sc if c.controlType == SimplePagedResultsControl.controlType ] if not pctrls: last_page = True i += 1 break cookie = pctrls[0].cookie if not cookie: last_page = True i += 1 break sc.cookie = cookie i += 1 # Data mangle # We don't need Full DN or registers with empty DN search_results = purge_results(search_results) ret_json = get_response_template() ret_json['page'] = i ret_json['quant'] = len(search_results) ret_json['last'] = last_page ret_json['data'] = search_results return ret_json
def paginated_query(connect, attributes, base, scope, search_flt, result_class=None): # how many users to search for in each page, # this depends on the server maximum setting # (default highest value is 1000) page_size = 500 req_ctrl = SimplePagedResultsControl(criticality=True, size=page_size, cookie="") msgid = connect.search_ext( base=base, scope=scope, filterstr=search_flt, attrlist=attributes, serverctrls=[req_ctrl], ) total_results = [] pages = 0 while True: # loop over all of the pages using the same cookie, # otherwise the search will fail pages += 1 rtype, rdata, rmsgid, serverctrls = connect.result3(msgid) for user in rdata: if not result_class: total_results.append(user) else: total_results.append(result_class(user)) pctrls = [ c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType ] if pctrls: if pctrls[ 0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = connect.search_ext( base=base, scope=scope, filterstr=search_flt, attrlist=attributes, serverctrls=[req_ctrl], ) else: break else: break return total_results
def _paged_search(self, base_dn, scope, search_filter, attrs): conn = self.connect() # Get paged results to prevent exceeding server size limit page_size = 1000 if PYTHON_LDAP_24: lc = SimplePagedResultsControl(size=page_size, cookie='') else: lc = SimplePagedResultsControl(LDAP_CONTROL_PAGED_RESULTS, True, (page_size, ''),) is_last_page = False results = [] while not is_last_page: msgid = conn.search_ext(base_dn, scope, search_filter, attrs, serverctrls=[lc]) rtype, rdata, rmsgid, serverctrls = conn.result3(msgid) pctrls = [c for c in serverctrls if c.controlType == LDAP_CONTROL_PAGED_RESULTS] results.extend(rdata) if pctrls: if PYTHON_LDAP_24: cookie = pctrls[0].cookie if cookie: lc.cookie = cookie else: is_last_page = True else: cookie = pctrls[0].controlValue[1] if cookie: # lc.controlValue seems to have been mutable at some # point, now it's a tuple. cv = list(lc.controlValue) cv[1] = cookie lc.controlValue = tuple(cv) else: is_last_page = True else: is_last_page = True if results: # If the search returned an empty result, page controls # aren't included - so don't produce a bogus warning logger.warn( "Server ignores paged results control (RFC 2696).") return results
def _search(self, basedn='', scope=ldap.SCOPE_SUBTREE, filter='', timeout=-1, sizelimit=0): if not self._handle: self._open() result = [] serverctrls = None clientctrls = None paged = SimplePagedResultsControl( criticality=False, size=self.pagesize, cookie='' ) paged_ctrls = {SimplePagedResultsControl.controlType: SimplePagedResultsControl} page = 0 while True: serverctrls = [paged] id = self._handle.search_ext( basedn, scope, filterstr=filter, attrlist=None, attrsonly=0, serverctrls=serverctrls, clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit ) (rtype, rdata, rmsgid, serverctrls) = self._handle.result3( id, resp_ctrl_classes=paged_ctrls ) result.extend(rdata) paged.size = 0 paged.cookie = cookie = None for sc in serverctrls: if sc.controlType == SimplePagedResultsControl.controlType: cookie = sc.cookie if cookie: paged.cookie = cookie paged.size = self.pagesize break if not cookie: break page += 1 return result
def get_ldap_list_membership(l, search_strings, list_name): results = set() known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl, } req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='') while True: msgid = l.search_ext( search_strings['user_search_string'], ldap.SCOPE_SUBTREE, serverctrls=[req_ctrl], attrlist=[search_strings['user_mail_field']], filterstr=search_strings['user_membership_filter'] % escape_filter_chars(list_name)) retries = 0 rdata = None while retries < 5: try: rtype, rdata, rmsgid, serverctrls = l.result3( msgid, timeout=ldap_timeout, resp_ctrl_classes=known_ldap_resp_ctrls) except Exception: logger.exception('Error getting ldap list memberships') retries += 1 else: break # ldap sometimees returns good but does not return rdata if rdata is None: break for data in rdata: member = data[1].get(search_strings['user_mail_field'], [None])[0] if isinstance(member, bytes): member = member.decode('utf-8') results |= {member} pctrls = [ c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType ] if not pctrls: # Paging not supported logger.debug('LDAP pagination not supported') break cookie = pctrls[0].cookie if not cookie: break req_ctrl.cookie = cookie return results
def get_ldap_lists(l, search_strings, parent_list=None): results = set() known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl, } req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='') if parent_list: filterstr = search_strings[ 'get_all_sub_lists_filter'] % escape_filter_chars(parent_list) else: filterstr = search_strings['get_all_lists_filter'] while True: msgid = l.search_ext(search_strings['list_search_string'], ldap.SCOPE_SUBTREE, serverctrls=[req_ctrl], attrlist=(search_strings['list_cn_field'], search_strings['list_name_field']), filterstr=filterstr) rtype, rdata, rmsgid, serverctrls = l.result3( msgid, timeout=ldap_timeout, resp_ctrl_classes=known_ldap_resp_ctrls) for (dn, data) in rdata: cn_field = data[search_strings['list_cn_field']][0] name_field = data[search_strings['list_name_field']][0] if isinstance(cn_field, bytes): cn_field = cn_field.decode('utf-8') if isinstance(name_field, bytes): name_field = name_field.decode('utf-8') results |= {(cn_field, name_field)} pctrls = [ c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType ] if not pctrls: # Paging not supported break cookie = pctrls[0].cookie if not cookie: break req_ctrl.cookie = cookie return results
def _paged_search_ext_s(self, base, scope, filterstr='(objectClass=*)', attrlist=None, attrsonly=0, serverctrls=None, clientctrls=None, timeout=-1, sizelimit=0, page_size=10): """ Behaves similarly to LDAPObject.search_ext_s() but internally uses the simple paged results control to retrieve search results in chunks. Taken from the python-ldap paged_search_ext_s.py demo, showing how to use the paged results control: https://bitbucket.org/jaraco/python-ldap/ """ request_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='') results = [] while True: msgid = self.conn.search_ext(base, scope, filterstr=filterstr, attrlist=attrlist, attrsonly=attrsonly, serverctrls=(serverctrls or []) + [request_ctrl], clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit) result_type, result_data, result_msgid, result_ctrls = self.conn.result3( msgid) results.extend(result_data) # Extract the simple paged results response control paged_ctrls = [ c for c in result_ctrls if c.controlType == SimplePagedResultsControl.controlType ] if paged_ctrls and paged_ctrls[0].cookie: # Copy cookie from response control to request control request_ctrl.cookie = paged_ctrls[0].cookie else: break return results
def _paged_search(self, basedn, searchfilter, attrlist=None, pagesize=100, sizelimit=0): """ Performs a pages search against the LDAP server. Code lifted from: https://gist.github.com/mattfahrner/c228ead9c516fc322d3a """ # Initialize the LDAP controls for paging. Note that we pass '' # for the cookie because on first iteration, it starts out empty. controls = SimplePagedResultsControl(True, size=pagesize, cookie='') # Do searches until we run out of pages to get from the LDAP server. results = [] while True: # Send search request. msgid = self.connection.search_ext( basedn, ldap.SCOPE_SUBTREE, searchfilter, attrlist, serverctrls=[controls], sizelimit=sizelimit ) rtype, rdata, rmsgid, serverctrls = self.connection.result3(msgid) # Each "rdata" is a tuple of the form (dn, attrs), where dn is # a string containing the DN (distinguished name) of the entry, # and attrs is a dictionary containing the attributes associated # with the entry. The keys of attrs are strings, and the associated # values are lists of strings. for dn, attrs in rdata: # AD returns an rdata at the end that is a reference that we want to ignore if type(attrs) == dict: results.append((dn, attrs)) # Get cookie for the next request. paged_controls = self._get_pctrls(serverctrls) if not paged_controls: self.logger.warning( f'paged_search.rfc2696_control_ignored searchfilter={searchfilter}' f'attrlist={",".join(attrlist)} pagesize={pagesize} sizelimit={sizelimit}' ) break # Push cookie back into the main controls. controls.cookie = paged_controls[0].cookie # If there is no cookie, we're done! if not paged_controls[0].cookie: break return results
def paged_search(self, search_dn, filter, scope, results_processor, attributes=None): if not attributes: attributes = ['*'] page_control = SimplePagedResultsControl(True, self.PAGE_SIZE, '') serverctrls = [page_control] msgid = self.connection.search_ext( search_dn, scope, filter, attrlist=attributes, serverctrls=serverctrls ) page = 0 records = 0 while True: page += 1 try: result_type, results, result_msg, serverctrls = self.connection.result3(msgid=msgid, timeout=self.LDAP_TIMEOUT) records += len(results) results_processor(results) pagectrls = [ c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType ] if pagectrls: if pagectrls[0].cookie: page_control.cookie = pagectrls[0].cookie msgid = self.connection.search_ext( search_dn, scope, filter, attrlist=attributes, serverctrls=[page_control] ) else: break else: break except ldap.LDAPError, e: print e
def test_lookup_with_pagination(udm): """Test serverctrls of ldap server""" pytest.skip('FIXME??? Bug #49638: git:63ba30a2040') from ldap.controls import SimplePagedResultsControl from ldap.controls.sss import SSSRequestControl name = uts.random_username() dns = [ udm.create_user(username=name + str(i), wait_for_replication=False, check_for_drs_replication=False, wait_for=False)[0] for i in range(1, 8) ] print(('Created users:', dns)) univention.admin.modules.update() lo = univention.uldap.getMachineConnection() res = {} page_size = 2 pctrl = SimplePagedResultsControl(True, size=page_size, cookie='') sctrl = SSSRequestControl(ordering_rules=['uid:caseIgnoreOrderingMatch']) users = univention.admin.modules.get('users/user') ctrls = [sctrl, pctrl] entries = [] while True: entries.append([ x.dn for x in users.lookup(None, lo, 'username=%s*' % (name, ), serverctrls=ctrls, response=res) ]) print(('Found', entries[-1])) for control in res['ctrls']: if control.controlType == SimplePagedResultsControl.controlType: pctrl.cookie = control.cookie if not pctrl.cookie: break assert len(entries[-1]) == page_size found = [] for entry in entries: found.extend(entry) assert sorted(found) == sorted(dns)
def paged_search_ext_s(self,base,scope,filterstr='(objectClass=*)',attrlist=None,attrsonly=0,serverctrls=None,clientctrls=None,timeout=-1,sizelimit=0,criticality=True,page_size=1000): """ Behaves exactly like LDAPObject.search_ext_s() but internally uses the simple paged results control to retrieve search results in chunks. This is non-sense for really large results sets which you would like to process one-by-one """ req_ctrl = SimplePagedResultsControl(criticality,size=page_size,cookie='') # Send first search request msgid = self.search_ext( base, scope, filterstr, attrlist=attrlist, serverctrls=(serverctrls or [])+[req_ctrl] ) result_pages = 0 all_results = [] while True: rtype, rdata, rmsgid, rctrls = self.result3(msgid) all_results.extend(rdata) result_pages += 1 # Extract the simple paged results response control pctrls = [ c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType ] if pctrls: if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = self.search_ext( base, scope, filterstr, attrlist=attrlist, serverctrls=(serverctrls or [])+[req_ctrl] ) else: break return result_pages,all_results
def _search(self, search_flt, attrs, limit=None, path_prefix=None): """ Query this academic calendar for records matching the search filter. Only looks one level deep in the tree to improve performance, so make sure you pass the dn that you want to look directly inside of as the `path` parameter. Returns a list of records. Each record is a dictionary containing the requested attributes. search_flt -- LDAP-style search filter attrs -- attributes to be retrieved limit -- max number of records to return path -- extra LDAP dn: is prepended to this object's basedn ie the LDAP directory to look from relative to the root """ if limit is None: limit = float('inf') if path_prefix is None: path = self._basedn else: path = '{}{}'.format(path_prefix, self._basedn) page_control = SimplePagedResultsControl( criticality=True, size=RemoteLDAPDatabase.PAGE_SIZE, cookie='') results = [] pages_retrieved = 0 first_pass = True while pages_retrieved*RemoteLDAPDatabase.PAGE_SIZE < limit \ and page_control.cookie \ or first_pass: first_pass = False try: msgid = self._client.search_ext(path, ldap.SCOPE_ONELEVEL, search_flt, attrlist=attrs, serverctrls=[page_control]) except: raise pages_retrieved += 1 _, result_data, msgid, serverctrls = self._client.result3(msgid) page_control.cookie = serverctrls[0].cookie results += extract_results_from_ldap_data(result_data) return results
def search(self, base_dn=None, scope=ldap.SCOPE_SUBTREE, filter='(objectClass=*)', attrs=[]): conn = self.connect() if conn is None: return [] if base_dn is None: base_dn = self.base_dn # Get paged results to prevent exceeding server size limit page_size = 1000 if PYTHON_LDAP_24: lc = SimplePagedResultsControl(size=page_size, cookie='') else: lc = SimplePagedResultsControl(LDAP_CONTROL_PAGED_RESULTS, True, (page_size, ''),) is_last_page = False results = [] while not is_last_page: msgid = conn.search_ext(base_dn, scope, filter, attrs, serverctrls=[lc]) rtype, rdata, rmsgid, serverctrls = conn.result3(msgid) results.extend(rdata) pctrls = [c for c in serverctrls if c.controlType == LDAP_CONTROL_PAGED_RESULTS] if pctrls: if PYTHON_LDAP_24: cookie = pctrls[0].cookie if cookie: lc.cookie = cookie else: is_last_page = True else: cookie = pctrls[0].controlValue[1] if cookie: lc.controlValue[1] = cookie else: is_last_page = True else: is_last_page = True logger.warn("Server ignores paged results control (RFC 2696).") return results
def _search(self, search_flt, attrs, limit=None, path_prefix=None): """ Query this academic calendar for records matching the search filter. Only looks one level deep in the tree to improve performance, so make sure you pass the dn that you want to look directly inside of as the `path` parameter. Returns a list of records. Each record is a dictionary containing the requested attributes. search_flt -- LDAP-style search filter attrs -- attributes to be retrieved limit -- max number of records to return path -- extra LDAP dn: is prepended to this object's basedn ie the LDAP directory to look from relative to the root """ if limit is None: limit = float('inf') if path_prefix is None: path = self._basedn else: path = '{}{}'.format(path_prefix, self._basedn) page_control = SimplePagedResultsControl( criticality=True, size=RemoteLDAPDatabase.PAGE_SIZE, cookie='' ) results = [] pages_retrieved = 0 first_pass = True while pages_retrieved*RemoteLDAPDatabase.PAGE_SIZE < limit \ and page_control.cookie \ or first_pass: first_pass = False try: msgid = self._client.search_ext(path, ldap.SCOPE_ONELEVEL, search_flt, attrlist=attrs, serverctrls=[page_control]) except: raise pages_retrieved += 1 _, result_data, msgid, serverctrls = self._client.result3(msgid) page_control.cookie = serverctrls[0].cookie results += extract_results_from_ldap_data(result_data) return results
def query(self, base, scope, params, fixed_rdn=None): ocs = [ "(objectClass=%s)" % o.strip() for o in params['objectClasses'].split(",") ] fltr = "(&" + "".join(ocs) + (ldap.filter.filter_format( "(%s)", [fixed_rdn]) if fixed_rdn else "") + ")" result = [] with self.lh.get_handle() as l, self.lock: self.log.debug( "LDAP Search using filter '{filter}'".format(filter=fltr)) lc = SimplePagedResultsControl(criticality=True, size=self.page_size, cookie='') ix = 0 while True: msgid = l.search_ext(base, ldap.SCOPE_ONELEVEL, fltr, attrlist=[self.uuid_entry], serverctrls=[lc]) res = l.result3(msgid=msgid) result_type, results, unused_msgid, serverctrls = res for resultdata in results: ix += 1 if resultdata[0] is not None: result.append(resultdata) cookie = None for serverctrl in serverctrls: if serverctrl.controlType == ldap.controls.libldap.SimplePagedResultsControl.controlType: cookie = serverctrl.cookie if cookie: lc.cookie = cookie break if not cookie: break return [x for x in dict(result).keys()]
def get_ldap_list_membership(l, search_strings, list_name): results = set() known_ldap_resp_ctrls = { SimplePagedResultsControl.controlType: SimplePagedResultsControl, } req_ctrl = SimplePagedResultsControl(True, size=ldap_pagination_size, cookie='') while True: msgid = l.search_ext( search_strings['user_search_string'], ldap.SCOPE_SUBTREE, serverctrls=[req_ctrl], attrlist=[search_strings['user_mail_field']], filterstr=search_strings['user_membership_filter'] % escape_filter_chars(list_name)) rtype, rdata, rmsgid, serverctrls = l.result3( msgid, timeout=ldap_timeout, resp_ctrl_classes=known_ldap_resp_ctrls) for data in rdata: member = data[1].get(search_strings['user_mail_field'], [None])[0] if isinstance(member, bytes): member = member.decode('utf-8') results |= {member} pctrls = [ c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType ] if not pctrls: # Paging not supported break cookie = pctrls[0].cookie if not cookie: break req_ctrl.cookie = cookie return results
def _make_paged_query( conn, search_base, search_scope, ad_query, attr_list, page_size ): """ Makes paged query to LDAP. Default max page size for LDAP is 1000. """ result = [] page_result_control = SimplePagedResultsControl( size=page_size, cookie='' ) msgid = conn.search_ext( search_base, search_scope, ad_query, attr_list, serverctrls=[page_result_control], ) while True: r_type, r_data, r_msgid, serverctrls = conn.result3(msgid) result.extend(r_data) if serverctrls: if serverctrls[0].cookie: page_result_control.size = page_size page_result_control.cookie = serverctrls[0].cookie msgid = conn.search_ext( search_base, search_scope, ad_query, attr_list, serverctrls=[page_result_control], ) else: break return result
def getSambaUsers(filter, attrs, base_dn): print("getSambaUsers") global ldap_connection ldapUsers = [] page_control = SimplePagedResultsControl(True, size=1000, cookie='') response = ldap_connection.search_ext(base_dn, ldap.SCOPE_SUBTREE, filter, attrs, serverctrls=[page_control]) try: pages = 0 while True: pages += 1 rtype, rdata, rmsgid, serverctrls = ldap_connection.result3( response) ldapUsers.extend(rdata) controls = [ control for control in serverctrls if control.controlType == SimplePagedResultsControl.controlType ] if not controls: print('The server ignores RFC 2696 control') break if not controls[0].cookie: break page_control.cookie = controls[0].cookie response = ldap_connection.search_ext(base_dn, ldap.SCOPE_SUBTREE, filter, attrs, serverctrls=[page_control]) except: print() # dönen dictionary array'indeki verileri for dönerek daha basit bir hale getirir. Key olarak kullanılar dn'leri siler results = [entry for dn, entry in ldapUsers if isinstance(entry, dict)] return results
def _run_ldap_query(self, query): with LDAPConnectionManager() as conn: lc = SimplePagedResultsControl( size=LDAP_RESULTS_PAGE_SIZE, cookie='' ) msgid = conn.search_ext( settings.AUTH_LDAP_USER_SEARCH_BASE, ldap.SCOPE_SUBTREE, query, serverctrls=[lc], ) page_num = 0 while True: page_num += 1 r_type, r_data, r_msgid, serverctrls = conn.result3(msgid) logger.info('Pack of {} users loaded (page {})'.format( LDAP_RESULTS_PAGE_SIZE, page_num, )) for item in r_data: yield item if serverctrls: if serverctrls[0].cookie: lc.size = LDAP_RESULTS_PAGE_SIZE lc.cookie = serverctrls[0].cookie msgid = conn.search_ext( settings.AUTH_LDAP_USER_SEARCH_BASE, ldap.SCOPE_SUBTREE, query, serverctrls=[lc], ) else: break else: logger.error( 'LDAP::_run_ldap_query\tQuery: Server ignores RFC 2696 ' 'control' ) sys.exit(1)
def ldap_paged_async_search(base, scope, filt, columns): ldap_log(' PAGED ASYNC SEARCH') page_size = config.ldap_connection.get('page_size', 100) if ldap_compat: lc = SimplePagedResultsControl(size = page_size, cookie = '') else: lc = SimplePagedResultsControl( LDAP_CONTROL_PAGED_RESULTS, True, (page_size, '') ) results = [] while True: # issue the ldap search command (async) msgid = ldap_connection.search_ext(base, scope, filt, columns, serverctrls = [lc]) unused_code, response, unused_msgid, serverctrls = ldap_connection.result3( msgid = msgid, timeout = config.ldap_connection.get('response_timeout', 5) ) for result in response: results.append(result) # Mark current position in pagination control for next loop cookie = None for serverctrl in serverctrls: if serverctrl.controlType == LDAP_CONTROL_PAGED_RESULTS: if ldap_compat: cookie = serverctrl.cookie if cookie: lc.cookie = cookie else: cookie = serverctrl.controlValue[1] if cookie: lc.controlValue = (page_size, cookie) break if not cookie: break return results
def _run_ldap_query(self, query): self.conn = get_ldap() lc = SimplePagedResultsControl(size=LDAP_RESULTS_PAGE_SIZE, cookie='') msgid = self.conn.search_ext( settings.AUTH_LDAP_USER_SEARCH_BASE, ldap.SCOPE_SUBTREE, query, serverctrls=[lc], ) page_num = 0 while True: page_num += 1 r_type, r_data, r_msgid, serverctrls = self.conn.result3(msgid) print "Pack of %s users loaded (page %s)" % ( LDAP_RESULTS_PAGE_SIZE, page_num, ) for item in r_data: yield item if serverctrls: if serverctrls[0].cookie: lc.size = LDAP_RESULTS_PAGE_SIZE lc.cookie = serverctrls[0].cookie msgid = self.conn.search_ext( settings.AUTH_LDAP_USER_SEARCH_BASE, ldap.SCOPE_SUBTREE, query, serverctrls=[lc], ) else: break else: logger.error( 'LDAP::_run_ldap_query\tQuery: %s\t' 'Server ignores RFC 2696 control', ) sys.exit(-1) self._disconnect()
def doSearch(self, searchFilter=None, retrieveAttributes=None): """ Searches for records in an ldap server, Research is filtered and the result is a selection of individual fields of the returned records. :param searchFilter: It is the filter to search the LDAP server. The filter must be written in the LDAP filter format. If the filter is not specified, the default filter is used (defined in the instance of the class). If the filter contains (@DefaultFilter) as a filter, the filter passed is added to the default filter. :param retrieveAttributes: This is the list of the attributes returned by the search for each record. The list of attributes is made up of a string with the list of attributes separated by commas. If attributes are not specified, the default attributes (defined in the instance of the class are returned). If the list of attributes including @DefaultAttribute attribute, passed attributes are added to the default attributes. :return: Return a list of dictionaries, a dictionary for each record. The value attached to each key in turn is a list of strings. """ if searchFilter: if '(@DefaultFilter)' in searchFilter: searchFilter = '(' + self.searchFilter[1:-1] + searchFilter[1:-1].replace('(@DefaultFilter)', '') + ')' else: searchFilter = self.searchFilter if retrieveAttributes: retrieveAttributes = [str(attribute).strip() for attribute in retrieveAttributes.split(',')] if '@DefaultAttribute' in retrieveAttributes: retrieveAttributes = list(set(self.userAttr + retrieveAttributes)) retrieveAttributes.remove('@DefaultAttribute') else: retrieveAttributes = self.userAttr page_size = 50 ldap.set_option(ldap.OPT_REFERRALS, 0) req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='') ldap_user = self.doLogin(user=self.searchUser, password=self.searchPassword, mode='Search') if ldap_user: try: ldap_result_id = self.ldapClient.search_ext(self.baseDN, ldap.SCOPE_SUBTREE, searchFilter, attrlist=retrieveAttributes, serverctrls=[req_ctrl]) result_set = [] result_pages = 0 while True: rtype, rdata, rmsgid, rctrls = self.ldapClient.result3(ldap_result_id) result_set.extend(rdata) result_pages += 1 pctrls = [c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType] if pctrls: if pctrls[0].cookie: req_ctrl.cookie = pctrls[0].cookie ldap_result_id = self.ldapClient.search_ext(self.baseDN, ldap.SCOPE_SUBTREE, searchFilter, attrlist=retrieveAttributes, serverctrls=[req_ctrl]) else: break except ldap.LDAPError, e: print e return None finally:
def test_ticket48808(topology, test_user): log.info('Run multiple paging controls on a single connection') users_num = 100 page_size = 30 users_list = add_users(topology, users_num) search_flt = r'(uid=test*)' searchreq_attrlist = ['dn', 'sn'] log.info('Set user bind') topology.standalone.simple_bind_s(TEST_USER_DN, TEST_USER_PWD) log.info('Create simple paged results control instance') req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='') controls = [req_ctrl] for ii in xrange(3): log.info('Iteration %d' % ii) msgid = topology.standalone.search_ext(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, search_flt, searchreq_attrlist, serverctrls=controls) rtype, rdata, rmsgid, rctrls = topology.standalone.result3(msgid) pctrls = [ c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType ] req_ctrl.cookie = pctrls[0].cookie msgid = topology.standalone.search_ext(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, search_flt, searchreq_attrlist, serverctrls=controls) log.info('Set Directory Manager bind back') topology.standalone.simple_bind_s(DN_DM, PASSWORD) del_users(topology, users_list) log.info('Abandon the search') users_num = 10 page_size = 0 users_list = add_users(topology, users_num) search_flt = r'(uid=test*)' searchreq_attrlist = ['dn', 'sn'] log.info('Set user bind') topology.standalone.simple_bind_s(TEST_USER_DN, TEST_USER_PWD) log.info('Create simple paged results control instance') req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='') controls = [req_ctrl] msgid = topology.standalone.search_ext(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, search_flt, searchreq_attrlist, serverctrls=controls) rtype, rdata, rmsgid, rctrls = topology.standalone.result3(msgid) pctrls = [ c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType ] assert not pctrls[0].cookie log.info('Set Directory Manager bind back') topology.standalone.simple_bind_s(DN_DM, PASSWORD) del_users(topology, users_list) log.info("Search should fail with 'nsPagedSizeLimit = 5'" "and 'nsslapd-pagedsizelimit = 15' with 10 users") conf_attr = '15' user_attr = '5' expected_rs = ldap.SIZELIMIT_EXCEEDED users_num = 10 page_size = 10 users_list = add_users(topology, users_num) search_flt = r'(uid=test*)' searchreq_attrlist = ['dn', 'sn'] conf_attr_bck = change_conf_attr(topology, DN_CONFIG, 'nsslapd-pagedsizelimit', conf_attr) user_attr_bck = change_conf_attr(topology, TEST_USER_DN, 'nsPagedSizeLimit', user_attr) log.info('Set user bind') topology.standalone.simple_bind_s(TEST_USER_DN, TEST_USER_PWD) log.info('Create simple paged results control instance') req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='') controls = [req_ctrl] log.info('Expect to fail with SIZELIMIT_EXCEEDED') with pytest.raises(expected_rs): all_results = paged_search(topology, controls, search_flt, searchreq_attrlist) log.info('Set Directory Manager bind back') topology.standalone.simple_bind_s(DN_DM, PASSWORD) del_users(topology, users_list) change_conf_attr(topology, DN_CONFIG, 'nsslapd-pagedsizelimit', conf_attr_bck) change_conf_attr(topology, TEST_USER_DN, 'nsPagedSizeLimit', user_attr_bck) log.info("Search should pass with 'nsPagedSizeLimit = 15'" "and 'nsslapd-pagedsizelimit = 5' with 10 users") conf_attr = '5' user_attr = '15' users_num = 10 page_size = 10 users_list = add_users(topology, users_num) search_flt = r'(uid=test*)' searchreq_attrlist = ['dn', 'sn'] conf_attr_bck = change_conf_attr(topology, DN_CONFIG, 'nsslapd-pagedsizelimit', conf_attr) user_attr_bck = change_conf_attr(topology, TEST_USER_DN, 'nsPagedSizeLimit', user_attr) log.info('Set user bind') topology.standalone.simple_bind_s(TEST_USER_DN, TEST_USER_PWD) log.info('Create simple paged results control instance') req_ctrl = SimplePagedResultsControl(True, size=page_size, cookie='') controls = [req_ctrl] log.info('Search should PASS') all_results = paged_search(topology, controls, search_flt, searchreq_attrlist) log.info('%d results' % len(all_results)) assert len(all_results) == len(users_list) log.info('Set Directory Manager bind back') topology.standalone.simple_bind_s(DN_DM, PASSWORD) del_users(topology, users_list) change_conf_attr(topology, DN_CONFIG, 'nsslapd-pagedsizelimit', conf_attr_bck) change_conf_attr(topology, TEST_USER_DN, 'nsPagedSizeLimit', user_attr_bck)
retrieveAttributes = ['dNSHostName', 'distinguishedName'] # searchFilter. Options can include CN=, etc searchFilter = "objectClass=computer" pg_ctrl = SimplePagedResultsControl(CRITICALITY, PAGE_SIZE, COOKIE) while first_pass or pg_ctrl.cookie: first_pass = False try: msgid = l.search_ext(baseDN, searchScope, searchFilter, retrieveAttributes, serverctrls=[pg_ctrl]) except ldap.LDAPError, e: print e result_type, data, msgid, serverctrls = l.result3(msgid) # print "data is: ", data # print "serverctrls value is: ", serverctrls # pg_ctrl.cookie = serverctrls[0].cookie if len(serverctrls) > 0: pg_ctrl.cookie = serverctrls[0].cookie results += [i[0] for i in data] # convert results to a string as re.findall will not accept a List results_str = str(results) # Active Directory # The regular expression to pull only the distinguishedName attribute from the LDAP query is: AD_reg = "'CN=([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])," # open our csv file that was dumped from our LDAP computer objects lookup # read the contents of the file # use a findall function to parse through file searching for content based on the regex output = re.findall(AD_reg, results_str ,flags=0) # print the results #open file and export results file = open('ldap_out.csv','w') #Loop all computer hostnames and deploy Agent MSI
print ("Getting page %d" % (pages)) rtype, rdata, rmsgid, serverctrls = l.result3(msgid,resp_ctrl_classes=known_ldap_resp_ctrls) print ('%d results' % len(rdata)) print ('serverctrls=',pprint.pprint(serverctrls)) # pprint.pprint(rdata) pctrls = [ c for c in serverctrls if c.controlType == SimplePagedResultsControl.controlType ] if pctrls: print ('pctrls[0].size',repr(pctrls[0].size)) print ('pctrls[0].cookie',repr(pctrls[0].cookie)) if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = l.search_ext( base, ldap.SCOPE_SUBTREE, search_flt, attrlist=searchreq_attrlist, serverctrls=[req_ctrl] ) else: break else: print ("Warning: Server ignores RFC 2696 control.") break l.unbind_s()
def discover(self, args): import omero ldap = self.__import_ldap__() c = self.ctx.conn(args) iconfig = c.sf.getConfigService() iadmin = c.sf.getAdminService() iquery = c.sf.getQueryService() LDAP_PROPERTIES = """ omero.ldap.urls omero.ldap.username omero.ldap.password omero.ldap.base omero.ldap.user_filter omero.ldap.user_mapping omero.ldap.group_filter omero.ldap.group_mapping omero.ldap.new_user_group """.split() cfg = dict() for key in LDAP_PROPERTIES: cfg[key.split(".")[-1]] = iconfig.getConfigValue(key) urls = args.urls and args.urls or cfg["urls"] basedn = args.base and args.base or cfg["base"] for url in urls.split(","): self.ctx.err("Connecting to %s..." % url) ld = ldap.initialize(url) ld.simple_bind_s(cfg['username'], cfg['password']) user_filter = cfg["user_filter"] user_mapping = cfg["user_mapping"] user_mapping = user_mapping.split(",") omeName_mapping = None for um in user_mapping: parts = um.split("=") if parts[0] == "omeName": omeName_mapping = parts[1] from ldap.controls import SimplePagedResultsControl cookie = '' # This is the limit for Active Directory, 1000. However # the LDAP connection has a sizeLimit that overrides # this value if the page_size exceeds it so it is safe # to enter pretty much anything here when using paged results. page_size = 1000 results = [] first = True page_control = SimplePagedResultsControl(False, page_size, cookie) while first or page_control.cookie: first = False try: msgid = ld.search_ext( basedn, ldap.SCOPE_SUBTREE, user_filter, serverctrls=[page_control] ) except: self.ctx.die(1, "Failed to execute LDAP search") result_type, results, msgid, serverctrls = ld.result3(msgid) if serverctrls: page_control.cookie = serverctrls[0].cookie user_names = set() user_dns = {} for dn, entry in results: omeName = entry[omeName_mapping] if isinstance(omeName, (list, tuple)): if len(omeName) == 1: omeName = omeName[0] user_names.add(omeName) user_dns[omeName] = dn else: self.ctx.err("Failed to unwrap omeName: %s" % omeName) continue if not user_names: continue # Early exit! from omero.rtypes import rlist from omero.rtypes import rstring from omero.rtypes import unwrap params = omero.sys.ParametersI() params.add("names", rlist([rstring(x) for x in user_names])) id_names = unwrap(iquery.projection( "select id, omeName from Experimenter " "where omeName in (:names)", params)) for eid, omeName in id_names: try: olddn = iadmin.lookupLdapAuthExperimenter(eid) dn = user_dns[omeName] except omero.ApiUsageException: continue # Unknown user if olddn: if olddn.lower() != dn.lower(): self.ctx.err("Found different DN for %s: %s" % (omeName, olddn)) else: self.ctx.dbg("DN already set for %s: %s" % (omeName, olddn)) else: if args.commands: self.ctx.out("%s ldap setdn %s %s" % (sys.argv[0], omeName, dn)) else: self.ctx.out("Experimenter:%s\tomeName=%s\t%s" % (eid, omeName, dn))
def paged_search_ext_s(self,base,scope,filterstr='(objectClass=*)',attrlist=None,attrsonly=0,serverctrls=None,clientctrls=None,timeout=-1,sizelimit=0): """ Behaves exactly like LDAPObject.search_ext_s() but internally uses the simple paged results control to retrieve search results in chunks. This is non-sense for really large results sets which you would like to process one-by-one """ while True: # loop for reconnecting if necessary req_ctrl = SimplePagedResultsControl(True,size=self.page_size,cookie='') try: # Send first search request msgid = self.search_ext( base, scope, filterstr=filterstr, attrlist=attrlist, attrsonly=attrsonly, serverctrls=(serverctrls or [])+[req_ctrl], clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit ) result_pages = 0 all_results = [] while True: rtype, rdata, rmsgid, rctrls = self.result3(msgid) all_results.extend(rdata) result_pages += 1 # Extract the simple paged results response control pctrls = [ c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType ] if pctrls: if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = self.search_ext( base, scope, filterstr=filterstr, attrlist=attrlist, attrsonly=attrsonly, serverctrls=(serverctrls or [])+[req_ctrl], clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit ) else: break # no more pages available except ldap.SERVER_DOWN,e: try: self.reconnect(self._uri) except AttributeError: raise e else: return result_pages,all_results
def paged_search_ext_s(self,base,scope,filterstr='(objectClass=*)',attrlist=None,attrsonly=0,serverctrls=None,clientctrls=None,timeout=-1,sizelimit=0, page_size=1000): """ Behaves exactly like LDAPObject.search_ext_s() but internally uses the simple paged results control to retrieve search results in chunks. Taken on 2013-07-10 from : https://bitbucket.org/jaraco/python-ldap/src/f208b6338a28/Demo/paged_search_ext_s.py Converted to a generator Added range support based on : http://stackoverflow.com/a/1711862/591064 """ req_ctrl = SimplePagedResultsControl(True,size=page_size,cookie='') # Send first search request msgid = self.search_ext( base, scope, filterstr=filterstr, attrlist=attrlist, attrsonly=attrsonly, serverctrls=(serverctrls or [])+[req_ctrl], clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit ) while True: rtype, rdata, rmsgid, rctrls = self.result3(msgid) #singleresult is a list of pyldap tuple in the form [(dn, dict),(dn,dict)] for singleresult in rdata: #Range support idea found on StackOverflow #http://stackoverflow.com/a/1711862/591064# dn = singleresult[0] for attrname in singleresult[1]: if ';range=' in attrname: # # parse range attr # actual_attrname, range_stmt = attrname.split(';') bound_lower, bound_upper = [ int(x) for x in range_stmt.split('=')[1].split('-') ] step = bound_upper - bound_lower + 1 while True: attr_next = '%s;range=%d-%d' % ( actual_attrname, bound_lower, bound_upper ) dn, attrs = self.search_s( dn, ldap.SCOPE_BASE, attrlist = [attr_next])[0] assert len(attrs) == 1 ret_attrname = attrs.keys()[0] singleresult[1][actual_attrname].extend(attrs[ret_attrname]) if ret_attrname.endswith('-*'): break bound_lower = bound_upper + 1 bound_upper += step yield singleresult # Extract the simple paged results response control pctrls = [ c for c in rctrls if c.controlType == SimplePagedResultsControl.controlType ] if pctrls: if pctrls[0].cookie: # Copy cookie from response control to request control req_ctrl.cookie = pctrls[0].cookie msgid = self.search_ext( base, scope, filterstr=filterstr, attrlist=attrlist, attrsonly=attrsonly, serverctrls=(serverctrls or [])+[req_ctrl], clientctrls=clientctrls, timeout=timeout, sizelimit=sizelimit ) else: break