def test_returns_populated_groups(self, get_group): conn = Mock() # The following tom-foolery with returns and side_effect is to make sure # we don't have to bother with setting up get_group correctly, and so # we assume it works correctly to mock it here. returns = [[sentinel.testuser1, sentinel.testuser2], [sentinel.testuser3, sentinel.testuser4], [sentinel.testuser5, sentinel.testuser6]] expected = [sentinel.testuser1, sentinel.testuser2, sentinel.testuser3, sentinel.testuser4, sentinel.testuser5, sentinel.testuser6, ] returned_groups = [MagicMock(), MagicMock(), MagicMock()] for group in returned_groups: group.__iter__.return_value = returns.pop(0) def side_effect(*args): result = returned_groups.pop(0) return result config = {'groups': self.multiple_test_groups, 'dir_guid_source': 'user_guid_source', 'dir_username_source': 'user_source', 'dir_fname_source': 'fname_source', 'dir_lname_source': 'lname_source', } get_group.side_effect = side_effect self.assertEqual(ldap_source.collect_groups(conn, config), expected)
def run_db_repair(config, db_conn): """Repairs the current user DB and billing API versus LDAP.""" # TODO: figure out what to do when email addresses *don't* match. log = logging.getLogger("run_db_repair") # Collect the users from LDAP, and insert into a temporary table. ldap_conn = ldap_source.OMLDAPConnection(config["dir_uri"], config["dir_base_dn"], config["dir_user"], config["dir_password"]) log.info("Collecting LDAP groups") ldap_users = ldap_source.collect_groups(ldap_conn, config) cur = db_conn.cursor() cur.execute("CREATE TEMPORARY TABLE ldap_users (LIKE users) ON COMMIT DROP;") cur.execute("ALTER TABLE ldap_users DROP COLUMN avatar_id;") cur.execute("ALTER TABLE ldap_users DROP COLUMN enabled;") cur.executemany("INSERT INTO ldap_users (uniqueid, email, givenname, surname, group_id) VALUES (%(uniqueid)s, %(email)s, %(firstname)s, %(lastname)s, %(group_id)s);", ldap_users) # Collect the users from the SpiderOak Accounts API, and insert into # a temporary table. log.info("Collecting SpiderOak user details") api = account_mgr.get_api(config) spider_users = api.list_users() for spider_user in spider_users: first_name, sep, last_name = spider_user['name'].strip().partition(' ') if not last_name: last_name = ' ' spider_user['firstname'] = first_name spider_user['lastname'] = last_name cur = db_conn.cursor() cur.execute("CREATE TEMPORARY TABLE spider_users (LIKE users) ON COMMIT DROP;") cur.execute("ALTER TABLE spider_users DROP COLUMN uniqueid;") cur.executemany("INSERT INTO spider_users " "(avatar_id, email, givenname, surname, group_id, enabled) VALUES " "(%(avatar_id)s, %(email)s, %(firstname)s, %(lastname)s, " "%(group_id)s, %(enabled)s);", spider_users) cur.execute("SELECT email, count(email) as occurences from ldap_users group by email having ( count(email) > 1 )") for row in cur.fetchall(): log.error("---> Duplicate user %s found %d times in LDAP query!", row[0], row[1]) # Clear out the current database. cur.execute("DELETE FROM users;") log.info("Inserting joined fields into the database") # Insert rows into users where email addresses match. cur.execute("INSERT INTO users " "SELECT l.uniqueid, s.email, s.avatar_id, s.givenname, " "s.surname, s.group_id, s.enabled " "FROM ldap_users l JOIN spider_users AS s ON l.email = s.email ") db_conn.commit()
def test_calls_appropriate_args(self, get_group): conn = Mock() test_group = MagicMock() test_group.__iter__.return_value = [] get_group.return_value = test_group config = {'groups': self.single_test_group, 'dir_guid_source': 'user_guid_source', 'dir_username_source': 'user_source', 'dir_fname_source': 'fname_source', 'dir_lname_source': 'lname_source', } ldap_source.collect_groups(conn, config) ldap_source.LdapGroup.get_group.assert_called_with( conn, config, self.single_test_group[0]['ldap_id'], self.single_test_group[0]['group_id'])
def test_calls_appropriate_args(self, get_group): conn = Mock() test_group = MagicMock() test_group.__iter__.return_value = [] get_group.return_value = test_group config = { 'groups': self.single_test_group, 'dir_guid_source': 'user_guid_source', 'dir_username_source': 'user_source', 'dir_fname_source': 'fname_source', 'dir_lname_source': 'lname_source', } ldap_source.collect_groups(conn, config) ldap_source.LdapGroup.get_group.assert_called_with( conn, config, self.single_test_group[0]['ldap_id'], self.single_test_group[0]['group_id'])
def test_returns_empty_groups(self, get_group): conn = Mock() test_group = MagicMock() test_group.__iter__.return_value = [] get_group.return_value = test_group # ldap_source.LdapGroup.get_group.return_value = test_group config = {'groups': self.multiple_test_groups, 'dir_guid_source': 'user_guid_source', 'dir_username_source': 'user_source', 'dir_fname_source': 'fname_source', 'dir_lname_source': 'lname_source',} self.assertEqual(len(ldap_source.collect_groups(conn, config)), 0)
def test_returns_empty_groups(self, get_group): conn = Mock() test_group = MagicMock() test_group.__iter__.return_value = [] get_group.return_value = test_group # ldap_source.LdapGroup.get_group.return_value = test_group config = { 'groups': self.multiple_test_groups, 'dir_guid_source': 'user_guid_source', 'dir_username_source': 'user_source', 'dir_fname_source': 'fname_source', 'dir_lname_source': 'lname_source', } self.assertEqual(len(ldap_source.collect_groups(conn, config)), 0)
def run_group_management(config, db_conn): """ Resolves differences between the LDAP and our idea of the SpiderOak user DB. :param config: configuration dict. Should be the standard OpenManage setup. :param user_source: UserSource object to pull users from. :param db_conn: DB connection object """ log = logging.getLogger('run_group_management') # First step, collect the users from the LDAP groups. ldap_conn = ldap_source.OMLDAPConnection(config["dir_uri"], config["dir_base_dn"], config["dir_user"], config["dir_password"]) ldap_users = ldap_source.collect_groups(ldap_conn, config) change_groups = _calculate_changes_against_db(db_conn, ldap_users) runner = account_runner.AccountRunner(config, db_conn) runner.runall(change_groups) db_conn.commit()
def run_group_management(config, db_conn): """ Resolves differences between the LDAP and our idea of the SpiderOak user DB. :param config: configuration dict. Should be the standard OpenManage setup. :param user_source: UserSource object to pull users from. :param db_conn: DB connection object """ log = logging.getLogger('run_group_management') # First step, collect the users from the LDAP groups. ldap_conn = ldap_source.OMLDAPConnection(config["dir_uri"], config["dir_base_dn"], config["dir_user"], config["dir_password"]) ldap_users = ldap_source.collect_groups(ldap_conn, config) change_groups = _calculate_changes_against_db(db_conn, config, ldap_users) runner = account_runner.AccountRunner(config, db_conn) runner.runall(change_groups) db_conn.commit()
def run_db_repair(config, db_conn): """Repairs the current user DB and billing API versus LDAP.""" # TODO: figure out what to do when email addresses *don't* match. # Collect the users from LDAP, and insert into a temporary table. ldap_conn = ldap_source.OMLDAPConnection(config["dir_uri"], config["dir_base_dn"], config["dir_user"], config["dir_password"]) ldap_users = ldap_source.collect_groups(ldap_conn, config) cur = db_conn.cursor() cur.execute("CREATE TEMPORARY TABLE ldap_users (LIKE users) ON COMMIT DROP;") cur.execute("ALTER TABLE ldap_users DROP COLUMN avatar_id;") cur.execute("ALTER TABLE ldap_users DROP COLUMN enabled;") cur.executemany("INSERT INTO ldap_users (uniqueid, email, givenname, surname, group_id) VALUES (%(uniqueid)s, %(email)s, %(firstname)s, %(lastname)s, %(group_id)s);", ldap_users) # Collect the users from the SpiderOak BillingAPI, and insert into # a temporary table. spider_users = api_interface.fetch_users() cur = db_conn.cursor() cur.execute("CREATE TEMPORARY TABLE spider_users (LIKE users) ON COMMIT DROP;") cur.execute("ALTER TABLE spider_users DROP COLUMN uniqueid;") cur.executemany("INSERT INTO spider_users " "(avatar_id, email, givenname, surname, group_id, enabled) VALUES " "(%(avatar_id)s, %(email)s, %(firstname)s, %(lastname)s, " "%(group_id)s, %(enabled)s);", spider_users) # Clear out the current database. cur.execute("DELETE FROM users;") # Insert rows into users where email addresses match. cur.execute("INSERT INTO users " "SELECT l.uniqueid, s.email, s.avatar_id, s.givenname, " "s.surname, s.group_id, s.enabled " "FROM ldap_users l JOIN spider_users AS s ON l.email = s.email ") db_conn.commit()
def test_returns_populated_groups(self, get_group): conn = Mock() # The following tom-foolery with returns and side_effect is to make sure # we don't have to bother with setting up get_group correctly, and so # we assume it works correctly to mock it here. returns = [[sentinel.testuser1, sentinel.testuser2], [sentinel.testuser3, sentinel.testuser4], [sentinel.testuser5, sentinel.testuser6]] expected = [ sentinel.testuser1, sentinel.testuser2, sentinel.testuser3, sentinel.testuser4, sentinel.testuser5, sentinel.testuser6, ] returned_groups = [MagicMock(), MagicMock(), MagicMock()] for group in returned_groups: group.__iter__.return_value = returns.pop(0) def side_effect(*args): result = returned_groups.pop(0) return result config = { 'groups': self.multiple_test_groups, 'dir_guid_source': 'user_guid_source', 'dir_username_source': 'user_source', 'dir_fname_source': 'fname_source', 'dir_lname_source': 'lname_source', } get_group.side_effect = side_effect self.assertEqual(ldap_source.collect_groups(conn, config), expected)
def run_db_repair(config, db_conn): """Repairs the current user DB and billing API versus LDAP.""" # TODO: figure out what to do when email addresses *don't* match. log = logging.getLogger("run_db_repair") # Collect the users from LDAP, and insert into a temporary table. ldap_conn = ldap_source.OMLDAPConnection(config["dir_uri"], config["dir_base_dn"], config["dir_user"], config["dir_password"]) log.info("Collecting LDAP groups") ldap_users = ldap_source.collect_groups(ldap_conn, config) cur = db_conn.cursor() cur.execute("CREATE TEMPORARY TABLE ldap_users (LIKE users) ON COMMIT DROP;") cur.execute("ALTER TABLE ldap_users DROP COLUMN avatar_id;") cur.execute("ALTER TABLE ldap_users DROP COLUMN enabled;") cur.executemany("INSERT INTO ldap_users (uniqueid, email, givenname, surname, group_id) VALUES (%(uniqueid)s, %(email)s, %(firstname)s, %(lastname)s, %(group_id)s);", ldap_users) # Collect the users from the SpiderOak Accounts API, and insert into # a temporary table. log.info("Collecting SpiderOak user details") api = account_mgr.get_api(config) spider_users = api.list_users() for spider_user in spider_users: first_name, sep, last_name = spider_user['name'].strip().partition(' ') if not last_name: last_name = ' ' spider_user['firstname'] = first_name spider_user['lastname'] = last_name cur = db_conn.cursor() cur.execute("CREATE TEMPORARY TABLE spider_users (LIKE users) ON COMMIT DROP;") cur.execute("ALTER TABLE spider_users DROP COLUMN uniqueid;") cur.executemany("INSERT INTO spider_users " "(avatar_id, email, givenname, surname, group_id, enabled) VALUES " "(%(avatar_id)s, %(email)s, %(firstname)s, %(lastname)s, " "%(group_id)s, %(enabled)s);", spider_users) # Delete duplicate rows cur.execute("DELETE FROM ldap_users " "WHERE ctid NOT IN " "(SELECT MAX(l.ctid) " "FROM ldap_users l " "GROUP BY l.email)" ) # Clear out the current database. cur.execute("DELETE FROM users;") log.info("Inserting joined fields into the database") # Insert rows into users where email addresses match. cur.execute("INSERT INTO users " "SELECT l.uniqueid, s.email, s.avatar_id, s.givenname, " "s.surname, s.group_id, s.enabled " "FROM ldap_users l JOIN spider_users AS s ON l.email = s.email ") # Collect the list of users who are NOT in the LDAP # There are two types of users not in the LDAP sync groups we're looking through: # 1. Users who exist in the LDAP still, but not anymore in a monitored group # 2. Users who do not at all exist in the LDAP. # # Users in the first group we can enter back into the user sync database as disabled, # as we can locate some form of unique ID from the LDAP to put in the sync DB. The # second group needs to be just disabled on the Accounts API side. Note that users # in this second group will have to have the whole DB rebuilt if they reappear on the LDAP # and wish to continue using the same account. cur.execute("SELECT s.email, s.avatar_id, s.givenname, s.surname, s.group_id, s.enabled " "FROM spider_users s " "LEFT OUTER JOIN ldap_users l USING (email) " "WHERE l.email IS NULL") orphans = cur.fetchall() # We only care about ldap users here orphans = [x for x in orphans \ if get_config_group(config, x[4])["user_source"] == 'ldap'] # "found_orphans" are the users who exist *somewhere* in the LDAP. lost_orphans do not. found_orphans = _run_disabled_users_for_repair(ldap_conn, config, cur.description, orphans) found_emails = [y['email'] for y in found_orphans] lost_orphans = [x for x in orphans if x[0] not in found_emails] # Put the found orphans in the DB. cur.executemany("INSERT INTO users " "(avatar_id, email, givenname, surname, group_id, enabled, uniqueid) " "VALUES (%(avatar_id)s, %(email)s, %(givenname)s, %(surname)s, " " %(group_id)s, %(enabled)s, %(uniqueid)s);", found_orphans) db_conn.commit() # ...and disable the lost orphans. We don't care about already disabled lost orphans, # we want to only disable orphans who are enabled so they can be rounded up and # deleted. for orphan in lost_orphans: if orphan[5]: # If the user is enabled then disable them. api.edit_user(orphan[0], dict(enabled=False))