def test_returns_populated_groups(self, get_group):
conn = Mock()
# The following tom-foolery with returns and side_effect is to make sure
# we don't have to bother with setting up get_group correctly, and so
# we assume it works correctly to mock it here.
returns = [[sentinel.testuser1, sentinel.testuser2],
[sentinel.testuser3, sentinel.testuser4],
[sentinel.testuser5, sentinel.testuser6]]
expected = [sentinel.testuser1, sentinel.testuser2,
sentinel.testuser3, sentinel.testuser4,
sentinel.testuser5, sentinel.testuser6,
]
returned_groups = [MagicMock(), MagicMock(), MagicMock()]
for group in returned_groups:
group.__iter__.return_value = returns.pop(0)
def side_effect(*args):
result = returned_groups.pop(0)
return result
config = {'groups': self.multiple_test_groups,
'dir_guid_source': 'user_guid_source',
'dir_username_source': 'user_source',
'dir_fname_source': 'fname_source',
'dir_lname_source': 'lname_source', }
get_group.side_effect = side_effect
self.assertEqual(ldap_source.collect_groups(conn, config),
expected)
def run_db_repair(config, db_conn):
"""Repairs the current user DB and billing API versus LDAP."""
# TODO: figure out what to do when email addresses *don't* match.
log = logging.getLogger("run_db_repair")
# Collect the users from LDAP, and insert into a temporary table.
ldap_conn = ldap_source.OMLDAPConnection(config["dir_uri"],
config["dir_base_dn"],
config["dir_user"],
config["dir_password"])
log.info("Collecting LDAP groups")
ldap_users = ldap_source.collect_groups(ldap_conn, config)
cur = db_conn.cursor()
cur.execute("CREATE TEMPORARY TABLE ldap_users (LIKE users) ON COMMIT DROP;")
cur.execute("ALTER TABLE ldap_users DROP COLUMN avatar_id;")
cur.execute("ALTER TABLE ldap_users DROP COLUMN enabled;")
cur.executemany("INSERT INTO ldap_users (uniqueid, email, givenname, surname, group_id) VALUES (%(uniqueid)s, %(email)s, %(firstname)s, %(lastname)s, %(group_id)s);",
ldap_users)
# Collect the users from the SpiderOak Accounts API, and insert into
# a temporary table.
log.info("Collecting SpiderOak user details")
api = account_mgr.get_api(config)
spider_users = api.list_users()
for spider_user in spider_users:
first_name, sep, last_name = spider_user['name'].strip().partition(' ')
if not last_name:
last_name = ' '
spider_user['firstname'] = first_name
spider_user['lastname'] = last_name
cur = db_conn.cursor()
cur.execute("CREATE TEMPORARY TABLE spider_users (LIKE users) ON COMMIT DROP;")
cur.execute("ALTER TABLE spider_users DROP COLUMN uniqueid;")
cur.executemany("INSERT INTO spider_users "
"(avatar_id, email, givenname, surname, group_id, enabled) VALUES "
"(%(avatar_id)s, %(email)s, %(firstname)s, %(lastname)s, "
"%(group_id)s, %(enabled)s);",
spider_users)
cur.execute("SELECT email, count(email) as occurences from ldap_users group by email having ( count(email) > 1 )")
for row in cur.fetchall():
log.error("---> Duplicate user %s found %d times in LDAP query!", row[0], row[1])
# Clear out the current database.
cur.execute("DELETE FROM users;")
log.info("Inserting joined fields into the database")
# Insert rows into users where email addresses match.
cur.execute("INSERT INTO users "
"SELECT l.uniqueid, s.email, s.avatar_id, s.givenname, "
"s.surname, s.group_id, s.enabled "
"FROM ldap_users l JOIN spider_users AS s ON l.email = s.email ")
db_conn.commit()
def test_calls_appropriate_args(self, get_group):
conn = Mock()
test_group = MagicMock()
test_group.__iter__.return_value = []
get_group.return_value = test_group
config = {'groups': self.single_test_group,
'dir_guid_source': 'user_guid_source',
'dir_username_source': 'user_source',
'dir_fname_source': 'fname_source',
'dir_lname_source': 'lname_source', }
ldap_source.collect_groups(conn, config)
ldap_source.LdapGroup.get_group.assert_called_with(
conn, config,
self.single_test_group[0]['ldap_id'],
self.single_test_group[0]['group_id'])
def test_calls_appropriate_args(self, get_group):
conn = Mock()
test_group = MagicMock()
test_group.__iter__.return_value = []
get_group.return_value = test_group
config = {
'groups': self.single_test_group,
'dir_guid_source': 'user_guid_source',
'dir_username_source': 'user_source',
'dir_fname_source': 'fname_source',
'dir_lname_source': 'lname_source',
}
ldap_source.collect_groups(conn, config)
ldap_source.LdapGroup.get_group.assert_called_with(
conn, config, self.single_test_group[0]['ldap_id'],
self.single_test_group[0]['group_id'])
def test_returns_empty_groups(self, get_group):
conn = Mock()
test_group = MagicMock()
test_group.__iter__.return_value = []
get_group.return_value = test_group
# ldap_source.LdapGroup.get_group.return_value = test_group
config = {'groups': self.multiple_test_groups,
'dir_guid_source': 'user_guid_source',
'dir_username_source': 'user_source',
'dir_fname_source': 'fname_source',
'dir_lname_source': 'lname_source',}
self.assertEqual(len(ldap_source.collect_groups(conn, config)),
0)
def test_returns_empty_groups(self, get_group):
conn = Mock()
test_group = MagicMock()
test_group.__iter__.return_value = []
get_group.return_value = test_group
# ldap_source.LdapGroup.get_group.return_value = test_group
config = {
'groups': self.multiple_test_groups,
'dir_guid_source': 'user_guid_source',
'dir_username_source': 'user_source',
'dir_fname_source': 'fname_source',
'dir_lname_source': 'lname_source',
}
self.assertEqual(len(ldap_source.collect_groups(conn, config)), 0)
def run_group_management(config, db_conn):
"""
Resolves differences between the LDAP and our idea of the SpiderOak user DB.
:param config: configuration dict. Should be the standard OpenManage setup.
:param user_source: UserSource object to pull users from.
:param db_conn: DB connection object
"""
log = logging.getLogger('run_group_management')
# First step, collect the users from the LDAP groups.
ldap_conn = ldap_source.OMLDAPConnection(config["dir_uri"], config["dir_base_dn"], config["dir_user"], config["dir_password"])
ldap_users = ldap_source.collect_groups(ldap_conn, config)
change_groups = _calculate_changes_against_db(db_conn, ldap_users)
runner = account_runner.AccountRunner(config, db_conn)
runner.runall(change_groups)
db_conn.commit()
def run_group_management(config, db_conn):
"""
Resolves differences between the LDAP and our idea of the SpiderOak user DB.
:param config: configuration dict. Should be the standard OpenManage setup.
:param user_source: UserSource object to pull users from.
:param db_conn: DB connection object
"""
log = logging.getLogger('run_group_management')
# First step, collect the users from the LDAP groups.
ldap_conn = ldap_source.OMLDAPConnection(config["dir_uri"], config["dir_base_dn"], config["dir_user"], config["dir_password"])
ldap_users = ldap_source.collect_groups(ldap_conn, config)
change_groups = _calculate_changes_against_db(db_conn, config, ldap_users)
runner = account_runner.AccountRunner(config, db_conn)
runner.runall(change_groups)
db_conn.commit()
def run_db_repair(config, db_conn):
"""Repairs the current user DB and billing API versus LDAP."""
# TODO: figure out what to do when email addresses *don't* match.
# Collect the users from LDAP, and insert into a temporary table.
ldap_conn = ldap_source.OMLDAPConnection(config["dir_uri"],
config["dir_base_dn"],
config["dir_user"],
config["dir_password"])
ldap_users = ldap_source.collect_groups(ldap_conn, config)
cur = db_conn.cursor()
cur.execute("CREATE TEMPORARY TABLE ldap_users (LIKE users) ON COMMIT DROP;")
cur.execute("ALTER TABLE ldap_users DROP COLUMN avatar_id;")
cur.execute("ALTER TABLE ldap_users DROP COLUMN enabled;")
cur.executemany("INSERT INTO ldap_users (uniqueid, email, givenname, surname, group_id) VALUES (%(uniqueid)s, %(email)s, %(firstname)s, %(lastname)s, %(group_id)s);",
ldap_users)
# Collect the users from the SpiderOak BillingAPI, and insert into
# a temporary table.
spider_users = api_interface.fetch_users()
cur = db_conn.cursor()
cur.execute("CREATE TEMPORARY TABLE spider_users (LIKE users) ON COMMIT DROP;")
cur.execute("ALTER TABLE spider_users DROP COLUMN uniqueid;")
cur.executemany("INSERT INTO spider_users "
"(avatar_id, email, givenname, surname, group_id, enabled) VALUES "
"(%(avatar_id)s, %(email)s, %(firstname)s, %(lastname)s, "
"%(group_id)s, %(enabled)s);",
spider_users)
# Clear out the current database.
cur.execute("DELETE FROM users;")
# Insert rows into users where email addresses match.
cur.execute("INSERT INTO users "
"SELECT l.uniqueid, s.email, s.avatar_id, s.givenname, "
"s.surname, s.group_id, s.enabled "
"FROM ldap_users l JOIN spider_users AS s ON l.email = s.email ")
db_conn.commit()
def test_returns_populated_groups(self, get_group):
conn = Mock()
# The following tom-foolery with returns and side_effect is to make sure
# we don't have to bother with setting up get_group correctly, and so
# we assume it works correctly to mock it here.
returns = [[sentinel.testuser1, sentinel.testuser2],
[sentinel.testuser3, sentinel.testuser4],
[sentinel.testuser5, sentinel.testuser6]]
expected = [
sentinel.testuser1,
sentinel.testuser2,
sentinel.testuser3,
sentinel.testuser4,
sentinel.testuser5,
sentinel.testuser6,
]
returned_groups = [MagicMock(), MagicMock(), MagicMock()]
for group in returned_groups:
group.__iter__.return_value = returns.pop(0)
def side_effect(*args):
result = returned_groups.pop(0)
return result
config = {
'groups': self.multiple_test_groups,
'dir_guid_source': 'user_guid_source',
'dir_username_source': 'user_source',
'dir_fname_source': 'fname_source',
'dir_lname_source': 'lname_source',
}
get_group.side_effect = side_effect
self.assertEqual(ldap_source.collect_groups(conn, config), expected)
def run_db_repair(config, db_conn):
"""Repairs the current user DB and billing API versus LDAP."""
# TODO: figure out what to do when email addresses *don't* match.
log = logging.getLogger("run_db_repair")
# Collect the users from LDAP, and insert into a temporary table.
ldap_conn = ldap_source.OMLDAPConnection(config["dir_uri"],
config["dir_base_dn"],
config["dir_user"],
config["dir_password"])
log.info("Collecting LDAP groups")
ldap_users = ldap_source.collect_groups(ldap_conn, config)
cur = db_conn.cursor()
cur.execute("CREATE TEMPORARY TABLE ldap_users (LIKE users) ON COMMIT DROP;")
cur.execute("ALTER TABLE ldap_users DROP COLUMN avatar_id;")
cur.execute("ALTER TABLE ldap_users DROP COLUMN enabled;")
cur.executemany("INSERT INTO ldap_users (uniqueid, email, givenname, surname, group_id) VALUES (%(uniqueid)s, %(email)s, %(firstname)s, %(lastname)s, %(group_id)s);",
ldap_users)
# Collect the users from the SpiderOak Accounts API, and insert into
# a temporary table.
log.info("Collecting SpiderOak user details")
api = account_mgr.get_api(config)
spider_users = api.list_users()
for spider_user in spider_users:
first_name, sep, last_name = spider_user['name'].strip().partition(' ')
if not last_name:
last_name = ' '
spider_user['firstname'] = first_name
spider_user['lastname'] = last_name
cur = db_conn.cursor()
cur.execute("CREATE TEMPORARY TABLE spider_users (LIKE users) ON COMMIT DROP;")
cur.execute("ALTER TABLE spider_users DROP COLUMN uniqueid;")
cur.executemany("INSERT INTO spider_users "
"(avatar_id, email, givenname, surname, group_id, enabled) VALUES "
"(%(avatar_id)s, %(email)s, %(firstname)s, %(lastname)s, "
"%(group_id)s, %(enabled)s);",
spider_users)
# Delete duplicate rows
cur.execute("DELETE FROM ldap_users "
"WHERE ctid NOT IN "
"(SELECT MAX(l.ctid) "
"FROM ldap_users l "
"GROUP BY l.email)"
)
# Clear out the current database.
cur.execute("DELETE FROM users;")
log.info("Inserting joined fields into the database")
# Insert rows into users where email addresses match.
cur.execute("INSERT INTO users "
"SELECT l.uniqueid, s.email, s.avatar_id, s.givenname, "
"s.surname, s.group_id, s.enabled "
"FROM ldap_users l JOIN spider_users AS s ON l.email = s.email ")
# Collect the list of users who are NOT in the LDAP
# There are two types of users not in the LDAP sync groups we're looking through:
# 1. Users who exist in the LDAP still, but not anymore in a monitored group
# 2. Users who do not at all exist in the LDAP.
#
# Users in the first group we can enter back into the user sync database as disabled,
# as we can locate some form of unique ID from the LDAP to put in the sync DB. The
# second group needs to be just disabled on the Accounts API side. Note that users
# in this second group will have to have the whole DB rebuilt if they reappear on the LDAP
# and wish to continue using the same account.
cur.execute("SELECT s.email, s.avatar_id, s.givenname, s.surname, s.group_id, s.enabled "
"FROM spider_users s "
"LEFT OUTER JOIN ldap_users l USING (email) "
"WHERE l.email IS NULL")
orphans = cur.fetchall()
# We only care about ldap users here
orphans = [x for x in orphans \
if get_config_group(config, x[4])["user_source"] == 'ldap']
# "found_orphans" are the users who exist *somewhere* in the LDAP. lost_orphans do not.
found_orphans = _run_disabled_users_for_repair(ldap_conn, config, cur.description, orphans)
found_emails = [y['email'] for y in found_orphans]
lost_orphans = [x for x in orphans if x[0] not in found_emails]
# Put the found orphans in the DB.
cur.executemany("INSERT INTO users "
"(avatar_id, email, givenname, surname, group_id, enabled, uniqueid) "
"VALUES (%(avatar_id)s, %(email)s, %(givenname)s, %(surname)s, "
" %(group_id)s, %(enabled)s, %(uniqueid)s);",
found_orphans)
db_conn.commit()
# ...and disable the lost orphans. We don't care about already disabled lost orphans,
# we want to only disable orphans who are enabled so they can be rounded up and
# deleted.
for orphan in lost_orphans:
if orphan[5]: # If the user is enabled then disable them.
api.edit_user(orphan[0], dict(enabled=False))
def run_db_repair(config, db_conn):
"""Repairs the current user DB and billing API versus LDAP."""
# TODO: figure out what to do when email addresses *don't* match.
log = logging.getLogger("run_db_repair")
# Collect the users from LDAP, and insert into a temporary table.
ldap_conn = ldap_source.OMLDAPConnection(config["dir_uri"],
config["dir_base_dn"],
config["dir_user"],
config["dir_password"])
log.info("Collecting LDAP groups")
ldap_users = ldap_source.collect_groups(ldap_conn, config)
cur = db_conn.cursor()
cur.execute("CREATE TEMPORARY TABLE ldap_users (LIKE users) ON COMMIT DROP;")
cur.execute("ALTER TABLE ldap_users DROP COLUMN avatar_id;")
cur.execute("ALTER TABLE ldap_users DROP COLUMN enabled;")
cur.executemany("INSERT INTO ldap_users (uniqueid, email, givenname, surname, group_id) VALUES (%(uniqueid)s, %(email)s, %(firstname)s, %(lastname)s, %(group_id)s);",
ldap_users)
# Collect the users from the SpiderOak Accounts API, and insert into
# a temporary table.
log.info("Collecting SpiderOak user details")
api = account_mgr.get_api(config)
spider_users = api.list_users()
for spider_user in spider_users:
first_name, sep, last_name = spider_user['name'].strip().partition(' ')
if not last_name:
last_name = ' '
spider_user['firstname'] = first_name
spider_user['lastname'] = last_name
cur = db_conn.cursor()
cur.execute("CREATE TEMPORARY TABLE spider_users (LIKE users) ON COMMIT DROP;")
cur.execute("ALTER TABLE spider_users DROP COLUMN uniqueid;")
cur.executemany("INSERT INTO spider_users "
"(avatar_id, email, givenname, surname, group_id, enabled) VALUES "
"(%(avatar_id)s, %(email)s, %(firstname)s, %(lastname)s, "
"%(group_id)s, %(enabled)s);",
spider_users)
# Delete duplicate rows
cur.execute("DELETE FROM ldap_users "
"WHERE ctid NOT IN "
"(SELECT MAX(l.ctid) "
"FROM ldap_users l "
"GROUP BY l.email)"
)
# Clear out the current database.
cur.execute("DELETE FROM users;")
log.info("Inserting joined fields into the database")
# Insert rows into users where email addresses match.
cur.execute("INSERT INTO users "
"SELECT l.uniqueid, s.email, s.avatar_id, s.givenname, "
"s.surname, s.group_id, s.enabled "
"FROM ldap_users l JOIN spider_users AS s ON l.email = s.email ")
# Collect the list of users who are NOT in the LDAP
# There are two types of users not in the LDAP sync groups we're looking through:
# 1. Users who exist in the LDAP still, but not anymore in a monitored group
# 2. Users who do not at all exist in the LDAP.
#
# Users in the first group we can enter back into the user sync database as disabled,
# as we can locate some form of unique ID from the LDAP to put in the sync DB. The
# second group needs to be just disabled on the Accounts API side. Note that users
# in this second group will have to have the whole DB rebuilt if they reappear on the LDAP
# and wish to continue using the same account.
cur.execute("SELECT s.email, s.avatar_id, s.givenname, s.surname, s.group_id, s.enabled "
"FROM spider_users s "
"LEFT OUTER JOIN ldap_users l USING (email) "
"WHERE l.email IS NULL")
orphans = cur.fetchall()
# We only care about ldap users here
orphans = [x for x in orphans \
if get_config_group(config, x[4])["user_source"] == 'ldap']
# "found_orphans" are the users who exist *somewhere* in the LDAP. lost_orphans do not.
found_orphans = _run_disabled_users_for_repair(ldap_conn, config, cur.description, orphans)
found_emails = [y['email'] for y in found_orphans]
lost_orphans = [x for x in orphans if x[0] not in found_emails]
# Put the found orphans in the DB.
cur.executemany("INSERT INTO users "
"(avatar_id, email, givenname, surname, group_id, enabled, uniqueid) "
"VALUES (%(avatar_id)s, %(email)s, %(givenname)s, %(surname)s, "
" %(group_id)s, %(enabled)s, %(uniqueid)s);",
found_orphans)
db_conn.commit()
# ...and disable the lost orphans. We don't care about already disabled lost orphans,
# we want to only disable orphans who are enabled so they can be rounded up and
# deleted.
for orphan in lost_orphans:
if orphan[5]: # If the user is enabled then disable them.
api.edit_user(orphan[0], dict(enabled=False))