def test_sync_altered_groups(self, mock_add_or_update):
"""Test the sync_altered accounts function"""
mock_client = mock.MagicMock()
test_group = mkGroup(test_vo_1)
mock_client.allgroups.modified[1].get.return_value = (200, [test_vo_1])
mock_client.vo[test_group.vsc_id].get.return_value = (200, test_vo_1)
mock_add_or_update.return_value = UPDATED
ldapsyncer = LdapSyncer(mock_client)
groups = ldapsyncer.sync_altered_groups(1)
self.assertEqual(groups, {'error': set([]), 'new': set([]), 'updated': set([test_group.vsc_id])})
ldap_attrs = {'status': ['active'], 'scratchDirectory': ['/user/scratch/gent/gvo000/gvo00003'],
'dataDirectory': ['/user/data/gent/gvo000/gvo00003'], 'cn': 'gvo00003', 'institute': ['gent'],
'memberUid': ['vsc40075'], 'moderator': ['vsc40075'], 'gidNumber': ['2640010'],
'fairshare': ['100'], 'description': ['VO']}
mock_add_or_update.assert_called_with(VscLdapGroup, test_group.vsc_id, ldap_attrs, True)
# raise a 404 error on getting the VO to trigger a non vo group add instead of a vo add as above
test_group = mkGroup(test_usergroup_1)
mock_client.allgroups.modified[1].get.return_value = (200, [test_usergroup_1])
mock_client.vo[test_group.vsc_id].get.side_effect = HTTPError("mock_url", 404, "Not Found", "mock_headers", None)
groups = ldapsyncer.sync_altered_groups(1)
self.assertEqual(groups, {'error': set([]), 'new': set([]), 'updated': set([test_group.vsc_id])})
ldap_attrs = {'status': ['active'], 'cn': 'vsc40075', 'institute': ['gent'], 'memberUid': ['vsc40075'],
'moderator': ['vsc40075'], 'gidNumber': ['2540075']}
mock_add_or_update.assert_called_with(VscLdapGroup, test_group.vsc_id, ldap_attrs, True)
def test_usergroup_instantiation(self):
mock_client = mock.MagicMock()
test_account = mkVscAccount(test_account_1)
mock_client.account[test_account.vsc_id] = mock.MagicMock()
mock_client.account[test_account.vsc_id].get.return_value = (
200, test_account_1)
mock_client.account[test_account.vsc_id].usergroup.get.return_value = (
200, test_usergroup_1)
accountpageuser = user.VscAccountPageUser(test_account.vsc_id,
rest_client=mock_client)
self.assertEqual(accountpageuser.usergroup,
mkUserGroup(test_usergroup_1))
mock_client.account[test_account.vsc_id].get.return_value = (
200, test_account_2)
mock_client.group[test_account_2].get.return_value = (
200, test_admin_group_1)
accountpageuser = user.VscAccountPageUser(test_account.vsc_id,
mock_client)
self.assertEqual(accountpageuser.usergroup,
mkGroup(test_admin_group_1))
def test_sync_group_without_moderators(self, mock_add_or_update):
"""Test the synchronisation of the group without active mods."""
mock_client = mock.MagicMock()
test_group = mkGroup(test_group_1)
mock_client.allgroups.modified[1].get.return_value = (200,
[test_group_1])
mock_client.vo[test_group.vsc_id].get.side_effect = HTTPError(
None, 404, 'not found', None, None)
mock_add_or_update.return_value = UPDATED
ldapsyncer = LdapSyncer(mock_client)
groups = ldapsyncer.sync_altered_groups(1)
self.assertEqual(groups, {
'error': set([]),
'new': set([]),
'updated': set([test_group.vsc_id])
})
ldap_attrs = {
'status': ['inactive'],
'cn': 'gtestgroup',
'institute': ['gent'],
'memberUid': ['vsc40075'],
'moderator': ['vsc40003'],
'gidNumber': ['2640011'],
}
mock_add_or_update.assert_called_with(VscLdapGroup, test_group.vsc_id,
ldap_attrs, True)
def usergroup(self):
if not self._cache['usergroup']:
if self.person.institute_login in ('x_admin', 'admin', 'voadmin'):
# TODO to be removed when magic site admin usergroups are purged from code
self._cache['usergroup'] = mkGroup((self.rest_client.group[self.user_id].get())[1])
else:
self._cache['usergroup'] = mkUserGroup((self.rest_client.account[self.user_id].usergroup.get()[1]))
return self._cache['usergroup']
def sync_altered_groups(self, last, dry_run=True):
"""
Synchronise altered groups back to LDAP.
This also includes usergroups
"""
changed_groups = [
mkGroup(a) for a in self.client.allgroups.modified[last].get()[1]
]
logging.info("Found %d modified groups in the range %s until %s" %
(len(changed_groups),
datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"),
self.now.strftime("%Y%m%d%H%M%SZ")))
logging.debug("Modified groups: %s",
[g.vsc_id for g in changed_groups])
groups = {
NEW: set(),
UPDATED: set(),
ERROR: set(),
}
for group in changed_groups:
vo = False
try:
vo = mkVo(self.client.vo[group.vsc_id].get()[1])
except HTTPError as err:
# if a 404 occured, the group is not an VO, so we skip this. Otherwise something else went wrong.
if err.code != 404:
raise
ldap_attributes = {
'cn': str(group.vsc_id),
'institute': [str(group.institute['site'])],
'gidNumber': ["%d" % (group.vsc_id_number, )],
'moderator': [str(m) for m in group.moderators],
'memberUid': [str(a) for a in group.members],
'status': [str(group.status)],
}
if vo:
ldap_attributes['fairshare'] = ["%d" % (vo.fairshare, )]
ldap_attributes['description'] = [str(vo.description)]
ldap_attributes['dataDirectory'] = [str(vo.data_path)]
ldap_attributes['scratchDirectory'] = [str(vo.scratch_path)]
# vsc40024 is moderator for all institute vo's
if vo.vsc_id in INSTITUTE_VOS_GENT.values():
ldap_attributes['moderator'] = ['vsc40024']
logging.debug("Proposed changes for group %s: %s", group.vsc_id,
ldap_attributes)
result = self.add_or_update(VscLdapGroup, group.vsc_id,
ldap_attributes, dry_run)
groups[result].add(group.vsc_id)
return groups
def sync_altered_groups(self, last, dry_run=True):
"""
Synchronise altered groups back to LDAP.
This also includes usergroups
"""
changed_groups = [
mkGroup(a) for a in self.client.allgroups.modified[last].get()[1]
]
logging.info("Found %d modified groups in the range %s until %s" %
(len(changed_groups),
datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"),
self.now.strftime("%Y%m%d%H%M%SZ")))
logging.debug("Modified groups: %s",
[g.vsc_id for g in changed_groups])
groups = {
NEW: set(),
UPDATED: set(),
ERROR: set(),
}
for group in changed_groups:
# General group attributes
group_moderators = [str(m) for m in group.moderators]
institute_name = str(group.institute['name'])
ldap_attributes = {
'cn': str(group.vsc_id),
'institute': [institute_name],
'gidNumber': ["%d" % (group.vsc_id_number, )],
'moderator': group_moderators,
'status': [str(group.status)],
}
# Only set memberUid if there are actually any members in the group
# Addition of new group records in LDAP will fail with empty memberUid
# Existing LDAP records of groups that become empty will lose memberUid
# ldap.modlist.modifyModlist (vsc-ldap) will delete any attributes that are missing in the new record
if group.members:
ldap_attributes['memberUid'] = [str(a) for a in group.members]
# VO attributes
try:
vo = mkVo(self.client.vo[group.vsc_id].get()[1])
except HTTPError as err:
# if a 404 occured, the group is not an VO, so we skip this. Otherwise something else went wrong.
if err.code != 404:
logging.raiseException(
"Retrieval of group VO failed for unexpected reasons")
else:
# Group is a VO
ldap_attributes['fairshare'] = ["%d" % (vo.fairshare, )]
ldap_attributes['description'] = [str(vo.description)]
ldap_attributes['dataDirectory'] = [str(vo.data_path)]
ldap_attributes['scratchDirectory'] = [str(vo.scratch_path)]
# Set institute moderator for main VOs
if vo.vsc_id in DEFAULT_VOS_ALL:
ldap_attributes['moderator'] = [
VSC_CONFIG.vo_group_mods[group.institute['name']]
]
logging.info("Using VO moderator %s for VO %s",
ldap_attributes['moderator'], group.vsc_id)
if not ldap_attributes['moderator']:
ldap_attributes['moderator'] = [
str(VSC_CONFIG.backup_group_mods[group.institute['name']])
]
logging.info("Using backup moderator %s for group %s",
ldap_attributes['moderator'], group.vsc_id)
logging.debug("Proposed changes for group %s: %s", group.vsc_id,
ldap_attributes)
result = self.add_or_update(VscLdapGroup, group.vsc_id,
ldap_attributes, dry_run)
groups[result].add(group.vsc_id)
return groups
def sync_altered_accounts(self, last, dry_run=True):
"""
Add new users to the LDAP and update altered users. This does not include usergroups.
this does include pubkeys
@type last: datetime
@return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered.
"""
sync_accounts = [
mkVscAccount(a)
for a in self.client.account.modified[last].get()[1]
]
accounts = {
NEW: set(),
UPDATED: set(),
ERROR: set(),
}
logging.info("Found %d modified accounts in the range %s until %s" %
(len(sync_accounts),
datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"),
self.now.strftime("%Y%m%d%H%M%SZ")))
logging.debug("Modified accounts: %s",
[a.vsc_id for a in sync_accounts])
for account in sync_accounts:
try:
if account.person.institute_login in ('x_admin', 'admin',
'voadmin'):
# TODO to be removed when magic site admin usergroups are purged from code
usergroup = mkGroup(
(self.client.group[account.vsc_id].get())[1])
else:
usergroup = mkUserGroup(
self.client.account[account.vsc_id].usergroup.get()[1])
except HTTPError:
logging.error("No corresponding UserGroup for user %s" %
(account.vsc_id, ))
continue
gecos = ensure_ascii_string(account.person.gecos)
logging.debug('fetching public key')
public_keys = [
ensure_ascii_string(x.pubkey)
for x in self.client.get_public_keys(account.vsc_id)
]
if not public_keys:
public_keys = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING]
LDAP_STATE_MAPPER = {'forceinactive': 'inactive'}
ldap_attributes = {
'cn':
str(account.vsc_id),
'uidNumber': ["%s" % (account.vsc_id_number, )],
'gecos': [gecos],
'mail': [str(account.email)],
'institute': [str(account.person.institute['name'])],
'instituteLogin': [str(account.person.institute_login)],
'uid': [str(account.vsc_id)],
'homeDirectory': [str(account.home_directory)],
'dataDirectory': [str(account.data_directory)],
'scratchDirectory': [str(account.scratch_directory)],
'pubkey':
public_keys,
'gidNumber': [str(usergroup.vsc_id_number)],
'loginShell': [str(account.login_shell)],
'researchField': [str(account.research_field[0])],
'status': [
LDAP_STATE_MAPPER.get(str(account.status),
str(account.status))
],
'homeQuota': ["1"],
'dataQuota': ["1"],
'scratchQuota': ["1"],
}
logging.debug('fetching quota')
quotas = self.client.account[account.vsc_id].quota.get()[1]
for quota in quotas:
for stype in ['home', 'data', 'scratch']:
# only gent sets filesets for vo's, so not gvo is user. (other institutes is empty or "None"
if quota['storage']['storage_type'] == stype and not quota[
'fileset'].startswith('gvo'):
ldap_attributes['%sQuota' %
stype] = ["%d" % quota["hard"]]
result = self.add_or_update(VscLdapUser, account.vsc_id,
ldap_attributes, dry_run)
accounts[result].add(account.vsc_id)
return accounts