def test_usergroup_instantiation(self):
mock_client = mock.MagicMock()
test_account = mkVscAccount(test_account_1)
mock_client.account[test_account.vsc_id] = mock.MagicMock()
mock_client.account[test_account.vsc_id].get.return_value = (
200, test_account_1)
mock_client.account[test_account.vsc_id].usergroup.get.return_value = (
200, test_usergroup_1)
accountpageuser = user.VscAccountPageUser(test_account.vsc_id,
rest_client=mock_client)
self.assertEqual(accountpageuser.usergroup,
mkUserGroup(test_usergroup_1))
mock_client.account[test_account.vsc_id].get.return_value = (
200, test_account_2)
mock_client.group[test_account_2].get.return_value = (
200, test_admin_group_1)
accountpageuser = user.VscAccountPageUser(test_account.vsc_id,
mock_client)
self.assertEqual(accountpageuser.usergroup,
mkGroup(test_admin_group_1))
def _create_fileset(self,
filesystem_name,
path,
parent_fileset=None,
fileset_name=None,
group_owner_id=None):
"""Create a fileset for the VO on the data filesystem.
- creates the fileset if it does not already exist
- sets ownership to the first (active) VO moderator, or to nobody if there is no moderator
- sets group ownership to the supplied value (group_owner_id) or if that is missing to the
vsc_id of the VO owning the fileset
The parent_fileset is used to support older (< 3.5.x) GPFS setups still present in our system
"""
self.gpfs.list_filesets()
if not fileset_name:
fileset_name = self.vo.vsc_id
if group_owner_id:
fileset_group_owner_id = group_owner_id
else:
fileset_group_owner_id = self.vo.vsc_id_number
if not self.gpfs.get_fileset_info(filesystem_name, fileset_name):
logging.info(
"Creating new fileset on %s with name %s and path %s" %
(filesystem_name, fileset_name, path))
base_dir_hierarchy = os.path.dirname(path)
self.gpfs.make_dir(base_dir_hierarchy)
# HACK to support versions older than 3.5 in our setup
if parent_fileset is None:
self.gpfs.make_fileset(path, fileset_name)
else:
self.gpfs.make_fileset(path, fileset_name, parent_fileset)
else:
logging.info(
"Fileset %s already exists for VO %s ... not creating again.",
fileset_name, self.vo.vsc_id)
self.gpfs.chmod(0o770, path)
try:
moderator = mkVscAccount(
self.rest_client.account[self.vo.moderators[0]].get()[1])
except HTTPError:
logging.exception(
"Cannot obtain moderator information from account page, setting ownership to nobody"
)
self.gpfs.chown(
pwd.getpwnam('nobody').pw_uid, fileset_group_owner_id, path)
except IndexError:
logging.error("There is no moderator available for VO %s" %
(self.vo.vsc_id, ))
self.gpfs.chown(
pwd.getpwnam('nobody').pw_uid, fileset_group_owner_id, path)
else:
self.gpfs.chown(moderator.vsc_id_number, fileset_group_owner_id,
path)
def test_init_quota(self):
tests = [(test_account_1, test_quota_1, GENT, 'vsc400'),
(test_account_3, test_quota_2, BRUSSEL, 'vsc100')]
for account, quota, site, fileset in tests:
mock_client = mock.MagicMock()
test_account = mkVscAccount(account)
mock_client.account[test_account.vsc_id].quota.get.return_value = (
200, quota)
accountpageuser = user.VscTier2AccountpageUser(
test_account.vsc_id,
storage=config.VscStorage(),
rest_client=mock_client,
account=test_account,
host_institute=site)
self.assertEqual(accountpageuser.user_home_quota, [
q['hard'] for q in quota if q['storage']['name'] == 'VSC_HOME'
and q['fileset'] == fileset
][0])
self.assertEqual(accountpageuser.user_data_quota, [
q['hard'] for q in quota if q['storage']['name'] == 'VSC_DATA'
and q['fileset'] == fileset
][0])
def test_get_institute_prefix(self):
test_account = mkVscAccount(test_account_1)
mock_client = mock.MagicMock()
accountpageuser = user.VscAccountPageUser(test_account.vsc_id, rest_client=mock_client, account=test_account)
self.assertEqual(accountpageuser.get_institute_prefix(), 'g')
def update_user_status(user, client):
"""
Change the status of the user's account in the account page to active.
The usergroup status is always in sync with thte accounts status
"""
if user.dry_run:
log.info("User %s has account status %s. Dry-run, not changing anything", user.user_id, user.account.status)
return
if user.account.status not in (NEW, MODIFIED, MODIFY):
log.info("Account %s has status %s, not changing" % (user.user_id, user.account.status))
return
payload = {"status": ACTIVE}
try:
response_account = client.account[user.user_id].patch(body=payload)
except HTTPError as err:
log.error("Account %s and UserGroup %s status were not changed", user.user_id, user.user_id)
raise UserStatusUpdateError("Account %s status was not changed - received HTTP code %d" % err.code)
else:
account = mkVscAccount(response_account[1])
if account.status == ACTIVE:
log.info("Account %s status changed to %s" % (user.user_id, ACTIVE))
else:
log.error("Account %s status was not changed", user.user_id)
raise UserStatusUpdateError("Account %s status was not changed, still at %s" %
(user.user_id, account.status))
def test_create_scratch_dir_tier2_user(self,
mock_client,
mock_gpfsoperations,
):
test_account = mkVscAccount(test_account_1)
accountpageuser = user.VscTier2AccountpageUser(test_account.vsc_id, rest_client=mock_client, account=test_account, host_institute=GENT)
accountpageuser.create_scratch_dir('VSC_SCRATCH_KYUKON')
def test_homeonscratch_instantiation(self):
mock_client = mock.MagicMock()
test_account = mkVscAccount(test_account_1)
mock_client.account[test_account.vsc_id].home_on_scratch.get.return_value = (200, test_hos_1)
accountpageuser = user.VscAccountPageUser(test_account.vsc_id, rest_client=mock_client)
self.assertEqual(accountpageuser.home_on_scratch, [mkVscHomeOnScratch(h) for h in test_hos_1])
def test_pubkeys_instantiation(self):
mock_client = mock.MagicMock()
test_account = mkVscAccount(test_account_1)
mock_client.account[test_account.vsc_id].pubkey.get.return_value = (200, test_pubkeys_1)
accountpageuser = user.VscAccountPageUser(test_account.vsc_id, rest_client=mock_client)
self.assertEqual(set(accountpageuser.pubkeys), set([mkVscAccountPubkey(p) for p in test_pubkeys_1]))
def test_init_quota(self):
mock_client = mock.MagicMock()
test_account = mkVscAccount(test_account_1)
mock_client.account[test_account.vsc_id].quota.get.return_value = (200, test_quota_1)
accountpageuser = user.VscTier2AccountpageUser(test_account.vsc_id, rest_client=mock_client, account=test_account, host_institute=GENT)
self.assertEqual(accountpageuser.user_home_quota, [q['hard'] for q in test_quota_1 if q['storage']['name'] == 'VSC_HOME' and q['fileset'] == 'vsc400'][0])
self.assertEqual(accountpageuser.user_data_quota, [q['hard'] for q in test_quota_1 if q['storage']['name'] == 'VSC_DATA' and q['fileset'] == 'vsc400'][0])
def set_up_accountpageuser(account, quota, site):
mock_client = mock.MagicMock()
test_account = mkVscAccount(account)
mock_client.account[test_account.vsc_id].quota.get.return_value = (
200, quota)
return user.VscTier2AccountpageUser(test_account.vsc_id,
storage=config.VscStorage(),
rest_client=mock_client,
account=test_account,
host_institute=site)
def test_person_instantiation(self):
mock_client = mock.MagicMock()
test_account = mkVscAccount(test_account_1)
accountpageuser = user.VscAccountPageUser(test_account.vsc_id, rest_client=mock_client, account=test_account)
self.assertEqual(accountpageuser.person, test_account.person)
mock_client.account[test_account.vsc_id].get.return_value = (200, test_account_1)
accountpageuser = user.VscAccountPageUser(test_account.vsc_id, mock_client)
self.assertEqual(accountpageuser.person, test_account.person)
def test_get_institute_prefix(self):
tests = [(test_account_1, 'g'), (test_account_3, 'b')]
for account, prefix in tests:
test_account = mkVscAccount(account)
mock_client = mock.MagicMock()
accountpageuser = user.VscAccountPageUser(test_account.vsc_id,
rest_client=mock_client,
account=test_account)
self.assertEqual(accountpageuser.get_institute_prefix(), prefix)
def test_create_scratch_dir_tier2_user(self, mock_client,
mock_gpfsoperations):
test_accounts = [(test_account_1, GENT), (test_account_3, BRUSSEL)]
for account, site in test_accounts:
test_account = mkVscAccount(account)
accountpageuser = user.VscTier2AccountpageUser(
test_account.vsc_id,
storage=config.VscStorage(),
rest_client=mock_client,
account=test_account,
host_institute=site)
accountpageuser.create_scratch_dir(VSC_PRODUCTION_SCRATCH[site][0])
def test_sync_altered_accounts_unicode_pubkey(self, mock_add_or_update):
"""Test the sync_altered accounts function with a pubkey containing unicode"""
mock_client = mock.MagicMock()
test_account = mkVscAccount(test_account_1)
mock_client.account[test_account.vsc_id] = mock.MagicMock()
mock_client.account.modified[1].get.return_value = (200,
[test_account_1])
mock_client.account[test_account.vsc_id].usergroup.get.return_value = (
200, test_usergroup_1)
mock_client.get_public_keys.return_value = [
mkVscAccountPubkey(p) for p in test_unicode_pubkeys
]
mock_client.account[test_account.vsc_id].quota.get.return_value = (
200, test_quota)
mock_add_or_update.return_value = UPDATED
ldapsyncer = LdapSyncer(mock_client)
accounts = ldapsyncer.sync_altered_accounts(1)
self.assertEqual(
accounts, {
'error': set([]),
'new': set([]),
'updated': set([test_account.vsc_id])
})
if is_py2():
expected_key = 'some pubkey \\[email protected]\\u201d'
else:
expected_key = 'some pubkey \\xe2\\x80\\[email protected]\\xe2\\x80\\x9d'
ldap_attrs = {
'status': ['active'],
'scratchDirectory': ['/user/scratch/gent/vsc400/vsc40075'],
'dataDirectory': ['/user/data/gent/vsc400/vsc40075'],
'cn': 'vsc40075',
'homeQuota': ['5242880'],
'institute': ['gent'],
'loginShell': ['/bin/bash'],
'uidNumber': ['2540075'],
'researchField': ['Bollocks'],
'gidNumber': ['2540075'],
'gecos': ['Foo Bar'],
'dataQuota': ['1'],
'homeDirectory': ['/user/home/gent/vsc400/vsc40075'],
'mail': ['*****@*****.**'],
'scratchQuota': ['1'],
'pubkey': [expected_key],
'instituteLogin': ['foobar'],
'uid': ['vsc40075']
}
mock_add_or_update.assert_called_with(VscLdapUser, test_account.vsc_id,
ldap_attrs, True)
def test_sync_forceinactive_account(self, mock_add_or_update):
"""Test the sync_altered accounts function"""
mock_client = mock.MagicMock()
test_account = mkVscAccount(test_account_4)
mock_client.account[test_account.vsc_id] = mock.MagicMock()
mock_client.account.modified[1].get.return_value = (200, [test_account_4])
mock_client.account[test_account.vsc_id].usergroup.get.return_value = (200, test_usergroup_1)
mock_client.get_public_keys.return_value = [mkVscAccountPubkey(p) for p in test_pubkeys_1]
mock_client.account[test_account.vsc_id].quota.get.return_value = (200, test_quota)
mock_add_or_update.return_value = UPDATED
ldapsyncer = LdapSyncer(mock_client)
accounts = ldapsyncer.sync_altered_accounts(1)
self.assertEqual(accounts, {'error': set([]), 'new': set([]), 'updated': set([test_account.vsc_id])})
ldap_attrs = {'status': ['inactive'], 'scratchDirectory': ['/scratch/brussel/vsc100/vsc10001'], 'dataDirectory': ['/data/brussel/vsc100/vsc10001'], 'cn': 'vsc10004', 'homeQuota': ['5242880'], 'institute': ['brussel'], 'loginShell': ['/bin/bash'], 'uidNumber': ['2510004'], 'researchField': ['Dinges'], 'gidNumber': ['2540075'], 'gecos': ['Foo Bar'], 'dataQuota': ['1'], 'homeDirectory': ['/user/brussel/vsc100/vsc10001'], 'mail': ['*****@*****.**'], 'scratchQuota': ['1'], 'pubkey': ['pubkey1', 'pubkey2'], 'instituteLogin': ['fooby'], 'uid': ['vsc10004']}
mock_add_or_update.assert_called_with(VscLdapUser, test_account.vsc_id, ldap_attrs, True)
def test_account_instantiation(self):
test_accounts = [test_account_1, test_account_3]
for account in test_accounts:
mock_client = mock.MagicMock()
test_account = mkVscAccount(account)
accountpageuser = user.VscAccountPageUser(test_account.vsc_id,
rest_client=mock_client,
account=test_account)
self.assertEqual(accountpageuser.account, test_account)
mock_client.account[test_account.vsc_id].get.return_value = (
200, account)
accountpageuser = user.VscAccountPageUser(test_account.vsc_id,
mock_client)
self.assertEqual(accountpageuser.account, test_account)
def sync_altered_accounts(self, last, dry_run=True):
"""
Add new users to the LDAP and update altered users. This does not include usergroups.
this does include pubkeys
@type last: datetime
@return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered.
"""
sync_accounts = [
mkVscAccount(a)
for a in self.client.account.modified[last].get()[1]
]
accounts = {
NEW: set(),
UPDATED: set(),
ERROR: set(),
}
logging.info("Found %d modified accounts in the range %s until %s" %
(len(sync_accounts),
datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"),
self.now.strftime("%Y%m%d%H%M%SZ")))
logging.debug("Modified accounts: %s",
[a.vsc_id for a in sync_accounts])
for account in sync_accounts:
try:
usergroup = mkUserGroup(
self.client.account[account.vsc_id].usergroup.get()[1])
except HTTPError:
logging.error("No corresponding UserGroup for user %s" %
(account.vsc_id, ))
continue
gecos = ensure_ascii_string(account.person.gecos)
logging.debug('fetching public key')
public_keys = [
ensure_ascii_string(x.pubkey)
for x in self.client.get_public_keys(account.vsc_id)
]
if not public_keys:
public_keys = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING]
LDAP_STATE_MAPPER = {'forceinactive': 'inactive'}
ldap_attributes = {
'cn':
str(account.vsc_id),
'uidNumber': ["%s" % (account.vsc_id_number, )],
'gecos': [gecos],
'mail': [str(account.email)],
'institute': [str(account.person.institute['name'])],
'instituteLogin': [str(account.person.institute_login)],
'uid': [str(account.vsc_id)],
'homeDirectory': [str(account.home_directory)],
'dataDirectory': [str(account.data_directory)],
'scratchDirectory': [str(account.scratch_directory)],
'pubkey':
public_keys,
'gidNumber': [str(usergroup.vsc_id_number)],
'loginShell': [str(account.login_shell)],
'researchField': [str(account.research_field[0])],
'status': [
LDAP_STATE_MAPPER.get(str(account.status),
str(account.status))
],
'homeQuota': ["1"],
'dataQuota': ["1"],
'scratchQuota': ["1"],
}
logging.debug('fetching quota')
quotas = self.client.account[account.vsc_id].quota.get()[1]
for quota in quotas:
for stype in ['home', 'data', 'scratch']:
# only gent sets filesets for vo's, so not gvo is user. (other institutes is empty or "None"
if quota['storage']['storage_type'] == stype and not quota[
'fileset'].startswith('gvo'):
ldap_attributes['%sQuota' %
stype] = ["%d" % quota["hard"]]
result = self.add_or_update(VscLdapUser, account.vsc_id,
ldap_attributes, dry_run)
accounts[result].add(account.vsc_id)
return accounts
def account(self):
if not self._cache['account']:
self._cache['account'] = mkVscAccount((self.rest_client.account[self.user_id].get())[1])
return self._cache['account']