def test_usergroup_instantiation(self): mock_client = mock.MagicMock() test_account = mkVscAccount(test_account_1) mock_client.account[test_account.vsc_id] = mock.MagicMock() mock_client.account[test_account.vsc_id].get.return_value = ( 200, test_account_1) mock_client.account[test_account.vsc_id].usergroup.get.return_value = ( 200, test_usergroup_1) accountpageuser = user.VscAccountPageUser(test_account.vsc_id, rest_client=mock_client) self.assertEqual(accountpageuser.usergroup, mkUserGroup(test_usergroup_1)) mock_client.account[test_account.vsc_id].get.return_value = ( 200, test_account_2) mock_client.group[test_account_2].get.return_value = ( 200, test_admin_group_1) accountpageuser = user.VscAccountPageUser(test_account.vsc_id, mock_client) self.assertEqual(accountpageuser.usergroup, mkGroup(test_admin_group_1))
def _create_fileset(self, filesystem_name, path, parent_fileset=None, fileset_name=None, group_owner_id=None): """Create a fileset for the VO on the data filesystem. - creates the fileset if it does not already exist - sets ownership to the first (active) VO moderator, or to nobody if there is no moderator - sets group ownership to the supplied value (group_owner_id) or if that is missing to the vsc_id of the VO owning the fileset The parent_fileset is used to support older (< 3.5.x) GPFS setups still present in our system """ self.gpfs.list_filesets() if not fileset_name: fileset_name = self.vo.vsc_id if group_owner_id: fileset_group_owner_id = group_owner_id else: fileset_group_owner_id = self.vo.vsc_id_number if not self.gpfs.get_fileset_info(filesystem_name, fileset_name): logging.info( "Creating new fileset on %s with name %s and path %s" % (filesystem_name, fileset_name, path)) base_dir_hierarchy = os.path.dirname(path) self.gpfs.make_dir(base_dir_hierarchy) # HACK to support versions older than 3.5 in our setup if parent_fileset is None: self.gpfs.make_fileset(path, fileset_name) else: self.gpfs.make_fileset(path, fileset_name, parent_fileset) else: logging.info( "Fileset %s already exists for VO %s ... not creating again.", fileset_name, self.vo.vsc_id) self.gpfs.chmod(0o770, path) try: moderator = mkVscAccount( self.rest_client.account[self.vo.moderators[0]].get()[1]) except HTTPError: logging.exception( "Cannot obtain moderator information from account page, setting ownership to nobody" ) self.gpfs.chown( pwd.getpwnam('nobody').pw_uid, fileset_group_owner_id, path) except IndexError: logging.error("There is no moderator available for VO %s" % (self.vo.vsc_id, )) self.gpfs.chown( pwd.getpwnam('nobody').pw_uid, fileset_group_owner_id, path) else: self.gpfs.chown(moderator.vsc_id_number, fileset_group_owner_id, path)
def test_init_quota(self): tests = [(test_account_1, test_quota_1, GENT, 'vsc400'), (test_account_3, test_quota_2, BRUSSEL, 'vsc100')] for account, quota, site, fileset in tests: mock_client = mock.MagicMock() test_account = mkVscAccount(account) mock_client.account[test_account.vsc_id].quota.get.return_value = ( 200, quota) accountpageuser = user.VscTier2AccountpageUser( test_account.vsc_id, storage=config.VscStorage(), rest_client=mock_client, account=test_account, host_institute=site) self.assertEqual(accountpageuser.user_home_quota, [ q['hard'] for q in quota if q['storage']['name'] == 'VSC_HOME' and q['fileset'] == fileset ][0]) self.assertEqual(accountpageuser.user_data_quota, [ q['hard'] for q in quota if q['storage']['name'] == 'VSC_DATA' and q['fileset'] == fileset ][0])
def test_get_institute_prefix(self): test_account = mkVscAccount(test_account_1) mock_client = mock.MagicMock() accountpageuser = user.VscAccountPageUser(test_account.vsc_id, rest_client=mock_client, account=test_account) self.assertEqual(accountpageuser.get_institute_prefix(), 'g')
def update_user_status(user, client): """ Change the status of the user's account in the account page to active. The usergroup status is always in sync with thte accounts status """ if user.dry_run: log.info("User %s has account status %s. Dry-run, not changing anything", user.user_id, user.account.status) return if user.account.status not in (NEW, MODIFIED, MODIFY): log.info("Account %s has status %s, not changing" % (user.user_id, user.account.status)) return payload = {"status": ACTIVE} try: response_account = client.account[user.user_id].patch(body=payload) except HTTPError as err: log.error("Account %s and UserGroup %s status were not changed", user.user_id, user.user_id) raise UserStatusUpdateError("Account %s status was not changed - received HTTP code %d" % err.code) else: account = mkVscAccount(response_account[1]) if account.status == ACTIVE: log.info("Account %s status changed to %s" % (user.user_id, ACTIVE)) else: log.error("Account %s status was not changed", user.user_id) raise UserStatusUpdateError("Account %s status was not changed, still at %s" % (user.user_id, account.status))
def test_create_scratch_dir_tier2_user(self, mock_client, mock_gpfsoperations, ): test_account = mkVscAccount(test_account_1) accountpageuser = user.VscTier2AccountpageUser(test_account.vsc_id, rest_client=mock_client, account=test_account, host_institute=GENT) accountpageuser.create_scratch_dir('VSC_SCRATCH_KYUKON')
def test_homeonscratch_instantiation(self): mock_client = mock.MagicMock() test_account = mkVscAccount(test_account_1) mock_client.account[test_account.vsc_id].home_on_scratch.get.return_value = (200, test_hos_1) accountpageuser = user.VscAccountPageUser(test_account.vsc_id, rest_client=mock_client) self.assertEqual(accountpageuser.home_on_scratch, [mkVscHomeOnScratch(h) for h in test_hos_1])
def test_pubkeys_instantiation(self): mock_client = mock.MagicMock() test_account = mkVscAccount(test_account_1) mock_client.account[test_account.vsc_id].pubkey.get.return_value = (200, test_pubkeys_1) accountpageuser = user.VscAccountPageUser(test_account.vsc_id, rest_client=mock_client) self.assertEqual(set(accountpageuser.pubkeys), set([mkVscAccountPubkey(p) for p in test_pubkeys_1]))
def test_init_quota(self): mock_client = mock.MagicMock() test_account = mkVscAccount(test_account_1) mock_client.account[test_account.vsc_id].quota.get.return_value = (200, test_quota_1) accountpageuser = user.VscTier2AccountpageUser(test_account.vsc_id, rest_client=mock_client, account=test_account, host_institute=GENT) self.assertEqual(accountpageuser.user_home_quota, [q['hard'] for q in test_quota_1 if q['storage']['name'] == 'VSC_HOME' and q['fileset'] == 'vsc400'][0]) self.assertEqual(accountpageuser.user_data_quota, [q['hard'] for q in test_quota_1 if q['storage']['name'] == 'VSC_DATA' and q['fileset'] == 'vsc400'][0])
def set_up_accountpageuser(account, quota, site): mock_client = mock.MagicMock() test_account = mkVscAccount(account) mock_client.account[test_account.vsc_id].quota.get.return_value = ( 200, quota) return user.VscTier2AccountpageUser(test_account.vsc_id, storage=config.VscStorage(), rest_client=mock_client, account=test_account, host_institute=site)
def test_person_instantiation(self): mock_client = mock.MagicMock() test_account = mkVscAccount(test_account_1) accountpageuser = user.VscAccountPageUser(test_account.vsc_id, rest_client=mock_client, account=test_account) self.assertEqual(accountpageuser.person, test_account.person) mock_client.account[test_account.vsc_id].get.return_value = (200, test_account_1) accountpageuser = user.VscAccountPageUser(test_account.vsc_id, mock_client) self.assertEqual(accountpageuser.person, test_account.person)
def test_get_institute_prefix(self): tests = [(test_account_1, 'g'), (test_account_3, 'b')] for account, prefix in tests: test_account = mkVscAccount(account) mock_client = mock.MagicMock() accountpageuser = user.VscAccountPageUser(test_account.vsc_id, rest_client=mock_client, account=test_account) self.assertEqual(accountpageuser.get_institute_prefix(), prefix)
def test_create_scratch_dir_tier2_user(self, mock_client, mock_gpfsoperations): test_accounts = [(test_account_1, GENT), (test_account_3, BRUSSEL)] for account, site in test_accounts: test_account = mkVscAccount(account) accountpageuser = user.VscTier2AccountpageUser( test_account.vsc_id, storage=config.VscStorage(), rest_client=mock_client, account=test_account, host_institute=site) accountpageuser.create_scratch_dir(VSC_PRODUCTION_SCRATCH[site][0])
def test_sync_altered_accounts_unicode_pubkey(self, mock_add_or_update): """Test the sync_altered accounts function with a pubkey containing unicode""" mock_client = mock.MagicMock() test_account = mkVscAccount(test_account_1) mock_client.account[test_account.vsc_id] = mock.MagicMock() mock_client.account.modified[1].get.return_value = (200, [test_account_1]) mock_client.account[test_account.vsc_id].usergroup.get.return_value = ( 200, test_usergroup_1) mock_client.get_public_keys.return_value = [ mkVscAccountPubkey(p) for p in test_unicode_pubkeys ] mock_client.account[test_account.vsc_id].quota.get.return_value = ( 200, test_quota) mock_add_or_update.return_value = UPDATED ldapsyncer = LdapSyncer(mock_client) accounts = ldapsyncer.sync_altered_accounts(1) self.assertEqual( accounts, { 'error': set([]), 'new': set([]), 'updated': set([test_account.vsc_id]) }) if is_py2(): expected_key = 'some pubkey \\[email protected]\\u201d' else: expected_key = 'some pubkey \\xe2\\x80\\[email protected]\\xe2\\x80\\x9d' ldap_attrs = { 'status': ['active'], 'scratchDirectory': ['/user/scratch/gent/vsc400/vsc40075'], 'dataDirectory': ['/user/data/gent/vsc400/vsc40075'], 'cn': 'vsc40075', 'homeQuota': ['5242880'], 'institute': ['gent'], 'loginShell': ['/bin/bash'], 'uidNumber': ['2540075'], 'researchField': ['Bollocks'], 'gidNumber': ['2540075'], 'gecos': ['Foo Bar'], 'dataQuota': ['1'], 'homeDirectory': ['/user/home/gent/vsc400/vsc40075'], 'mail': ['*****@*****.**'], 'scratchQuota': ['1'], 'pubkey': [expected_key], 'instituteLogin': ['foobar'], 'uid': ['vsc40075'] } mock_add_or_update.assert_called_with(VscLdapUser, test_account.vsc_id, ldap_attrs, True)
def test_sync_forceinactive_account(self, mock_add_or_update): """Test the sync_altered accounts function""" mock_client = mock.MagicMock() test_account = mkVscAccount(test_account_4) mock_client.account[test_account.vsc_id] = mock.MagicMock() mock_client.account.modified[1].get.return_value = (200, [test_account_4]) mock_client.account[test_account.vsc_id].usergroup.get.return_value = (200, test_usergroup_1) mock_client.get_public_keys.return_value = [mkVscAccountPubkey(p) for p in test_pubkeys_1] mock_client.account[test_account.vsc_id].quota.get.return_value = (200, test_quota) mock_add_or_update.return_value = UPDATED ldapsyncer = LdapSyncer(mock_client) accounts = ldapsyncer.sync_altered_accounts(1) self.assertEqual(accounts, {'error': set([]), 'new': set([]), 'updated': set([test_account.vsc_id])}) ldap_attrs = {'status': ['inactive'], 'scratchDirectory': ['/scratch/brussel/vsc100/vsc10001'], 'dataDirectory': ['/data/brussel/vsc100/vsc10001'], 'cn': 'vsc10004', 'homeQuota': ['5242880'], 'institute': ['brussel'], 'loginShell': ['/bin/bash'], 'uidNumber': ['2510004'], 'researchField': ['Dinges'], 'gidNumber': ['2540075'], 'gecos': ['Foo Bar'], 'dataQuota': ['1'], 'homeDirectory': ['/user/brussel/vsc100/vsc10001'], 'mail': ['*****@*****.**'], 'scratchQuota': ['1'], 'pubkey': ['pubkey1', 'pubkey2'], 'instituteLogin': ['fooby'], 'uid': ['vsc10004']} mock_add_or_update.assert_called_with(VscLdapUser, test_account.vsc_id, ldap_attrs, True)
def test_account_instantiation(self): test_accounts = [test_account_1, test_account_3] for account in test_accounts: mock_client = mock.MagicMock() test_account = mkVscAccount(account) accountpageuser = user.VscAccountPageUser(test_account.vsc_id, rest_client=mock_client, account=test_account) self.assertEqual(accountpageuser.account, test_account) mock_client.account[test_account.vsc_id].get.return_value = ( 200, account) accountpageuser = user.VscAccountPageUser(test_account.vsc_id, mock_client) self.assertEqual(accountpageuser.account, test_account)
def sync_altered_accounts(self, last, dry_run=True): """ Add new users to the LDAP and update altered users. This does not include usergroups. this does include pubkeys @type last: datetime @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. """ sync_accounts = [ mkVscAccount(a) for a in self.client.account.modified[last].get()[1] ] accounts = { NEW: set(), UPDATED: set(), ERROR: set(), } logging.info("Found %d modified accounts in the range %s until %s" % (len(sync_accounts), datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"), self.now.strftime("%Y%m%d%H%M%SZ"))) logging.debug("Modified accounts: %s", [a.vsc_id for a in sync_accounts]) for account in sync_accounts: try: usergroup = mkUserGroup( self.client.account[account.vsc_id].usergroup.get()[1]) except HTTPError: logging.error("No corresponding UserGroup for user %s" % (account.vsc_id, )) continue gecos = ensure_ascii_string(account.person.gecos) logging.debug('fetching public key') public_keys = [ ensure_ascii_string(x.pubkey) for x in self.client.get_public_keys(account.vsc_id) ] if not public_keys: public_keys = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING] LDAP_STATE_MAPPER = {'forceinactive': 'inactive'} ldap_attributes = { 'cn': str(account.vsc_id), 'uidNumber': ["%s" % (account.vsc_id_number, )], 'gecos': [gecos], 'mail': [str(account.email)], 'institute': [str(account.person.institute['name'])], 'instituteLogin': [str(account.person.institute_login)], 'uid': [str(account.vsc_id)], 'homeDirectory': [str(account.home_directory)], 'dataDirectory': [str(account.data_directory)], 'scratchDirectory': [str(account.scratch_directory)], 'pubkey': public_keys, 'gidNumber': [str(usergroup.vsc_id_number)], 'loginShell': [str(account.login_shell)], 'researchField': [str(account.research_field[0])], 'status': [ LDAP_STATE_MAPPER.get(str(account.status), str(account.status)) ], 'homeQuota': ["1"], 'dataQuota': ["1"], 'scratchQuota': ["1"], } logging.debug('fetching quota') quotas = self.client.account[account.vsc_id].quota.get()[1] for quota in quotas: for stype in ['home', 'data', 'scratch']: # only gent sets filesets for vo's, so not gvo is user. (other institutes is empty or "None" if quota['storage']['storage_type'] == stype and not quota[ 'fileset'].startswith('gvo'): ldap_attributes['%sQuota' % stype] = ["%d" % quota["hard"]] result = self.add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) accounts[result].add(account.vsc_id) return accounts
def account(self): if not self._cache['account']: self._cache['account'] = mkVscAccount((self.rest_client.account[self.user_id].get())[1]) return self._cache['account']