def test_process_gent_institute_vo(self, mock_storage, mock_client): test_vo_id = "gvo00012" Options = namedtuple("Options", ['dry_run']) options = Options(dry_run=False) mc = mock_client.return_value mc.vo = mock.MagicMock() date = "20321231" mc.vo['gvo00012'].member.modified[date].get.return_value = ( 200, [{ u'broken': False, u'create_timestamp': u'2014-04-23T09:11:22.460Z', u'data_directory': u'/user/leuven/data/vsc400/vsc40075', u'email': u'*****@*****.**', u'home_directory': u'/user/leuven/home/vsc400/vsc40075', u'login_shell': u'/bin/bash', u'person': { u'gecos': u'Andy Georges', u'institute': {u'site': u'leuven'}, u'institute_login': u'ageorges' }, u'research_field': [u'Computer systems, architectures, networks', u'nwo'], u'scratch_directory': u'/user/leuven/scratch/vsc400/vsc40075', u'status': u'active', u'vsc_id': u'vsc40075', u'vsc_id_number': 2540075, }]) for storage_name in (VSC_HOME, VSC_DATA) + GENT_PRODUCTION_SCRATCH: with mock.patch('vsc.administration.vo.VscTier2AccountpageUser', autospec=True) as mock_user: with mock.patch('vsc.administration.vo.update_vo_status') as mock_update_vo_status: with mock.patch.object(vo.VscTier2AccountpageVo, 'create_scratch_fileset') as mock_cr_s_fileset: with mock.patch.object(vo.VscTier2AccountpageVo, 'set_scratch_quota') as mock_s_s_quota: with mock.patch.object(vo.VscTier2AccountpageVo, 'create_data_fileset') as mock_cr_d_fileset: with mock.patch.object(vo.VscTier2AccountpageVo, 'set_data_quota') as mock_s_d_quota: with mock.patch.object(vo.VscTier2AccountpageVo, 'set_member_data_quota') as mock_s_m_d_quota: with mock.patch.object(vo.VscTier2AccountpageVo, 'create_member_data_dir') as mock_cr_m_d_dir: with mock.patch.object(vo.VscTier2AccountpageVo, 'set_member_scratch_quota') as mock_s_m_s_quota: with mock.patch.object(vo.VscTier2AccountpageVo, 'create_member_scratch_dir') as mock_cr_m_s_dir: mock_user.return_value = mock.MagicMock() ok, errors = vo.process_vos(options, [test_vo_id], storage_name, mc, "99991231") self.assertEqual(errors, {}) mock_cr_s_fileset.assert_not_called() mock_s_s_quota.assert_not_called() mock_cr_d_fileset.assert_not_called() mock_s_d_quota.assert_not_called() mock_update_vo_status.assert_not_called() mock_s_m_d_quota.assert_not_called() mock_cr_m_d_dir.assert_not_called()
def test_process_gent_institute_vo_data_share(self, mock_storage, mock_client): test_vo_id = "gvo03442" Options = namedtuple("Options", ['dry_run']) options = Options(dry_run=False) mc = mock_client.return_value mc.vo = mock.MagicMock() v = mock.MagicMock() mc.vo[test_vo_id].get.return_value = v for storage_name in (VSC_DATA_SHARED,): with mock.patch("vsc.administration.vo.VscTier2AccountpageVo.data_sharing", new_callable=mock.PropertyMock) as mock_data_sharing: with mock.patch("vsc.administration.vo.VscTier2AccountpageVo.sharing_group", new_callable=mock.PropertyMock) as mock_sharing_group: with mock.patch('vsc.administration.vo.update_vo_status') as mock_update_vo_status: with mock.patch.object(vo.VscTier2AccountpageVo, 'create_scratch_fileset') as mock_cr_s_fileset: with mock.patch.object(vo.VscTier2AccountpageVo, 'set_scratch_quota') as mock_s_s_quota: with mock.patch.object(vo.VscTier2AccountpageVo, 'create_data_fileset') as mock_cr_d_fileset: with mock.patch.object(vo.VscTier2AccountpageVo, 'set_data_quota') as mock_s_d_quota: with mock.patch.object(vo.VscTier2AccountpageVo, 'create_data_shared_fileset') as mock_cr_d_shared_fileset: with mock.patch.object(vo.VscTier2AccountpageVo, 'set_data_shared_quota') as mock_s_d_shared_quota: with mock.patch.object(vo.VscTier2AccountpageVo, 'set_member_data_quota') as mock_s_m_d_quota: with mock.patch.object(vo.VscTier2AccountpageVo, 'create_member_data_dir') as mock_cr_m_d_dir: with mock.patch.object(vo.VscTier2AccountpageVo, 'set_member_scratch_quota') as mock_s_m_s_quota: with mock.patch.object(vo.VscTier2AccountpageVo, 'create_member_scratch_dir') as mock_cr_m_s_dir: mock_data_sharing.return_value = True mock_sharing_group.return_value = VscAutogroup( vsc_id=test_vo_id.replace('gvo', 'gvos'), status='active', vsc_id_number=123456, institute='Gent', members=['vsc40075'], description="test autogroup" ) ok, errors = vo.process_vos(options, [test_vo_id], storage_name, mc, "99991231") self.assertEqual(errors, {}) mock_cr_s_fileset.assert_not_called() mock_s_s_quota.assert_not_called() mock_cr_d_fileset.assert_not_called() mock_s_d_quota.assert_not_called() mock_cr_d_shared_fileset.assert_called() mock_s_d_shared_quota.assert_called() mock_update_vo_status.assert_not_called() mock_s_m_d_quota.assert_not_called() mock_cr_m_d_dir.assert_not_called()
def main(): """ Main script. - build the filter - fetches the users - process the users - write the new timestamp if everything went OK - write the nagios check file """ options = { 'nagios-check-interval-threshold': NAGIOS_CHECK_INTERVAL_THRESHOLD, 'storage': ('storage systems on which to deploy users and vos', None, 'extend', []), 'user': ('process users', None, 'store_true', False), 'vo': ('process vos', None, 'store_true', False), 'access_token': ('OAuth2 token to access the account page REST API', None, 'store', None), 'account_page_url': ('URL of the account page where we can find the REST API', None, 'store', None), 'host_institute': ('Name of the institute where this script is being run', str, 'store', GENT), } opts = ExtendedSimpleOption(options) stats = {} try: now = datetime.utcnow() client = AccountpageClient(token=opts.options.access_token, url=opts.options.account_page_url + "/api/") try: last_timestamp = read_timestamp(SYNC_TIMESTAMP_FILENAME) except Exception: logger.exception("Something broke reading the timestamp from %s" % SYNC_TIMESTAMP_FILENAME) last_timestamp = "200901010000Z" logger.info("Last recorded timestamp was %s" % (last_timestamp)) last_timestamp = convert_to_unix_timestamp(last_timestamp) (users_ok, users_fail) = ([], []) (quota_ok, quota_fail) = ([], []) if opts.options.user: ugent_changed_accounts = client.account.institute['gent'].modified[ last_timestamp].get()[1] logger.info( "Found %d UGent accounts that have changed in the accountpage since %s" % (len(ugent_changed_accounts), last_timestamp)) ugent_accounts = [u['vsc_id'] for u in ugent_changed_accounts] ugent_accounts = nub(ugent_accounts) for storage_name in opts.options.storage: (users_ok, users_fail) = process_users(opts.options, ugent_accounts, storage_name, client, opts.options.host_institute) stats["%s_users_sync" % (storage_name, )] = len(users_ok) stats["%s_users_sync_fail" % (storage_name, )] = len(users_fail) stats["%s_users_sync_fail_warning" % (storage_name, )] = STORAGE_USERS_LIMIT_WARNING stats["%s_users_sync_fail_critical" % (storage_name, )] = STORAGE_USERS_LIMIT_CRITICAL for storage_name in opts.options.storage: storage_changed_quota = [ mkVscUserSizeQuota(q) for q in client.quota.user. storage[storage_name].modified[last_timestamp].get()[1] ] storage_changed_quota = [ q for q in storage_changed_quota if q.fileset.startswith('vsc') ] logger.info( "Found %d accounts that have changed quota on storage %s in the accountpage since %s", len(storage_changed_quota), storage_name, last_timestamp) (quota_ok, quota_fail) = process_users_quota( opts.options, storage_changed_quota, storage_name, client, opts.options.host_institute) stats["%s_quota_sync" % (storage_name, )] = len(quota_ok) stats["%s_quota_sync_fail" % (storage_name, )] = len(quota_fail) stats["%s_quota_sync_fail_warning" % (storage_name, )] = STORAGE_QUOTA_LIMIT_WARNING stats["%s_quota_sync_fail_critical" % (storage_name, )] = STORAGE_QUOTA_LIMIT_CRITICAL (vos_ok, vos_fail) = ([], []) if opts.options.vo: ugent_changed_vos = client.vo.modified[last_timestamp].get()[1] ugent_changed_vo_quota = client.quota.vo.modified[ last_timestamp].get()[1] ugent_vos = sorted( set([v['vsc_id'] for v in ugent_changed_vos] + [ v['virtual_organisation'] for v in ugent_changed_vo_quota ])) logger.info( "Found %d UGent VOs that have changed in the accountpage since %s" % (len(ugent_changed_vos), last_timestamp)) logger.info( "Found %d UGent VOs that have changed quota in the accountpage since %s" % (len(ugent_changed_vo_quota), last_timestamp)) logger.debug( "Found the following UGent VOs: {vos}".format(vos=ugent_vos)) for storage_name in opts.options.storage: (vos_ok, vos_fail) = process_vos(opts.options, ugent_vos, storage_name, client, last_timestamp, opts.options.host_institute) stats["%s_vos_sync" % (storage_name, )] = len(vos_ok) stats["%s_vos_sync_fail" % (storage_name, )] = len(vos_fail) stats["%s_vos_sync_fail_warning" % (storage_name, )] = STORAGE_VO_LIMIT_WARNING stats["%s_vos_sync_fail_critical" % (storage_name, )] = STORAGE_VO_LIMIT_CRITICAL if not (users_fail or quota_fail or vos_fail): (_, ldap_timestamp) = convert_timestamp(now) if not opts.options.dry_run: write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp) except Exception as err: logger.exception("critical exception caught: %s" % (err)) opts.critical("Script failed in a horrible way") sys.exit(NAGIOS_EXIT_CRITICAL) opts.epilogue("UGent users and VOs synchronised", stats)
def main(): """ Main script. - build the filter - fetches the users - process the users - write the new timestamp if everything went OK - write the nagios check file """ options = { 'nagios-check-interval-threshold': NAGIOS_CHECK_INTERVAL_THRESHOLD, 'storage': ('storage systems on which to deploy users and vos', None, 'extend', []), 'user': ('process users', None, 'store_true', False), 'vo': ('process vos', None, 'store_true', False), 'access_token': ('OAuth2 token to access the account page REST API', None, 'store', None), 'account_page_url': ('URL of the account page where we can find the REST API', None, 'store', None), 'host_institute': ('Name of the institute where this script is being run', str, 'store', GENT), 'start_timestamp': ('Timestamp to start the sync from', str, 'store', None), } opts = ExtendedSimpleOption(options) stats = {} (last_timestamp, start_time) = retrieve_timestamp_with_default( SYNC_TIMESTAMP_FILENAME, start_timestamp=opts.options.start_timestamp) logging.info("Using timestamp %s", last_timestamp) logging.info("Using startime %s", start_time) try: client = AccountpageClient(token=opts.options.access_token, url=opts.options.account_page_url + "/api/") institute = opts.options.host_institute (users_ok, users_fail) = ([], []) (quota_ok, quota_fail) = ([], []) if opts.options.user: changed_accounts = client.account.institute[institute].modified[ last_timestamp].get()[1] logging.info( "Found %d %s accounts that have changed in the accountpage since %s" % (len(changed_accounts), institute, last_timestamp)) accounts = nub([u['vsc_id'] for u in changed_accounts]) for storage_name in opts.options.storage: (users_ok, users_fail) = process_users(opts.options, accounts, storage_name, client, institute) stats["%s_users_sync" % (storage_name, )] = len(users_ok) stats["%s_users_sync_fail" % (storage_name, )] = len(users_fail) stats["%s_users_sync_fail_warning" % (storage_name, )] = STORAGE_USERS_LIMIT_WARNING stats["%s_users_sync_fail_critical" % (storage_name, )] = STORAGE_USERS_LIMIT_CRITICAL for storage_name in opts.options.storage: storage_changed_quota = [ mkVscUserSizeQuota(q) for q in client.quota.user. storage[storage_name].modified[last_timestamp].get()[1] ] storage_changed_quota = [ q for q in storage_changed_quota if q.fileset.startswith('vsc') ] logging.info( "Found %d accounts that have changed quota on storage %s in the accountpage since %s", len(storage_changed_quota), storage_name, last_timestamp) (quota_ok, quota_fail) = process_users_quota( opts.options, storage_changed_quota, storage_name, client, institute) stats["%s_quota_sync" % (storage_name, )] = len(quota_ok) stats["%s_quota_sync_fail" % (storage_name, )] = len(quota_fail) stats["%s_quota_sync_fail_warning" % (storage_name, )] = STORAGE_QUOTA_LIMIT_WARNING stats["%s_quota_sync_fail_critical" % (storage_name, )] = STORAGE_QUOTA_LIMIT_CRITICAL (vos_ok, vos_fail) = ([], []) if opts.options.vo: changed_vos = client.vo.institute[institute].modified[ last_timestamp].get()[1] changed_vo_quota = client.quota.vo.modified[last_timestamp].get( )[1] vos = sorted( set([v['vsc_id'] for v in changed_vos] + [v['virtual_organisation'] for v in changed_vo_quota])) logging.info( "Found %d %s VOs that have changed in the accountpage since %s" % (len(changed_vos), institute, last_timestamp)) logging.info( "Found %d %s VOs that have changed quota in the accountpage since %s" % (len(changed_vo_quota), institute, last_timestamp)) logging.debug("Found the following {institute} VOs: {vos}".format( institute=institute, vos=vos)) for storage_name in opts.options.storage: (vos_ok, vos_fail) = process_vos(opts.options, vos, storage_name, client, last_timestamp, institute) stats["%s_vos_sync" % (storage_name, )] = len(vos_ok) stats["%s_vos_sync_fail" % (storage_name, )] = len(vos_fail) stats["%s_vos_sync_fail_warning" % (storage_name, )] = STORAGE_VO_LIMIT_WARNING stats["%s_vos_sync_fail_critical" % (storage_name, )] = STORAGE_VO_LIMIT_CRITICAL if not (users_fail or quota_fail or vos_fail) and not opts.options.dry_run: (_, ldap_timestamp) = convert_timestamp(start_time) write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp) except Exception as err: logger.exception("critical exception caught: %s" % (err)) opts.critical("Script failed in a horrible way") sys.exit(NAGIOS_EXIT_CRITICAL) opts.epilogue("%s users and VOs synchronised" % institute, stats)
def test_process_brussel_default_vo_gent_user(self, mock_posix, mock_gpfs, mock_client): """Test for a vsc4 account in the default Brussel VO""" test_vo_id = "bvo00003" date = "203012310000" Options = namedtuple("Options", ["dry_run"]) options = Options(dry_run=False) # first mock all the calls to the accountpage mc = mock_client.return_value mc.account = mock.MagicMock() mc.vo = mock.MagicMock() account_1 = { "vsc_id": "vsc40002", "status": "active", "isactive": True, "force_active": False, "expiry_date": None, "grace_until": None, "vsc_id_number": 2540002, "home_directory": "/user/gent/400/vsc40002", "data_directory": "/data/gent/400/vsc40002", "scratch_directory": "/scratch/gent/400/vsc40002", "login_shell": "/bin/bash", "broken": False, "email": "*****@*****.**", "research_field": ["unknown", "unknown"], "create_timestamp": "2014-04-23T09:11:21.168000Z", "person": { "gecos": "Stijn Deweirdt", "institute": {"name": "gent"}, "institute_login": "******", "institute_affiliation": "unknown", "realeppn": "*****@*****.**", }, "home_on_scratch": False, } mc.account[account_1["vsc_id"]].get.return_value = (200, account_1) mc.vo[test_vo_id].member.modified[date].get.return_value = (200, [account_1]) mc.vo[test_vo_id].get.return_value = ( 200, { "vsc_id": "bvo00003", "status": "active", "vsc_id_number": 2610008, "institute": {"name": "brussel"}, "fairshare": 100, "data_path": "/data/brussel/vo/000/bvo00003", "scratch_path": "/scratch/brussel/vo/000/bvo00003", "description": "default gent VO at brussel", "members": [ "vsc40001", "vsc40002", "vsc40016", "vsc40023", "vsc40075", "vsc40485", "vsc41041", "vsc41420", ], "moderators": [], }, ) mc.vo[test_vo_id].quota.get.return_value = ( 200, [ { "virtual_organisation": "bvo00003", "storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"}, "fileset": "bvo00003", "hard": 104857600, } ], ) mc.account[account_1["vsc_id"]].quota.get.return_value = ( 200, [ { "user": "******", "storage": {"institute": "gent", "name": "VSC_SCRATCH_DELCATTY", "storage_type": "scratch"}, "fileset": "vsc400", "hard": 262144000, }, { "user": "******", "storage": {"institute": "gent", "name": "VSC_DATA", "storage_type": "data"}, "fileset": "vsc400", "hard": 26214400, }, { "user": "******", "storage": {"institute": "gent", "name": "VSC_SCRATCH_GENGAR", "storage_type": "scratch"}, "fileset": "vsc400", "hard": 26214400, }, { "user": "******", "storage": {"institute": "gent", "name": "VSC_SCRATCH_GULPIN", "storage_type": "scratch"}, "fileset": "vsc400", "hard": 26214400, }, { "user": "******", "storage": {"institute": "gent", "name": "VSC_HOME", "storage_type": "home"}, "fileset": "vsc400", "hard": 6291456, }, { "user": "******", "storage": {"institute": "gent", "name": "VSC_SCRATCH_DELCATTY", "storage_type": "scratch"}, "fileset": "gvo00001", "hard": 262144000, }, { "user": "******", "storage": {"institute": "gent", "name": "VSC_DATA", "storage_type": "data"}, "fileset": "gvo00001", "hard": 52428800, }, { "user": "******", "storage": {"institute": "gent", "name": "VSC_SCRATCH_GENGAR", "storage_type": "scratch"}, "fileset": "gvo00001", "hard": 104857600, }, { "user": "******", "storage": {"institute": "gent", "name": "VSC_SCRATCH_GULPIN", "storage_type": "scratch"}, "fileset": "gvo00001", "hard": 104857600, }, { "user": "******", "storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"}, "fileset": "bvo00003", "hard": 52428800, }, ], ) # This shouldn't do anything ok, errors = vo.process_vos(options, [test_vo_id], VSC_HOME, mc, date, host_institute=BRUSSEL) self.assertEqual(errors, {}) self.assertEqual(ok, {}) self.assertEqual(mock_gpfs.mock_calls, [mock.call()]) self.assertEqual(mock_posix.mock_calls, [mock.call()]) # VSC_DATA should also not do anything mock_gpfs.reset_mock() mock_posix.reset_mock() ok, errors = vo.process_vos(options, [test_vo_id], VSC_DATA, mc, date, host_institute=BRUSSEL) self.assertEqual(errors, {}) self.assertEqual(ok, {}) self.assertEqual(mock_gpfs.mock_calls, [mock.call()]) self.assertEqual(mock_posix.mock_calls, [mock.call()]) # VSC_SCRATCH: this should allocate space mock_gpfs.reset_mock() mock_posix.reset_mock() ok, errors = vo.process_vos( options, [test_vo_id], VSC_PRODUCTION_SCRATCH[BRUSSEL][0], mc, date, host_institute=BRUSSEL ) self.assertEqual(errors, {}) self.assertEqual(ok, {'bvo00003': ['vsc40002']}) self.assertEqual(mock_posix.mock_calls, [mock.call()]) mock_gpfs.return_value.list_filesets.assert_called_with() mock_gpfs.return_value.get_fileset_info.assert_called_with("theiascratch", "bvo00003") mock_gpfs.return_value.chmod.assert_called_with(504, "/theia/scratch/brussel/vo/000/bvo00003") mock_gpfs.return_value.chown.assert_called_with(99, 2610008, "/theia/scratch/brussel/vo/000/bvo00003") mock_gpfs.return_value.set_fileset_quota.assert_called_with( 102005473280, "/theia/scratch/brussel/vo/000/bvo00003", "bvo00003", 107374182400 ) mock_gpfs.return_value.set_fileset_grace.assert_called_with("/theia/scratch/brussel/vo/000/bvo00003", 604800) mock_gpfs.return_value.create_stat_directory.assert_called_with( "/theia/scratch/brussel/vo/000/bvo00003/vsc40002", 448, 2540002, 1, override_permissions=False )
def test_process_brussel_default_vo(self, mock_posix, mock_gpfs, mock_client): """Test for a vsc1 account in the default Brussel VO""" test_vo_id = "bvo00001" date = "203012310000" Options = namedtuple("Options", ["dry_run"]) options = Options(dry_run=False) # first mock all the calls to the accountpage mc = mock_client.return_value mc.account = mock.MagicMock() mc.vo = mock.MagicMock() account_1 = { "vsc_id": "vsc10002", "status": "active", "isactive": True, "force_active": False, "expiry_date": None, "grace_until": None, "vsc_id_number": 2510002, "home_directory": "/user/brussel/100/vsc10002", "data_directory": "/data/brussel/100/vsc10002", "scratch_directory": "/scratch/brussel/100/vsc10002", "login_shell": "/bin/bash", "broken": False, "email": "*****@*****.**", "research_field": ["Chemistry", "nwo"], "create_timestamp": "2020-02-24T13:39:27.219855Z", "person": { "gecos": "Samuel Moors", "institute": {"name": "brussel"}, "institute_login": "******", "institute_affiliation": "staff", "realeppn": "*****@*****.**", }, "home_on_scratch": False, } mc.account[account_1["vsc_id"]].get.return_value = (200, account_1) mc.vo[test_vo_id].member.modified[date].get.return_value = (200, [account_1]) mc.vo[test_vo_id].get.return_value = ( 200, { "vsc_id": "bvo00001", "status": "active", "vsc_id_number": 2610006, "institute": {"name": "brussel"}, "fairshare": 100, "data_path": "/data/brussel/vo/000/bvo00001", "scratch_path": "/scratch/brussel/vo/000/bvo00001", "description": "default brussel VO at brussel", "members": ["vsc10002", "vsc10004"], "moderators": [], }, ) mc.vo[test_vo_id].quota.get.return_value = ( 200, [ { "virtual_organisation": "bvo00001", "storage": {"institute": "brussel", "name": "VSC_DATA", "storage_type": "data"}, "fileset": "bvo00001", "hard": 104857600, }, { "virtual_organisation": "bvo00001", "storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"}, "fileset": "bvo00001", "hard": 104857600, }, ], ) mc.account[account_1["vsc_id"]].quota.get.return_value = ( 200, [ { "user": "******", "storage": {"institute": "gent", "name": "VSC_DATA", "storage_type": "data"}, "fileset": "gvo00016", "hard": 131072000, }, { "user": "******", "storage": {"institute": "gent", "name": "VSC_SCRATCH_DELCATTY", "storage_type": "scratch"}, "fileset": "gvo00016", "hard": 131072000, }, { "user": "******", "storage": {"institute": "brussel", "name": "VSC_DATA", "storage_type": "data"}, "fileset": "bvo00001", "hard": 52428800, }, { "user": "******", "storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"}, "fileset": "bvo00001", "hard": 52428800, }, { "user": "******", "storage": {"institute": "brussel", "name": "VSC_HOME", "storage_type": "home"}, "fileset": "vsc100", "hard": 12582912, }, { "user": "******", "storage": {"institute": "brussel", "name": "VSC_DATA", "storage_type": "data"}, "fileset": "vsc100", "hard": 104857600, }, { "user": "******", "storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"}, "fileset": "vsc100", "hard": 104857600, }, ], ) # This shouldn't do anything ok, errors = vo.process_vos(options, [test_vo_id], VSC_HOME, mc, date, host_institute=BRUSSEL) self.assertEqual(errors, {}) self.assertEqual(ok, {}) self.assertEqual(mock_gpfs.mock_calls, [mock.call()]) self.assertEqual(mock_posix.mock_calls, [mock.call()]) # VSC_DATA should also not do anything mock_gpfs.reset_mock() mock_posix.reset_mock() ok, errors = vo.process_vos(options, [test_vo_id], VSC_DATA, mc, date, host_institute=BRUSSEL) self.assertEqual(errors, {}) self.assertEqual(ok, {}) self.assertEqual(mock_gpfs.mock_calls, [mock.call()]) self.assertEqual(mock_posix.mock_calls, [mock.call()]) # VSC_SCRATCH should also not do anything mock_gpfs.reset_mock() mock_posix.reset_mock() ok, errors = vo.process_vos( options, [test_vo_id], VSC_PRODUCTION_SCRATCH[BRUSSEL][0], mc, date, host_institute=BRUSSEL ) self.assertEqual(errors, {}) self.assertEqual(ok, {}) self.assertEqual(mock_gpfs.mock_calls, [mock.call()]) self.assertEqual(mock_posix.mock_calls, [mock.call()])
def test_process_brussel_vo(self, mock_posix, mock_gpfs, mock_client): """Test to see deploying a Brussel VO works fine""" test_vo_id = "bvo00005" date = "203112310000" Options = namedtuple("Options", ["dry_run"]) options = Options(dry_run=False) # first mock all the calls to the accountpage mc = mock_client.return_value mc.account = mock.MagicMock() mc.vo = mock.MagicMock() account_1 = { "vsc_id": "vsc10001", "status": "active", "isactive": True, "force_active": False, "expiry_date": None, "grace_until": None, "vsc_id_number": 2510001, "home_directory": "/user/brussel/100/vsc10001", "data_directory": "/data/brussel/100/vsc10001", "scratch_directory": "/scratch/brussel/100/vsc10001", "login_shell": "/bin/bash", "broken": False, "email": "*****@*****.**", "research_field": ["Physics", "nwo"], "create_timestamp": "2018-11-13T14:27:53.394000Z", "person": { "gecos": "Ward Poelmans", "institute": {"name": "brussel"}, "institute_login": "******", "institute_affiliation": "staff", "realeppn": "*****@*****.**", }, "home_on_scratch": False, } mc.account[account_1["vsc_id"]].get.return_value = (200, account_1) mc.vo[test_vo_id].member.modified[date].get.return_value = (200, [account_1]) mc.vo[test_vo_id].get.return_value = ( 200, { "vsc_id": "bvo00005", "status": "active", "vsc_id_number": 2610010, "institute": {"name": "brussel"}, "fairshare": 100, "data_path": "/data/brussel/vo/000/bvo00005", "scratch_path": "/scratch/brussel/vo/000/bvo00005", "description": "hpcvub", "members": ["vsc10001", "vsc10003"], "moderators": ["vsc10001"], }, ) mc.vo[test_vo_id].quota.get.return_value = ( 200, [ { "virtual_organisation": "bvo00005", "storage": {"institute": "brussel", "name": "VSC_DATA", "storage_type": "data"}, "fileset": "bvo00005", "hard": 104857600, }, { "virtual_organisation": "bvo00005", "storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"}, "fileset": "bvo00005", "hard": 104857600, }, ], ) mc.account[account_1["vsc_id"]].quota.get.return_value = ( 200, [ { "user": "******", "storage": {"institute": "brussel", "name": "VSC_HOME", "storage_type": "home"}, "fileset": "vsc100", "hard": 12582912, }, { "user": "******", "storage": {"institute": "brussel", "name": "VSC_DATA", "storage_type": "data"}, "fileset": "vsc100", "hard": 26214400, }, { "user": "******", "storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"}, "fileset": "vsc100", "hard": 26214400, }, { "user": "******", "storage": {"institute": "gent", "name": "VSC_DATA", "storage_type": "data"}, "fileset": "gvo00016", "hard": 131072000, }, { "user": "******", "storage": {"institute": "gent", "name": "VSC_SCRATCH_DELCATTY", "storage_type": "scratch"}, "fileset": "gvo00016", "hard": 131072000, }, { "user": "******", "storage": {"institute": "brussel", "name": "VSC_DATA", "storage_type": "data"}, "fileset": "bvo00005", "hard": 52428800, }, { "user": "******", "storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"}, "fileset": "bvo00005", "hard": 52428800, }, ], ) # This shouldn't do anything ok, errors = vo.process_vos(options, [test_vo_id], VSC_HOME, mc, date, host_institute=BRUSSEL) self.assertEqual(errors, {}) self.assertEqual(ok, {}) self.assertEqual(mock_gpfs.mock_calls, [mock.call()]) self.assertEqual(mock_posix.mock_calls, [mock.call()]) # VSC_DATA test mock_gpfs.reset_mock() mock_posix.reset_mock() ok, errors = vo.process_vos(options, [test_vo_id], VSC_DATA, mc, date, host_institute=BRUSSEL) self.assertEqual(errors, {}) self.assertEqual(ok, {"bvo00005": ["vsc10001"]}) self.assertEqual(mock_posix.mock_calls, [mock.call()]) mock_gpfs.return_value.list_filesets.assert_called() mock_gpfs.return_value.get_fileset_info.assert_called_with("theiadata", "bvo00005") mock_gpfs.return_value.chmod.assert_called_with(504, "/theia/data/brussel/vo/000/bvo00005") mock_gpfs.return_value.chown.assert_called_with(2510001, 2610010, "/theia/data/brussel/vo/000/bvo00005") mock_gpfs.return_value.set_fileset_quota.assert_called_with( 204010946560, "/theia/data/brussel/vo/000/bvo00005", "bvo00005", 214748364800 ) mock_gpfs.return_value.set_fileset_grace.assert_called_with("/theia/data/brussel/vo/000/bvo00005", 604800) mock_gpfs.return_value.set_user_quota.assert_called_with( hard=107374182400, obj="/theia/data/brussel/vo/000/bvo00005", soft=102005473280, user=2510001 ) mock_gpfs.return_value.create_stat_directory.assert_called_with( "/theia/data/brussel/vo/000/bvo00005/vsc10001", 448, 2510001, 1, override_permissions=False ) # VSC_SCRATCH test mock_gpfs.reset_mock() mock_posix.reset_mock() ok, errors = vo.process_vos( options, [test_vo_id], VSC_PRODUCTION_SCRATCH[BRUSSEL][0], mc, date, host_institute=BRUSSEL ) self.assertEqual(errors, {}) self.assertEqual(ok, {"bvo00005": ["vsc10001"]}) self.assertEqual(mock_posix.mock_calls, [mock.call()]) mock_gpfs.return_value.list_filesets.assert_called_with() mock_gpfs.return_value.get_fileset_info.assert_called_with("theiascratch", "bvo00005") mock_gpfs.return_value.chmod.assert_called_with(504, "/theia/scratch/brussel/vo/000/bvo00005") mock_gpfs.return_value.chown.assert_called_with(2510001, 2610010, "/theia/scratch/brussel/vo/000/bvo00005") mock_gpfs.return_value.set_fileset_quota.assert_called_with( 102005473280, "/theia/scratch/brussel/vo/000/bvo00005", "bvo00005", 107374182400 ) mock_gpfs.return_value.set_fileset_grace.assert_called_with("/theia/scratch/brussel/vo/000/bvo00005", 604800) mock_gpfs.return_value.set_user_quota.assert_called_with( hard=53687091200, obj="/theia/scratch/brussel/vo/000/bvo00005", soft=51002736640, user=2510001 ) mock_gpfs.return_value.create_stat_directory.assert_called_with( "/theia/scratch/brussel/vo/000/bvo00005/vsc10001", 448, 2510001, 1, override_permissions=False )