def test_process_regular_users(self, mock_client): test_account_ids = ['vsc40075', 'vsc40123', 'vsc40039'] Options = namedtuple("Options", ['dry_run']) options = Options(dry_run=False) mock_client.return_value = mock.MagicMock() for storage_name in (VSC_HOME, VSC_DATA, VSC_SCRATCH_DELCATTY, VSC_SCRATCH_PHANPY): with mock.patch('vsc.administration.user.VscTier2AccountpageUser', autospec=True) as mock_user: with mock.patch('vsc.administration.user.update_user_status') as mock_update_user_status: mock_user.return_value = mock.MagicMock() mock_user_instance = mock_user.return_value user.process_users(options, test_account_ids, storage_name, mock_client) mock_user_instance.set_scratch_quota.assert_not_called() mock_user_instance.set_home_quota.assert_not_called() mock_user_instance.set_data_quota.assert_not_called() if storage_name in (VSC_HOME): mock_user_instance.create_scratch_dir.assert_not_called() mock_user_instance.create_data_dir.assert_not_called() self.assertEqual(mock_user_instance.create_home_dir.called, True) self.assertEqual(mock_user_instance.populate_home_dir.called, True) self.assertEqual(mock_update_user_status.called, True) if storage_name in (VSC_DATA,): mock_user_instance.create_home_dir.assert_not_called() mock_user_instance.populate_home_dir.assert_not_called() mock_update_user_status.assert_not_called() mock_user_instance.create_scratch_dir.assert_not_called() self.assertEqual(mock_user_instance.create_data_dir.called, True) if storage_name in (VSC_SCRATCH_KYUKON,): mock_user_instance.create_home_dir.assert_not_called() mock_user_instance.populate_home_dir.assert_not_called() mock_update_user_status.assert_not_called() mock_user_instance.create_data_dir.assert_not_called() self.assertEqual(mock_user_instance.create_scratch_dir.called, True)
def main(): """ Main script. - build the filter - fetches the users - process the users - write the new timestamp if everything went OK - write the nagios check file """ options = { 'nagios-check-interval-threshold': NAGIOS_CHECK_INTERVAL_THRESHOLD, 'storage': ('storage systems on which to deploy users and vos', None, 'extend', []), 'user': ('process users', None, 'store_true', False), 'vo': ('process vos', None, 'store_true', False), 'access_token': ('OAuth2 token to access the account page REST API', None, 'store', None), 'account_page_url': ('URL of the account page where we can find the REST API', None, 'store', None), 'host_institute': ('Name of the institute where this script is being run', str, 'store', GENT), } opts = ExtendedSimpleOption(options) stats = {} try: now = datetime.utcnow() client = AccountpageClient(token=opts.options.access_token, url=opts.options.account_page_url + "/api/") try: last_timestamp = read_timestamp(SYNC_TIMESTAMP_FILENAME) except Exception: logger.exception("Something broke reading the timestamp from %s" % SYNC_TIMESTAMP_FILENAME) last_timestamp = "200901010000Z" logger.info("Last recorded timestamp was %s" % (last_timestamp)) last_timestamp = convert_to_unix_timestamp(last_timestamp) (users_ok, users_fail) = ([], []) (quota_ok, quota_fail) = ([], []) if opts.options.user: ugent_changed_accounts = client.account.institute['gent'].modified[ last_timestamp].get()[1] logger.info( "Found %d UGent accounts that have changed in the accountpage since %s" % (len(ugent_changed_accounts), last_timestamp)) ugent_accounts = [u['vsc_id'] for u in ugent_changed_accounts] ugent_accounts = nub(ugent_accounts) for storage_name in opts.options.storage: (users_ok, users_fail) = process_users(opts.options, ugent_accounts, storage_name, client, opts.options.host_institute) stats["%s_users_sync" % (storage_name, )] = len(users_ok) stats["%s_users_sync_fail" % (storage_name, )] = len(users_fail) stats["%s_users_sync_fail_warning" % (storage_name, )] = STORAGE_USERS_LIMIT_WARNING stats["%s_users_sync_fail_critical" % (storage_name, )] = STORAGE_USERS_LIMIT_CRITICAL for storage_name in opts.options.storage: storage_changed_quota = [ mkVscUserSizeQuota(q) for q in client.quota.user. storage[storage_name].modified[last_timestamp].get()[1] ] storage_changed_quota = [ q for q in storage_changed_quota if q.fileset.startswith('vsc') ] logger.info( "Found %d accounts that have changed quota on storage %s in the accountpage since %s", len(storage_changed_quota), storage_name, last_timestamp) (quota_ok, quota_fail) = process_users_quota( opts.options, storage_changed_quota, storage_name, client, opts.options.host_institute) stats["%s_quota_sync" % (storage_name, )] = len(quota_ok) stats["%s_quota_sync_fail" % (storage_name, )] = len(quota_fail) stats["%s_quota_sync_fail_warning" % (storage_name, )] = STORAGE_QUOTA_LIMIT_WARNING stats["%s_quota_sync_fail_critical" % (storage_name, )] = STORAGE_QUOTA_LIMIT_CRITICAL (vos_ok, vos_fail) = ([], []) if opts.options.vo: ugent_changed_vos = client.vo.modified[last_timestamp].get()[1] ugent_changed_vo_quota = client.quota.vo.modified[ last_timestamp].get()[1] ugent_vos = sorted( set([v['vsc_id'] for v in ugent_changed_vos] + [ v['virtual_organisation'] for v in ugent_changed_vo_quota ])) logger.info( "Found %d UGent VOs that have changed in the accountpage since %s" % (len(ugent_changed_vos), last_timestamp)) logger.info( "Found %d UGent VOs that have changed quota in the accountpage since %s" % (len(ugent_changed_vo_quota), last_timestamp)) logger.debug( "Found the following UGent VOs: {vos}".format(vos=ugent_vos)) for storage_name in opts.options.storage: (vos_ok, vos_fail) = process_vos(opts.options, ugent_vos, storage_name, client, last_timestamp, opts.options.host_institute) stats["%s_vos_sync" % (storage_name, )] = len(vos_ok) stats["%s_vos_sync_fail" % (storage_name, )] = len(vos_fail) stats["%s_vos_sync_fail_warning" % (storage_name, )] = STORAGE_VO_LIMIT_WARNING stats["%s_vos_sync_fail_critical" % (storage_name, )] = STORAGE_VO_LIMIT_CRITICAL if not (users_fail or quota_fail or vos_fail): (_, ldap_timestamp) = convert_timestamp(now) if not opts.options.dry_run: write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp) except Exception as err: logger.exception("critical exception caught: %s" % (err)) opts.critical("Script failed in a horrible way") sys.exit(NAGIOS_EXIT_CRITICAL) opts.epilogue("UGent users and VOs synchronised", stats)
def test_process_regular_users(self, mock_client): test_accounts = [(['vsc40075', 'vsc40123', 'vsc40039'], GENT), (['vsc10001'], BRUSSEL)] Options = namedtuple("Options", ['dry_run']) options = Options(dry_run=False) mock_client.return_value = mock.MagicMock() for accounts, site in test_accounts: for storage_name in ( VSC_HOME, VSC_DATA, ) + VSC_PRODUCTION_SCRATCH[site]: with mock.patch( 'vsc.administration.user.VscTier2AccountpageUser', autospec=True) as mock_user: with mock.patch( 'vsc.administration.user.update_user_status' ) as mock_update_user_status: mock_user.return_value = mock.MagicMock() mock_user_instance = mock_user.return_value user.process_users(options, accounts, storage_name, mock_client, host_institute=site) mock_user_instance.set_scratch_quota.assert_not_called( ) mock_user_instance.set_home_quota.assert_not_called() mock_user_instance.set_data_quota.assert_not_called() if storage_name in (VSC_HOME, ): mock_user_instance.create_scratch_dir.assert_not_called( ) mock_user_instance.create_data_dir.assert_not_called( ) self.assertEqual( mock_user_instance.create_home_dir.called, True) self.assertEqual( mock_user_instance.populate_home_dir.called, True) self.assertEqual(mock_update_user_status.called, True) if storage_name in (VSC_DATA, ): mock_user_instance.create_home_dir.assert_not_called( ) mock_user_instance.populate_home_dir.assert_not_called( ) mock_update_user_status.assert_not_called() mock_user_instance.create_scratch_dir.assert_not_called( ) self.assertEqual( mock_user_instance.create_data_dir.called, True) if storage_name in VSC_PRODUCTION_SCRATCH[site]: mock_user_instance.create_home_dir.assert_not_called( ) mock_user_instance.populate_home_dir.assert_not_called( ) mock_update_user_status.assert_not_called() mock_user_instance.create_data_dir.assert_not_called( ) self.assertEqual( mock_user_instance.create_scratch_dir.called, True)
def main(): """ Main script. - build the filter - fetches the users - process the users - write the new timestamp if everything went OK - write the nagios check file """ options = { 'nagios-check-interval-threshold': NAGIOS_CHECK_INTERVAL_THRESHOLD, 'storage': ('storage systems on which to deploy users and vos', None, 'extend', []), 'user': ('process users', None, 'store_true', False), 'vo': ('process vos', None, 'store_true', False), 'access_token': ('OAuth2 token to access the account page REST API', None, 'store', None), 'account_page_url': ('URL of the account page where we can find the REST API', None, 'store', None), 'host_institute': ('Name of the institute where this script is being run', str, 'store', GENT), 'start_timestamp': ('Timestamp to start the sync from', str, 'store', None), } opts = ExtendedSimpleOption(options) stats = {} (last_timestamp, start_time) = retrieve_timestamp_with_default( SYNC_TIMESTAMP_FILENAME, start_timestamp=opts.options.start_timestamp) logging.info("Using timestamp %s", last_timestamp) logging.info("Using startime %s", start_time) try: client = AccountpageClient(token=opts.options.access_token, url=opts.options.account_page_url + "/api/") institute = opts.options.host_institute (users_ok, users_fail) = ([], []) (quota_ok, quota_fail) = ([], []) if opts.options.user: changed_accounts = client.account.institute[institute].modified[ last_timestamp].get()[1] logging.info( "Found %d %s accounts that have changed in the accountpage since %s" % (len(changed_accounts), institute, last_timestamp)) accounts = nub([u['vsc_id'] for u in changed_accounts]) for storage_name in opts.options.storage: (users_ok, users_fail) = process_users(opts.options, accounts, storage_name, client, institute) stats["%s_users_sync" % (storage_name, )] = len(users_ok) stats["%s_users_sync_fail" % (storage_name, )] = len(users_fail) stats["%s_users_sync_fail_warning" % (storage_name, )] = STORAGE_USERS_LIMIT_WARNING stats["%s_users_sync_fail_critical" % (storage_name, )] = STORAGE_USERS_LIMIT_CRITICAL for storage_name in opts.options.storage: storage_changed_quota = [ mkVscUserSizeQuota(q) for q in client.quota.user. storage[storage_name].modified[last_timestamp].get()[1] ] storage_changed_quota = [ q for q in storage_changed_quota if q.fileset.startswith('vsc') ] logging.info( "Found %d accounts that have changed quota on storage %s in the accountpage since %s", len(storage_changed_quota), storage_name, last_timestamp) (quota_ok, quota_fail) = process_users_quota( opts.options, storage_changed_quota, storage_name, client, institute) stats["%s_quota_sync" % (storage_name, )] = len(quota_ok) stats["%s_quota_sync_fail" % (storage_name, )] = len(quota_fail) stats["%s_quota_sync_fail_warning" % (storage_name, )] = STORAGE_QUOTA_LIMIT_WARNING stats["%s_quota_sync_fail_critical" % (storage_name, )] = STORAGE_QUOTA_LIMIT_CRITICAL (vos_ok, vos_fail) = ([], []) if opts.options.vo: changed_vos = client.vo.institute[institute].modified[ last_timestamp].get()[1] changed_vo_quota = client.quota.vo.modified[last_timestamp].get( )[1] vos = sorted( set([v['vsc_id'] for v in changed_vos] + [v['virtual_organisation'] for v in changed_vo_quota])) logging.info( "Found %d %s VOs that have changed in the accountpage since %s" % (len(changed_vos), institute, last_timestamp)) logging.info( "Found %d %s VOs that have changed quota in the accountpage since %s" % (len(changed_vo_quota), institute, last_timestamp)) logging.debug("Found the following {institute} VOs: {vos}".format( institute=institute, vos=vos)) for storage_name in opts.options.storage: (vos_ok, vos_fail) = process_vos(opts.options, vos, storage_name, client, last_timestamp, institute) stats["%s_vos_sync" % (storage_name, )] = len(vos_ok) stats["%s_vos_sync_fail" % (storage_name, )] = len(vos_fail) stats["%s_vos_sync_fail_warning" % (storage_name, )] = STORAGE_VO_LIMIT_WARNING stats["%s_vos_sync_fail_critical" % (storage_name, )] = STORAGE_VO_LIMIT_CRITICAL if not (users_fail or quota_fail or vos_fail) and not opts.options.dry_run: (_, ldap_timestamp) = convert_timestamp(start_time) write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp) except Exception as err: logger.exception("critical exception caught: %s" % (err)) opts.critical("Script failed in a horrible way") sys.exit(NAGIOS_EXIT_CRITICAL) opts.epilogue("%s users and VOs synchronised" % institute, stats)