def test_process_regular_users(self, mock_client):
test_account_ids = ['vsc40075', 'vsc40123', 'vsc40039']
Options = namedtuple("Options", ['dry_run'])
options = Options(dry_run=False)
mock_client.return_value = mock.MagicMock()
for storage_name in (VSC_HOME, VSC_DATA, VSC_SCRATCH_DELCATTY, VSC_SCRATCH_PHANPY):
with mock.patch('vsc.administration.user.VscTier2AccountpageUser', autospec=True) as mock_user:
with mock.patch('vsc.administration.user.update_user_status') as mock_update_user_status:
mock_user.return_value = mock.MagicMock()
mock_user_instance = mock_user.return_value
user.process_users(options, test_account_ids, storage_name, mock_client)
mock_user_instance.set_scratch_quota.assert_not_called()
mock_user_instance.set_home_quota.assert_not_called()
mock_user_instance.set_data_quota.assert_not_called()
if storage_name in (VSC_HOME):
mock_user_instance.create_scratch_dir.assert_not_called()
mock_user_instance.create_data_dir.assert_not_called()
self.assertEqual(mock_user_instance.create_home_dir.called, True)
self.assertEqual(mock_user_instance.populate_home_dir.called, True)
self.assertEqual(mock_update_user_status.called, True)
if storage_name in (VSC_DATA,):
mock_user_instance.create_home_dir.assert_not_called()
mock_user_instance.populate_home_dir.assert_not_called()
mock_update_user_status.assert_not_called()
mock_user_instance.create_scratch_dir.assert_not_called()
self.assertEqual(mock_user_instance.create_data_dir.called, True)
if storage_name in (VSC_SCRATCH_KYUKON,):
mock_user_instance.create_home_dir.assert_not_called()
mock_user_instance.populate_home_dir.assert_not_called()
mock_update_user_status.assert_not_called()
mock_user_instance.create_data_dir.assert_not_called()
self.assertEqual(mock_user_instance.create_scratch_dir.called, True)
def main():
"""
Main script.
- build the filter
- fetches the users
- process the users
- write the new timestamp if everything went OK
- write the nagios check file
"""
options = {
'nagios-check-interval-threshold':
NAGIOS_CHECK_INTERVAL_THRESHOLD,
'storage': ('storage systems on which to deploy users and vos', None,
'extend', []),
'user': ('process users', None, 'store_true', False),
'vo': ('process vos', None, 'store_true', False),
'access_token': ('OAuth2 token to access the account page REST API',
None, 'store', None),
'account_page_url':
('URL of the account page where we can find the REST API', None,
'store', None),
'host_institute':
('Name of the institute where this script is being run', str, 'store',
GENT),
}
opts = ExtendedSimpleOption(options)
stats = {}
try:
now = datetime.utcnow()
client = AccountpageClient(token=opts.options.access_token,
url=opts.options.account_page_url + "/api/")
try:
last_timestamp = read_timestamp(SYNC_TIMESTAMP_FILENAME)
except Exception:
logger.exception("Something broke reading the timestamp from %s" %
SYNC_TIMESTAMP_FILENAME)
last_timestamp = "200901010000Z"
logger.info("Last recorded timestamp was %s" % (last_timestamp))
last_timestamp = convert_to_unix_timestamp(last_timestamp)
(users_ok, users_fail) = ([], [])
(quota_ok, quota_fail) = ([], [])
if opts.options.user:
ugent_changed_accounts = client.account.institute['gent'].modified[
last_timestamp].get()[1]
logger.info(
"Found %d UGent accounts that have changed in the accountpage since %s"
% (len(ugent_changed_accounts), last_timestamp))
ugent_accounts = [u['vsc_id'] for u in ugent_changed_accounts]
ugent_accounts = nub(ugent_accounts)
for storage_name in opts.options.storage:
(users_ok,
users_fail) = process_users(opts.options, ugent_accounts,
storage_name, client,
opts.options.host_institute)
stats["%s_users_sync" % (storage_name, )] = len(users_ok)
stats["%s_users_sync_fail" %
(storage_name, )] = len(users_fail)
stats["%s_users_sync_fail_warning" %
(storage_name, )] = STORAGE_USERS_LIMIT_WARNING
stats["%s_users_sync_fail_critical" %
(storage_name, )] = STORAGE_USERS_LIMIT_CRITICAL
for storage_name in opts.options.storage:
storage_changed_quota = [
mkVscUserSizeQuota(q) for q in client.quota.user.
storage[storage_name].modified[last_timestamp].get()[1]
]
storage_changed_quota = [
q for q in storage_changed_quota
if q.fileset.startswith('vsc')
]
logger.info(
"Found %d accounts that have changed quota on storage %s in the accountpage since %s",
len(storage_changed_quota), storage_name, last_timestamp)
(quota_ok, quota_fail) = process_users_quota(
opts.options, storage_changed_quota, storage_name, client,
opts.options.host_institute)
stats["%s_quota_sync" % (storage_name, )] = len(quota_ok)
stats["%s_quota_sync_fail" %
(storage_name, )] = len(quota_fail)
stats["%s_quota_sync_fail_warning" %
(storage_name, )] = STORAGE_QUOTA_LIMIT_WARNING
stats["%s_quota_sync_fail_critical" %
(storage_name, )] = STORAGE_QUOTA_LIMIT_CRITICAL
(vos_ok, vos_fail) = ([], [])
if opts.options.vo:
ugent_changed_vos = client.vo.modified[last_timestamp].get()[1]
ugent_changed_vo_quota = client.quota.vo.modified[
last_timestamp].get()[1]
ugent_vos = sorted(
set([v['vsc_id'] for v in ugent_changed_vos] + [
v['virtual_organisation'] for v in ugent_changed_vo_quota
]))
logger.info(
"Found %d UGent VOs that have changed in the accountpage since %s"
% (len(ugent_changed_vos), last_timestamp))
logger.info(
"Found %d UGent VOs that have changed quota in the accountpage since %s"
% (len(ugent_changed_vo_quota), last_timestamp))
logger.debug(
"Found the following UGent VOs: {vos}".format(vos=ugent_vos))
for storage_name in opts.options.storage:
(vos_ok, vos_fail) = process_vos(opts.options, ugent_vos,
storage_name, client,
last_timestamp,
opts.options.host_institute)
stats["%s_vos_sync" % (storage_name, )] = len(vos_ok)
stats["%s_vos_sync_fail" % (storage_name, )] = len(vos_fail)
stats["%s_vos_sync_fail_warning" %
(storage_name, )] = STORAGE_VO_LIMIT_WARNING
stats["%s_vos_sync_fail_critical" %
(storage_name, )] = STORAGE_VO_LIMIT_CRITICAL
if not (users_fail or quota_fail or vos_fail):
(_, ldap_timestamp) = convert_timestamp(now)
if not opts.options.dry_run:
write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp)
except Exception as err:
logger.exception("critical exception caught: %s" % (err))
opts.critical("Script failed in a horrible way")
sys.exit(NAGIOS_EXIT_CRITICAL)
opts.epilogue("UGent users and VOs synchronised", stats)
def test_process_regular_users(self, mock_client):
test_accounts = [(['vsc40075', 'vsc40123', 'vsc40039'], GENT),
(['vsc10001'], BRUSSEL)]
Options = namedtuple("Options", ['dry_run'])
options = Options(dry_run=False)
mock_client.return_value = mock.MagicMock()
for accounts, site in test_accounts:
for storage_name in (
VSC_HOME,
VSC_DATA,
) + VSC_PRODUCTION_SCRATCH[site]:
with mock.patch(
'vsc.administration.user.VscTier2AccountpageUser',
autospec=True) as mock_user:
with mock.patch(
'vsc.administration.user.update_user_status'
) as mock_update_user_status:
mock_user.return_value = mock.MagicMock()
mock_user_instance = mock_user.return_value
user.process_users(options,
accounts,
storage_name,
mock_client,
host_institute=site)
mock_user_instance.set_scratch_quota.assert_not_called(
)
mock_user_instance.set_home_quota.assert_not_called()
mock_user_instance.set_data_quota.assert_not_called()
if storage_name in (VSC_HOME, ):
mock_user_instance.create_scratch_dir.assert_not_called(
)
mock_user_instance.create_data_dir.assert_not_called(
)
self.assertEqual(
mock_user_instance.create_home_dir.called,
True)
self.assertEqual(
mock_user_instance.populate_home_dir.called,
True)
self.assertEqual(mock_update_user_status.called,
True)
if storage_name in (VSC_DATA, ):
mock_user_instance.create_home_dir.assert_not_called(
)
mock_user_instance.populate_home_dir.assert_not_called(
)
mock_update_user_status.assert_not_called()
mock_user_instance.create_scratch_dir.assert_not_called(
)
self.assertEqual(
mock_user_instance.create_data_dir.called,
True)
if storage_name in VSC_PRODUCTION_SCRATCH[site]:
mock_user_instance.create_home_dir.assert_not_called(
)
mock_user_instance.populate_home_dir.assert_not_called(
)
mock_update_user_status.assert_not_called()
mock_user_instance.create_data_dir.assert_not_called(
)
self.assertEqual(
mock_user_instance.create_scratch_dir.called,
True)
def main():
"""
Main script.
- build the filter
- fetches the users
- process the users
- write the new timestamp if everything went OK
- write the nagios check file
"""
options = {
'nagios-check-interval-threshold':
NAGIOS_CHECK_INTERVAL_THRESHOLD,
'storage': ('storage systems on which to deploy users and vos', None,
'extend', []),
'user': ('process users', None, 'store_true', False),
'vo': ('process vos', None, 'store_true', False),
'access_token': ('OAuth2 token to access the account page REST API',
None, 'store', None),
'account_page_url':
('URL of the account page where we can find the REST API', None,
'store', None),
'host_institute':
('Name of the institute where this script is being run', str, 'store',
GENT),
'start_timestamp':
('Timestamp to start the sync from', str, 'store', None),
}
opts = ExtendedSimpleOption(options)
stats = {}
(last_timestamp, start_time) = retrieve_timestamp_with_default(
SYNC_TIMESTAMP_FILENAME, start_timestamp=opts.options.start_timestamp)
logging.info("Using timestamp %s", last_timestamp)
logging.info("Using startime %s", start_time)
try:
client = AccountpageClient(token=opts.options.access_token,
url=opts.options.account_page_url + "/api/")
institute = opts.options.host_institute
(users_ok, users_fail) = ([], [])
(quota_ok, quota_fail) = ([], [])
if opts.options.user:
changed_accounts = client.account.institute[institute].modified[
last_timestamp].get()[1]
logging.info(
"Found %d %s accounts that have changed in the accountpage since %s"
% (len(changed_accounts), institute, last_timestamp))
accounts = nub([u['vsc_id'] for u in changed_accounts])
for storage_name in opts.options.storage:
(users_ok, users_fail) = process_users(opts.options, accounts,
storage_name, client,
institute)
stats["%s_users_sync" % (storage_name, )] = len(users_ok)
stats["%s_users_sync_fail" %
(storage_name, )] = len(users_fail)
stats["%s_users_sync_fail_warning" %
(storage_name, )] = STORAGE_USERS_LIMIT_WARNING
stats["%s_users_sync_fail_critical" %
(storage_name, )] = STORAGE_USERS_LIMIT_CRITICAL
for storage_name in opts.options.storage:
storage_changed_quota = [
mkVscUserSizeQuota(q) for q in client.quota.user.
storage[storage_name].modified[last_timestamp].get()[1]
]
storage_changed_quota = [
q for q in storage_changed_quota
if q.fileset.startswith('vsc')
]
logging.info(
"Found %d accounts that have changed quota on storage %s in the accountpage since %s",
len(storage_changed_quota), storage_name, last_timestamp)
(quota_ok, quota_fail) = process_users_quota(
opts.options, storage_changed_quota, storage_name, client,
institute)
stats["%s_quota_sync" % (storage_name, )] = len(quota_ok)
stats["%s_quota_sync_fail" %
(storage_name, )] = len(quota_fail)
stats["%s_quota_sync_fail_warning" %
(storage_name, )] = STORAGE_QUOTA_LIMIT_WARNING
stats["%s_quota_sync_fail_critical" %
(storage_name, )] = STORAGE_QUOTA_LIMIT_CRITICAL
(vos_ok, vos_fail) = ([], [])
if opts.options.vo:
changed_vos = client.vo.institute[institute].modified[
last_timestamp].get()[1]
changed_vo_quota = client.quota.vo.modified[last_timestamp].get(
)[1]
vos = sorted(
set([v['vsc_id'] for v in changed_vos] +
[v['virtual_organisation'] for v in changed_vo_quota]))
logging.info(
"Found %d %s VOs that have changed in the accountpage since %s"
% (len(changed_vos), institute, last_timestamp))
logging.info(
"Found %d %s VOs that have changed quota in the accountpage since %s"
% (len(changed_vo_quota), institute, last_timestamp))
logging.debug("Found the following {institute} VOs: {vos}".format(
institute=institute, vos=vos))
for storage_name in opts.options.storage:
(vos_ok, vos_fail) = process_vos(opts.options, vos,
storage_name, client,
last_timestamp, institute)
stats["%s_vos_sync" % (storage_name, )] = len(vos_ok)
stats["%s_vos_sync_fail" % (storage_name, )] = len(vos_fail)
stats["%s_vos_sync_fail_warning" %
(storage_name, )] = STORAGE_VO_LIMIT_WARNING
stats["%s_vos_sync_fail_critical" %
(storage_name, )] = STORAGE_VO_LIMIT_CRITICAL
if not (users_fail or quota_fail
or vos_fail) and not opts.options.dry_run:
(_, ldap_timestamp) = convert_timestamp(start_time)
write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp)
except Exception as err:
logger.exception("critical exception caught: %s" % (err))
opts.critical("Script failed in a horrible way")
sys.exit(NAGIOS_EXIT_CRITICAL)
opts.epilogue("%s users and VOs synchronised" % institute, stats)
