def test_process_gent_institute_vo(self, mock_storage, mock_client):
test_vo_id = "gvo00012"
Options = namedtuple("Options", ['dry_run'])
options = Options(dry_run=False)
mc = mock_client.return_value
mc.vo = mock.MagicMock()
date = "20321231"
mc.vo['gvo00012'].member.modified[date].get.return_value = (
200, [{
u'broken': False,
u'create_timestamp': u'2014-04-23T09:11:22.460Z',
u'data_directory': u'/user/leuven/data/vsc400/vsc40075',
u'email': u'*****@*****.**',
u'home_directory': u'/user/leuven/home/vsc400/vsc40075',
u'login_shell': u'/bin/bash',
u'person': {
u'gecos': u'Andy Georges',
u'institute': {u'site': u'leuven'},
u'institute_login': u'ageorges'
},
u'research_field': [u'Computer systems, architectures, networks', u'nwo'],
u'scratch_directory': u'/user/leuven/scratch/vsc400/vsc40075',
u'status': u'active',
u'vsc_id': u'vsc40075',
u'vsc_id_number': 2540075,
}])
for storage_name in (VSC_HOME, VSC_DATA) + GENT_PRODUCTION_SCRATCH:
with mock.patch('vsc.administration.vo.VscTier2AccountpageUser', autospec=True) as mock_user:
with mock.patch('vsc.administration.vo.update_vo_status') as mock_update_vo_status:
with mock.patch.object(vo.VscTier2AccountpageVo, 'create_scratch_fileset') as mock_cr_s_fileset:
with mock.patch.object(vo.VscTier2AccountpageVo, 'set_scratch_quota') as mock_s_s_quota:
with mock.patch.object(vo.VscTier2AccountpageVo, 'create_data_fileset') as mock_cr_d_fileset:
with mock.patch.object(vo.VscTier2AccountpageVo, 'set_data_quota') as mock_s_d_quota:
with mock.patch.object(vo.VscTier2AccountpageVo, 'set_member_data_quota') as mock_s_m_d_quota:
with mock.patch.object(vo.VscTier2AccountpageVo, 'create_member_data_dir') as mock_cr_m_d_dir:
with mock.patch.object(vo.VscTier2AccountpageVo, 'set_member_scratch_quota') as mock_s_m_s_quota:
with mock.patch.object(vo.VscTier2AccountpageVo, 'create_member_scratch_dir') as mock_cr_m_s_dir:
mock_user.return_value = mock.MagicMock()
ok, errors = vo.process_vos(options, [test_vo_id], storage_name, mc, "99991231")
self.assertEqual(errors, {})
mock_cr_s_fileset.assert_not_called()
mock_s_s_quota.assert_not_called()
mock_cr_d_fileset.assert_not_called()
mock_s_d_quota.assert_not_called()
mock_update_vo_status.assert_not_called()
mock_s_m_d_quota.assert_not_called()
mock_cr_m_d_dir.assert_not_called()
def test_process_gent_institute_vo_data_share(self, mock_storage, mock_client):
test_vo_id = "gvo03442"
Options = namedtuple("Options", ['dry_run'])
options = Options(dry_run=False)
mc = mock_client.return_value
mc.vo = mock.MagicMock()
v = mock.MagicMock()
mc.vo[test_vo_id].get.return_value = v
for storage_name in (VSC_DATA_SHARED,):
with mock.patch("vsc.administration.vo.VscTier2AccountpageVo.data_sharing", new_callable=mock.PropertyMock) as mock_data_sharing:
with mock.patch("vsc.administration.vo.VscTier2AccountpageVo.sharing_group", new_callable=mock.PropertyMock) as mock_sharing_group:
with mock.patch('vsc.administration.vo.update_vo_status') as mock_update_vo_status:
with mock.patch.object(vo.VscTier2AccountpageVo, 'create_scratch_fileset') as mock_cr_s_fileset:
with mock.patch.object(vo.VscTier2AccountpageVo, 'set_scratch_quota') as mock_s_s_quota:
with mock.patch.object(vo.VscTier2AccountpageVo, 'create_data_fileset') as mock_cr_d_fileset:
with mock.patch.object(vo.VscTier2AccountpageVo, 'set_data_quota') as mock_s_d_quota:
with mock.patch.object(vo.VscTier2AccountpageVo, 'create_data_shared_fileset') as mock_cr_d_shared_fileset:
with mock.patch.object(vo.VscTier2AccountpageVo, 'set_data_shared_quota') as mock_s_d_shared_quota:
with mock.patch.object(vo.VscTier2AccountpageVo, 'set_member_data_quota') as mock_s_m_d_quota:
with mock.patch.object(vo.VscTier2AccountpageVo, 'create_member_data_dir') as mock_cr_m_d_dir:
with mock.patch.object(vo.VscTier2AccountpageVo, 'set_member_scratch_quota') as mock_s_m_s_quota:
with mock.patch.object(vo.VscTier2AccountpageVo, 'create_member_scratch_dir') as mock_cr_m_s_dir:
mock_data_sharing.return_value = True
mock_sharing_group.return_value = VscAutogroup(
vsc_id=test_vo_id.replace('gvo', 'gvos'),
status='active',
vsc_id_number=123456,
institute='Gent',
members=['vsc40075'],
description="test autogroup"
)
ok, errors = vo.process_vos(options, [test_vo_id], storage_name, mc, "99991231")
self.assertEqual(errors, {})
mock_cr_s_fileset.assert_not_called()
mock_s_s_quota.assert_not_called()
mock_cr_d_fileset.assert_not_called()
mock_s_d_quota.assert_not_called()
mock_cr_d_shared_fileset.assert_called()
mock_s_d_shared_quota.assert_called()
mock_update_vo_status.assert_not_called()
mock_s_m_d_quota.assert_not_called()
mock_cr_m_d_dir.assert_not_called()
def main():
"""
Main script.
- build the filter
- fetches the users
- process the users
- write the new timestamp if everything went OK
- write the nagios check file
"""
options = {
'nagios-check-interval-threshold':
NAGIOS_CHECK_INTERVAL_THRESHOLD,
'storage': ('storage systems on which to deploy users and vos', None,
'extend', []),
'user': ('process users', None, 'store_true', False),
'vo': ('process vos', None, 'store_true', False),
'access_token': ('OAuth2 token to access the account page REST API',
None, 'store', None),
'account_page_url':
('URL of the account page where we can find the REST API', None,
'store', None),
'host_institute':
('Name of the institute where this script is being run', str, 'store',
GENT),
}
opts = ExtendedSimpleOption(options)
stats = {}
try:
now = datetime.utcnow()
client = AccountpageClient(token=opts.options.access_token,
url=opts.options.account_page_url + "/api/")
try:
last_timestamp = read_timestamp(SYNC_TIMESTAMP_FILENAME)
except Exception:
logger.exception("Something broke reading the timestamp from %s" %
SYNC_TIMESTAMP_FILENAME)
last_timestamp = "200901010000Z"
logger.info("Last recorded timestamp was %s" % (last_timestamp))
last_timestamp = convert_to_unix_timestamp(last_timestamp)
(users_ok, users_fail) = ([], [])
(quota_ok, quota_fail) = ([], [])
if opts.options.user:
ugent_changed_accounts = client.account.institute['gent'].modified[
last_timestamp].get()[1]
logger.info(
"Found %d UGent accounts that have changed in the accountpage since %s"
% (len(ugent_changed_accounts), last_timestamp))
ugent_accounts = [u['vsc_id'] for u in ugent_changed_accounts]
ugent_accounts = nub(ugent_accounts)
for storage_name in opts.options.storage:
(users_ok,
users_fail) = process_users(opts.options, ugent_accounts,
storage_name, client,
opts.options.host_institute)
stats["%s_users_sync" % (storage_name, )] = len(users_ok)
stats["%s_users_sync_fail" %
(storage_name, )] = len(users_fail)
stats["%s_users_sync_fail_warning" %
(storage_name, )] = STORAGE_USERS_LIMIT_WARNING
stats["%s_users_sync_fail_critical" %
(storage_name, )] = STORAGE_USERS_LIMIT_CRITICAL
for storage_name in opts.options.storage:
storage_changed_quota = [
mkVscUserSizeQuota(q) for q in client.quota.user.
storage[storage_name].modified[last_timestamp].get()[1]
]
storage_changed_quota = [
q for q in storage_changed_quota
if q.fileset.startswith('vsc')
]
logger.info(
"Found %d accounts that have changed quota on storage %s in the accountpage since %s",
len(storage_changed_quota), storage_name, last_timestamp)
(quota_ok, quota_fail) = process_users_quota(
opts.options, storage_changed_quota, storage_name, client,
opts.options.host_institute)
stats["%s_quota_sync" % (storage_name, )] = len(quota_ok)
stats["%s_quota_sync_fail" %
(storage_name, )] = len(quota_fail)
stats["%s_quota_sync_fail_warning" %
(storage_name, )] = STORAGE_QUOTA_LIMIT_WARNING
stats["%s_quota_sync_fail_critical" %
(storage_name, )] = STORAGE_QUOTA_LIMIT_CRITICAL
(vos_ok, vos_fail) = ([], [])
if opts.options.vo:
ugent_changed_vos = client.vo.modified[last_timestamp].get()[1]
ugent_changed_vo_quota = client.quota.vo.modified[
last_timestamp].get()[1]
ugent_vos = sorted(
set([v['vsc_id'] for v in ugent_changed_vos] + [
v['virtual_organisation'] for v in ugent_changed_vo_quota
]))
logger.info(
"Found %d UGent VOs that have changed in the accountpage since %s"
% (len(ugent_changed_vos), last_timestamp))
logger.info(
"Found %d UGent VOs that have changed quota in the accountpage since %s"
% (len(ugent_changed_vo_quota), last_timestamp))
logger.debug(
"Found the following UGent VOs: {vos}".format(vos=ugent_vos))
for storage_name in opts.options.storage:
(vos_ok, vos_fail) = process_vos(opts.options, ugent_vos,
storage_name, client,
last_timestamp,
opts.options.host_institute)
stats["%s_vos_sync" % (storage_name, )] = len(vos_ok)
stats["%s_vos_sync_fail" % (storage_name, )] = len(vos_fail)
stats["%s_vos_sync_fail_warning" %
(storage_name, )] = STORAGE_VO_LIMIT_WARNING
stats["%s_vos_sync_fail_critical" %
(storage_name, )] = STORAGE_VO_LIMIT_CRITICAL
if not (users_fail or quota_fail or vos_fail):
(_, ldap_timestamp) = convert_timestamp(now)
if not opts.options.dry_run:
write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp)
except Exception as err:
logger.exception("critical exception caught: %s" % (err))
opts.critical("Script failed in a horrible way")
sys.exit(NAGIOS_EXIT_CRITICAL)
opts.epilogue("UGent users and VOs synchronised", stats)
def main():
"""
Main script.
- build the filter
- fetches the users
- process the users
- write the new timestamp if everything went OK
- write the nagios check file
"""
options = {
'nagios-check-interval-threshold':
NAGIOS_CHECK_INTERVAL_THRESHOLD,
'storage': ('storage systems on which to deploy users and vos', None,
'extend', []),
'user': ('process users', None, 'store_true', False),
'vo': ('process vos', None, 'store_true', False),
'access_token': ('OAuth2 token to access the account page REST API',
None, 'store', None),
'account_page_url':
('URL of the account page where we can find the REST API', None,
'store', None),
'host_institute':
('Name of the institute where this script is being run', str, 'store',
GENT),
'start_timestamp':
('Timestamp to start the sync from', str, 'store', None),
}
opts = ExtendedSimpleOption(options)
stats = {}
(last_timestamp, start_time) = retrieve_timestamp_with_default(
SYNC_TIMESTAMP_FILENAME, start_timestamp=opts.options.start_timestamp)
logging.info("Using timestamp %s", last_timestamp)
logging.info("Using startime %s", start_time)
try:
client = AccountpageClient(token=opts.options.access_token,
url=opts.options.account_page_url + "/api/")
institute = opts.options.host_institute
(users_ok, users_fail) = ([], [])
(quota_ok, quota_fail) = ([], [])
if opts.options.user:
changed_accounts = client.account.institute[institute].modified[
last_timestamp].get()[1]
logging.info(
"Found %d %s accounts that have changed in the accountpage since %s"
% (len(changed_accounts), institute, last_timestamp))
accounts = nub([u['vsc_id'] for u in changed_accounts])
for storage_name in opts.options.storage:
(users_ok, users_fail) = process_users(opts.options, accounts,
storage_name, client,
institute)
stats["%s_users_sync" % (storage_name, )] = len(users_ok)
stats["%s_users_sync_fail" %
(storage_name, )] = len(users_fail)
stats["%s_users_sync_fail_warning" %
(storage_name, )] = STORAGE_USERS_LIMIT_WARNING
stats["%s_users_sync_fail_critical" %
(storage_name, )] = STORAGE_USERS_LIMIT_CRITICAL
for storage_name in opts.options.storage:
storage_changed_quota = [
mkVscUserSizeQuota(q) for q in client.quota.user.
storage[storage_name].modified[last_timestamp].get()[1]
]
storage_changed_quota = [
q for q in storage_changed_quota
if q.fileset.startswith('vsc')
]
logging.info(
"Found %d accounts that have changed quota on storage %s in the accountpage since %s",
len(storage_changed_quota), storage_name, last_timestamp)
(quota_ok, quota_fail) = process_users_quota(
opts.options, storage_changed_quota, storage_name, client,
institute)
stats["%s_quota_sync" % (storage_name, )] = len(quota_ok)
stats["%s_quota_sync_fail" %
(storage_name, )] = len(quota_fail)
stats["%s_quota_sync_fail_warning" %
(storage_name, )] = STORAGE_QUOTA_LIMIT_WARNING
stats["%s_quota_sync_fail_critical" %
(storage_name, )] = STORAGE_QUOTA_LIMIT_CRITICAL
(vos_ok, vos_fail) = ([], [])
if opts.options.vo:
changed_vos = client.vo.institute[institute].modified[
last_timestamp].get()[1]
changed_vo_quota = client.quota.vo.modified[last_timestamp].get(
)[1]
vos = sorted(
set([v['vsc_id'] for v in changed_vos] +
[v['virtual_organisation'] for v in changed_vo_quota]))
logging.info(
"Found %d %s VOs that have changed in the accountpage since %s"
% (len(changed_vos), institute, last_timestamp))
logging.info(
"Found %d %s VOs that have changed quota in the accountpage since %s"
% (len(changed_vo_quota), institute, last_timestamp))
logging.debug("Found the following {institute} VOs: {vos}".format(
institute=institute, vos=vos))
for storage_name in opts.options.storage:
(vos_ok, vos_fail) = process_vos(opts.options, vos,
storage_name, client,
last_timestamp, institute)
stats["%s_vos_sync" % (storage_name, )] = len(vos_ok)
stats["%s_vos_sync_fail" % (storage_name, )] = len(vos_fail)
stats["%s_vos_sync_fail_warning" %
(storage_name, )] = STORAGE_VO_LIMIT_WARNING
stats["%s_vos_sync_fail_critical" %
(storage_name, )] = STORAGE_VO_LIMIT_CRITICAL
if not (users_fail or quota_fail
or vos_fail) and not opts.options.dry_run:
(_, ldap_timestamp) = convert_timestamp(start_time)
write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp)
except Exception as err:
logger.exception("critical exception caught: %s" % (err))
opts.critical("Script failed in a horrible way")
sys.exit(NAGIOS_EXIT_CRITICAL)
opts.epilogue("%s users and VOs synchronised" % institute, stats)
def test_process_brussel_default_vo_gent_user(self, mock_posix, mock_gpfs, mock_client):
"""Test for a vsc4 account in the default Brussel VO"""
test_vo_id = "bvo00003"
date = "203012310000"
Options = namedtuple("Options", ["dry_run"])
options = Options(dry_run=False)
# first mock all the calls to the accountpage
mc = mock_client.return_value
mc.account = mock.MagicMock()
mc.vo = mock.MagicMock()
account_1 = {
"vsc_id": "vsc40002",
"status": "active",
"isactive": True,
"force_active": False,
"expiry_date": None,
"grace_until": None,
"vsc_id_number": 2540002,
"home_directory": "/user/gent/400/vsc40002",
"data_directory": "/data/gent/400/vsc40002",
"scratch_directory": "/scratch/gent/400/vsc40002",
"login_shell": "/bin/bash",
"broken": False,
"email": "*****@*****.**",
"research_field": ["unknown", "unknown"],
"create_timestamp": "2014-04-23T09:11:21.168000Z",
"person": {
"gecos": "Stijn Deweirdt",
"institute": {"name": "gent"},
"institute_login": "******",
"institute_affiliation": "unknown",
"realeppn": "*****@*****.**",
},
"home_on_scratch": False,
}
mc.account[account_1["vsc_id"]].get.return_value = (200, account_1)
mc.vo[test_vo_id].member.modified[date].get.return_value = (200, [account_1])
mc.vo[test_vo_id].get.return_value = (
200,
{
"vsc_id": "bvo00003",
"status": "active",
"vsc_id_number": 2610008,
"institute": {"name": "brussel"},
"fairshare": 100,
"data_path": "/data/brussel/vo/000/bvo00003",
"scratch_path": "/scratch/brussel/vo/000/bvo00003",
"description": "default gent VO at brussel",
"members": [
"vsc40001",
"vsc40002",
"vsc40016",
"vsc40023",
"vsc40075",
"vsc40485",
"vsc41041",
"vsc41420",
],
"moderators": [],
},
)
mc.vo[test_vo_id].quota.get.return_value = (
200,
[
{
"virtual_organisation": "bvo00003",
"storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"},
"fileset": "bvo00003",
"hard": 104857600,
}
],
)
mc.account[account_1["vsc_id"]].quota.get.return_value = (
200,
[
{
"user": "******",
"storage": {"institute": "gent", "name": "VSC_SCRATCH_DELCATTY", "storage_type": "scratch"},
"fileset": "vsc400",
"hard": 262144000,
},
{
"user": "******",
"storage": {"institute": "gent", "name": "VSC_DATA", "storage_type": "data"},
"fileset": "vsc400",
"hard": 26214400,
},
{
"user": "******",
"storage": {"institute": "gent", "name": "VSC_SCRATCH_GENGAR", "storage_type": "scratch"},
"fileset": "vsc400",
"hard": 26214400,
},
{
"user": "******",
"storage": {"institute": "gent", "name": "VSC_SCRATCH_GULPIN", "storage_type": "scratch"},
"fileset": "vsc400",
"hard": 26214400,
},
{
"user": "******",
"storage": {"institute": "gent", "name": "VSC_HOME", "storage_type": "home"},
"fileset": "vsc400",
"hard": 6291456,
},
{
"user": "******",
"storage": {"institute": "gent", "name": "VSC_SCRATCH_DELCATTY", "storage_type": "scratch"},
"fileset": "gvo00001",
"hard": 262144000,
},
{
"user": "******",
"storage": {"institute": "gent", "name": "VSC_DATA", "storage_type": "data"},
"fileset": "gvo00001",
"hard": 52428800,
},
{
"user": "******",
"storage": {"institute": "gent", "name": "VSC_SCRATCH_GENGAR", "storage_type": "scratch"},
"fileset": "gvo00001",
"hard": 104857600,
},
{
"user": "******",
"storage": {"institute": "gent", "name": "VSC_SCRATCH_GULPIN", "storage_type": "scratch"},
"fileset": "gvo00001",
"hard": 104857600,
},
{
"user": "******",
"storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"},
"fileset": "bvo00003",
"hard": 52428800,
},
],
)
# This shouldn't do anything
ok, errors = vo.process_vos(options, [test_vo_id], VSC_HOME, mc, date, host_institute=BRUSSEL)
self.assertEqual(errors, {})
self.assertEqual(ok, {})
self.assertEqual(mock_gpfs.mock_calls, [mock.call()])
self.assertEqual(mock_posix.mock_calls, [mock.call()])
# VSC_DATA should also not do anything
mock_gpfs.reset_mock()
mock_posix.reset_mock()
ok, errors = vo.process_vos(options, [test_vo_id], VSC_DATA, mc, date, host_institute=BRUSSEL)
self.assertEqual(errors, {})
self.assertEqual(ok, {})
self.assertEqual(mock_gpfs.mock_calls, [mock.call()])
self.assertEqual(mock_posix.mock_calls, [mock.call()])
# VSC_SCRATCH: this should allocate space
mock_gpfs.reset_mock()
mock_posix.reset_mock()
ok, errors = vo.process_vos(
options, [test_vo_id], VSC_PRODUCTION_SCRATCH[BRUSSEL][0], mc, date, host_institute=BRUSSEL
)
self.assertEqual(errors, {})
self.assertEqual(ok, {'bvo00003': ['vsc40002']})
self.assertEqual(mock_posix.mock_calls, [mock.call()])
mock_gpfs.return_value.list_filesets.assert_called_with()
mock_gpfs.return_value.get_fileset_info.assert_called_with("theiascratch", "bvo00003")
mock_gpfs.return_value.chmod.assert_called_with(504, "/theia/scratch/brussel/vo/000/bvo00003")
mock_gpfs.return_value.chown.assert_called_with(99, 2610008, "/theia/scratch/brussel/vo/000/bvo00003")
mock_gpfs.return_value.set_fileset_quota.assert_called_with(
102005473280, "/theia/scratch/brussel/vo/000/bvo00003", "bvo00003", 107374182400
)
mock_gpfs.return_value.set_fileset_grace.assert_called_with("/theia/scratch/brussel/vo/000/bvo00003", 604800)
mock_gpfs.return_value.create_stat_directory.assert_called_with(
"/theia/scratch/brussel/vo/000/bvo00003/vsc40002", 448, 2540002, 1, override_permissions=False
)
def test_process_brussel_default_vo(self, mock_posix, mock_gpfs, mock_client):
"""Test for a vsc1 account in the default Brussel VO"""
test_vo_id = "bvo00001"
date = "203012310000"
Options = namedtuple("Options", ["dry_run"])
options = Options(dry_run=False)
# first mock all the calls to the accountpage
mc = mock_client.return_value
mc.account = mock.MagicMock()
mc.vo = mock.MagicMock()
account_1 = {
"vsc_id": "vsc10002",
"status": "active",
"isactive": True,
"force_active": False,
"expiry_date": None,
"grace_until": None,
"vsc_id_number": 2510002,
"home_directory": "/user/brussel/100/vsc10002",
"data_directory": "/data/brussel/100/vsc10002",
"scratch_directory": "/scratch/brussel/100/vsc10002",
"login_shell": "/bin/bash",
"broken": False,
"email": "*****@*****.**",
"research_field": ["Chemistry", "nwo"],
"create_timestamp": "2020-02-24T13:39:27.219855Z",
"person": {
"gecos": "Samuel Moors",
"institute": {"name": "brussel"},
"institute_login": "******",
"institute_affiliation": "staff",
"realeppn": "*****@*****.**",
},
"home_on_scratch": False,
}
mc.account[account_1["vsc_id"]].get.return_value = (200, account_1)
mc.vo[test_vo_id].member.modified[date].get.return_value = (200, [account_1])
mc.vo[test_vo_id].get.return_value = (
200,
{
"vsc_id": "bvo00001",
"status": "active",
"vsc_id_number": 2610006,
"institute": {"name": "brussel"},
"fairshare": 100,
"data_path": "/data/brussel/vo/000/bvo00001",
"scratch_path": "/scratch/brussel/vo/000/bvo00001",
"description": "default brussel VO at brussel",
"members": ["vsc10002", "vsc10004"],
"moderators": [],
},
)
mc.vo[test_vo_id].quota.get.return_value = (
200,
[
{
"virtual_organisation": "bvo00001",
"storage": {"institute": "brussel", "name": "VSC_DATA", "storage_type": "data"},
"fileset": "bvo00001",
"hard": 104857600,
},
{
"virtual_organisation": "bvo00001",
"storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"},
"fileset": "bvo00001",
"hard": 104857600,
},
],
)
mc.account[account_1["vsc_id"]].quota.get.return_value = (
200,
[
{
"user": "******",
"storage": {"institute": "gent", "name": "VSC_DATA", "storage_type": "data"},
"fileset": "gvo00016",
"hard": 131072000,
},
{
"user": "******",
"storage": {"institute": "gent", "name": "VSC_SCRATCH_DELCATTY", "storage_type": "scratch"},
"fileset": "gvo00016",
"hard": 131072000,
},
{
"user": "******",
"storage": {"institute": "brussel", "name": "VSC_DATA", "storage_type": "data"},
"fileset": "bvo00001",
"hard": 52428800,
},
{
"user": "******",
"storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"},
"fileset": "bvo00001",
"hard": 52428800,
},
{
"user": "******",
"storage": {"institute": "brussel", "name": "VSC_HOME", "storage_type": "home"},
"fileset": "vsc100",
"hard": 12582912,
},
{
"user": "******",
"storage": {"institute": "brussel", "name": "VSC_DATA", "storage_type": "data"},
"fileset": "vsc100",
"hard": 104857600,
},
{
"user": "******",
"storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"},
"fileset": "vsc100",
"hard": 104857600,
},
],
)
# This shouldn't do anything
ok, errors = vo.process_vos(options, [test_vo_id], VSC_HOME, mc, date, host_institute=BRUSSEL)
self.assertEqual(errors, {})
self.assertEqual(ok, {})
self.assertEqual(mock_gpfs.mock_calls, [mock.call()])
self.assertEqual(mock_posix.mock_calls, [mock.call()])
# VSC_DATA should also not do anything
mock_gpfs.reset_mock()
mock_posix.reset_mock()
ok, errors = vo.process_vos(options, [test_vo_id], VSC_DATA, mc, date, host_institute=BRUSSEL)
self.assertEqual(errors, {})
self.assertEqual(ok, {})
self.assertEqual(mock_gpfs.mock_calls, [mock.call()])
self.assertEqual(mock_posix.mock_calls, [mock.call()])
# VSC_SCRATCH should also not do anything
mock_gpfs.reset_mock()
mock_posix.reset_mock()
ok, errors = vo.process_vos(
options, [test_vo_id], VSC_PRODUCTION_SCRATCH[BRUSSEL][0], mc, date, host_institute=BRUSSEL
)
self.assertEqual(errors, {})
self.assertEqual(ok, {})
self.assertEqual(mock_gpfs.mock_calls, [mock.call()])
self.assertEqual(mock_posix.mock_calls, [mock.call()])
def test_process_brussel_vo(self, mock_posix, mock_gpfs, mock_client):
"""Test to see deploying a Brussel VO works fine"""
test_vo_id = "bvo00005"
date = "203112310000"
Options = namedtuple("Options", ["dry_run"])
options = Options(dry_run=False)
# first mock all the calls to the accountpage
mc = mock_client.return_value
mc.account = mock.MagicMock()
mc.vo = mock.MagicMock()
account_1 = {
"vsc_id": "vsc10001",
"status": "active",
"isactive": True,
"force_active": False,
"expiry_date": None,
"grace_until": None,
"vsc_id_number": 2510001,
"home_directory": "/user/brussel/100/vsc10001",
"data_directory": "/data/brussel/100/vsc10001",
"scratch_directory": "/scratch/brussel/100/vsc10001",
"login_shell": "/bin/bash",
"broken": False,
"email": "*****@*****.**",
"research_field": ["Physics", "nwo"],
"create_timestamp": "2018-11-13T14:27:53.394000Z",
"person": {
"gecos": "Ward Poelmans",
"institute": {"name": "brussel"},
"institute_login": "******",
"institute_affiliation": "staff",
"realeppn": "*****@*****.**",
},
"home_on_scratch": False,
}
mc.account[account_1["vsc_id"]].get.return_value = (200, account_1)
mc.vo[test_vo_id].member.modified[date].get.return_value = (200, [account_1])
mc.vo[test_vo_id].get.return_value = (
200,
{
"vsc_id": "bvo00005",
"status": "active",
"vsc_id_number": 2610010,
"institute": {"name": "brussel"},
"fairshare": 100,
"data_path": "/data/brussel/vo/000/bvo00005",
"scratch_path": "/scratch/brussel/vo/000/bvo00005",
"description": "hpcvub",
"members": ["vsc10001", "vsc10003"],
"moderators": ["vsc10001"],
},
)
mc.vo[test_vo_id].quota.get.return_value = (
200,
[
{
"virtual_organisation": "bvo00005",
"storage": {"institute": "brussel", "name": "VSC_DATA", "storage_type": "data"},
"fileset": "bvo00005",
"hard": 104857600,
},
{
"virtual_organisation": "bvo00005",
"storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"},
"fileset": "bvo00005",
"hard": 104857600,
},
],
)
mc.account[account_1["vsc_id"]].quota.get.return_value = (
200,
[
{
"user": "******",
"storage": {"institute": "brussel", "name": "VSC_HOME", "storage_type": "home"},
"fileset": "vsc100",
"hard": 12582912,
},
{
"user": "******",
"storage": {"institute": "brussel", "name": "VSC_DATA", "storage_type": "data"},
"fileset": "vsc100",
"hard": 26214400,
},
{
"user": "******",
"storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"},
"fileset": "vsc100",
"hard": 26214400,
},
{
"user": "******",
"storage": {"institute": "gent", "name": "VSC_DATA", "storage_type": "data"},
"fileset": "gvo00016",
"hard": 131072000,
},
{
"user": "******",
"storage": {"institute": "gent", "name": "VSC_SCRATCH_DELCATTY", "storage_type": "scratch"},
"fileset": "gvo00016",
"hard": 131072000,
},
{
"user": "******",
"storage": {"institute": "brussel", "name": "VSC_DATA", "storage_type": "data"},
"fileset": "bvo00005",
"hard": 52428800,
},
{
"user": "******",
"storage": {"institute": "brussel", "name": "VSC_SCRATCH_THEIA", "storage_type": "scratch"},
"fileset": "bvo00005",
"hard": 52428800,
},
],
)
# This shouldn't do anything
ok, errors = vo.process_vos(options, [test_vo_id], VSC_HOME, mc, date, host_institute=BRUSSEL)
self.assertEqual(errors, {})
self.assertEqual(ok, {})
self.assertEqual(mock_gpfs.mock_calls, [mock.call()])
self.assertEqual(mock_posix.mock_calls, [mock.call()])
# VSC_DATA test
mock_gpfs.reset_mock()
mock_posix.reset_mock()
ok, errors = vo.process_vos(options, [test_vo_id], VSC_DATA, mc, date, host_institute=BRUSSEL)
self.assertEqual(errors, {})
self.assertEqual(ok, {"bvo00005": ["vsc10001"]})
self.assertEqual(mock_posix.mock_calls, [mock.call()])
mock_gpfs.return_value.list_filesets.assert_called()
mock_gpfs.return_value.get_fileset_info.assert_called_with("theiadata", "bvo00005")
mock_gpfs.return_value.chmod.assert_called_with(504, "/theia/data/brussel/vo/000/bvo00005")
mock_gpfs.return_value.chown.assert_called_with(2510001, 2610010, "/theia/data/brussel/vo/000/bvo00005")
mock_gpfs.return_value.set_fileset_quota.assert_called_with(
204010946560, "/theia/data/brussel/vo/000/bvo00005", "bvo00005", 214748364800
)
mock_gpfs.return_value.set_fileset_grace.assert_called_with("/theia/data/brussel/vo/000/bvo00005", 604800)
mock_gpfs.return_value.set_user_quota.assert_called_with(
hard=107374182400, obj="/theia/data/brussel/vo/000/bvo00005", soft=102005473280, user=2510001
)
mock_gpfs.return_value.create_stat_directory.assert_called_with(
"/theia/data/brussel/vo/000/bvo00005/vsc10001", 448, 2510001, 1, override_permissions=False
)
# VSC_SCRATCH test
mock_gpfs.reset_mock()
mock_posix.reset_mock()
ok, errors = vo.process_vos(
options, [test_vo_id], VSC_PRODUCTION_SCRATCH[BRUSSEL][0], mc, date, host_institute=BRUSSEL
)
self.assertEqual(errors, {})
self.assertEqual(ok, {"bvo00005": ["vsc10001"]})
self.assertEqual(mock_posix.mock_calls, [mock.call()])
mock_gpfs.return_value.list_filesets.assert_called_with()
mock_gpfs.return_value.get_fileset_info.assert_called_with("theiascratch", "bvo00005")
mock_gpfs.return_value.chmod.assert_called_with(504, "/theia/scratch/brussel/vo/000/bvo00005")
mock_gpfs.return_value.chown.assert_called_with(2510001, 2610010, "/theia/scratch/brussel/vo/000/bvo00005")
mock_gpfs.return_value.set_fileset_quota.assert_called_with(
102005473280, "/theia/scratch/brussel/vo/000/bvo00005", "bvo00005", 107374182400
)
mock_gpfs.return_value.set_fileset_grace.assert_called_with("/theia/scratch/brussel/vo/000/bvo00005", 604800)
mock_gpfs.return_value.set_user_quota.assert_called_with(
hard=53687091200, obj="/theia/scratch/brussel/vo/000/bvo00005", soft=51002736640, user=2510001
)
mock_gpfs.return_value.create_stat_directory.assert_called_with(
"/theia/scratch/brussel/vo/000/bvo00005/vsc10001", 448, 2510001, 1, override_permissions=False
)
