def slurm_vo_accounts(account_page_vos, slurm_account_info, clusters):
"""Check for the presence of the new/changed VOs in the slurm account list.
@returns: list of sacctmgr commands to add the accounts for VOs if needed
"""
commands = []
for cluster in clusters:
cluster_accounts = [
acct.Account for acct in slurm_account_info
if acct and acct.Cluster == cluster
]
for vo in account_page_vos:
if vo.vsc_id in INSTITUTE_VOS_GENT.values():
continue
if vo.vsc_id not in cluster_accounts:
commands.append(
create_add_account_command(
account=vo.vsc_id,
parent=vo.institute['site'],
cluster=cluster,
organisation=vo.institute['site']))
return commands
def slurm_institute_accounts(slurm_account_info, clusters):
"""Check for the presence of the institutes and their default VOs in the slurm account list.
@returns: list of sacctmgr commands to add the accounts to the clusters if needed
"""
commands = []
for cluster in clusters:
cluster_accounts = [
acct.Account for acct in slurm_account_info
if acct and acct.Cluster == cluster
]
for (inst, vo) in INSTITUTE_VOS_GENT.items():
if inst not in cluster_accounts:
commands.append(
create_add_account_command(account=inst,
parent=None,
cluster=cluster,
organisation=inst))
if vo not in cluster_accounts:
commands.append(
create_add_account_command(account=vo,
parent=inst,
cluster=cluster,
organisation=inst))
return commands
def set_member_scratch_quota(self, storage_name, member):
"""Set the quota on the scratch FS for the member in the VO fileset.
@type member: VscTier2AccountpageUser instance
The user can have up to half of the VO quota.
FIXME: This should probably be some variable in a config setting instance
"""
if not self.vo_scratch_quota:
logging.warning("Not setting VO %s member %s scratch quota: no VO quota info available" %
(self.vo.vsc_id, member.account.vsc_id))
return
if self.vo.vsc_id in INSTITUTE_VOS_GENT.values():
logging.warning("Not setting VO %s member %s scratch quota: No VO member quota for this VO",
member.account.vsc_id, self.vo.vsc_id)
return
if member.vo_scratch_quota:
quota = filter(lambda q: q.storage['name'] in (storage_name,) and q.fileset in (self.vo_id,),
member.vo_scratch_quota)
if quota:
logging.info("Setting the scratch quota for VO %s member %s to %d GiB on %s",
self.vo.vsc_id, member.account.vsc_id, quota[0].hard / 1024 / 1024, storage_name)
self._set_member_quota(storage_name, self._scratch_path(storage_name), member, quota[0].hard)
else:
logging.error("No VO %s scratch quota for member %s on %s after filter (all %s)",
self.vo.vsc_id, member.account.vsc_id, storage_name, member.vo_scratch_quota)
else:
logging.error("No VO %s scratch quota set for member %s on %s",
self.vo.vsc_id, member.account.vsc_id, storage_name)
def set_member_data_quota(self, member):
"""Set the quota on the data FS for the member in the VO fileset.
@type member: VscTier2AccountPageUser instance
The user can have up to half of the VO quota.
FIXME: This should probably be some variable in a config setting instance
"""
if not self.vo_data_quota:
logging.warning("Not setting VO %s member %s data quota: no VO data quota info available" %
(VSC_DATA, self.vo.vsc_id, member.account.vsc_id))
return
if self.vo.vsc_id in INSTITUTE_VOS_GENT.values():
logging.warning("Not setting VO %s member %s data quota: No VO member quota for this VO",
member.account.vsc_id, self.vo.vsc_id)
return
if member.vo_data_quota:
# users having belonged to multiple VOs have multiple quota on VSC_DATA, so we
# only need to deploy the quota for the VO the user currently belongs to.
quota = [q for q in member.vo_data_quota
if q.fileset == self.vo.vsc_id and not q.storage['name'].endswith(SHARED)]
if len(quota) > 1:
logging.exception("Cannot set data quota for member %s with multiple quota instances %s" % (
member, quota))
raise
else:
logging.info("Setting the data quota for VO %s member %s to %d KiB" %
(self.vo.vsc_id, member.account.vsc_id, quota[0].hard))
self._set_member_quota(VSC_DATA, self._data_path(), member, quota[0].hard)
else:
logging.error("No VO %s data quota set for member %s" % (self.vo.vsc_id, member.account.vsc_id))
def sync_altered_groups(self, last, dry_run=True):
"""
Synchronise altered groups back to LDAP.
This also includes usergroups
"""
changed_groups = [
mkGroup(a) for a in self.client.allgroups.modified[last].get()[1]
]
logging.info("Found %d modified groups in the range %s until %s" %
(len(changed_groups),
datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"),
self.now.strftime("%Y%m%d%H%M%SZ")))
logging.debug("Modified groups: %s",
[g.vsc_id for g in changed_groups])
groups = {
NEW: set(),
UPDATED: set(),
ERROR: set(),
}
for group in changed_groups:
vo = False
try:
vo = mkVo(self.client.vo[group.vsc_id].get()[1])
except HTTPError as err:
# if a 404 occured, the group is not an VO, so we skip this. Otherwise something else went wrong.
if err.code != 404:
raise
ldap_attributes = {
'cn': str(group.vsc_id),
'institute': [str(group.institute['site'])],
'gidNumber': ["%d" % (group.vsc_id_number, )],
'moderator': [str(m) for m in group.moderators],
'memberUid': [str(a) for a in group.members],
'status': [str(group.status)],
}
if vo:
ldap_attributes['fairshare'] = ["%d" % (vo.fairshare, )]
ldap_attributes['description'] = [str(vo.description)]
ldap_attributes['dataDirectory'] = [str(vo.data_path)]
ldap_attributes['scratchDirectory'] = [str(vo.scratch_path)]
# vsc40024 is moderator for all institute vo's
if vo.vsc_id in INSTITUTE_VOS_GENT.values():
ldap_attributes['moderator'] = ['vsc40024']
logging.debug("Proposed changes for group %s: %s", group.vsc_id,
ldap_attributes)
result = self.add_or_update(VscLdapGroup, group.vsc_id,
ldap_attributes, dry_run)
groups[result].add(group.vsc_id)
return groups
def process_vos(options,
vo_ids,
storage_name,
client,
datestamp,
host_institute=None):
"""Process the virtual organisations.
- make the fileset per VO
- set the quota for the complete fileset
- set the quota on a per-user basis for all VO members
"""
listm = Monoid([], lambda xs, ys: xs + ys)
ok_vos = MonoidDict(copy.deepcopy(listm))
error_vos = MonoidDict(copy.deepcopy(listm))
for vo_id in sorted(vo_ids):
vo = VscTier2AccountpageVo(vo_id, rest_client=client)
vo.dry_run = options.dry_run
try:
if storage_name in [VSC_HOME]:
continue
if storage_name in [VSC_DATA
] and vo_id not in INSTITUTE_VOS_GENT.values():
vo.create_data_fileset()
vo.set_data_quota()
update_vo_status(vo, client)
if storage_name in [
VSC_DATA_SHARED
] and vo_id not in INSTITUTE_VOS_GENT.values() and vo.data_sharing:
vo.create_data_shared_fileset()
vo.set_data_shared_quota()
if vo_id == INSTITUTE_VOS_GENT[GENT]:
logging.info("Not deploying default VO %s members" % (vo_id, ))
continue
if storage_name in GENT_PRODUCTION_SCRATCH:
vo.create_scratch_fileset(storage_name)
vo.set_scratch_quota(storage_name)
if vo_id in INSTITUTE_VOS_GENT.values() and storage_name in (
VSC_HOME, VSC_DATA):
logging.info("Not deploying default VO %s members on %s",
vo_id, storage_name)
continue
modified_member_list = client.vo[
vo.vo_id].member.modified[datestamp].get()
factory = lambda vid: VscTier2AccountpageUser(vid,
rest_client=client,
host_institute=
host_institute,
use_user_cache=True)
modified_members = [
factory(a["vsc_id"]) for a in modified_member_list[1]
]
for member in modified_members:
try:
member.dry_run = options.dry_run
if storage_name in [VSC_DATA]:
vo.set_member_data_quota(
member) # half of the VO quota
vo.create_member_data_dir(member)
if storage_name in GENT_PRODUCTION_SCRATCH:
vo.set_member_scratch_quota(
storage_name, member) # half of the VO quota
vo.create_member_scratch_dir(storage_name, member)
ok_vos[vo.vo_id] = [member.account.vsc_id]
except Exception:
logging.exception(
"Failure at setting up the member %s of VO %s on %s" %
(member.account.vsc_id, vo.vo_id, storage_name))
error_vos[vo.vo_id] = [member.account.vsc_id]
except Exception:
logging.exception(
"Something went wrong setting up the VO %s on the storage %s" %
(vo.vo_id, storage_name))
error_vos[vo.vo_id] = vo.members
return (ok_vos, error_vos)