def slurm_vo_accounts(account_page_vos, slurm_account_info, clusters): """Check for the presence of the new/changed VOs in the slurm account list. @returns: list of sacctmgr commands to add the accounts for VOs if needed """ commands = [] for cluster in clusters: cluster_accounts = [ acct.Account for acct in slurm_account_info if acct and acct.Cluster == cluster ] for vo in account_page_vos: if vo.vsc_id in INSTITUTE_VOS_GENT.values(): continue if vo.vsc_id not in cluster_accounts: commands.append( create_add_account_command( account=vo.vsc_id, parent=vo.institute['site'], cluster=cluster, organisation=vo.institute['site'])) return commands
def slurm_institute_accounts(slurm_account_info, clusters): """Check for the presence of the institutes and their default VOs in the slurm account list. @returns: list of sacctmgr commands to add the accounts to the clusters if needed """ commands = [] for cluster in clusters: cluster_accounts = [ acct.Account for acct in slurm_account_info if acct and acct.Cluster == cluster ] for (inst, vo) in INSTITUTE_VOS_GENT.items(): if inst not in cluster_accounts: commands.append( create_add_account_command(account=inst, parent=None, cluster=cluster, organisation=inst)) if vo not in cluster_accounts: commands.append( create_add_account_command(account=vo, parent=inst, cluster=cluster, organisation=inst)) return commands
def set_member_scratch_quota(self, storage_name, member): """Set the quota on the scratch FS for the member in the VO fileset. @type member: VscTier2AccountpageUser instance The user can have up to half of the VO quota. FIXME: This should probably be some variable in a config setting instance """ if not self.vo_scratch_quota: logging.warning("Not setting VO %s member %s scratch quota: no VO quota info available" % (self.vo.vsc_id, member.account.vsc_id)) return if self.vo.vsc_id in INSTITUTE_VOS_GENT.values(): logging.warning("Not setting VO %s member %s scratch quota: No VO member quota for this VO", member.account.vsc_id, self.vo.vsc_id) return if member.vo_scratch_quota: quota = filter(lambda q: q.storage['name'] in (storage_name,) and q.fileset in (self.vo_id,), member.vo_scratch_quota) if quota: logging.info("Setting the scratch quota for VO %s member %s to %d GiB on %s", self.vo.vsc_id, member.account.vsc_id, quota[0].hard / 1024 / 1024, storage_name) self._set_member_quota(storage_name, self._scratch_path(storage_name), member, quota[0].hard) else: logging.error("No VO %s scratch quota for member %s on %s after filter (all %s)", self.vo.vsc_id, member.account.vsc_id, storage_name, member.vo_scratch_quota) else: logging.error("No VO %s scratch quota set for member %s on %s", self.vo.vsc_id, member.account.vsc_id, storage_name)
def set_member_data_quota(self, member): """Set the quota on the data FS for the member in the VO fileset. @type member: VscTier2AccountPageUser instance The user can have up to half of the VO quota. FIXME: This should probably be some variable in a config setting instance """ if not self.vo_data_quota: logging.warning("Not setting VO %s member %s data quota: no VO data quota info available" % (VSC_DATA, self.vo.vsc_id, member.account.vsc_id)) return if self.vo.vsc_id in INSTITUTE_VOS_GENT.values(): logging.warning("Not setting VO %s member %s data quota: No VO member quota for this VO", member.account.vsc_id, self.vo.vsc_id) return if member.vo_data_quota: # users having belonged to multiple VOs have multiple quota on VSC_DATA, so we # only need to deploy the quota for the VO the user currently belongs to. quota = [q for q in member.vo_data_quota if q.fileset == self.vo.vsc_id and not q.storage['name'].endswith(SHARED)] if len(quota) > 1: logging.exception("Cannot set data quota for member %s with multiple quota instances %s" % ( member, quota)) raise else: logging.info("Setting the data quota for VO %s member %s to %d KiB" % (self.vo.vsc_id, member.account.vsc_id, quota[0].hard)) self._set_member_quota(VSC_DATA, self._data_path(), member, quota[0].hard) else: logging.error("No VO %s data quota set for member %s" % (self.vo.vsc_id, member.account.vsc_id))
def sync_altered_groups(self, last, dry_run=True): """ Synchronise altered groups back to LDAP. This also includes usergroups """ changed_groups = [ mkGroup(a) for a in self.client.allgroups.modified[last].get()[1] ] logging.info("Found %d modified groups in the range %s until %s" % (len(changed_groups), datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"), self.now.strftime("%Y%m%d%H%M%SZ"))) logging.debug("Modified groups: %s", [g.vsc_id for g in changed_groups]) groups = { NEW: set(), UPDATED: set(), ERROR: set(), } for group in changed_groups: vo = False try: vo = mkVo(self.client.vo[group.vsc_id].get()[1]) except HTTPError as err: # if a 404 occured, the group is not an VO, so we skip this. Otherwise something else went wrong. if err.code != 404: raise ldap_attributes = { 'cn': str(group.vsc_id), 'institute': [str(group.institute['site'])], 'gidNumber': ["%d" % (group.vsc_id_number, )], 'moderator': [str(m) for m in group.moderators], 'memberUid': [str(a) for a in group.members], 'status': [str(group.status)], } if vo: ldap_attributes['fairshare'] = ["%d" % (vo.fairshare, )] ldap_attributes['description'] = [str(vo.description)] ldap_attributes['dataDirectory'] = [str(vo.data_path)] ldap_attributes['scratchDirectory'] = [str(vo.scratch_path)] # vsc40024 is moderator for all institute vo's if vo.vsc_id in INSTITUTE_VOS_GENT.values(): ldap_attributes['moderator'] = ['vsc40024'] logging.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) result = self.add_or_update(VscLdapGroup, group.vsc_id, ldap_attributes, dry_run) groups[result].add(group.vsc_id) return groups
def process_vos(options, vo_ids, storage_name, client, datestamp, host_institute=None): """Process the virtual organisations. - make the fileset per VO - set the quota for the complete fileset - set the quota on a per-user basis for all VO members """ listm = Monoid([], lambda xs, ys: xs + ys) ok_vos = MonoidDict(copy.deepcopy(listm)) error_vos = MonoidDict(copy.deepcopy(listm)) for vo_id in sorted(vo_ids): vo = VscTier2AccountpageVo(vo_id, rest_client=client) vo.dry_run = options.dry_run try: if storage_name in [VSC_HOME]: continue if storage_name in [VSC_DATA ] and vo_id not in INSTITUTE_VOS_GENT.values(): vo.create_data_fileset() vo.set_data_quota() update_vo_status(vo, client) if storage_name in [ VSC_DATA_SHARED ] and vo_id not in INSTITUTE_VOS_GENT.values() and vo.data_sharing: vo.create_data_shared_fileset() vo.set_data_shared_quota() if vo_id == INSTITUTE_VOS_GENT[GENT]: logging.info("Not deploying default VO %s members" % (vo_id, )) continue if storage_name in GENT_PRODUCTION_SCRATCH: vo.create_scratch_fileset(storage_name) vo.set_scratch_quota(storage_name) if vo_id in INSTITUTE_VOS_GENT.values() and storage_name in ( VSC_HOME, VSC_DATA): logging.info("Not deploying default VO %s members on %s", vo_id, storage_name) continue modified_member_list = client.vo[ vo.vo_id].member.modified[datestamp].get() factory = lambda vid: VscTier2AccountpageUser(vid, rest_client=client, host_institute= host_institute, use_user_cache=True) modified_members = [ factory(a["vsc_id"]) for a in modified_member_list[1] ] for member in modified_members: try: member.dry_run = options.dry_run if storage_name in [VSC_DATA]: vo.set_member_data_quota( member) # half of the VO quota vo.create_member_data_dir(member) if storage_name in GENT_PRODUCTION_SCRATCH: vo.set_member_scratch_quota( storage_name, member) # half of the VO quota vo.create_member_scratch_dir(storage_name, member) ok_vos[vo.vo_id] = [member.account.vsc_id] except Exception: logging.exception( "Failure at setting up the member %s of VO %s on %s" % (member.account.vsc_id, vo.vo_id, storage_name)) error_vos[vo.vo_id] = [member.account.vsc_id] except Exception: logging.exception( "Something went wrong setting up the VO %s on the storage %s" % (vo.vo_id, storage_name)) error_vos[vo.vo_id] = vo.members return (ok_vos, error_vos)