class VscTier2AccountpageVo(VscAccountPageVo): """Class representing a VO in the VSC. A VO is a special kind of group, identified mainly by its name. """ def __init__(self, vo_id, storage=None, rest_client=None, host_institute=GENT): """Initialise""" super(VscTier2AccountpageVo, self).__init__(vo_id, rest_client) self.vo_id = vo_id self.vsc = VSC() self.host_institute = host_institute if not storage: self.storage = VscStorage() else: self.storage = storage self.gpfs = GpfsOperations() self.posix = PosixOperations() self.dry_run = False self._vo_data_quota_cache = None self._vo_data_shared_quota_cache = None self._vo_scratch_quota_cache = None self._institute_quota_cache = None self._sharing_group_cache = None @property def _institute_quota(self): if not self._institute_quota_cache: all_quota = [mkVscVoSizeQuota(q) for q in whenHTTPErrorRaise(self.rest_client.vo[self.vo.vsc_id].quota.get, "Could not get quotata from accountpage for VO %s" % self.vo.vsc_id)[1]] self._institute_quota_cache = [q for q in all_quota if q.storage['institute'] == self.host_institute] return self._institute_quota_cache def _get_institute_data_quota(self): return [q for q in self._institute_quota if q.storage['storage_type'] == DATA_KEY] def _get_institute_non_shared_data_quota(self): return [q.hard for q in self._get_institute_data_quota() if not q.storage['name'].endswith(STORAGE_SHARED_SUFFIX)] def _get_institute_shared_data_quota(self): return [q.hard for q in self._get_institute_data_quota() if q.storage['name'].endswith(STORAGE_SHARED_SUFFIX)] @property def vo_data_quota(self): if not self._vo_data_quota_cache: self._vo_data_quota_cache = self._get_institute_non_shared_data_quota() if not self._vo_data_quota_cache: self._vo_data_quota_cache = [self.storage[VSC_DATA].quota_vo] return self._vo_data_quota_cache[0] # there can be only one @property def vo_data_shared_quota(self): if not self._vo_data_shared_quota_cache: try: self._vo_data_shared_quota_cache = self._get_institute_shared_data_quota()[0] except IndexError: return None return self._vo_data_shared_quota_cache @property def vo_scratch_quota(self): if not self._vo_scratch_quota_cache: self._vo_scratch_quota_cache = [q for q in self._institute_quota if q.storage['storage_type'] == SCRATCH_KEY] return self._vo_scratch_quota_cache @property def sharing_group(self): if not self.data_sharing: return None if not self._sharing_group_cache: group_name = self.vo.vsc_id.replace(VO_PREFIX_BY_INSTITUTE[self.vo.institute['name']], VO_SHARED_PREFIX_BY_INSTITUTE[self.vo.institute['name']]) self._sharing_group_cache = mkVscAutogroup( whenHTTPErrorRaise(self.rest_client.autogroup[group_name].get, "Could not get autogroup %s details" % group_name)[1]) return self._sharing_group_cache @property def data_sharing(self): return self.vo_data_shared_quota is not None def members(self): """Return a list with all the VO members in it.""" return self.vo.members def _get_path(self, storage, mount_point="gpfs"): """Get the path for the (if any) user directory on the given storage.""" (path, _) = self.storage.path_templates[self.host_institute][storage]['vo'](self.vo.vsc_id) if mount_point == "login": mount_path = self.storage[self.host_institute][storage].login_mount_point elif mount_point == "gpfs": mount_path = self.storage[self.host_institute][storage].gpfs_mount_point else: logging.error("mount_point (%s)is not login or gpfs", mount_point) raise Exception() return os.path.join(mount_path, path) def _data_path(self, mount_point="gpfs"): """Return the path to the VO data fileset on GPFS""" return self._get_path(VSC_DATA, mount_point) def _data_shared_path(self, mount_point="gpfs"): """Return the path the VO shared data fileset on GPFS""" return self._get_path(VSC_DATA_SHARED, mount_point) def _scratch_path(self, storage, mount_point="gpfs"): """Return the path to the VO scratch fileset on GPFS. @type storage: string @param storage: name of the storage we are looking at. """ return self._get_path(storage, mount_point) def _create_fileset(self, filesystem_name, path, parent_fileset=None, fileset_name=None, group_owner_id=None): """Create a fileset for the VO on the data filesystem. - creates the fileset if it does not already exist - sets ownership to the first (active) VO moderator, or to nobody if there is no moderator - sets group ownership to the supplied value (group_owner_id) or if that is missing to the vsc_id of the VO owning the fileset The parent_fileset is used to support older (< 3.5.x) GPFS setups still present in our system """ self.gpfs.list_filesets() if not fileset_name: fileset_name = self.vo.vsc_id if group_owner_id: fileset_group_owner_id = group_owner_id else: fileset_group_owner_id = self.vo.vsc_id_number if not self.gpfs.get_fileset_info(filesystem_name, fileset_name): logging.info("Creating new fileset on %s with name %s and path %s", filesystem_name, fileset_name, path) base_dir_hierarchy = os.path.dirname(path) self.gpfs.make_dir(base_dir_hierarchy) # HACK to support versions older than 3.5 in our setup if parent_fileset is None: self.gpfs.make_fileset(path, fileset_name) else: self.gpfs.make_fileset(path, fileset_name, parent_fileset) else: logging.info("Fileset %s already exists for VO %s ... not creating again.", fileset_name, self.vo.vsc_id) self.gpfs.chmod(0o770, path) try: moderator = mkVscAccount(self.rest_client.account[self.vo.moderators[0]].get()[1]) except HTTPError: logging.exception("Cannot obtain moderator information from account page, setting ownership to nobody") self.gpfs.chown(pwd.getpwnam('nobody').pw_uid, fileset_group_owner_id, path) except IndexError: logging.error("There is no moderator available for VO %s", self.vo.vsc_id) self.gpfs.chown(pwd.getpwnam('nobody').pw_uid, fileset_group_owner_id, path) else: self.gpfs.chown(moderator.vsc_id_number, fileset_group_owner_id, path) def create_data_fileset(self): """Create the VO's directory on the HPC data filesystem. Always set the quota.""" path = self._data_path() try: fs = self.storage[self.host_institute][VSC_DATA].filesystem except AttributeError: logging.exception("Trying to access non-existent attribute 'filesystem' in the data storage instance") except KeyError: logging.exception("Trying to access non-existent field %s in the data storage dictionary", VSC_DATA) self._create_fileset(fs, path) def create_data_shared_fileset(self): """Create a VO directory for sharing data on the HPC data filesystem. Always set the quota.""" path = self._data_shared_path() msg = "Trying to access non-existent" try: fs = self.storage[self.host_institute][VSC_DATA_SHARED].filesystem except AttributeError: logging.exception("%s attribute 'filesystem' in the shared data storage instance", msg) except KeyError: logging.exception("%s field %s in the shared data storage dictionary", msg, VSC_DATA_SHARED) self._create_fileset(fs, path, fileset_name=self.sharing_group.vsc_id, group_owner_id=self.sharing_group.vsc_id_number) def create_scratch_fileset(self, storage_name): """Create the VO's directory on the HPC data filesystem. Always set the quota.""" msg = "Trying to access non-existent" try: path = self._scratch_path(storage_name) if self.storage[self.host_institute][storage_name].version >= (3, 5, 0, 0): self._create_fileset(self.storage[self.host_institute][storage_name].filesystem, path) else: self._create_fileset(self.storage[self.host_institute][storage_name].filesystem, path, 'root') except AttributeError: logging.exception("%s attribute 'filesystem' in the scratch storage instance", msg) except KeyError: logging.exception("%s field %s in the scratch storage dictionary", msg, storage_name) def _create_vo_dir(self, path): """Create a user owned directory on the GPFS.""" self.gpfs.make_dir(path) def _set_quota(self, storage_name, path, quota, fileset_name=None): """Set FILESET quota on the FS for the VO fileset. @type quota: int @param quota: soft quota limit expressed in KiB """ if not fileset_name: fileset_name = self.vo.vsc_id try: # expressed in bytes, retrieved in KiB from the backend hard = quota * 1024 * self.storage[self.host_institute][storage_name].data_replication_factor soft = int(hard * self.vsc.quota_soft_fraction) # LDAP information is expressed in KiB, GPFS wants bytes. self.gpfs.set_fileset_quota(soft, path, fileset_name, hard) self.gpfs.set_fileset_grace(path, self.vsc.vo_storage_grace_time) # 7 days except GpfsOperationError: logging.exception("Unable to set quota on path %s", path) raise def set_data_quota(self): """Set FILESET quota on the data FS for the VO fileset.""" if self.vo_data_quota: self._set_quota(VSC_DATA, self._data_path(), int(self.vo_data_quota)) else: self._set_quota(VSC_DATA, self._data_path(), 16 * 1024) def set_data_shared_quota(self): """Set FILESET quota on the data FS for the VO fileset.""" if self.vo_data_shared_quota: self._set_quota( VSC_DATA_SHARED, self._data_shared_path(), int(self.vo_data_shared_quota), fileset_name=self.vo.vsc_id.replace( VO_PREFIX_BY_INSTITUTE[self.vo.institute["name"]], VO_SHARED_PREFIX_BY_INSTITUTE[self.vo.institute["name"]], ), ) def set_scratch_quota(self, storage_name): """Set FILESET quota on the scratch FS for the VO fileset.""" quota = [q for q in self.vo_scratch_quota if q.storage['name'] in (storage_name,)] if not quota: logging.error("No VO %s scratch quota information available for %s", self.vo.vsc_id, storage_name) logging.info("Setting default VO %s scratch quota on storage %s to %d", self.vo.vsc_id, storage_name, self.storage[storage_name].quota_vo) self._set_quota(storage_name, self._scratch_path(storage_name), self.storage[storage_name].quota_vo) return elif len(quota) > 1: logging.exception("Cannot set scratch quota for %s with multiple quota instances %s", storage_name, quota) raise logging.info("Setting VO %s quota on storage %s to %d", self.vo.vsc_id, storage_name, quota[0].hard) self._set_quota(storage_name, self._scratch_path(storage_name), quota[0].hard) def _set_member_quota(self, storage_name, path, member, quota): """Set USER quota on the FS for the VO fileset @type member: VscTier2AccountpageUser @type quota: integer (hard value) """ try: hard = quota * 1024 * self.storage[self.host_institute][storage_name].data_replication_factor soft = int(hard * self.vsc.quota_soft_fraction) self.gpfs.set_user_quota(soft=soft, user=int(member.account.vsc_id_number), obj=path, hard=hard) except GpfsOperationError: logging.exception("Unable to set USR quota for member %s on path %s", member.account.vsc_id, path) raise def set_member_data_quota(self, member): """Set the quota on the data FS for the member in the VO fileset. @type member: VscTier2AccountPageUser instance The user can have up to half of the VO quota. FIXME: This should probably be some variable in a config setting instance """ if not self.vo_data_quota: logging.warning("Not setting VO %s member %s data quota: no VO data quota info available", self.vo.vsc_id, member.account.vsc_id) return if self.vo.vsc_id in DEFAULT_VOS_ALL: logging.warning("Not setting VO %s member %s data quota: No VO member quota for this VO", member.account.vsc_id, self.vo.vsc_id) return if member.vo_data_quota: # users having belonged to multiple VOs have multiple quota on VSC_DATA, so we # only need to deploy the quota for the VO the user currently belongs to. quota = [q for q in member.vo_data_quota if q.fileset == self.vo.vsc_id and not q.storage['name'].endswith(STORAGE_SHARED_SUFFIX)] if len(quota) > 1: logging.exception("Cannot set data quota for member %s with multiple quota instances %s", member, quota) raise else: logging.info("Setting the data quota for VO %s member %s to %d KiB", self.vo.vsc_id, member.account.vsc_id, quota[0].hard) self._set_member_quota(VSC_DATA, self._data_path(), member, quota[0].hard) else: logging.error("No VO %s data quota set for member %s", self.vo.vsc_id, member.account.vsc_id) def set_member_scratch_quota(self, storage_name, member): """Set the quota on the scratch FS for the member in the VO fileset. @type member: VscTier2AccountpageUser instance The user can have up to half of the VO quota. FIXME: This should probably be some variable in a config setting instance """ if not self.vo_scratch_quota: logging.warning("Not setting VO %s member %s scratch quota: no VO quota info available", self.vo.vsc_id, member.account.vsc_id) return if self.vo.vsc_id in DEFAULT_VOS_ALL: logging.warning("Not setting VO %s member %s scratch quota: No VO member quota for this VO", member.account.vsc_id, self.vo.vsc_id) return if member.vo_scratch_quota: quota = [q for q in member.vo_scratch_quota if q.storage['name'] in (storage_name,) and q.fileset in (self.vo_id,)] if quota: logging.info("Setting the scratch quota for VO %s member %s to %d GiB on %s", self.vo.vsc_id, member.account.vsc_id, quota[0].hard / 1024 / 1024, storage_name) self._set_member_quota(storage_name, self._scratch_path(storage_name), member, quota[0].hard) else: logging.error("No VO %s scratch quota for member %s on %s after filter (all %s)", self.vo.vsc_id, member.account.vsc_id, storage_name, member.vo_scratch_quota) else: logging.error("No VO %s scratch quota set for member %s on %s", self.vo.vsc_id, member.account.vsc_id, storage_name) def _create_member_dir(self, member, target): """Create a member-owned directory in the VO fileset.""" self.gpfs.create_stat_directory( target, 0o700, int(member.account.vsc_id_number), int(member.usergroup.vsc_id_number), # we should not override permissions on an existing dir where users may have changed them override_permissions=False) def create_member_data_dir(self, member): """Create a directory on data in the VO fileset that is owned by the member with name $VSC_DATA_VO/<vscid>.""" target = os.path.join(self._data_path(), member.user_id) self._create_member_dir(member, target) def create_member_scratch_dir(self, storage_name, member): """Create a directory on scratch in the VO fileset that is owned by the member with name $VSC_SCRATCH_VO/<vscid>.""" target = os.path.join(self._scratch_path(storage_name), member.user_id) self._create_member_dir(member, target) def __setattr__(self, name, value): """Override the setting of an attribute: - dry_run: set this here and in the gpfs and posix instance fields. - otherwise, call super's __setattr__() """ if name == 'dry_run': self.gpfs.dry_run = value self.posix.dry_run = value super(VscTier2AccountpageVo, self).__setattr__(name, value)
class VscTier2AccountpageUser(VscAccountPageUser): """ A user on each of our Tier-2 system using the account page REST API to retrieve its information. """ def __init__(self, user_id, storage=None, pickle_storage='VSC_SCRATCH_KYUKON', rest_client=None, account=None, pubkeys=None, host_institute=None, use_user_cache=False): """ Initialisation. @type vsc_user_id: string representing the user's VSC ID (vsc[0-9]{5}) """ super(VscTier2AccountpageUser, self).__init__(user_id, rest_client, account=account, pubkeys=pubkeys, use_user_cache=use_user_cache) self.pickle_storage = pickle_storage if not storage: self.storage = VscStorage() else: self.storage = storage self.vsc = VSC() self.gpfs = GpfsOperations() # Only used when needed self.posix = PosixOperations() self.host_institute = host_institute def _init_cache(self, **kwargs): super(VscTier2AccountpageUser, self)._init_cache(**kwargs) self._cache['quota'] = {} @property def user_home_quota(self): if not self._cache['quota']: self._init_quota_cache() return self._cache['quota']['home'] @property def user_data_quota(self): if not self._cache['quota']: self._init_quota_cache() return self._cache['quota']['data'] @property def user_scratch_quota(self): if not self._cache['quota']: self._init_quota_cache() return self._cache['quota']['scratch'] @property def vo_data_quota(self): if not self._cache['quota']: self._init_quota_cache() return self._cache['quota']['vo']['data'] @property def vo_scratch_quota(self): if not self._cache['quota']: self._init_quota_cache() return self._cache['quota']['vo']['scratch'] def _init_quota_cache(self): if self.host_institute is None: logging.warn("_init_quota_cache with host_institute None") all_quota = [mkVscUserSizeQuota(q) for q in self.rest_client.account[self.user_id].quota.get()[1]] # we no longer set defaults, since we do not want to accidentally revert people to some default # that is lower than their actual quota if the accountpage goes down in between retrieving the users # and fetching the quota institute_quota = [q for q in all_quota if q.storage['institute'] == self.host_institute] fileset_name = self.vsc.user_grouping_fileset(self.account.vsc_id) def user_proposition(quota, storage_type): return quota.fileset == fileset_name and quota.storage['storage_type'] == storage_type # Non-UGent users who have quota in Gent, e.g., in a VO, should not have these set if self.person.institute['site'] == self.host_institute: self._cache['quota']['home'] = [q.hard for q in institute_quota if user_proposition(q, 'home')][0] self._cache['quota']['data'] = [q.hard for q in institute_quota if user_proposition(q, 'data') and not q.storage['name'].endswith('SHARED')][0] self._cache['quota']['scratch'] = filter(lambda q: user_proposition(q, 'scratch'), institute_quota) else: self._cache['quota']['home'] = None self._cache['quota']['data'] = None self._cache['quota']['scratch'] = None fileset_name = 'gvo' def user_vo_proposition(quota, storage_type): return quota.fileset.startswith(fileset_name) and quota.storage['storage_type'] == storage_type self._cache['quota']['vo'] = {} self._cache['quota']['vo']['data'] = [q for q in institute_quota if user_vo_proposition(q, 'data')] self._cache['quota']['vo']['scratch'] = [q for q in institute_quota if user_vo_proposition(q, 'scratch')] def pickle_path(self): """Provide the location where to store pickle files for this user. This location is the user'path on the pickle_storage specified when creating a VscTier2AccountpageUser instance. """ (path, _) = self.storage.path_templates[GENT][self.pickle_storage]['user'](self.account.vsc_id) return os.path.join(self.storage[self.pickle_storage].gpfs_mount_point, path) def _create_grouping_fileset(self, filesystem_name, path, fileset_name): """Create a fileset for a group of 100 user accounts - creates the fileset if it does not already exist """ self.gpfs.list_filesets() logging.info("Trying to create the grouping fileset %s with link path %s", fileset_name, path) if not self.gpfs.get_fileset_info(filesystem_name, fileset_name): logging.info("Creating new fileset on %s with name %s and path %s", filesystem_name, fileset_name, path) base_dir_hierarchy = os.path.dirname(path) self.gpfs.make_dir(base_dir_hierarchy) self.gpfs.make_fileset(path, fileset_name) else: logging.info("Fileset %s already exists for user group of %s ... not creating again.", fileset_name, self.account.vsc_id) self.gpfs.chmod(0o755, path) def _get_mount_path(self, storage_name, mount_point): """Get the mount point for the location we're running""" if mount_point == "login": mount_path = self.storage[storage_name].login_mount_point elif mount_point == "gpfs": mount_path = self.storage[storage_name].gpfs_mount_point else: logging.error("mount_point (%s) is not login or gpfs", mount_point) raise Exception("mount_point (%s) is not designated as gpfs or login" % (mount_point,)) return mount_path def _get_path(self, storage_name, mount_point="gpfs"): """Get the path for the (if any) user directory on the given storage_name.""" (path, _) = self.storage.path_templates[GENT][storage_name]['user'](self.account.vsc_id) return os.path.join(self._get_mount_path(storage_name, mount_point), path) def _get_grouping_path(self, storage_name, mount_point="gpfs"): """Get the path and the fileset for the user group directory (and associated fileset).""" (path, fileset) = self.storage.path_templates[GENT][storage_name]['user'](self.account.vsc_id) return (os.path.join(self._get_mount_path(storage_name, mount_point), os.path.dirname(path)), fileset) def _home_path(self, mount_point="gpfs"): """Return the path to the home dir.""" return self._get_path(VSC_HOME, mount_point) def _data_path(self, mount_point="gpfs"): """Return the path to the data dir.""" return self._get_path(VSC_DATA, mount_point) def _scratch_path(self, storage_name, mount_point="gpfs"): """Return the path to the scratch dir""" return self._get_path(storage_name, mount_point) def _grouping_home_path(self, mount_point="gpfs"): """Return the path to the grouping fileset for the users on data.""" return self._get_grouping_path(VSC_HOME, mount_point) def _grouping_data_path(self, mount_point="gpfs"): """Return the path to the grouping fileset for the users on data.""" return self._get_grouping_path(VSC_DATA, mount_point) def _grouping_scratch_path(self, storage_name, mount_point="gpfs"): """Return the path to the grouping fileset for the users on the given scratch filesystem.""" return self._get_grouping_path(storage_name, mount_point) def _create_user_dir(self, grouping_f, path_f, storage_name): """Create the directories and files for some user location. @type grouping: function that yields the grouping path for the location. @type path: function that yields the actual path for the location. """ try: (grouping_path, fileset) = grouping_f() self._create_grouping_fileset(self.storage[storage_name].filesystem, grouping_path, fileset) path = path_f() if self.gpfs.is_symlink(path): logging.warning("Trying to make a user dir, but a symlink already exists at %s", path) return create_stat_directory( path, 0o700, int(self.account.vsc_id_number), int(self.usergroup.vsc_id_number), self.gpfs ) except Exception: logging.exception("Could not create dir %s for user %s", path, self.account.vsc_id) raise def create_home_dir(self): """Create all required files in the (future) user's home directory.""" self._create_user_dir(self._grouping_home_path, self._home_path, VSC_HOME) def create_data_dir(self): """Create the user's directory on the HPC data filesystem.""" self._create_user_dir(self._grouping_data_path, self._data_path, VSC_DATA) def create_scratch_dir(self, storage_name): """Create the user's directory on the given scratch filesystem.""" self._create_user_dir( lambda: self._grouping_scratch_path(storage_name), lambda: self._scratch_path(storage_name), storage_name) def _set_quota(self, storage_name, path, hard): """Set the given quota on the target path. @type path: path into a GPFS mount @type hard: hard limit """ if not hard: logging.error("No user quota set for %s", storage_name) return quota = hard * 1024 * self.storage[storage_name].data_replication_factor soft = int(self.vsc.quota_soft_fraction * quota) logging.info("Setting quota for %s on %s to %d", storage_name, path, quota) # LDAP information is expressed in KiB, GPFS wants bytes. self.gpfs.set_user_quota(soft, int(self.account.vsc_id_number), path, quota) self.gpfs.set_user_grace(path, self.vsc.user_storage_grace_time) # 7 days def set_home_quota(self): """Set USR quota on the home FS in the user fileset.""" path = self._home_path() hard = self.user_home_quota self._set_quota(VSC_HOME, path, hard) def set_data_quota(self): """Set USR quota on the data FS in the user fileset.""" (path, _) = self._grouping_data_path() hard = self.user_data_quota self._set_quota(VSC_DATA, path, hard) def set_scratch_quota(self, storage_name): """Set USR quota on the scratch FS in the user fileset.""" quota = filter(lambda q: q.storage['name'] in (storage_name,), self.user_scratch_quota) if not quota: logging.error("No scratch quota information available for %s", storage_name) return if self.storage[storage_name].user_grouping_fileset: (path, _) = self._grouping_scratch_path(storage_name) else: # Hack; this should actually become the link path of the fileset # that contains the path (the file, not the followed symlink) path = os.path.normpath(os.path.join(self._scratch_path(storage_name), '..')) self._set_quota(storage_name, path, quota[0].hard) def populate_home_dir(self): """Store the required files in the user's home directory. Does not overwrite files that may contain user defined content. """ path = self._home_path() self.gpfs.populate_home_dir(int(self.account.vsc_id_number), int(self.usergroup.vsc_id_number), path, [p.pubkey for p in self.pubkeys]) def __setattr__(self, name, value): """Override the setting of an attribute: - dry_run: set this here and in the gpfs and posix instance fields. - otherwise, call super's __setattr__() """ if name == 'dry_run': self.gpfs.dry_run = value self.posix.dry_run = value super(VscTier2AccountpageUser, self).__setattr__(name, value)