def process_user_quota(storage,
gpfs,
storage_name,
filesystem,
quota_map,
user_map,
client,
dry_run=False,
institute=GENT):
"""
Wrapper around the new function to keep the old behaviour intact.
"""
del filesystem
del gpfs
exceeding_users = []
path_template = storage.path_templates[institute][storage_name]
vsc = VSC()
logging.info("Logging user quota to account page")
logging.debug("Considering the following quota items for pushing: %s",
quota_map)
with DjangoPusher(storage_name, client, QUOTA_USER_KIND,
dry_run) as pusher:
for (user_id, quota) in quota_map.items():
user_institute = vsc.user_id_to_institute(int(user_id))
if user_institute != institute:
continue
user_name = user_map.get(int(user_id), None)
if not user_name:
try:
user_name = getpwuid(int(user_id)).pw_name
except KeyError:
continue
fileset_name = path_template['user'](user_name)[1]
fileset_re = '^(vsc[1-4]|%s|%s|%s)' % (
VO_PREFIX_BY_SITE[institute],
VO_SHARED_PREFIX_BY_SITE[institute], fileset_name)
for (fileset, quota_) in quota.quota_map.items():
if re.search(fileset_re, fileset):
pusher.push_quota(user_name, fileset, quota_)
if quota.exceeds():
exceeding_users.append((user_name, quota))
return exceeding_users
def __init__(self,
vo_id,
storage=None,
rest_client=None,
host_institute=GENT):
"""Initialise"""
super(VscTier2AccountpageVo, self).__init__(vo_id, rest_client)
self.vo_id = vo_id
self.vsc = VSC()
self.host_institute = host_institute
if not storage:
self.storage = VscStorage()
else:
self.storage = storage
self.gpfs = GpfsOperations()
self.posix = PosixOperations()
self.dry_run = False
self._vo_data_quota_cache = None
self._vo_data_shared_quota_cache = None
self._vo_scratch_quota_cache = None
self._institute_quota_cache = None
self._sharing_group_cache = None
def __init__(self,
user_id,
storage=None,
pickle_storage=None,
rest_client=None,
account=None,
pubkeys=None,
host_institute=None,
use_user_cache=False):
"""
Initialisation.
@type vsc_user_id: string representing the user's VSC ID (vsc[0-9]{5})
"""
super(VscTier2AccountpageUser,
self).__init__(user_id,
rest_client,
account=account,
pubkeys=pubkeys,
use_user_cache=use_user_cache)
# Move to vsc-config?
default_pickle_storage = {
GENT: VSC_SCRATCH_KYUKON,
BRUSSEL: VSC_SCRATCH_THEIA,
}
if host_institute is None:
host_institute = GENT
self.host_institute = host_institute
if pickle_storage is None:
pickle_storage = default_pickle_storage[host_institute]
self.pickle_storage = pickle_storage
if storage is None:
storage = VscStorage()
self.institute_path_templates = storage.path_templates[
self.host_institute]
self.institute_storage = storage[self.host_institute]
self.vsc = VSC()
self.gpfs = GpfsOperations() # Only used when needed
self.posix = PosixOperations()
def __init__(self, user_id, storage=None, pickle_storage='VSC_SCRATCH_KYUKON', rest_client=None,
account=None, pubkeys=None, host_institute=None, use_user_cache=False):
"""
Initialisation.
@type vsc_user_id: string representing the user's VSC ID (vsc[0-9]{5})
"""
super(VscTier2AccountpageUser, self).__init__(user_id, rest_client, account=account,
pubkeys=pubkeys, use_user_cache=use_user_cache)
self.pickle_storage = pickle_storage
if not storage:
self.storage = VscStorage()
else:
self.storage = storage
self.vsc = VSC()
self.gpfs = GpfsOperations() # Only used when needed
self.posix = PosixOperations()
self.host_institute = host_institute
from ldap import LDAPError
from vsc.accountpage.wrappers import mkVscAccount, mkUserGroup, mkGroup, mkVo
from vsc.config.base import VSC, INSTITUTE_VOS_GENT
from vsc.ldap.entities import VscLdapUser, VscLdapGroup
from vsc.ldap.filters import CnFilter
from vsc.utils.py2vs3 import ensure_ascii_string
ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING = "THIS ACCOUNT HAS NO VALID PUBLIC KEYS"
DONE = 'done'
NEW = 'new'
UPDATED = 'updated'
ERROR = 'error'
VSC_CONFIG = VSC()
class LdapSyncer(object):
"""
This class implements a system for syncing changes from the accountpage api
to the vsc ldap
"""
def __init__(self, client):
"""
Create an ldap syncer, requires a RestClient client to get the information from
(typically AccountpageClient)
"""
self.client = client
self.now = datetime.utcnow().replace(tzinfo=timezone.utc)
class VscTier2AccountpageUser(VscAccountPageUser):
"""
A user on each of our Tier-2 system using the account page REST API
to retrieve its information.
"""
def __init__(self, user_id, storage=None, pickle_storage='VSC_SCRATCH_KYUKON', rest_client=None,
account=None, pubkeys=None, host_institute=None, use_user_cache=False):
"""
Initialisation.
@type vsc_user_id: string representing the user's VSC ID (vsc[0-9]{5})
"""
super(VscTier2AccountpageUser, self).__init__(user_id, rest_client, account=account,
pubkeys=pubkeys, use_user_cache=use_user_cache)
self.pickle_storage = pickle_storage
if not storage:
self.storage = VscStorage()
else:
self.storage = storage
self.vsc = VSC()
self.gpfs = GpfsOperations() # Only used when needed
self.posix = PosixOperations()
self.host_institute = host_institute
def _init_cache(self, **kwargs):
super(VscTier2AccountpageUser, self)._init_cache(**kwargs)
self._cache['quota'] = {}
@property
def user_home_quota(self):
if not self._cache['quota']:
self._init_quota_cache()
return self._cache['quota']['home']
@property
def user_data_quota(self):
if not self._cache['quota']:
self._init_quota_cache()
return self._cache['quota']['data']
@property
def user_scratch_quota(self):
if not self._cache['quota']:
self._init_quota_cache()
return self._cache['quota']['scratch']
@property
def vo_data_quota(self):
if not self._cache['quota']:
self._init_quota_cache()
return self._cache['quota']['vo']['data']
@property
def vo_scratch_quota(self):
if not self._cache['quota']:
self._init_quota_cache()
return self._cache['quota']['vo']['scratch']
def _init_quota_cache(self):
if self.host_institute is None:
logging.warn("_init_quota_cache with host_institute None")
all_quota = [mkVscUserSizeQuota(q) for q in self.rest_client.account[self.user_id].quota.get()[1]]
# we no longer set defaults, since we do not want to accidentally revert people to some default
# that is lower than their actual quota if the accountpage goes down in between retrieving the users
# and fetching the quota
institute_quota = [q for q in all_quota if q.storage['institute'] == self.host_institute]
fileset_name = self.vsc.user_grouping_fileset(self.account.vsc_id)
def user_proposition(quota, storage_type):
return quota.fileset == fileset_name and quota.storage['storage_type'] == storage_type
# Non-UGent users who have quota in Gent, e.g., in a VO, should not have these set
if self.person.institute['site'] == self.host_institute:
self._cache['quota']['home'] = [q.hard for q in institute_quota if user_proposition(q, 'home')][0]
self._cache['quota']['data'] = [q.hard for q in institute_quota
if user_proposition(q, 'data')
and not q.storage['name'].endswith('SHARED')][0]
self._cache['quota']['scratch'] = filter(lambda q: user_proposition(q, 'scratch'), institute_quota)
else:
self._cache['quota']['home'] = None
self._cache['quota']['data'] = None
self._cache['quota']['scratch'] = None
fileset_name = 'gvo'
def user_vo_proposition(quota, storage_type):
return quota.fileset.startswith(fileset_name) and quota.storage['storage_type'] == storage_type
self._cache['quota']['vo'] = {}
self._cache['quota']['vo']['data'] = [q for q in institute_quota if user_vo_proposition(q, 'data')]
self._cache['quota']['vo']['scratch'] = [q for q in institute_quota if user_vo_proposition(q, 'scratch')]
def pickle_path(self):
"""Provide the location where to store pickle files for this user.
This location is the user'path on the pickle_storage specified when creating
a VscTier2AccountpageUser instance.
"""
(path, _) = self.storage.path_templates[GENT][self.pickle_storage]['user'](self.account.vsc_id)
return os.path.join(self.storage[self.pickle_storage].gpfs_mount_point, path)
def _create_grouping_fileset(self, filesystem_name, path, fileset_name):
"""Create a fileset for a group of 100 user accounts
- creates the fileset if it does not already exist
"""
self.gpfs.list_filesets()
logging.info("Trying to create the grouping fileset %s with link path %s", fileset_name, path)
if not self.gpfs.get_fileset_info(filesystem_name, fileset_name):
logging.info("Creating new fileset on %s with name %s and path %s", filesystem_name, fileset_name, path)
base_dir_hierarchy = os.path.dirname(path)
self.gpfs.make_dir(base_dir_hierarchy)
self.gpfs.make_fileset(path, fileset_name)
else:
logging.info("Fileset %s already exists for user group of %s ... not creating again.",
fileset_name, self.account.vsc_id)
self.gpfs.chmod(0o755, path)
def _get_mount_path(self, storage_name, mount_point):
"""Get the mount point for the location we're running"""
if mount_point == "login":
mount_path = self.storage[storage_name].login_mount_point
elif mount_point == "gpfs":
mount_path = self.storage[storage_name].gpfs_mount_point
else:
logging.error("mount_point (%s) is not login or gpfs", mount_point)
raise Exception("mount_point (%s) is not designated as gpfs or login" % (mount_point,))
return mount_path
def _get_path(self, storage_name, mount_point="gpfs"):
"""Get the path for the (if any) user directory on the given storage_name."""
(path, _) = self.storage.path_templates[GENT][storage_name]['user'](self.account.vsc_id)
return os.path.join(self._get_mount_path(storage_name, mount_point), path)
def _get_grouping_path(self, storage_name, mount_point="gpfs"):
"""Get the path and the fileset for the user group directory (and associated fileset)."""
(path, fileset) = self.storage.path_templates[GENT][storage_name]['user'](self.account.vsc_id)
return (os.path.join(self._get_mount_path(storage_name, mount_point), os.path.dirname(path)), fileset)
def _home_path(self, mount_point="gpfs"):
"""Return the path to the home dir."""
return self._get_path(VSC_HOME, mount_point)
def _data_path(self, mount_point="gpfs"):
"""Return the path to the data dir."""
return self._get_path(VSC_DATA, mount_point)
def _scratch_path(self, storage_name, mount_point="gpfs"):
"""Return the path to the scratch dir"""
return self._get_path(storage_name, mount_point)
def _grouping_home_path(self, mount_point="gpfs"):
"""Return the path to the grouping fileset for the users on data."""
return self._get_grouping_path(VSC_HOME, mount_point)
def _grouping_data_path(self, mount_point="gpfs"):
"""Return the path to the grouping fileset for the users on data."""
return self._get_grouping_path(VSC_DATA, mount_point)
def _grouping_scratch_path(self, storage_name, mount_point="gpfs"):
"""Return the path to the grouping fileset for the users on the given scratch filesystem."""
return self._get_grouping_path(storage_name, mount_point)
def _create_user_dir(self, grouping_f, path_f, storage_name):
"""Create the directories and files for some user location.
@type grouping: function that yields the grouping path for the location.
@type path: function that yields the actual path for the location.
"""
try:
(grouping_path, fileset) = grouping_f()
self._create_grouping_fileset(self.storage[storage_name].filesystem, grouping_path, fileset)
path = path_f()
if self.gpfs.is_symlink(path):
logging.warning("Trying to make a user dir, but a symlink already exists at %s", path)
return
create_stat_directory(
path,
0o700,
int(self.account.vsc_id_number),
int(self.usergroup.vsc_id_number),
self.gpfs
)
except Exception:
logging.exception("Could not create dir %s for user %s", path, self.account.vsc_id)
raise
def create_home_dir(self):
"""Create all required files in the (future) user's home directory."""
self._create_user_dir(self._grouping_home_path, self._home_path, VSC_HOME)
def create_data_dir(self):
"""Create the user's directory on the HPC data filesystem."""
self._create_user_dir(self._grouping_data_path, self._data_path, VSC_DATA)
def create_scratch_dir(self, storage_name):
"""Create the user's directory on the given scratch filesystem."""
self._create_user_dir(
lambda: self._grouping_scratch_path(storage_name),
lambda: self._scratch_path(storage_name),
storage_name)
def _set_quota(self, storage_name, path, hard):
"""Set the given quota on the target path.
@type path: path into a GPFS mount
@type hard: hard limit
"""
if not hard:
logging.error("No user quota set for %s", storage_name)
return
quota = hard * 1024 * self.storage[storage_name].data_replication_factor
soft = int(self.vsc.quota_soft_fraction * quota)
logging.info("Setting quota for %s on %s to %d", storage_name, path, quota)
# LDAP information is expressed in KiB, GPFS wants bytes.
self.gpfs.set_user_quota(soft, int(self.account.vsc_id_number), path, quota)
self.gpfs.set_user_grace(path, self.vsc.user_storage_grace_time) # 7 days
def set_home_quota(self):
"""Set USR quota on the home FS in the user fileset."""
path = self._home_path()
hard = self.user_home_quota
self._set_quota(VSC_HOME, path, hard)
def set_data_quota(self):
"""Set USR quota on the data FS in the user fileset."""
(path, _) = self._grouping_data_path()
hard = self.user_data_quota
self._set_quota(VSC_DATA, path, hard)
def set_scratch_quota(self, storage_name):
"""Set USR quota on the scratch FS in the user fileset."""
quota = filter(lambda q: q.storage['name'] in (storage_name,), self.user_scratch_quota)
if not quota:
logging.error("No scratch quota information available for %s", storage_name)
return
if self.storage[storage_name].user_grouping_fileset:
(path, _) = self._grouping_scratch_path(storage_name)
else:
# Hack; this should actually become the link path of the fileset
# that contains the path (the file, not the followed symlink)
path = os.path.normpath(os.path.join(self._scratch_path(storage_name), '..'))
self._set_quota(storage_name, path, quota[0].hard)
def populate_home_dir(self):
"""Store the required files in the user's home directory.
Does not overwrite files that may contain user defined content.
"""
path = self._home_path()
self.gpfs.populate_home_dir(int(self.account.vsc_id_number),
int(self.usergroup.vsc_id_number),
path,
[p.pubkey for p in self.pubkeys])
def __setattr__(self, name, value):
"""Override the setting of an attribute:
- dry_run: set this here and in the gpfs and posix instance fields.
- otherwise, call super's __setattr__()
"""
if name == 'dry_run':
self.gpfs.dry_run = value
self.posix.dry_run = value
super(VscTier2AccountpageUser, self).__setattr__(name, value)
