def list_filesets(self, devices=None, filesetnames=None, update=False):
"""Get all the filesets for one or more specific devices
@type devices: list of devices (if string: 1 device; if None: all found devices)
@type filesetnames: report only on specific filesets (if string: 1 filesetname)
set self.gpfslocalfilesets is dict with
key = filesystemName value is dict with
key = id value is dict
key = remaining header entries and corresponding values
"""
if not update and self.gpfslocalfilesets:
return self.gpfslocalfilesets
opts = []
if devices is None:
# get all devices from all filesystems
if self.gpfslocalfilesystems is None:
self.list_filesystems()
devices = self.gpfslocalfilesystems.keys()
else:
if isinstance(devices, str):
devices = [devices]
if filesetnames is not None:
if isinstance(filesetnames, str):
filesetnames = [filesetnames]
filesetnamestxt = ','.join(filesetnames)
opts.append(filesetnamestxt)
self.log.debug("Looking up filesets for devices %s" % (devices))
listm = Monoid([], lambda xs, ys: xs + ys)
info = MonoidDict(listm)
for device in devices:
opts_ = copy.deepcopy(opts)
opts_.insert(1, device)
res = self._executeY('mmlsfileset', opts_)
# for v3.5 filesystemName:filesetName:id:rootInode:status:path:parentId:created:inodes:dataInKB:comment:filesetMode:afmTarget:afmState:afmMode:afmFileLookupRefreshInterval:afmFileOpenRefreshInterval:afmDirLookupRefreshInterval:afmDirOpenRefreshInterval:afmAsyncDelay:reserved:afmExpirationTimeout:afmRPO:afmLastPSnapId:inodeSpace:isInodeSpaceOwner:maxInodes:allocInodes:inodeSpaceMask:afmShowHomeSnapshots:afmNumReadThreads:afmNumReadGWs:afmReadBufferSize:afmWriteBufferSize:afmReadSparseThreshold:afmParallelReadChunkSize:afmParallelReadThreshold:snapId:
self.log.debug("list_filesets res keys = %s " % (res.keys()))
for (key, value) in res.items():
info[key] = value
datakeys = info.keys()
datakeys.remove('filesystemName')
datakeys.remove('id')
fss = nub(info.get('filesystemName', []))
res = dict([(fs, {}) for fs in fss]) # build structure
for idx, (fs, qid) in enumerate(zip(info['filesystemName'], info['id'])):
details = dict([(k, info[k][idx]) for k in datakeys])
res[fs][qid] = details
self.gpfslocalfilesets = res
return res
def _assemble_fields(self, fields, out):
"""Assemble executeY output fields """
# do we have multiple field counts?
field_counts = [i for (i, _) in fields]
if len(nub(field_counts)) > 1:
maximum_field_count = max(field_counts)
description_field_count = field_counts[0]
for (field_count, line) in fields[1:]:
if field_count == description_field_count:
continue
elif field_count < description_field_count:
self.log.debug(
"Description length %s greater then %s. Adding whitespace. (names %s, row %s)",
maximum_field_count, field_count, fields[0][6:],
line[6:])
line.extend([''] * (maximum_field_count - field_count))
else:
# try to fix the line
self.log.info(
"Line has too many fields (%d > %d), trying to fix %s",
field_count, description_field_count, line)
fixed_lines = self.fixup_executeY_line(
line, description_field_count)
i = fields.index((field_count, line))
fields[i:i + 1] = map(lambda fs: (len(fs), fs),
fixed_lines)
# assemble result
listm = Monoid([], lambda xs, ys: xs + ys
) # not exactly the fastest mappend for lists ...
res = MonoidDict(listm)
try:
for index, name in enumerate(fields[0][1][6:]):
if name != '':
for (_, line) in fields[1:]:
res[name] = [line[6 + index]]
except IndexError:
self.log.raiseException(
"Failed to regroup data %s (from output %s)" % (fields, out))
return res
def process_vos(options, vo_ids, storage_name, client, datestamp, host_institute=GENT):
"""Process the virtual organisations.
- make the fileset per VO
- set the quota for the complete fileset
- set the quota on a per-user basis for all VO members
"""
listm = Monoid([], lambda xs, ys: xs + ys)
ok_vos = MonoidDict(copy.deepcopy(listm))
error_vos = MonoidDict(copy.deepcopy(listm))
for vo_id in sorted(vo_ids):
vo = VscTier2AccountpageVo(vo_id, rest_client=client, host_institute=host_institute)
vo.dry_run = options.dry_run
try:
if storage_name in [VSC_HOME]:
continue
if storage_name in [VSC_DATA] and vo_id not in DEFAULT_VOS_ALL:
vo.create_data_fileset()
vo.set_data_quota()
update_vo_status(vo)
if storage_name in [VSC_DATA_SHARED] and vo_id not in DEFAULT_VOS_ALL and vo.data_sharing:
vo.create_data_shared_fileset()
vo.set_data_shared_quota()
if vo_id == INSTITUTE_VOS_BY_INSTITUTE[host_institute][host_institute]:
logging.info("Not deploying default VO %s members", vo_id)
continue
if storage_name in VSC_PRODUCTION_SCRATCH[host_institute]:
vo.create_scratch_fileset(storage_name)
vo.set_scratch_quota(storage_name)
if vo_id in DEFAULT_VOS_ALL and storage_name in (VSC_HOME, VSC_DATA):
logging.info("Not deploying default VO %s members on %s", vo_id, storage_name)
continue
modified_member_list = client.vo[vo.vo_id].member.modified[datestamp].get()
factory = lambda vid: VscTier2AccountpageUser(vid,
rest_client=client,
host_institute=host_institute,
use_user_cache=True)
modified_members = [factory(a["vsc_id"]) for a in modified_member_list[1]]
for member in modified_members:
try:
member.dry_run = options.dry_run
if storage_name in [VSC_DATA]:
vo.set_member_data_quota(member) # half of the VO quota
vo.create_member_data_dir(member)
if storage_name in VSC_PRODUCTION_SCRATCH[host_institute]:
vo.set_member_scratch_quota(storage_name, member) # half of the VO quota
vo.create_member_scratch_dir(storage_name, member)
ok_vos[vo.vo_id] = [member.account.vsc_id]
except Exception:
logging.exception("Failure at setting up the member %s of VO %s on %s",
member.account.vsc_id, vo.vo_id, storage_name)
error_vos[vo.vo_id] = [member.account.vsc_id]
except Exception:
logging.exception("Something went wrong setting up the VO %s on the storage %s", vo.vo_id, storage_name)
error_vos[vo.vo_id] = vo.members
return (ok_vos, error_vos)
def list_quota(self, devices=None):
"""get quota info for all filesystems for all USR,GRP,FILESET
set self.gpfslocalquota to
dict: key = deviceName, value is
dict with key quotaType (USR | GRP | FILESET) value is dict with
key = id, value dict with
key = remaining header entries and corresponding values as a NamedTuple
- GPFS 3.5 has the following fields in the output lines of mmrepquota (colon separated)
- filesystemName
- quotaType
- id
- name
- blockUsage
- blockQuota
- blockLimit
- blockInDoubt
- blockGrace
- filesUsage
- filesQuota
- filesLimit
- filesInDoubt
- filesGrace
- remarks
- quota
- defQuota
- fid
- filesetname
- GPFS 3.5 also is able to list multiple e.g., USR lines in different filesets.
"""
if devices is None:
devices = self.list_filesystems().keys()
elif isinstance(devices, str):
devices = [devices]
listm = Monoid([], lambda xs, ys: xs + ys) # not exactly the fastest mappend for lists ...
info = MonoidDict(listm)
for device in devices:
res = self._executeY('mmrepquota', ['-n', device], prefix=True)
for (key, value) in res.items():
info[key] = value
datakeys = info.keys()
datakeys.remove('filesystemName')
datakeys.remove('quotaType')
datakeys.remove('id')
fss = nub(info.get('filesystemName', []))
self.log.debug("Found the following filesystem names: %s" % (fss))
quotatypes = nub(info.get('quotaType', []))
quotatypesstruct = dict([(qt, MonoidDict(Monoid([], lambda xs, ys: xs + ys))) for qt in quotatypes])
res = dict([(fs, copy.deepcopy(quotatypesstruct)) for fs in fss]) # build structure
for idx, (fs, qt, qid) in enumerate(zip(info['filesystemName'], info['quotaType'], info['id'])):
details = dict([(k, info[k][idx]) for k in datakeys])
if qt == 'FILESET':
# GPFS fileset quota have empty filesetName field
details['filesetname'] = details['name']
res[fs][qt][qid] = [GpfsQuota(**details)]
self.gpfslocalquotas = res
return res
def list_filesets(self, devices=None, filesetnames=None, update=False):
"""
Get all the filesets for one or more specific devices
@type devices: list of devices (if string: 1 device; if None: all found devices)
@type filesetnames: report only on specific filesets (if string: 1 filesetname)
set self.gpfslocalfilesets is dict with
key = filesystemName value is dict with
key = id value is dict
key = remaining header entries and corresponding values
"""
if not update and self.gpfslocalfilesets:
return self.gpfslocalfilesets
opts = []
if devices is None:
# get all devices from all filesystems
if self.gpfslocalfilesystems is None:
self.list_filesystems()
devices = self.gpfslocalfilesystems.keys()
else:
if isinstance(devices, str):
devices = [devices]
if filesetnames is not None:
if isinstance(filesetnames, str):
filesetnames = [filesetnames]
filesetnamestxt = ','.join(filesetnames)
opts.append(filesetnamestxt)
self.log.debug("Looking up filesets for devices %s", devices)
listm = Monoid([], lambda xs, ys: xs + ys)
info = MonoidDict(listm)
for device in devices:
opts_ = copy.deepcopy(opts)
opts_.insert(1, device)
res = self._executeY('mmlsfileset', opts_)
# for v3.5
# filesystemName:filesetName:id:rootInode:status:path:parentId:created:inodes:dataInKB:comment:
# filesetMode:afmTarget:afmState:afmMode:afmFileLookupRefreshInterval:afmFileOpenRefreshInterval:
# afmDirLookupRefreshInterval:afmDirOpenRefreshInterval:afmAsyncDelay:reserved:afmExpirationTimeout:afmRPO:
# afmLastPSnapId:inodeSpace:isInodeSpaceOwner:maxInodes:allocInodes:inodeSpaceMask:afmShowHomeSnapshots:
# afmNumReadThreads:afmNumReadGWs:afmReadBufferSize:afmWriteBufferSize:afmReadSparseThreshold:
# afmParallelReadChunkSize:afmParallelReadThreshold:snapId:
self.log.debug("list_filesets res keys = %s ", res.keys())
for (key, value) in res.items():
info[key] = value
datakeys = list(info.keys())
datakeys.remove('filesystemName')
datakeys.remove('id')
fss = nub(info.get('filesystemName', []))
res = dict([(fs, {}) for fs in fss]) # build structure
for idx, (fs, qid) in enumerate(zip(info['filesystemName'],
info['id'])):
details = dict([(k, info[k][idx]) for k in datakeys])
res[fs][qid] = details
self.gpfslocalfilesets = res
return res
def list_quota(self, devices=None):
"""get quota info for all filesystems for all USR,GRP,FILESET
set self.gpfslocalquota to
dict: key = deviceName, value is
dict with key quotaType (USR | GRP | FILESET) value is dict with
key = id, value dict with
key = remaining header entries and corresponding values as a NamedTuple
- GPFS 3.5 has the following fields in the output lines of mmrepquota (colon separated)
- filesystemName
- quotaType
- id
- name
- blockUsage
- blockQuota
- blockLimit
- blockInDoubt
- blockGrace
- filesUsage
- filesQuota
- filesLimit
- filesInDoubt
- filesGrace
- remarks
- quota
- defQuota
- fid
- filesetname
- GPFS 3.5 also is able to list multiple e.g., USR lines in different filesets.
"""
if devices is None:
devices = self.list_filesystems().keys()
elif isinstance(devices, str):
devices = [devices]
listm = Monoid([], lambda xs, ys: xs + ys
) # not exactly the fastest mappend for lists ...
info = MonoidDict(listm)
for device in devices:
res = self._executeY('mmrepquota', ['-n', device], prefix=True)
for (key, value) in res.items():
info[key] = value
datakeys = list(info.keys())
datakeys.remove('filesystemName')
datakeys.remove('quotaType')
datakeys.remove('id')
fss = nub(info.get('filesystemName', []))
self.log.debug("Found the following filesystem names: %s", fss)
quotatypes = nub(info.get('quotaType', []))
quotatypesstruct = dict([(qt,
MonoidDict(Monoid([],
lambda xs, ys: xs + ys)))
for qt in quotatypes])
res = dict([(fs, copy.deepcopy(quotatypesstruct))
for fs in fss]) # build structure
for idx, (fs, qt, qid) in enumerate(
zip(info['filesystemName'], info['quotaType'], info['id'])):
details = dict([(k, info[k][idx]) for k in datakeys])
if qt == 'FILESET':
# GPFS fileset quota have empty filesetName field
details['filesetname'] = details['name']
res[fs][qt][qid] = [GpfsQuota(**details)]
self.gpfslocalquotas = res
return res