def get_metadata_bulk(dids, vo='def', session=None):
"""
Get metadata for a list of dids
:param dids: A list of dids.
:param session: The database session in use.
"""
validate_schema(name='dids', obj=dids, vo=vo)
for entry in dids:
entry['scope'] = InternalScope(entry['scope'], vo=vo)
meta = did.get_metadata_bulk(dids)
for met in meta:
yield api_update_return_dict(met)
def get_metadata_bulk(dids, inherit=False, vo='def'):
"""
Get metadata for a list of dids
:param dids: A list of dids.
:param inherit: A boolean. If set to true, the metadata of the parent are concatenated.
:param vo: The VO to act on.
"""
validate_schema(name='dids', obj=dids, vo=vo)
for entry in dids:
entry['scope'] = InternalScope(entry['scope'], vo=vo)
meta = did.get_metadata_bulk(dids, inherit=inherit)
for met in meta:
yield api_update_return_dict(met)
def add_exception(dids, account, pattern, comments, expires_at, session=None):
"""
Add exceptions to Lifetime Model.
:param dids: The list of dids
:param account: The account of the requester.
:param pattern: The account.
:param comments: The comments associated to the exception.
:param expires_at: The expiration date of the exception.
:param session: The database session in use.
returns: A dictionary with id of the exceptions split by scope, datatype.
"""
from rucio.core.did import get_metadata_bulk
result = dict()
result['exceptions'] = dict()
try:
max_extension = config_get('lifetime_model',
'max_extension',
default=None,
session=session)
if max_extension:
if not expires_at:
expires_at = datetime.utcnow() + timedelta(days=max_extension)
else:
if isinstance(expires_at, string_types):
expires_at = str_to_date(expires_at)
if expires_at > datetime.utcnow() + timedelta(
days=max_extension):
expires_at = datetime.utcnow() + timedelta(
days=max_extension)
except (ConfigNotFound, ValueError, NoSectionError):
max_extension = None
try:
cutoff_date = config_get('lifetime_model',
'cutoff_date',
default=None,
session=session)
except (ConfigNotFound, NoSectionError):
raise UnsupportedOperation('Cannot submit exception at that date.')
try:
cutoff_date = datetime.strptime(cutoff_date, '%Y-%m-%d')
except ValueError:
raise UnsupportedOperation('Cannot submit exception at that date.')
if cutoff_date < datetime.utcnow():
raise UnsupportedOperation('Cannot submit exception at that date.')
did_group = dict()
not_affected = list()
list_dids = [(did['scope'], did['name']) for did in dids]
metadata = [meta for meta in get_metadata_bulk(dids=dids, session=session)]
for did in metadata:
scope, name, did_type = did['scope'], did['name'], did['did_type']
if (scope, name) in list_dids:
list_dids.remove((scope, name))
datatype = did.get('datatype', '')
eol_at = did.get('eol_at', None)
if eol_at and eol_at < cutoff_date:
if (scope, datatype) not in did_group:
did_group[(scope, datatype)] = [list(), 0]
did_group[(scope, datatype)][0].append({
'scope': scope,
'name': name,
'did_type': did_type
})
did_group[(scope, datatype)][1] += did['bytes'] or 0
else:
not_affected.append((scope, name, did_type))
for entry in did_group:
exception_id = __add_exception(did_group[entry][0],
account=account,
pattern=pattern,
comments=comments,
expires_at=expires_at,
estimated_volume=did_group[entry][1],
session=session)
result['exceptions'][exception_id] = did_group[entry][0]
result['unknown'] = [{
'scope': did[0],
'name': did[1],
'did_type': DIDType.DATASET
} for did in list_dids]
result['not_affected'] = [{
'scope': did[0],
'name': did[1],
'did_type': did[2]
} for did in not_affected]
return result