def add_exception(dids, account, pattern, comments, expires_at, session=None):
"""
Add exceptions to Lifetime Model.
:param dids: The list of dids
:param account: The account of the requester.
:param pattern: The account.
:param comments: The comments associated to the exception.
:param expires_at: The expiration date of the exception.
:param session: The database session in use.
returns: The id of the exception.
"""
exception_id = generate_uuid()
text = 'Account %s requested a lifetime extension for a list of DIDs that can be found below\n' % account
reason = comments
volume = None
lifetime = None
if comments.find('||||') > -1:
reason, volume = comments.split('||||')
text += 'The reason for the extension is "%s"\n' % reason
text += 'It represents %s datasets\n' % len(dids)
if volume:
text += 'The estimated physical volume is %s\n' % volume
if expires_at and isinstance(expires_at, string_types):
lifetime = str_to_date(expires_at)
text += 'The lifetime exception should expires on %s\n' % str(
expires_at)
elif isinstance(expires_at, datetime):
lifetime = expires_at
text += 'The lifetime exception should expires on %s\n' % str(
expires_at)
text += 'Link to approve or reject this request can be found at the end of the mail\n'
text += '\n'
text += 'DIDTYPE SCOPE NAME\n'
text += '\n'
truncated_message = False
for did in dids:
did_type = None
if 'did_type' in did:
if isinstance(did['did_type'], string_types):
did_type = DIDType.from_sym(did['did_type'])
else:
did_type = did['did_type']
new_exception = models.LifetimeExceptions(
id=exception_id,
scope=did['scope'],
name=did['name'],
did_type=did_type,
account=account,
pattern=pattern,
comments=reason,
state=LifetimeExceptionsState.WAITING,
expires_at=lifetime)
if len(text) < 3000:
text += '%s %s %s\n' % (str(did_type), did['scope'], did['name'])
else:
truncated_message = True
try:
new_exception.save(session=session, flush=False)
except IntegrityError as error:
if match('.*ORA-00001.*', str(error.args[0])) \
or match('.*IntegrityError.*UNIQUE constraint failed.*', str(error.args[0])) \
or match('.*1062.*Duplicate entry.*for key.*', str(error.args[0])) \
or match('.*IntegrityError.*columns? .*not unique.*', error.args[0]):
raise LifetimeExceptionDuplicate()
raise RucioException(error.args[0])
if truncated_message:
text += '...\n'
text += 'List too long. Truncated\n'
text += '\n'
text += 'Approve: https://rucio-ui.cern.ch/lifetime_exception?id=%s&action=approve\n' % str(
exception_id)
text += 'Deny: https://rucio-ui.cern.ch/lifetime_exception?id=%s&action=deny\n' % str(
exception_id)
approvers_email = get('lifetime_model',
'approvers_email',
default=[],
session=session)
if approvers_email:
approvers_email = approvers_email.split(',') # pylint: disable=no-member
add_message(event_type='email',
payload={
'body':
text,
'to':
approvers_email,
'subject':
'[RUCIO] Request to approve lifetime exception %s' %
str(exception_id)
},
session=session)
return exception_id
def list_dids(self, scope, filters, type='collection', ignore_case=False, limit=None,
offset=None, long=False, recursive=False, session=None):
"""
Search data identifiers
:param scope: the scope name.
:param filters: dictionary of attributes by which the results should be filtered.
:param type: the type of the did: all(container, dataset, file), collection(dataset or container), dataset, container, file.
:param ignore_case: ignore case distinctions.
:param limit: limit number.
:param offset: offset number.
:param long: Long format option to display more information for each DID.
:param session: The database session in use.
:param recursive: Recursively list DIDs content.
"""
types = ['all', 'collection', 'container', 'dataset', 'file']
if type not in types:
raise exception.UnsupportedOperation("Valid type are: %(types)s" % locals())
query = session.query(models.DataIdentifier.scope,
models.DataIdentifier.name,
models.DataIdentifier.did_type,
models.DataIdentifier.bytes,
models.DataIdentifier.length).\
filter(models.DataIdentifier.scope == scope)
# Exclude suppressed dids
query = query.filter(models.DataIdentifier.suppressed != true())
if type == 'all':
query = query.filter(or_(models.DataIdentifier.did_type == DIDType.CONTAINER,
models.DataIdentifier.did_type == DIDType.DATASET,
models.DataIdentifier.did_type == DIDType.FILE))
elif type.lower() == 'collection':
query = query.filter(or_(models.DataIdentifier.did_type == DIDType.CONTAINER,
models.DataIdentifier.did_type == DIDType.DATASET))
elif type.lower() == 'container':
query = query.filter(models.DataIdentifier.did_type == DIDType.CONTAINER)
elif type.lower() == 'dataset':
query = query.filter(models.DataIdentifier.did_type == DIDType.DATASET)
elif type.lower() == 'file':
query = query.filter(models.DataIdentifier.did_type == DIDType.FILE)
for (k, v) in filters.items():
if k not in ['created_before', 'created_after', 'length.gt', 'length.lt', 'length.lte', 'length.gte', 'length'] \
and not hasattr(models.DataIdentifier, k):
raise exception.KeyNotFound(k)
if isinstance(v, string_types) and ('*' in v or '%' in v):
if v in ('*', '%', u'*', u'%'):
continue
if session.bind.dialect.name == 'postgresql':
query = query.filter(getattr(models.DataIdentifier, k).
like(v.replace('*', '%').replace('_', '\_'), # NOQA: W605
escape='\\'))
else:
query = query.filter(getattr(models.DataIdentifier, k).
like(v.replace('*', '%').replace('_', '\_'), escape='\\')) # NOQA: W605
elif k == 'created_before':
created_before = str_to_date(v)
query = query.filter(models.DataIdentifier.created_at <= created_before)
elif k == 'created_after':
created_after = str_to_date(v)
query = query.filter(models.DataIdentifier.created_at >= created_after)
elif k == 'guid':
query = query.filter_by(guid=v).\
with_hint(models.ReplicaLock, "INDEX(DIDS_GUIDS_IDX)", 'oracle')
elif k == 'length.gt':
query = query.filter(models.DataIdentifier.length > v)
elif k == 'length.lt':
query = query.filter(models.DataIdentifier.length < v)
elif k == 'length.gte':
query = query.filter(models.DataIdentifier.length >= v)
elif k == 'length.lte':
query = query.filter(models.DataIdentifier.length <= v)
elif k == 'length':
query = query.filter(models.DataIdentifier.length == v)
else:
query = query.filter(getattr(models.DataIdentifier, k) == v)
if 'name' in filters:
if '*' in filters['name']:
query = query.\
with_hint(models.DataIdentifier, "NO_INDEX(dids(SCOPE,NAME))", 'oracle')
else:
query = query.\
with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle')
if limit:
query = query.limit(limit)
if recursive:
# Get attachted DIDs and save in list because query has to be finished before starting a new one in the recursion
collections_content = []
parent_scope = scope
from rucio.core.did import list_content
for scope, name, did_type, bytes, length in query.yield_per(100):
if (did_type == DIDType.CONTAINER or did_type == DIDType.DATASET):
collections_content += [did for did in list_content(scope=scope, name=name)]
# List DIDs again to use filter
for did in collections_content:
filters['name'] = did['name']
for result in self.list_dids(scope=did['scope'], filters=filters, recursive=True, type=type, limit=limit, offset=offset, long=long, session=session):
yield result
if long:
for scope, name, did_type, bytes, length in query.yield_per(5):
yield {'scope': scope,
'name': name,
'did_type': str(did_type),
'bytes': bytes,
'length': length}
else:
for scope, name, did_type, bytes, length in query.yield_per(5):
yield name
def add_exception(dids, account, pattern, comments, expires_at, session=None):
"""
Add exceptions to Lifetime Model.
:param dids: The list of dids
:param account: The account of the requester.
:param pattern: The account.
:param comments: The comments associated to the exception.
:param expires_at: The expiration date of the exception.
:param session: The database session in use.
returns: A dictionary with id of the exceptions split by scope, datatype.
"""
from rucio.core.did import get_metadata_bulk
result = dict()
result['exceptions'] = dict()
try:
max_extension = config_get('lifetime_model',
'max_extension',
default=None,
session=session)
if max_extension:
if not expires_at:
expires_at = datetime.utcnow() + timedelta(days=max_extension)
else:
if isinstance(expires_at, string_types):
expires_at = str_to_date(expires_at)
if expires_at > datetime.utcnow() + timedelta(
days=max_extension):
expires_at = datetime.utcnow() + timedelta(
days=max_extension)
except (ConfigNotFound, ValueError, NoSectionError):
max_extension = None
try:
cutoff_date = config_get('lifetime_model',
'cutoff_date',
default=None,
session=session)
except (ConfigNotFound, NoSectionError):
raise UnsupportedOperation('Cannot submit exception at that date.')
try:
cutoff_date = datetime.strptime(cutoff_date, '%Y-%m-%d')
except ValueError:
raise UnsupportedOperation('Cannot submit exception at that date.')
if cutoff_date < datetime.utcnow():
raise UnsupportedOperation('Cannot submit exception at that date.')
did_group = dict()
not_affected = list()
list_dids = [(did['scope'], did['name']) for did in dids]
metadata = [meta for meta in get_metadata_bulk(dids=dids, session=session)]
for did in metadata:
scope, name, did_type = did['scope'], did['name'], did['did_type']
if (scope, name) in list_dids:
list_dids.remove((scope, name))
datatype = did.get('datatype', '')
eol_at = did.get('eol_at', None)
if eol_at and eol_at < cutoff_date:
if (scope, datatype) not in did_group:
did_group[(scope, datatype)] = [list(), 0]
did_group[(scope, datatype)][0].append({
'scope': scope,
'name': name,
'did_type': did_type
})
did_group[(scope, datatype)][1] += did['bytes'] or 0
else:
not_affected.append((scope, name, did_type))
for entry in did_group:
exception_id = __add_exception(did_group[entry][0],
account=account,
pattern=pattern,
comments=comments,
expires_at=expires_at,
estimated_volume=did_group[entry][1],
session=session)
result['exceptions'][exception_id] = did_group[entry][0]
result['unknown'] = [{
'scope': did[0],
'name': did[1],
'did_type': DIDType.DATASET
} for did in list_dids]
result['not_affected'] = [{
'scope': did[0],
'name': did[1],
'did_type': did[2]
} for did in not_affected]
return result
