def add_exception(dids, account, pattern, comments, expires_at, session=None): """ Add exceptions to Lifetime Model. :param dids: The list of dids :param account: The account of the requester. :param pattern: The account. :param comments: The comments associated to the exception. :param expires_at: The expiration date of the exception. :param session: The database session in use. returns: The id of the exception. """ exception_id = generate_uuid() text = 'Account %s requested a lifetime extension for a list of DIDs that can be found below\n' % account reason = comments volume = None lifetime = None if comments.find('||||') > -1: reason, volume = comments.split('||||') text += 'The reason for the extension is "%s"\n' % reason text += 'It represents %s datasets\n' % len(dids) if volume: text += 'The estimated physical volume is %s\n' % volume if expires_at and isinstance(expires_at, string_types): lifetime = str_to_date(expires_at) text += 'The lifetime exception should expires on %s\n' % str( expires_at) elif isinstance(expires_at, datetime): lifetime = expires_at text += 'The lifetime exception should expires on %s\n' % str( expires_at) text += 'Link to approve or reject this request can be found at the end of the mail\n' text += '\n' text += 'DIDTYPE SCOPE NAME\n' text += '\n' truncated_message = False for did in dids: did_type = None if 'did_type' in did: if isinstance(did['did_type'], string_types): did_type = DIDType.from_sym(did['did_type']) else: did_type = did['did_type'] new_exception = models.LifetimeExceptions( id=exception_id, scope=did['scope'], name=did['name'], did_type=did_type, account=account, pattern=pattern, comments=reason, state=LifetimeExceptionsState.WAITING, expires_at=lifetime) if len(text) < 3000: text += '%s %s %s\n' % (str(did_type), did['scope'], did['name']) else: truncated_message = True try: new_exception.save(session=session, flush=False) except IntegrityError as error: if match('.*ORA-00001.*', str(error.args[0])) \ or match('.*IntegrityError.*UNIQUE constraint failed.*', str(error.args[0])) \ or match('.*1062.*Duplicate entry.*for key.*', str(error.args[0])) \ or match('.*IntegrityError.*columns? .*not unique.*', error.args[0]): raise LifetimeExceptionDuplicate() raise RucioException(error.args[0]) if truncated_message: text += '...\n' text += 'List too long. Truncated\n' text += '\n' text += 'Approve: https://rucio-ui.cern.ch/lifetime_exception?id=%s&action=approve\n' % str( exception_id) text += 'Deny: https://rucio-ui.cern.ch/lifetime_exception?id=%s&action=deny\n' % str( exception_id) approvers_email = get('lifetime_model', 'approvers_email', default=[], session=session) if approvers_email: approvers_email = approvers_email.split(',') # pylint: disable=no-member add_message(event_type='email', payload={ 'body': text, 'to': approvers_email, 'subject': '[RUCIO] Request to approve lifetime exception %s' % str(exception_id) }, session=session) return exception_id
def list_dids(self, scope, filters, type='collection', ignore_case=False, limit=None, offset=None, long=False, recursive=False, session=None): """ Search data identifiers :param scope: the scope name. :param filters: dictionary of attributes by which the results should be filtered. :param type: the type of the did: all(container, dataset, file), collection(dataset or container), dataset, container, file. :param ignore_case: ignore case distinctions. :param limit: limit number. :param offset: offset number. :param long: Long format option to display more information for each DID. :param session: The database session in use. :param recursive: Recursively list DIDs content. """ types = ['all', 'collection', 'container', 'dataset', 'file'] if type not in types: raise exception.UnsupportedOperation("Valid type are: %(types)s" % locals()) query = session.query(models.DataIdentifier.scope, models.DataIdentifier.name, models.DataIdentifier.did_type, models.DataIdentifier.bytes, models.DataIdentifier.length).\ filter(models.DataIdentifier.scope == scope) # Exclude suppressed dids query = query.filter(models.DataIdentifier.suppressed != true()) if type == 'all': query = query.filter(or_(models.DataIdentifier.did_type == DIDType.CONTAINER, models.DataIdentifier.did_type == DIDType.DATASET, models.DataIdentifier.did_type == DIDType.FILE)) elif type.lower() == 'collection': query = query.filter(or_(models.DataIdentifier.did_type == DIDType.CONTAINER, models.DataIdentifier.did_type == DIDType.DATASET)) elif type.lower() == 'container': query = query.filter(models.DataIdentifier.did_type == DIDType.CONTAINER) elif type.lower() == 'dataset': query = query.filter(models.DataIdentifier.did_type == DIDType.DATASET) elif type.lower() == 'file': query = query.filter(models.DataIdentifier.did_type == DIDType.FILE) for (k, v) in filters.items(): if k not in ['created_before', 'created_after', 'length.gt', 'length.lt', 'length.lte', 'length.gte', 'length'] \ and not hasattr(models.DataIdentifier, k): raise exception.KeyNotFound(k) if isinstance(v, string_types) and ('*' in v or '%' in v): if v in ('*', '%', u'*', u'%'): continue if session.bind.dialect.name == 'postgresql': query = query.filter(getattr(models.DataIdentifier, k). like(v.replace('*', '%').replace('_', '\_'), # NOQA: W605 escape='\\')) else: query = query.filter(getattr(models.DataIdentifier, k). like(v.replace('*', '%').replace('_', '\_'), escape='\\')) # NOQA: W605 elif k == 'created_before': created_before = str_to_date(v) query = query.filter(models.DataIdentifier.created_at <= created_before) elif k == 'created_after': created_after = str_to_date(v) query = query.filter(models.DataIdentifier.created_at >= created_after) elif k == 'guid': query = query.filter_by(guid=v).\ with_hint(models.ReplicaLock, "INDEX(DIDS_GUIDS_IDX)", 'oracle') elif k == 'length.gt': query = query.filter(models.DataIdentifier.length > v) elif k == 'length.lt': query = query.filter(models.DataIdentifier.length < v) elif k == 'length.gte': query = query.filter(models.DataIdentifier.length >= v) elif k == 'length.lte': query = query.filter(models.DataIdentifier.length <= v) elif k == 'length': query = query.filter(models.DataIdentifier.length == v) else: query = query.filter(getattr(models.DataIdentifier, k) == v) if 'name' in filters: if '*' in filters['name']: query = query.\ with_hint(models.DataIdentifier, "NO_INDEX(dids(SCOPE,NAME))", 'oracle') else: query = query.\ with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle') if limit: query = query.limit(limit) if recursive: # Get attachted DIDs and save in list because query has to be finished before starting a new one in the recursion collections_content = [] parent_scope = scope from rucio.core.did import list_content for scope, name, did_type, bytes, length in query.yield_per(100): if (did_type == DIDType.CONTAINER or did_type == DIDType.DATASET): collections_content += [did for did in list_content(scope=scope, name=name)] # List DIDs again to use filter for did in collections_content: filters['name'] = did['name'] for result in self.list_dids(scope=did['scope'], filters=filters, recursive=True, type=type, limit=limit, offset=offset, long=long, session=session): yield result if long: for scope, name, did_type, bytes, length in query.yield_per(5): yield {'scope': scope, 'name': name, 'did_type': str(did_type), 'bytes': bytes, 'length': length} else: for scope, name, did_type, bytes, length in query.yield_per(5): yield name
def add_exception(dids, account, pattern, comments, expires_at, session=None): """ Add exceptions to Lifetime Model. :param dids: The list of dids :param account: The account of the requester. :param pattern: The account. :param comments: The comments associated to the exception. :param expires_at: The expiration date of the exception. :param session: The database session in use. returns: A dictionary with id of the exceptions split by scope, datatype. """ from rucio.core.did import get_metadata_bulk result = dict() result['exceptions'] = dict() try: max_extension = config_get('lifetime_model', 'max_extension', default=None, session=session) if max_extension: if not expires_at: expires_at = datetime.utcnow() + timedelta(days=max_extension) else: if isinstance(expires_at, string_types): expires_at = str_to_date(expires_at) if expires_at > datetime.utcnow() + timedelta( days=max_extension): expires_at = datetime.utcnow() + timedelta( days=max_extension) except (ConfigNotFound, ValueError, NoSectionError): max_extension = None try: cutoff_date = config_get('lifetime_model', 'cutoff_date', default=None, session=session) except (ConfigNotFound, NoSectionError): raise UnsupportedOperation('Cannot submit exception at that date.') try: cutoff_date = datetime.strptime(cutoff_date, '%Y-%m-%d') except ValueError: raise UnsupportedOperation('Cannot submit exception at that date.') if cutoff_date < datetime.utcnow(): raise UnsupportedOperation('Cannot submit exception at that date.') did_group = dict() not_affected = list() list_dids = [(did['scope'], did['name']) for did in dids] metadata = [meta for meta in get_metadata_bulk(dids=dids, session=session)] for did in metadata: scope, name, did_type = did['scope'], did['name'], did['did_type'] if (scope, name) in list_dids: list_dids.remove((scope, name)) datatype = did.get('datatype', '') eol_at = did.get('eol_at', None) if eol_at and eol_at < cutoff_date: if (scope, datatype) not in did_group: did_group[(scope, datatype)] = [list(), 0] did_group[(scope, datatype)][0].append({ 'scope': scope, 'name': name, 'did_type': did_type }) did_group[(scope, datatype)][1] += did['bytes'] or 0 else: not_affected.append((scope, name, did_type)) for entry in did_group: exception_id = __add_exception(did_group[entry][0], account=account, pattern=pattern, comments=comments, expires_at=expires_at, estimated_volume=did_group[entry][1], session=session) result['exceptions'][exception_id] = did_group[entry][0] result['unknown'] = [{ 'scope': did[0], 'name': did[1], 'did_type': DIDType.DATASET } for did in list_dids] result['not_affected'] = [{ 'scope': did[0], 'name': did[1], 'did_type': did[2] } for did in not_affected] return result