def delete_metadata(self, scope, name, key, session=None):
"""
Delete a key from the metadata column
:param scope: the scope of did
:param name: the name of the did
:param key: the key to be deleted
"""
if not self.json_implemented(session):
raise NotImplementedError
try:
row = session.query(models.DidMeta).filter_by(scope=scope, name=name).one()
existing_meta = getattr(row, 'meta')
# Oracle returns a string instead of a dict
if session.bind.dialect.name in ['oracle', 'sqlite'] and existing_meta is not None:
existing_meta = json_lib.loads(existing_meta)
if key not in existing_meta:
raise exception.KeyNotFound(key)
existing_meta.pop(key, None)
row.meta = None
session.flush()
# Oracle insert takes a string as input
if session.bind.dialect.name in ['oracle', 'sqlite']:
existing_meta = json_lib.dumps(existing_meta)
row.meta = existing_meta
except NoResultFound:
raise exception.DataIdentifierNotFound("Key not found for data identifier '%(scope)s:%(name)s'" % locals())
def _coerce_filter_word_to_model_attribute(self,
word,
model_class,
strict=True):
"""
Attempts to coerce a filter word to an attribute of a <model_class>.
:param model_class: The word.
:param model_class: The SQL model class.
:returns: The coerced attribute if successful or the word if not.
"""
if isinstance(word, str):
if hasattr(model_class, word):
return getattr(model_class, word)
else:
if strict:
raise exception.KeyNotFound(
"'{}' keyword could not be coerced to model class attribute. Attribute not found."
.format(word))
return word
def list_dids(self, scope, filters, type='collection', ignore_case=False, limit=None,
offset=None, long=False, recursive=False, session=None):
"""
Search data identifiers
:param scope: the scope name.
:param filters: dictionary of attributes by which the results should be filtered.
:param type: the type of the did: all(container, dataset, file), collection(dataset or container), dataset, container, file.
:param ignore_case: ignore case distinctions.
:param limit: limit number.
:param offset: offset number.
:param long: Long format option to display more information for each DID.
:param session: The database session in use.
:param recursive: Recursively list DIDs content.
"""
types = ['all', 'collection', 'container', 'dataset', 'file']
if type not in types:
raise exception.UnsupportedOperation("Valid type are: %(types)s" % locals())
query = session.query(models.DataIdentifier.scope,
models.DataIdentifier.name,
models.DataIdentifier.did_type,
models.DataIdentifier.bytes,
models.DataIdentifier.length).\
filter(models.DataIdentifier.scope == scope)
# Exclude suppressed dids
query = query.filter(models.DataIdentifier.suppressed != true())
if type == 'all':
query = query.filter(or_(models.DataIdentifier.did_type == DIDType.CONTAINER,
models.DataIdentifier.did_type == DIDType.DATASET,
models.DataIdentifier.did_type == DIDType.FILE))
elif type.lower() == 'collection':
query = query.filter(or_(models.DataIdentifier.did_type == DIDType.CONTAINER,
models.DataIdentifier.did_type == DIDType.DATASET))
elif type.lower() == 'container':
query = query.filter(models.DataIdentifier.did_type == DIDType.CONTAINER)
elif type.lower() == 'dataset':
query = query.filter(models.DataIdentifier.did_type == DIDType.DATASET)
elif type.lower() == 'file':
query = query.filter(models.DataIdentifier.did_type == DIDType.FILE)
for (k, v) in filters.items():
if k not in ['created_before', 'created_after', 'length.gt', 'length.lt', 'length.lte', 'length.gte', 'length'] \
and not hasattr(models.DataIdentifier, k):
raise exception.KeyNotFound(k)
if isinstance(v, string_types) and ('*' in v or '%' in v):
if v in ('*', '%', u'*', u'%'):
continue
if session.bind.dialect.name == 'postgresql':
query = query.filter(getattr(models.DataIdentifier, k).
like(v.replace('*', '%').replace('_', '\_'), # NOQA: W605
escape='\\'))
else:
query = query.filter(getattr(models.DataIdentifier, k).
like(v.replace('*', '%').replace('_', '\_'), escape='\\')) # NOQA: W605
elif k == 'created_before':
created_before = str_to_date(v)
query = query.filter(models.DataIdentifier.created_at <= created_before)
elif k == 'created_after':
created_after = str_to_date(v)
query = query.filter(models.DataIdentifier.created_at >= created_after)
elif k == 'guid':
query = query.filter_by(guid=v).\
with_hint(models.ReplicaLock, "INDEX(DIDS_GUIDS_IDX)", 'oracle')
elif k == 'length.gt':
query = query.filter(models.DataIdentifier.length > v)
elif k == 'length.lt':
query = query.filter(models.DataIdentifier.length < v)
elif k == 'length.gte':
query = query.filter(models.DataIdentifier.length >= v)
elif k == 'length.lte':
query = query.filter(models.DataIdentifier.length <= v)
elif k == 'length':
query = query.filter(models.DataIdentifier.length == v)
else:
query = query.filter(getattr(models.DataIdentifier, k) == v)
if 'name' in filters:
if '*' in filters['name']:
query = query.\
with_hint(models.DataIdentifier, "NO_INDEX(dids(SCOPE,NAME))", 'oracle')
else:
query = query.\
with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle')
if limit:
query = query.limit(limit)
if recursive:
# Get attachted DIDs and save in list because query has to be finished before starting a new one in the recursion
collections_content = []
parent_scope = scope
from rucio.core.did import list_content
for scope, name, did_type, bytes, length in query.yield_per(100):
if (did_type == DIDType.CONTAINER or did_type == DIDType.DATASET):
collections_content += [did for did in list_content(scope=scope, name=name)]
# List DIDs again to use filter
for did in collections_content:
filters['name'] = did['name']
for result in self.list_dids(scope=did['scope'], filters=filters, recursive=True, type=type, limit=limit, offset=offset, long=long, session=session):
yield result
if long:
for scope, name, did_type, bytes, length in query.yield_per(5):
yield {'scope': scope,
'name': name,
'did_type': str(did_type),
'bytes': bytes,
'length': length}
else:
for scope, name, did_type, bytes, length in query.yield_per(5):
yield name