def set_metadata_bulk(self, scope, name, meta, recursive=False, session=None):
if not self.json_implemented(session):
raise NotImplementedError
if session.query(models.DataIdentifier).filter_by(scope=scope, name=name).one_or_none() is None:
raise exception.DataIdentifierNotFound("Data identifier '%s:%s' not found" % (scope, name))
row_did_meta = session.query(models.DidMeta).filter_by(scope=scope, name=name).scalar()
if row_did_meta is None:
# Add metadata column to new table (if not already present)
row_did_meta = models.DidMeta(scope=scope, name=name)
row_did_meta.save(session=session, flush=False)
existing_meta = {}
if hasattr(row_did_meta, 'meta'):
if row_did_meta.meta:
existing_meta = row_did_meta.meta
# Oracle returns a string instead of a dict
if session.bind.dialect.name in ['oracle', 'sqlite'] and existing_meta:
existing_meta = json_lib.loads(existing_meta)
for key, value in meta.items():
existing_meta[key] = value
row_did_meta.meta = None
session.flush()
# Oracle insert takes a string as input
if session.bind.dialect.name in ['oracle', 'sqlite']:
existing_meta = json_lib.dumps(existing_meta)
row_did_meta.meta = existing_meta
row_did_meta.save(session=session, flush=True)
def delete_metadata(self, scope, name, key, session=None):
"""
Delete a key from the metadata column
:param scope: the scope of did
:param name: the name of the did
:param key: the key to be deleted
"""
if not self.json_implemented(session):
raise NotImplementedError
try:
row = session.query(models.DidMeta).filter_by(scope=scope, name=name).one()
existing_meta = getattr(row, 'meta')
# Oracle returns a string instead of a dict
if session.bind.dialect.name in ['oracle', 'sqlite'] and existing_meta is not None:
existing_meta = json_lib.loads(existing_meta)
if key not in existing_meta:
raise exception.KeyNotFound(key)
existing_meta.pop(key, None)
row.meta = None
session.flush()
# Oracle insert takes a string as input
if session.bind.dialect.name in ['oracle', 'sqlite']:
existing_meta = json_lib.dumps(existing_meta)
row.meta = existing_meta
except NoResultFound:
raise exception.DataIdentifierNotFound("Key not found for data identifier '%(scope)s:%(name)s'" % locals())
def delete_metadata(self, scope, name, key, session=None):
"""
Delete a key from metadata.
:param scope: the scope of did
:param name: the name of the did
:param key: the key to be deleted
"""
meta = {key: ""}
try:
self.col.update_one({"_id": "{}:{}".format(scope.internal, name)},
{'$unset': meta})
except Exception as e:
raise exception.DataIdentifierNotFound(e)
def get_metadata(self, scope, name, session=None):
"""
Get data identifier metadata
:param scope: The scope name.
:param name: The data identifier name.
:param session: The database session in use.
"""
try:
row = session.query(models.DataIdentifier).filter_by(scope=scope, name=name).\
with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').one()
d = {}
for column in row.__table__.columns:
d[column.name] = getattr(row, column.name)
return d
except NoResultFound:
raise exception.DataIdentifierNotFound("Data identifier '%(scope)s:%(name)s' not found" % locals())
def get_metadata(self, scope, name, session=None):
"""
Get data identifier metadata (JSON)
:param scope: The scope name.
:param name: The data identifier name.
:param session: The database session in use.
"""
if not self.json_implemented(session):
raise NotImplementedError
try:
row = session.query(models.DidMeta).filter_by(scope=scope, name=name).one()
meta = getattr(row, 'meta')
return json_lib.loads(meta) if session.bind.dialect.name in ['oracle', 'sqlite'] else meta
except NoResultFound:
raise exception.DataIdentifierNotFound("No generic metadata found for '%(scope)s:%(name)s'" % locals())
def set_metadata(self, scope, name, key, value, recursive, session=None):
"""
Add or update the given metadata to the given did
:param scope: the scope of the did
:param name: the name of the did
:param meta: the metadata to be added or updated
"""
if not self.json_implemented(session):
raise NotImplementedError
try:
row_did = session.query(models.DataIdentifier).filter_by(
scope=scope, name=name).one()
row_did_meta = session.query(models.DidMeta).filter_by(
scope=scope, name=name).scalar()
if row_did_meta is None:
# Add metadata column to new table (if not already present)
row_did_meta = models.DidMeta(scope=scope, name=name)
row_did_meta.save(session=session, flush=False)
existing_meta = getattr(row_did_meta, 'meta')
# Oracle returns a string instead of a dict
if session.bind.dialect.name in ['oracle', 'sqlite'
] and existing_meta is not None:
existing_meta = json_lib.loads(existing_meta)
if existing_meta is None:
existing_meta = {}
# for k, v in iteritems(meta):
# existing_meta[k] = v
existing_meta[key] = value
row_did_meta.meta = None
session.flush()
# Oracle insert takes a string as input
if session.bind.dialect.name in ['oracle', 'sqlite']:
existing_meta = json_lib.dumps(existing_meta)
row_did_meta.meta = existing_meta
row_did_meta.save(session=session, flush=True)
except NoResultFound:
raise exception.DataIdentifierNotFound(
"Data identifier '%(scope)s:%(name)s' not found" % locals())
def get_metadata(self, scope, name, session=None):
"""
Get data identifier metadata.
:param scope: The scope name
:param name: The data identifier name
:param session: The database session in use
"""
# get first document with this did == _id
doc = self.col.find_one({"_id": "{}:{}".format(scope.internal, name)})
# pop immutable keys
for key in IMMUTABLE_KEYS:
if key in doc:
doc.pop(key)
if not doc:
raise exception.DataIdentifierNotFound(
"No metadata found for did '%(scope)s:%(name)s'" % locals())
return doc
def set_metadata_bulk(self, scope, name, meta, recursive=False, session=None):
did_query = session.query(models.DataIdentifier).with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').filter_by(scope=scope, name=name)
if did_query.one_or_none() is None:
raise exception.DataIdentifierNotFound("Data identifier '%s:%s' not found" % (scope, name))
remainder = {}
for key, value in meta.items():
if key == 'lifetime':
try:
expired_at = None
if value is not None:
expired_at = datetime.utcnow() + timedelta(seconds=float(value))
rowcount = did_query.update({'expired_at': expired_at}, synchronize_session='fetch')
except TypeError as error:
raise exception.InvalidValueForKey(error)
if not rowcount:
# check for did presence
raise exception.UnsupportedOperation('%s for %s:%s cannot be updated' % (key, scope, name))
elif key in ['guid', 'events']:
rowcount = did_query.filter_by(did_type=DIDType.FILE).update({key: value}, synchronize_session=False)
if not rowcount:
# check for did presence
raise exception.UnsupportedOperation('%s for %s:%s cannot be updated' % (key, scope, name))
session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False)
if key == 'events':
for parent_scope, parent_name in session.query(models.DataIdentifierAssociation.scope, models.DataIdentifierAssociation.name).filter_by(child_scope=scope, child_name=name):
events = session.query(func.sum(models.DataIdentifierAssociation.events)).filter_by(scope=parent_scope, name=parent_name).one()[0]
session.query(models.DataIdentifier).filter_by(scope=parent_scope, name=parent_name).update({'events': events}, synchronize_session=False)
elif key == 'adler32':
rowcount = did_query.filter_by(did_type=DIDType.FILE).update({key: value}, synchronize_session=False)
if not rowcount:
# check for did presence
raise exception.UnsupportedOperation('%s for %s:%s cannot be updated' % (key, scope, name))
session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False)
session.query(models.Request).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False)
session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False)
elif key == 'bytes':
rowcount = did_query.filter_by(did_type=DIDType.FILE).update({key: value}, synchronize_session=False)
if not rowcount:
# check for did presence
raise exception.UnsupportedOperation('%s for %s:%s cannot be updated' % (key, scope, name))
session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False)
session.query(models.Request).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False)
for account, bytes, rse_id, rule_id in session.query(models.ReplicaLock.account, models.ReplicaLock.bytes, models.ReplicaLock.rse_id, models.ReplicaLock.rule_id).filter_by(scope=scope, name=name):
session.query(models.ReplicaLock).filter_by(scope=scope, name=name, rule_id=rule_id, rse_id=rse_id).update({key: value}, synchronize_session=False)
account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=bytes, session=session)
account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=value, session=session)
for bytes, rse_id in session.query(models.RSEFileAssociation.bytes, models.RSEFileAssociation.rse_id).filter_by(scope=scope, name=name):
session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name, rse_id=rse_id).update({key: value}, synchronize_session=False)
rse_counter.decrease(rse_id=rse_id, files=1, bytes=bytes, session=session)
rse_counter.increase(rse_id=rse_id, files=1, bytes=value, session=session)
for parent_scope, parent_name in session.query(models.DataIdentifierAssociation.scope, models.DataIdentifierAssociation.name).filter_by(child_scope=scope, child_name=name):
values = {}
values['length'], values['bytes'], values['events'] = session.query(func.count(models.DataIdentifierAssociation.scope),
func.sum(models.DataIdentifierAssociation.bytes),
func.sum(models.DataIdentifierAssociation.events)).filter_by(scope=parent_scope, name=parent_name).one()
session.query(models.DataIdentifier).filter_by(scope=parent_scope, name=parent_name).update(values, synchronize_session=False)
session.query(models.DatasetLock).filter_by(scope=parent_scope, name=parent_name).update({'length': values['length'], 'bytes': values['bytes']}, synchronize_session=False)
else:
remainder[key] = value
if remainder:
try:
rowcount = did_query.update(remainder, synchronize_session='fetch')
except CompileError as error:
raise exception.InvalidMetadata(error)
except InvalidRequestError:
raise exception.InvalidMetadata("Some of the keys are not accepted: " + str(list(remainder.keys())))
if not rowcount:
raise exception.UnsupportedOperation('Some of the keys for %s:%s cannot be updated: %s' % (scope, name, str(list(remainder.keys()))))
# propagate metadata updates to child content
if recursive:
content_query = session.query(models.DataIdentifierAssociation.child_scope, models.DataIdentifierAssociation.child_name)
content_query = content_query.with_hint(models.DataIdentifierAssociation, "INDEX(CONTENTS CONTENTS_PK)", 'oracle').filter_by(scope=scope, name=name)
for child_scope, child_name in content_query:
try:
child_did_query = session.query(models.DataIdentifier).with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').filter_by(scope=child_scope, name=child_name)
child_did_query.update(remainder, synchronize_session='fetch')
except CompileError as error:
raise exception.InvalidMetadata(error)
except InvalidRequestError:
raise exception.InvalidMetadata("Some of the keys are not accepted recursively: " + str(list(remainder.keys())))
def set_metadata(self, scope, name, key, value, recursive=False, session=None):
"""
Add metadata to data identifier.
:param scope: The scope name.
:param name: The data identifier name.
:param key: the key.
:param value: the value.
:param did: The data identifier info.
:param recursive: Option to propagate the metadata change to content.
:param session: The database session in use.
"""
try:
rowcount = session.query(models.DataIdentifier).filter_by(scope=scope, name=name).\
with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').one()
except NoResultFound:
raise exception.DataIdentifierNotFound("Data identifier '%s:%s' not found" % (scope, name))
if key == 'lifetime':
try:
expired_at = None
if value is not None:
expired_at = datetime.utcnow() + timedelta(seconds=float(value))
rowcount = session.query(models.DataIdentifier).filter_by(scope=scope, name=name).update({'expired_at': expired_at}, synchronize_session='fetch')
except TypeError as error:
raise exception.InvalidValueForKey(error)
elif key in ['guid', 'events']:
rowcount = session.query(models.DataIdentifier).filter_by(scope=scope, name=name, did_type=DIDType.FILE).update({key: value}, synchronize_session=False)
session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False)
if key == 'events':
for parent_scope, parent_name in session.query(models.DataIdentifierAssociation.scope, models.DataIdentifierAssociation.name).filter_by(child_scope=scope, child_name=name):
events = session.query(func.sum(models.DataIdentifierAssociation.events)).filter_by(scope=parent_scope, name=parent_name).one()[0]
session.query(models.DataIdentifier).filter_by(scope=parent_scope, name=parent_name).update({'events': events}, synchronize_session=False)
elif key == 'adler32':
rowcount = session.query(models.DataIdentifier).filter_by(scope=scope, name=name, did_type=DIDType.FILE).update({key: value}, synchronize_session=False)
session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False)
session.query(models.Request).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False)
session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False)
elif key == 'bytes':
rowcount = session.query(models.DataIdentifier).filter_by(scope=scope, name=name, did_type=DIDType.FILE).update({key: value}, synchronize_session=False)
session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False)
session.query(models.Request).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False)
for account, bytes, rse_id, rule_id in session.query(models.ReplicaLock.account, models.ReplicaLock.bytes, models.ReplicaLock.rse_id, models.ReplicaLock.rule_id).filter_by(scope=scope, name=name):
session.query(models.ReplicaLock).filter_by(scope=scope, name=name, rule_id=rule_id, rse_id=rse_id).update({key: value}, synchronize_session=False)
account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=bytes, session=session)
account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=value, session=session)
for bytes, rse_id in session.query(models.RSEFileAssociation.bytes, models.RSEFileAssociation.rse_id).filter_by(scope=scope, name=name):
session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name, rse_id=rse_id).update({key: value}, synchronize_session=False)
rse_counter.decrease(rse_id=rse_id, files=1, bytes=bytes, session=session)
rse_counter.increase(rse_id=rse_id, files=1, bytes=value, session=session)
for parent_scope, parent_name in session.query(models.DataIdentifierAssociation.scope, models.DataIdentifierAssociation.name).filter_by(child_scope=scope, child_name=name):
values = {}
values['length'], values['bytes'], values['events'] = session.query(func.count(models.DataIdentifierAssociation.scope),
func.sum(models.DataIdentifierAssociation.bytes),
func.sum(models.DataIdentifierAssociation.events)).filter_by(scope=parent_scope, name=parent_name).one()
session.query(models.DataIdentifier).filter_by(scope=parent_scope, name=parent_name).update(values, synchronize_session=False)
session.query(models.DatasetLock).filter_by(scope=parent_scope, name=parent_name).update({'length': values['length'], 'bytes': values['bytes']}, synchronize_session=False)
else:
try:
rowcount = session.query(models.DataIdentifier).\
with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').\
filter_by(scope=scope, name=name).\
update({key: value}, synchronize_session='fetch')
except CompileError as error:
raise exception.InvalidMetadata(error)
except InvalidRequestError:
raise exception.InvalidMetadata("Key %s is not accepted" % key)
# propagate metadata updates to child content
if recursive:
content_query = session.query(models.DataIdentifierAssociation.child_scope,
models.DataIdentifierAssociation.child_name).\
with_hint(models.DataIdentifierAssociation,
"INDEX(CONTENTS CONTENTS_PK)", 'oracle').\
filter_by(scope=scope, name=name)
for child_scope, child_name in content_query:
try:
session.query(models.DataIdentifier).\
with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').\
filter_by(scope=child_scope, name=child_name).\
update({key: value}, synchronize_session='fetch')
except CompileError as error:
raise exception.InvalidMetadata(error)
except InvalidRequestError:
raise exception.InvalidMetadata("Key %s is not accepted" % key)
if not rowcount:
# check for did presence
raise exception.UnsupportedOperation('%(key)s for %(scope)s:%(name)s cannot be updated' % locals())
