def set_metadata_bulk(self, scope, name, meta, recursive=False, session=None): if not self.json_implemented(session): raise NotImplementedError if session.query(models.DataIdentifier).filter_by(scope=scope, name=name).one_or_none() is None: raise exception.DataIdentifierNotFound("Data identifier '%s:%s' not found" % (scope, name)) row_did_meta = session.query(models.DidMeta).filter_by(scope=scope, name=name).scalar() if row_did_meta is None: # Add metadata column to new table (if not already present) row_did_meta = models.DidMeta(scope=scope, name=name) row_did_meta.save(session=session, flush=False) existing_meta = {} if hasattr(row_did_meta, 'meta'): if row_did_meta.meta: existing_meta = row_did_meta.meta # Oracle returns a string instead of a dict if session.bind.dialect.name in ['oracle', 'sqlite'] and existing_meta: existing_meta = json_lib.loads(existing_meta) for key, value in meta.items(): existing_meta[key] = value row_did_meta.meta = None session.flush() # Oracle insert takes a string as input if session.bind.dialect.name in ['oracle', 'sqlite']: existing_meta = json_lib.dumps(existing_meta) row_did_meta.meta = existing_meta row_did_meta.save(session=session, flush=True)
def delete_metadata(self, scope, name, key, session=None): """ Delete a key from the metadata column :param scope: the scope of did :param name: the name of the did :param key: the key to be deleted """ if not self.json_implemented(session): raise NotImplementedError try: row = session.query(models.DidMeta).filter_by(scope=scope, name=name).one() existing_meta = getattr(row, 'meta') # Oracle returns a string instead of a dict if session.bind.dialect.name in ['oracle', 'sqlite'] and existing_meta is not None: existing_meta = json_lib.loads(existing_meta) if key not in existing_meta: raise exception.KeyNotFound(key) existing_meta.pop(key, None) row.meta = None session.flush() # Oracle insert takes a string as input if session.bind.dialect.name in ['oracle', 'sqlite']: existing_meta = json_lib.dumps(existing_meta) row.meta = existing_meta except NoResultFound: raise exception.DataIdentifierNotFound("Key not found for data identifier '%(scope)s:%(name)s'" % locals())
def delete_metadata(self, scope, name, key, session=None): """ Delete a key from metadata. :param scope: the scope of did :param name: the name of the did :param key: the key to be deleted """ meta = {key: ""} try: self.col.update_one({"_id": "{}:{}".format(scope.internal, name)}, {'$unset': meta}) except Exception as e: raise exception.DataIdentifierNotFound(e)
def get_metadata(self, scope, name, session=None): """ Get data identifier metadata :param scope: The scope name. :param name: The data identifier name. :param session: The database session in use. """ try: row = session.query(models.DataIdentifier).filter_by(scope=scope, name=name).\ with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').one() d = {} for column in row.__table__.columns: d[column.name] = getattr(row, column.name) return d except NoResultFound: raise exception.DataIdentifierNotFound("Data identifier '%(scope)s:%(name)s' not found" % locals())
def get_metadata(self, scope, name, session=None): """ Get data identifier metadata (JSON) :param scope: The scope name. :param name: The data identifier name. :param session: The database session in use. """ if not self.json_implemented(session): raise NotImplementedError try: row = session.query(models.DidMeta).filter_by(scope=scope, name=name).one() meta = getattr(row, 'meta') return json_lib.loads(meta) if session.bind.dialect.name in ['oracle', 'sqlite'] else meta except NoResultFound: raise exception.DataIdentifierNotFound("No generic metadata found for '%(scope)s:%(name)s'" % locals())
def set_metadata(self, scope, name, key, value, recursive, session=None): """ Add or update the given metadata to the given did :param scope: the scope of the did :param name: the name of the did :param meta: the metadata to be added or updated """ if not self.json_implemented(session): raise NotImplementedError try: row_did = session.query(models.DataIdentifier).filter_by( scope=scope, name=name).one() row_did_meta = session.query(models.DidMeta).filter_by( scope=scope, name=name).scalar() if row_did_meta is None: # Add metadata column to new table (if not already present) row_did_meta = models.DidMeta(scope=scope, name=name) row_did_meta.save(session=session, flush=False) existing_meta = getattr(row_did_meta, 'meta') # Oracle returns a string instead of a dict if session.bind.dialect.name in ['oracle', 'sqlite' ] and existing_meta is not None: existing_meta = json_lib.loads(existing_meta) if existing_meta is None: existing_meta = {} # for k, v in iteritems(meta): # existing_meta[k] = v existing_meta[key] = value row_did_meta.meta = None session.flush() # Oracle insert takes a string as input if session.bind.dialect.name in ['oracle', 'sqlite']: existing_meta = json_lib.dumps(existing_meta) row_did_meta.meta = existing_meta row_did_meta.save(session=session, flush=True) except NoResultFound: raise exception.DataIdentifierNotFound( "Data identifier '%(scope)s:%(name)s' not found" % locals())
def get_metadata(self, scope, name, session=None): """ Get data identifier metadata. :param scope: The scope name :param name: The data identifier name :param session: The database session in use """ # get first document with this did == _id doc = self.col.find_one({"_id": "{}:{}".format(scope.internal, name)}) # pop immutable keys for key in IMMUTABLE_KEYS: if key in doc: doc.pop(key) if not doc: raise exception.DataIdentifierNotFound( "No metadata found for did '%(scope)s:%(name)s'" % locals()) return doc
def set_metadata_bulk(self, scope, name, meta, recursive=False, session=None): did_query = session.query(models.DataIdentifier).with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').filter_by(scope=scope, name=name) if did_query.one_or_none() is None: raise exception.DataIdentifierNotFound("Data identifier '%s:%s' not found" % (scope, name)) remainder = {} for key, value in meta.items(): if key == 'lifetime': try: expired_at = None if value is not None: expired_at = datetime.utcnow() + timedelta(seconds=float(value)) rowcount = did_query.update({'expired_at': expired_at}, synchronize_session='fetch') except TypeError as error: raise exception.InvalidValueForKey(error) if not rowcount: # check for did presence raise exception.UnsupportedOperation('%s for %s:%s cannot be updated' % (key, scope, name)) elif key in ['guid', 'events']: rowcount = did_query.filter_by(did_type=DIDType.FILE).update({key: value}, synchronize_session=False) if not rowcount: # check for did presence raise exception.UnsupportedOperation('%s for %s:%s cannot be updated' % (key, scope, name)) session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False) if key == 'events': for parent_scope, parent_name in session.query(models.DataIdentifierAssociation.scope, models.DataIdentifierAssociation.name).filter_by(child_scope=scope, child_name=name): events = session.query(func.sum(models.DataIdentifierAssociation.events)).filter_by(scope=parent_scope, name=parent_name).one()[0] session.query(models.DataIdentifier).filter_by(scope=parent_scope, name=parent_name).update({'events': events}, synchronize_session=False) elif key == 'adler32': rowcount = did_query.filter_by(did_type=DIDType.FILE).update({key: value}, synchronize_session=False) if not rowcount: # check for did presence raise exception.UnsupportedOperation('%s for %s:%s cannot be updated' % (key, scope, name)) session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False) session.query(models.Request).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False) session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False) elif key == 'bytes': rowcount = did_query.filter_by(did_type=DIDType.FILE).update({key: value}, synchronize_session=False) if not rowcount: # check for did presence raise exception.UnsupportedOperation('%s for %s:%s cannot be updated' % (key, scope, name)) session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False) session.query(models.Request).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False) for account, bytes, rse_id, rule_id in session.query(models.ReplicaLock.account, models.ReplicaLock.bytes, models.ReplicaLock.rse_id, models.ReplicaLock.rule_id).filter_by(scope=scope, name=name): session.query(models.ReplicaLock).filter_by(scope=scope, name=name, rule_id=rule_id, rse_id=rse_id).update({key: value}, synchronize_session=False) account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=bytes, session=session) account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=value, session=session) for bytes, rse_id in session.query(models.RSEFileAssociation.bytes, models.RSEFileAssociation.rse_id).filter_by(scope=scope, name=name): session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name, rse_id=rse_id).update({key: value}, synchronize_session=False) rse_counter.decrease(rse_id=rse_id, files=1, bytes=bytes, session=session) rse_counter.increase(rse_id=rse_id, files=1, bytes=value, session=session) for parent_scope, parent_name in session.query(models.DataIdentifierAssociation.scope, models.DataIdentifierAssociation.name).filter_by(child_scope=scope, child_name=name): values = {} values['length'], values['bytes'], values['events'] = session.query(func.count(models.DataIdentifierAssociation.scope), func.sum(models.DataIdentifierAssociation.bytes), func.sum(models.DataIdentifierAssociation.events)).filter_by(scope=parent_scope, name=parent_name).one() session.query(models.DataIdentifier).filter_by(scope=parent_scope, name=parent_name).update(values, synchronize_session=False) session.query(models.DatasetLock).filter_by(scope=parent_scope, name=parent_name).update({'length': values['length'], 'bytes': values['bytes']}, synchronize_session=False) else: remainder[key] = value if remainder: try: rowcount = did_query.update(remainder, synchronize_session='fetch') except CompileError as error: raise exception.InvalidMetadata(error) except InvalidRequestError: raise exception.InvalidMetadata("Some of the keys are not accepted: " + str(list(remainder.keys()))) if not rowcount: raise exception.UnsupportedOperation('Some of the keys for %s:%s cannot be updated: %s' % (scope, name, str(list(remainder.keys())))) # propagate metadata updates to child content if recursive: content_query = session.query(models.DataIdentifierAssociation.child_scope, models.DataIdentifierAssociation.child_name) content_query = content_query.with_hint(models.DataIdentifierAssociation, "INDEX(CONTENTS CONTENTS_PK)", 'oracle').filter_by(scope=scope, name=name) for child_scope, child_name in content_query: try: child_did_query = session.query(models.DataIdentifier).with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').filter_by(scope=child_scope, name=child_name) child_did_query.update(remainder, synchronize_session='fetch') except CompileError as error: raise exception.InvalidMetadata(error) except InvalidRequestError: raise exception.InvalidMetadata("Some of the keys are not accepted recursively: " + str(list(remainder.keys())))
def set_metadata(self, scope, name, key, value, recursive=False, session=None): """ Add metadata to data identifier. :param scope: The scope name. :param name: The data identifier name. :param key: the key. :param value: the value. :param did: The data identifier info. :param recursive: Option to propagate the metadata change to content. :param session: The database session in use. """ try: rowcount = session.query(models.DataIdentifier).filter_by(scope=scope, name=name).\ with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').one() except NoResultFound: raise exception.DataIdentifierNotFound("Data identifier '%s:%s' not found" % (scope, name)) if key == 'lifetime': try: expired_at = None if value is not None: expired_at = datetime.utcnow() + timedelta(seconds=float(value)) rowcount = session.query(models.DataIdentifier).filter_by(scope=scope, name=name).update({'expired_at': expired_at}, synchronize_session='fetch') except TypeError as error: raise exception.InvalidValueForKey(error) elif key in ['guid', 'events']: rowcount = session.query(models.DataIdentifier).filter_by(scope=scope, name=name, did_type=DIDType.FILE).update({key: value}, synchronize_session=False) session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False) if key == 'events': for parent_scope, parent_name in session.query(models.DataIdentifierAssociation.scope, models.DataIdentifierAssociation.name).filter_by(child_scope=scope, child_name=name): events = session.query(func.sum(models.DataIdentifierAssociation.events)).filter_by(scope=parent_scope, name=parent_name).one()[0] session.query(models.DataIdentifier).filter_by(scope=parent_scope, name=parent_name).update({'events': events}, synchronize_session=False) elif key == 'adler32': rowcount = session.query(models.DataIdentifier).filter_by(scope=scope, name=name, did_type=DIDType.FILE).update({key: value}, synchronize_session=False) session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False) session.query(models.Request).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False) session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False) elif key == 'bytes': rowcount = session.query(models.DataIdentifier).filter_by(scope=scope, name=name, did_type=DIDType.FILE).update({key: value}, synchronize_session=False) session.query(models.DataIdentifierAssociation).filter_by(child_scope=scope, child_name=name, child_type=DIDType.FILE).update({key: value}, synchronize_session=False) session.query(models.Request).filter_by(scope=scope, name=name).update({key: value}, synchronize_session=False) for account, bytes, rse_id, rule_id in session.query(models.ReplicaLock.account, models.ReplicaLock.bytes, models.ReplicaLock.rse_id, models.ReplicaLock.rule_id).filter_by(scope=scope, name=name): session.query(models.ReplicaLock).filter_by(scope=scope, name=name, rule_id=rule_id, rse_id=rse_id).update({key: value}, synchronize_session=False) account_counter.decrease(rse_id=rse_id, account=account, files=1, bytes=bytes, session=session) account_counter.increase(rse_id=rse_id, account=account, files=1, bytes=value, session=session) for bytes, rse_id in session.query(models.RSEFileAssociation.bytes, models.RSEFileAssociation.rse_id).filter_by(scope=scope, name=name): session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name, rse_id=rse_id).update({key: value}, synchronize_session=False) rse_counter.decrease(rse_id=rse_id, files=1, bytes=bytes, session=session) rse_counter.increase(rse_id=rse_id, files=1, bytes=value, session=session) for parent_scope, parent_name in session.query(models.DataIdentifierAssociation.scope, models.DataIdentifierAssociation.name).filter_by(child_scope=scope, child_name=name): values = {} values['length'], values['bytes'], values['events'] = session.query(func.count(models.DataIdentifierAssociation.scope), func.sum(models.DataIdentifierAssociation.bytes), func.sum(models.DataIdentifierAssociation.events)).filter_by(scope=parent_scope, name=parent_name).one() session.query(models.DataIdentifier).filter_by(scope=parent_scope, name=parent_name).update(values, synchronize_session=False) session.query(models.DatasetLock).filter_by(scope=parent_scope, name=parent_name).update({'length': values['length'], 'bytes': values['bytes']}, synchronize_session=False) else: try: rowcount = session.query(models.DataIdentifier).\ with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').\ filter_by(scope=scope, name=name).\ update({key: value}, synchronize_session='fetch') except CompileError as error: raise exception.InvalidMetadata(error) except InvalidRequestError: raise exception.InvalidMetadata("Key %s is not accepted" % key) # propagate metadata updates to child content if recursive: content_query = session.query(models.DataIdentifierAssociation.child_scope, models.DataIdentifierAssociation.child_name).\ with_hint(models.DataIdentifierAssociation, "INDEX(CONTENTS CONTENTS_PK)", 'oracle').\ filter_by(scope=scope, name=name) for child_scope, child_name in content_query: try: session.query(models.DataIdentifier).\ with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').\ filter_by(scope=child_scope, name=child_name).\ update({key: value}, synchronize_session='fetch') except CompileError as error: raise exception.InvalidMetadata(error) except InvalidRequestError: raise exception.InvalidMetadata("Key %s is not accepted" % key) if not rowcount: # check for did presence raise exception.UnsupportedOperation('%(key)s for %(scope)s:%(name)s cannot be updated' % locals())