def document_delete(document, revision_id, bucket, session=None):
"""Delete a document
Creates a new document with the bare minimum information about the document
that is to be deleted, and then sets the appropriate deleted fields
:param document: document object/dict to be deleted
:param revision_id: id of the revision where the document is to be deleted
:param bucket: bucket object/dict where the document will be deleted from
:param session: Database session object.
:return: dict representation of deleted document
"""
session = session or get_session()
doc = models.Document()
# Store bare minimum information about the document.
doc['schema'] = document['schema']
doc['name'] = document['name']
doc['layer'] = document['layer']
doc['data'] = {}
doc['meta'] = document['metadata']
doc['data_hash'] = _make_hash({})
doc['metadata_hash'] = _make_hash({})
doc['bucket_id'] = bucket['id']
doc['revision_id'] = revision_id
# Save and mark the document as `deleted` in the database.
try:
doc.save(session=session)
except db_exception.DBDuplicateEntry:
raise errors.DuplicateDocumentExists(schema=doc['schema'],
layer=doc['layer'],
name=doc['name'],
bucket=bucket['name'])
doc.safe_delete(session=session)
return doc.to_dict()
def revision_rollback(revision_id, latest_revision, session=None):
"""Rollback the latest revision to revision specified by ``revision_id``.
Rolls back the latest revision to the revision specified by ``revision_id``
thereby creating a new, carbon-copy revision.
:param revision_id: Revision ID to which to rollback.
:param latest_revision: Dictionary representation of the latest revision
in the system.
:returns: The newly created revision.
"""
session = session or get_session()
latest_revision_hashes = [(d['data_hash'], d['metadata_hash'])
for d in latest_revision['documents']]
if latest_revision['id'] == revision_id:
LOG.debug('The revision being rolled back to is the current revision.'
'Expect no meaningful changes.')
if revision_id == 0:
# Placeholder revision as revision_id=0 doesn't exist.
orig_revision = {'documents': []}
else:
orig_revision = revision_get(revision_id, session=session)
# A mechanism for determining whether a particular document has changed
# between revisions. Keyed with the document_id, the value is True if
# it has changed, else False.
doc_diff = {}
for orig_doc in orig_revision['documents']:
if ((orig_doc['data_hash'], orig_doc['metadata_hash'])
not in latest_revision_hashes):
doc_diff[orig_doc['id']] = True
else:
doc_diff[orig_doc['id']] = False
# No changes have been made between the target revision to rollback to
# and the latest revision.
if set(doc_diff.values()) == set([False]):
LOG.debug('The revision being rolled back to has the same documents '
'as that of the current revision. Expect no meaningful '
'changes.')
# Create the new revision,
new_revision = models.Revision()
with session.begin():
new_revision.save(session=session)
# Create the documents for the revision.
for orig_document in orig_revision['documents']:
orig_document['revision_id'] = new_revision['id']
orig_document['meta'] = orig_document.pop('metadata')
new_document = models.Document()
new_document.update({
x: orig_document[x]
for x in ('name', 'meta', 'layer', 'data', 'data_hash',
'metadata_hash', 'schema', 'bucket_id')
})
new_document['revision_id'] = new_revision['id']
# If the document has changed, then use the revision_id of the new
# revision, otherwise use the original revision_id to preserve the
# revision history.
if doc_diff[orig_document['id']]:
new_document['orig_revision_id'] = new_revision['id']
else:
new_document['orig_revision_id'] = orig_revision['id']
with session.begin():
new_document.save(session=session)
new_revision = new_revision.to_dict()
new_revision['documents'] = _update_revision_history(
new_revision['documents'])
return new_revision
def _document_create(document):
model = models.Document()
model.update(document)
return model
def documents_create(bucket_name, documents, session=None):
"""Create a set of documents and associated bucket.
If no changes are detected, a new revision will not be created. This
allows services to periodically re-register their schemas without
creating unnecessary revisions.
:param bucket_name: The name of the bucket with which to associate created
documents.
:param documents: List of documents to be created.
:param session: Database session object.
:returns: List of created documents in dictionary format.
:raises DocumentExists: If the document already exists in the DB for any
bucket.
"""
session = session or get_session()
resp = []
with session.begin():
documents_to_create = _documents_create(bucket_name,
documents,
session=session)
# The documents to be deleted are computed by comparing the documents
# for the previous revision (if it exists) that belong to `bucket_name`
# with `documents`: the difference between the former and the latter.
document_history = [
d for d in revision_documents_get(bucket_name=bucket_name,
session=session)
]
documents_to_delete = [
h for h in document_history
if eng_utils.meta(h) not in [eng_utils.meta(d) for d in documents]
]
# Only create a revision if any docs have been created, changed or
# deleted.
if any([documents_to_create, documents_to_delete]):
revision = revision_create(session=session)
bucket = bucket_get_or_create(bucket_name, session=session)
if documents_to_delete:
LOG.debug('Deleting documents: %s.',
[eng_utils.meta(d) for d in documents_to_delete])
deleted_documents = []
for d in documents_to_delete:
doc = models.Document()
# Store bare minimum information about the document.
doc['schema'] = d['schema']
doc['name'] = d['name']
doc['layer'] = d['layer']
doc['data'] = {}
doc['meta'] = d['metadata']
doc['data_hash'] = _make_hash({})
doc['metadata_hash'] = _make_hash({})
doc['bucket_id'] = bucket['id']
doc['revision_id'] = revision['id']
# Save and mark the document as `deleted` in the database.
try:
doc.save(session=session)
except db_exception.DBDuplicateEntry:
raise errors.DuplicateDocumentExists(schema=doc['schema'],
layer=doc['layer'],
name=doc['name'],
bucket=bucket['name'])
doc.safe_delete(session=session)
deleted_documents.append(doc)
resp.append(doc.to_dict())
if documents_to_create:
LOG.debug('Creating documents: %s.',
[(d['schema'], d['layer'], d['name'])
for d in documents_to_create])
for doc in documents_to_create:
doc['bucket_id'] = bucket['id']
doc['revision_id'] = revision['id']
if not doc.get('orig_revision_id'):
doc['orig_revision_id'] = doc['revision_id']
try:
doc.save(session=session)
except db_exception.DBDuplicateEntry:
raise errors.DuplicateDocumentExists(schema=doc['schema'],
layer=doc['layer'],
name=doc['name'],
bucket=bucket['name'])
resp.append(doc.to_dict())
# NOTE(fmontei): The orig_revision_id is not copied into the
# revision_id for each created document, because the revision_id here
# should reference the just-created revision. In case the user needs
# the original revision_id, that is returned as well.
return resp
def _document_create(values):
document = models.Document()
with session.begin():
document.update(values)
return document
def documents_create(bucket_name, documents, validations=None,
session=None):
"""Create a set of documents and associated bucket.
If no changes are detected, a new revision will not be created. This
allows services to periodically re-register their schemas without
creating unnecessary revisions.
:param bucket_name: The name of the bucket with which to associate created
documents.
:param documents: List of documents to be created.
:param validation_policies: List of validation policies to be created.
:param session: Database session object.
:returns: List of created documents in dictionary format.
:raises DocumentExists: If the (document.schema, document.metadata.name)
already exists in another bucket.
"""
session = session or get_session()
documents_to_create = _documents_create(bucket_name, documents, session)
resp = []
# The documents to be deleted are computed by comparing the documents for
# the previous revision (if it exists) that belong to `bucket_name` with
# `documents`: the difference between the former and the latter.
document_history = [(d['schema'], d['name'])
for d in revision_documents_get(
bucket_name=bucket_name)]
documents_to_delete = [
h for h in document_history if h not in
[(d['schema'], d['metadata']['name']) for d in documents]]
# Only create a revision if any docs have been created, changed or deleted.
if any([documents_to_create, documents_to_delete]):
bucket = bucket_get_or_create(bucket_name)
revision = revision_create()
if validations:
for validation in validations:
validation_create(revision['id'], validation['name'],
validation)
if documents_to_delete:
LOG.debug('Deleting documents: %s.', documents_to_delete)
deleted_documents = []
for d in documents_to_delete:
doc = models.Document()
with session.begin():
# Store bare minimum information about the document.
doc['schema'] = d[0]
doc['name'] = d[1]
doc['data'] = {}
doc['_metadata'] = {}
doc['data_hash'] = _make_hash({})
doc['metadata_hash'] = _make_hash({})
doc['bucket_id'] = bucket['id']
doc['revision_id'] = revision['id']
# Save and mark the document as `deleted` in the database.
doc.save(session=session)
doc.safe_delete(session=session)
deleted_documents.append(doc)
resp.append(doc.to_dict())
if documents_to_create:
LOG.debug('Creating documents: %s.',
[(d['schema'], d['name']) for d in documents_to_create])
for doc in documents_to_create:
with session.begin():
doc['bucket_id'] = bucket['id']
doc['revision_id'] = revision['id']
doc.save(session=session)
resp.append(doc.to_dict())
# NOTE(fmontei): The orig_revision_id is not copied into the
# revision_id for each created document, because the revision_id here
# should reference the just-created revision. In case the user needs
# the original revision_id, that is returned as well.
return resp
def _document_create(document):
model = models.Document()
with session.begin():
model.update(document)
return model
def revision_rollback(revision_id, latest_revision, session=None):
"""Rollback the latest revision to revision specified by ``revision_id``.
Rolls back the latest revision to the revision specified by ``revision_id``
thereby creating a new, carbon-copy revision.
:param revision_id: Revision ID to which to rollback.
:param latest_revision: Dictionary representation of the latest revision
in the system.
:returns: The newly created revision.
"""
session = session or get_session()
latest_revision_hashes = [(d['data_hash'], d['metadata_hash'])
for d in latest_revision['documents']]
# If the rollback revision is the same as the latest revision, then there's
# no point in rolling back.
if latest_revision['id'] == revision_id:
raise errors.InvalidRollback(revision_id=revision_id)
orig_revision = revision_get(revision_id, session=session)
# A mechanism for determining whether a particular document has changed
# between revisions. Keyed with the document_id, the value is True if
# it has changed, else False.
doc_diff = {}
for orig_doc in orig_revision['documents']:
if ((orig_doc['data_hash'], orig_doc['metadata_hash'])
not in latest_revision_hashes):
doc_diff[orig_doc['id']] = True
else:
doc_diff[orig_doc['id']] = False
# If no changes have been made between the target revision to rollback to
# and the latest revision, raise an exception.
if set(doc_diff.values()) == set([False]):
raise errors.InvalidRollback(revision_id=revision_id)
# Create the new revision,
new_revision = models.Revision()
with session.begin():
new_revision.save(session=session)
# Create the documents for the revision.
for orig_document in orig_revision['documents']:
orig_document['revision_id'] = new_revision['id']
orig_document['_metadata'] = orig_document.pop('metadata')
new_document = models.Document()
new_document.update({
x: orig_document[x]
for x in ('name', '_metadata', 'data', 'data_hash',
'metadata_hash', 'schema', 'bucket_id')
})
new_document['revision_id'] = new_revision['id']
# If the document has changed, then use the revision_id of the new
# revision, otherwise use the original revision_id to preserve the
# revision history.
if doc_diff[orig_document['id']]:
new_document['orig_revision_id'] = new_revision['id']
else:
new_document['orig_revision_id'] = orig_revision['id']
with session.begin():
new_document.save(session=session)
new_revision = new_revision.to_dict()
new_revision['documents'] = _update_revision_history(
new_revision['documents'])
return new_revision
def revision_rollback(revision_id, latest_revision, session=None):
"""Rollback the latest revision to revision specified by ``revision_id``.
Rolls back the latest revision to the revision specified by ``revision_id``
thereby creating a new, carbon-copy revision.
:param revision_id: Revision ID to which to rollback.
:param latest_revision: Dictionary representation of the latest revision
in the system.
:returns: The newly created revision.
"""
session = session or get_session()
latest_revision_docs = revision_documents_get(latest_revision['id'],
session=session)
latest_revision_hashes = [(d['data_hash'], d['metadata_hash'])
for d in latest_revision_docs]
if latest_revision['id'] == revision_id:
LOG.debug('The revision being rolled back to is the current revision.'
'Expect no meaningful changes.')
if revision_id == 0:
# Delete all existing documents in all buckets
all_buckets = bucket_get_all(deleted=False)
bucket_names = [str(b['name']) for b in all_buckets]
revision = documents_delete_from_buckets_list(bucket_names,
session=session)
return revision.to_dict()
else:
# Sorting the documents so the documents in the new revision are in
# the same order as the previous revision to support stable testing
orig_revision_docs = sorted(revision_documents_get(revision_id,
session=session),
key=lambda d: d['id'])
# A mechanism for determining whether a particular document has changed
# between revisions. Keyed with the document_id, the value is True if
# it has changed, else False.
doc_diff = {}
# List of unique buckets that exist in this revision
unique_buckets = []
for orig_doc in orig_revision_docs:
if ((orig_doc['data_hash'], orig_doc['metadata_hash'])
not in latest_revision_hashes):
doc_diff[orig_doc['id']] = True
else:
doc_diff[orig_doc['id']] = False
if orig_doc['bucket_id'] not in unique_buckets:
unique_buckets.append(orig_doc['bucket_id'])
# We need to find which buckets did not exist at this revision
buckets_to_delete = []
all_buckets = bucket_get_all(deleted=False)
for bucket in all_buckets:
if bucket['id'] not in unique_buckets:
buckets_to_delete.append(str(bucket['name']))
# Create the new revision,
if len(buckets_to_delete) > 0:
new_revision = documents_delete_from_buckets_list(buckets_to_delete,
session=session)
else:
new_revision = models.Revision()
with session.begin():
new_revision.save(session=session)
# No changes have been made between the target revision to rollback to
# and the latest revision.
if set(doc_diff.values()) == set([False]):
LOG.debug('The revision being rolled back to has the same documents '
'as that of the current revision. Expect no meaningful '
'changes.')
# Create the documents for the revision.
for orig_document in orig_revision_docs:
orig_document['revision_id'] = new_revision['id']
orig_document['meta'] = orig_document.pop('metadata')
new_document = models.Document()
new_document.update({
x: orig_document[x]
for x in ('name', 'meta', 'layer', 'data', 'data_hash',
'metadata_hash', 'schema', 'bucket_id')
})
new_document['revision_id'] = new_revision['id']
# If the document has changed, then use the revision_id of the new
# revision, otherwise use the original revision_id to preserve the
# revision history.
if doc_diff[orig_document['id']]:
new_document['orig_revision_id'] = new_revision['id']
else:
new_document['orig_revision_id'] = revision_id
with session.begin():
new_document.save(session=session)
new_revision = new_revision.to_dict()
new_revision['documents'] = _update_revision_history(
new_revision['documents'])
return new_revision
