def can_update(self, metadata_type, allow_unsafe_updates=False):
"""
Check if metadata type can be updated. Return bool,safe_changes,unsafe_changes
Safe updates currently allow new search fields to be added, description to be changed.
:param datacube.model.MetadataType metadata_type: updated MetadataType
:param bool allow_unsafe_updates: Allow unsafe changes. Use with caution.
:rtype: bool,list[change],list[change]
"""
MetadataType.validate(metadata_type.definition)
existing = self.get_by_name(metadata_type.name)
if not existing:
raise ValueError('Unknown metadata type %s, cannot update – '
'did you intend to add it?' % metadata_type.name)
updates_allowed = {
('description',): changes.allow_any,
# You can add new fields safely but not modify existing ones.
('dataset',): changes.allow_extension,
('dataset', 'search_fields'): changes.allow_extension
}
doc_changes = get_doc_changes(existing.definition, jsonify_document(metadata_type.definition))
good_changes, bad_changes = changes.classify_changes(doc_changes, updates_allowed)
for offset, old_val, new_val in good_changes:
_LOG.info("Safe change in %s from %r to %r", _readable_offset(offset), old_val, new_val)
for offset, old_val, new_val in bad_changes:
_LOG.warning("Unsafe change in %s from %r to %r", _readable_offset(offset), old_val, new_val)
return allow_unsafe_updates or not bad_changes, good_changes, bad_changes
def can_update(self, product, allow_unsafe_updates=False):
"""
Check if product can be updated. Return bool,safe_changes,unsafe_changes
(An unsafe change is anything that may potentially make the product
incompatible with existing datasets of that type)
:param DatasetType product: Product to update
:param bool allow_unsafe_updates: Allow unsafe changes. Use with caution.
:rtype: bool,list[change],list[change]
"""
DatasetType.validate(product.definition)
existing = self.get_by_name(product.name)
if not existing:
raise ValueError(
'Unknown product %s, cannot update – did you intend to add it?'
% product.name)
updates_allowed = {
('description', ):
changes.allow_any,
('license', ):
changes.allow_any,
('metadata_type', ):
changes.allow_any,
# You can safely make the match rules looser but not tighter.
# Tightening them could exclude datasets already matched to the product.
# (which would make search results wrong)
(
'metadata', ):
changes.allow_truncation,
# Some old storage fields should not be in the product definition any more: allow removal.
('storage', 'chunking'):
changes.allow_removal,
('storage', 'driver'):
changes.allow_removal,
('storage', 'dimension_order'):
changes.allow_removal,
}
doc_changes = get_doc_changes(existing.definition,
jsonify_document(product.definition))
good_changes, bad_changes = changes.classify_changes(
doc_changes, updates_allowed)
for offset, old_val, new_val in good_changes:
_LOG.info("Safe change in %s from %r to %r",
_readable_offset(offset), old_val, new_val)
for offset, old_val, new_val in bad_changes:
_LOG.warning("Unsafe change in %s from %r to %r",
_readable_offset(offset), old_val, new_val)
return allow_unsafe_updates or not bad_changes, good_changes, bad_changes
def update(self, dataset: Dataset, updates_allowed=None):
"""
Update dataset metadata and location
:param Dataset dataset: Dataset to update
:param updates_allowed: Allowed updates
:rtype: Dataset
"""
existing = self.get(dataset.id)
can_update, safe_changes, unsafe_changes = self.can_update(
dataset, updates_allowed)
if not safe_changes and not unsafe_changes:
self._ensure_new_locations(dataset, existing)
_LOG.info("No changes detected for dataset %s", dataset.id)
return dataset
for offset, old_val, new_val in safe_changes:
_LOG.info("Safe change in %s from %r to %r",
_readable_offset(offset), old_val, new_val)
for offset, old_val, new_val in unsafe_changes:
_LOG.warning("Unsafe change in %s from %r to %r",
_readable_offset(offset), old_val, new_val)
if not can_update:
raise ValueError(f"Unsafe changes in {dataset.id}: " + (", ".join(
_readable_offset(offset) for offset, _, _ in unsafe_changes)))
_LOG.info("Updating dataset %s", dataset.id)
product = self.types.get_by_name(dataset.type.name)
with self._db.begin() as transaction:
if not transaction.update_dataset(
dataset.metadata_doc_without_lineage(), dataset.id,
product.id):
raise ValueError("Failed to update dataset %s..." % dataset.id)
self._ensure_new_locations(dataset, existing)
return dataset
def update(self,
metadata_type: MetadataType,
allow_unsafe_updates=False,
allow_table_lock=False):
"""
Update a metadata type from the document. Unsafe changes will throw a ValueError by default.
Safe updates currently allow new search fields to be added, description to be changed.
:param datacube.model.MetadataType metadata_type: updated MetadataType
:param bool allow_unsafe_updates: Allow unsafe changes. Use with caution.
:param allow_table_lock:
Allow an exclusive lock to be taken on the table while creating the indexes.
This will halt other user's requests until completed.
If false, creation will be slower and cannot be done in a transaction.
:rtype: datacube.model.MetadataType
"""
can_update, safe_changes, unsafe_changes = self.can_update(
metadata_type, allow_unsafe_updates)
if not safe_changes and not unsafe_changes:
_LOG.info("No changes detected for metadata type %s",
metadata_type.name)
return self.get_by_name(metadata_type.name)
if not can_update:
raise ValueError(f"Unsafe changes in {metadata_type.name}: " +
(", ".join(
_readable_offset(offset)
for offset, _, _ in unsafe_changes)))
_LOG.info("Updating metadata type %s", metadata_type.name)
with self._db.connect() as connection:
connection.update_metadata_type(
name=metadata_type.name,
definition=metadata_type.definition,
concurrently=not allow_table_lock)
self.get_by_name_unsafe.cache_clear()
self.get_unsafe.cache_clear()
return self.get_by_name(metadata_type.name)
def update(self,
product: DatasetType,
allow_unsafe_updates=False,
allow_table_lock=False):
"""
Update a product. Unsafe changes will throw a ValueError by default.
(An unsafe change is anything that may potentially make the product
incompatible with existing datasets of that type)
:param DatasetType product: Product to update
:param bool allow_unsafe_updates: Allow unsafe changes. Use with caution.
:param allow_table_lock:
Allow an exclusive lock to be taken on the table while creating the indexes.
This will halt other user's requests until completed.
If false, creation will be slower and cannot be done in a transaction.
:rtype: DatasetType
"""
can_update, safe_changes, unsafe_changes = self.can_update(
product, allow_unsafe_updates)
if not safe_changes and not unsafe_changes:
_LOG.info("No changes detected for product %s", product.name)
return self.get_by_name(product.name)
if not can_update:
raise ValueError(f"Unsafe changes in {product.name}: " +
(", ".join(
_readable_offset(offset)
for offset, _, _ in unsafe_changes)))
_LOG.info("Updating product %s", product.name)
existing = self.get_by_name(product.name)
changing_metadata_type = product.metadata_type.name != existing.metadata_type.name
if changing_metadata_type:
raise ValueError(
"Unsafe change: cannot (currently) switch metadata types for a product"
)
# TODO: Ask Jeremy WTF is going on here
# If the two metadata types declare the same field with different postgres expressions
# we can't safely change it.
# (Replacing the index would cause all existing users to have no effective index)
# for name, field in existing.metadata_type.dataset_fields.items():
# new_field = type_.metadata_type.dataset_fields.get(name)
# if new_field and (new_field.sql_expression != field.sql_expression):
# declare_unsafe(
# ('metadata_type',),
# 'Metadata type change results in incompatible index '
# 'for {!r} ({!r} → {!r})'.format(
# name, field.sql_expression, new_field.sql_expression
# )
# )
metadata_type = self.metadata_type_resource.get_by_name(
product.metadata_type.name)
# TODO: should we add metadata type here?
assert metadata_type, "TODO: should we add metadata type here?"
with self._db.connect() as conn:
conn.update_product(name=product.name,
metadata=product.metadata_doc,
metadata_type_id=metadata_type.id,
search_fields=metadata_type.dataset_fields,
definition=product.definition,
update_metadata_type=changing_metadata_type,
concurrently=not allow_table_lock)
self.get_by_name_unsafe.cache_clear()
self.get_unsafe.cache_clear()
return self.get_by_name(product.name)
