def can_update(self, dataset, updates_allowed=None): """ Check if dataset can be updated. Return bool,safe_changes,unsafe_changes :param datacube.model.Dataset dataset: Dataset to update :param dict updates_allowed: Allowed updates :rtype: bool,list[change],list[change] """ existing = self.get(dataset.id, include_sources=True) if not existing: raise ValueError( 'Unknown dataset %s, cannot update – did you intend to add it?' % dataset.id) if dataset.type.name != existing.type.name: raise ValueError( 'Changing product is not supported. From %s to %s in %s' % (existing.type.name, dataset.type.name, dataset.id)) # TODO: figure out (un)safe changes from metadata type? allowed = { # can always add more metadata tuple(): changes.allow_extension, } allowed.update(updates_allowed or {}) doc_changes = get_doc_changes(existing.metadata_doc, jsonify_document(dataset.metadata_doc)) good_changes, bad_changes = changes.classify_changes( doc_changes, allowed) return not bad_changes, good_changes, bad_changes
def can_update(self, product, allow_unsafe_updates=False): """ Check if product can be updated. Return bool,safe_changes,unsafe_changes (An unsafe change is anything that may potentially make the product incompatible with existing datasets of that type) :param datacube.model.DatasetType product: Product to update :param bool allow_unsafe_updates: Allow unsafe changes. Use with caution. :rtype: bool,list[change],list[change] """ DatasetType.validate(product.definition) existing = self.get_by_name(product.name) if not existing: raise ValueError('Unknown product %s, cannot update – did you intend to add it?' % product.name) updates_allowed = { ('description',): changes.allow_any, ('metadata_type',): changes.allow_any, # You can safely make the match rules looser but not tighter. # Tightening them could exclude datasets already matched to the product. # (which would make search results wrong) ('metadata',): changes.allow_truncation } doc_changes = get_doc_changes(existing.definition, jsonify_document(product.definition)) good_changes, bad_changes = changes.classify_changes(doc_changes, updates_allowed) return allow_unsafe_updates or not bad_changes, good_changes, bad_changes
def can_update(self, metadata_type, allow_unsafe_updates=False): """ Check if metadata type can be updated. Return bool,safe_changes,unsafe_changes Safe updates currently allow new search fields to be added, description to be changed. :param datacube.model.MetadataType metadata_type: updated MetadataType :param bool allow_unsafe_updates: Allow unsafe changes. Use with caution. :rtype: bool,list[change],list[change] """ MetadataType.validate(metadata_type.definition) existing = self.get_by_name(metadata_type.name) if not existing: raise ValueError( 'Unknown metadata type %s, cannot update – did you intend to add it?' % metadata_type.name) updates_allowed = { ('description', ): changes.allow_any, # You can add new fields safely but not modify existing ones. ( 'dataset', ): changes.allow_extension, ('dataset', 'search_fields'): changes.allow_extension } doc_changes = get_doc_changes( existing.definition, jsonify_document(metadata_type.definition)) good_changes, bad_changes = changes.classify_changes( doc_changes, updates_allowed) return allow_unsafe_updates or not bad_changes, good_changes, bad_changes
def can_update(self, product, allow_unsafe_updates=False): """ Check if product can be updated. Return bool,safe_changes,unsafe_changes (An unsafe change is anything that may potentially make the product incompatible with existing datasets of that type) :param DatasetType product: Product to update :param bool allow_unsafe_updates: Allow unsafe changes. Use with caution. :rtype: bool,list[change],list[change] """ DatasetType.validate(product.definition) existing = self.get_by_name(product.name) if not existing: raise ValueError( 'Unknown product %s, cannot update – did you intend to add it?' % product.name) updates_allowed = { ('description', ): changes.allow_any, ('license', ): changes.allow_any, ('metadata_type', ): changes.allow_any, # You can safely make the match rules looser but not tighter. # Tightening them could exclude datasets already matched to the product. # (which would make search results wrong) ( 'metadata', ): changes.allow_truncation, # Some old storage fields should not be in the product definition any more: allow removal. ('storage', 'chunking'): changes.allow_removal, ('storage', 'driver'): changes.allow_removal, ('storage', 'dimension_order'): changes.allow_removal, } doc_changes = get_doc_changes(existing.definition, jsonify_document(product.definition)) good_changes, bad_changes = changes.classify_changes( doc_changes, updates_allowed) for offset, old_val, new_val in good_changes: _LOG.info("Safe change in %s from %r to %r", _readable_offset(offset), old_val, new_val) for offset, old_val, new_val in bad_changes: _LOG.warning("Unsafe change in %s from %r to %r", _readable_offset(offset), old_val, new_val) return allow_unsafe_updates or not bad_changes, good_changes, bad_changes
def check_consistent(a, b): diffs = get_doc_changes(a, b) if len(diffs) == 0: return True, None def render_diff(offset, a, b): offset = '.'.join(map(str, offset)) return '{}: {!r}!={!r}'.format(offset, a, b) return False, ", ".join([render_diff(offset, a, b) for offset, a, b in diffs])
def make_output_type(index, config): source_type = index.products.get_by_name(config['source_type']) if not source_type: click.echo("Source DatasetType %s does not exist" % config['source_type']) click.get_current_context().exit(1) output_type = morph_dataset_type(source_type, config) _LOG.info('Created DatasetType %s', output_type.name) # Some storage fields should not be in the product definition, and should be removed. # To handle backwards compatibility for now, ignore them with custom rules, # rather than using the default checks done by index.products.add existing = index.products.get_by_name(output_type.name) backwards_compatible_fields = True if existing and backwards_compatible_fields: updates_allowed = { ('description', ): changes.allow_any, ('metadata_type', ): changes.allow_any, ('storage', 'chunking'): changes.allow_any, ('storage', 'driver'): changes.allow_any, ('storage', 'dimension_order'): changes.allow_any, ('metadata', ): changes.allow_truncation } doc_changes = changes.get_doc_changes( output_type.definition, datacube.utils.jsonify_document(existing.definition)) good_changes, bad_changes = changes.classify_changes( doc_changes, updates_allowed) if bad_changes: raise ValueError('{} differs from stored ({})'.format( output_type.name, ', '.join([ '{}: {!r}!={!r}'.format('.'.join(offset), v1, v2) for offset, v1, v2 in bad_changes ]))) output_type = index.products.update(output_type, allow_unsafe_updates=True) else: output_type = index.products.add(output_type) return source_type, output_type
def test_get_doc_changes_w_baseprefix(): rval = get_doc_changes({}, None, base_prefix=('a', )) assert rval == [(('a', ), {}, None)]
def test_get_doc_changes(v1, v2, expected): rval = get_doc_changes(v1, v2) assert rval == expected