def add(self, type_):
"""
Add a Product.
:param datacube.model.DatasetType type_: Product to add
:rtype: datacube.model.DatasetType
"""
DatasetType.validate(type_.definition)
existing = self.get_by_name(type_.name)
if existing:
check_doc_unchanged(existing.definition,
jsonify_document(type_.definition),
'Metadata Type {}'.format(type_.name))
else:
metadata_type = self.metadata_type_resource.get_by_name(
type_.metadata_type.name)
if metadata_type is None:
_LOG.warning('Adding metadata_type "%s" as it doesn\'t exist.',
type_.metadata_type.name)
metadata_type = self.metadata_type_resource.add(
type_.metadata_type)
with self._db.connect() as connection:
connection.add_dataset_type(
name=type_.name,
metadata=type_.metadata_doc,
metadata_type_id=metadata_type.id,
search_fields=metadata_type.dataset_fields,
definition=type_.definition)
return self.get_by_name(type_.name)
def add(self, metadata_type, allow_table_lock=False):
"""
:param datacube.model.MetadataType metadata_type:
:param allow_table_lock:
Allow an exclusive lock to be taken on the table while creating the indexes.
This will halt other user's requests until completed.
If false, creation will be slightly slower and cannot be done in a transaction.
:rtype: datacube.model.MetadataType
"""
# This column duplication is getting out of hand:
MetadataType.validate(metadata_type.definition)
existing = self.get_by_name(metadata_type.name)
if existing:
# They've passed us the same one again. Make sure it matches what is stored.
check_doc_unchanged(existing.definition,
jsonify_document(metadata_type.definition),
'Metadata Type {}'.format(metadata_type.name))
else:
with self._db.connect() as connection:
connection.add_metadata_type(
name=metadata_type.name,
definition=metadata_type.definition,
concurrently=not allow_table_lock)
return self.get_by_name(metadata_type.name)
def test_check_doc_unchanged(v1, v2, expected):
if expected != []:
with pytest.raises(DocumentMismatchError):
check_doc_unchanged(v1, v2, 'name')
else:
# No Error Raised
check_doc_unchanged(v1, v2, 'name')
def add(self, dataset, sources_policy='verify', **kwargs):
"""
Add ``dataset`` to the index. No-op if it is already present.
:param Dataset dataset: dataset to add
:param str sources_policy: how should source datasets included in this dataset be handled:
``verify``
Verify that each source exists in the index, and that they are identical.
``ensure``
Add source datasets to the index if they doesn't exist.
``skip``
don't attempt to index source datasets (use when sources are already indexed)
:rtype: Dataset
"""
if 'skip_sources' in kwargs and kwargs['skip_sources']:
warnings.warn(
'"skip_sources" is deprecated, use "sources_policy=\'skip\'"',
DeprecationWarning)
sources_policy = 'skip'
self._add_sources(dataset, sources_policy)
sources_tmp = dataset.type.dataset_reader(dataset.metadata_doc).sources
dataset.type.dataset_reader(dataset.metadata_doc).sources = {}
try:
_LOG.info('Indexing %s', dataset.id)
if not self._try_add(dataset):
existing = self.get(dataset.id)
if existing:
check_doc_unchanged(existing.metadata_doc,
jsonify_document(dataset.metadata_doc),
'Dataset {}'.format(dataset.id))
# reinsert attempt? try updating the location
if dataset.uris:
try:
with self._db.begin() as transaction:
transaction.ensure_dataset_locations(
dataset.id, dataset.uris)
except DuplicateRecordError as e:
_LOG.warning(str(e))
finally:
dataset.type.dataset_reader(
dataset.metadata_doc).sources = sources_tmp
return dataset
def add(self, dataset, skip_sources=False):
"""
Ensure a dataset is in the index. Add it if not present.
:param datacube.model.Dataset dataset: dataset to add
:param bool skip_sources: don't attempt to index source datasets (use when sources are already indexed)
:rtype: datacube.model.Dataset
"""
if skip_sources:
for source in dataset.sources.values():
if not self.has(source.id):
self.add(source, skip_sources=skip_sources)
else:
for source in dataset.sources.values():
self.add(source, skip_sources=skip_sources)
sources_tmp = dataset.type.dataset_reader(dataset.metadata_doc).sources
dataset.type.dataset_reader(dataset.metadata_doc).sources = {}
try:
_LOG.info('Indexing %s', dataset.id)
if not self._try_add(dataset):
existing = self.get(dataset.id)
if existing:
check_doc_unchanged(existing.metadata_doc,
jsonify_document(dataset.metadata_doc),
'Dataset {}'.format(dataset.id))
# reinsert attempt? try updating the location
if dataset.local_uri:
try:
with self._db.connect() as connection:
connection.ensure_dataset_location(
dataset.id, dataset.local_uri)
except DuplicateRecordError as e:
_LOG.warning(str(e))
finally:
dataset.type.dataset_reader(
dataset.metadata_doc).sources = sources_tmp
return dataset
def add(self, dataset, skip_sources=False, sources_policy='verify'):
"""
Ensure a dataset is in the index. Add it if not present.
:param datacube.model.Dataset dataset: dataset to add
:param str sources_policy: one of 'verify' - verify the metadata, 'ensure' - add if doesn't exist, 'skip' - skip
:param bool skip_sources: don't attempt to index source datasets (use when sources are already indexed)
:rtype: datacube.model.Dataset
"""
if skip_sources:
warnings.warn('"skip_sources" is deprecated, use "sources_policy"',
DeprecationWarning)
sources_policy = 'skip'
self._add_sources(dataset, sources_policy)
sources_tmp = dataset.type.dataset_reader(dataset.metadata_doc).sources
dataset.type.dataset_reader(dataset.metadata_doc).sources = {}
try:
_LOG.info('Indexing %s', dataset.id)
if not self._try_add(dataset):
existing = self.get(dataset.id)
if existing:
check_doc_unchanged(existing.metadata_doc,
jsonify_document(dataset.metadata_doc),
'Dataset {}'.format(dataset.id))
# reinsert attempt? try updating the location
if dataset.local_uri:
try:
with self._db.connect() as connection:
connection.ensure_dataset_location(
dataset.id, dataset.local_uri)
except DuplicateRecordError as e:
_LOG.warning(str(e))
finally:
dataset.type.dataset_reader(
dataset.metadata_doc).sources = sources_tmp
return dataset
def add(self, type_, allow_table_lock=False):
"""
Add a Product.
:param allow_table_lock:
Allow an exclusive lock to be taken on the table while creating the indexes.
This will halt other user's requests until completed.
If false, creation will be slightly slower and cannot be done in a transaction.
:param datacube.model.DatasetType type_: Product to add
:rtype: datacube.model.DatasetType
"""
DatasetType.validate(type_.definition)
existing = self.get_by_name(type_.name)
if existing:
check_doc_unchanged(existing.definition,
jsonify_document(type_.definition),
'Metadata Type {}'.format(type_.name))
else:
metadata_type = self.metadata_type_resource.get_by_name(
type_.metadata_type.name)
if metadata_type is None:
_LOG.warning('Adding metadata_type "%s" as it doesn\'t exist.',
type_.metadata_type.name)
metadata_type = self.metadata_type_resource.add(
type_.metadata_type, allow_table_lock=allow_table_lock)
with self._db.connect() as connection:
connection.add_dataset_type(
name=type_.name,
metadata=type_.metadata_doc,
metadata_type_id=metadata_type.id,
search_fields=metadata_type.dataset_fields,
definition=type_.definition,
concurrently=not allow_table_lock,
)
return self.get_by_name(type_.name)
def test_more_check_doc_unchanged():
# No exception raised
check_doc_unchanged({'a': 1}, {'a': 1}, 'Letters')
with pytest.raises(DocumentMismatchError, message='Letters differs from stored (a: 1!=2)'):
check_doc_unchanged({'a': 1}, {'a': 2}, 'Letters')
with pytest.raises(DocumentMismatchError, message='Letters differs from stored (a.b: 1!=2)'):
check_doc_unchanged({'a': {'b': 1}}, {'a': {'b': 2}}, 'Letters')
