def add(self, definition, allow_table_lock=False):
"""
:type definition: dict
:param allow_table_lock:
Allow an exclusive lock to be taken on the table while creating the indexes.
This will halt other user's requests until completed.
If false, creation will be slightly slower and cannot be done in a transaction.
:rtype: datacube.model.MetadataType
"""
# This column duplication is getting out of hand:
MetadataType.validate(definition)
name = definition['name']
existing = self._db.get_metadata_type_by_name(name)
if existing:
# They've passed us the same one again. Make sure it matches what is stored.
# TODO: Support for adding/updating search fields?
check_doc_unchanged(
existing.definition,
definition,
'Metadata Type {}'.format(name)
)
else:
self._db.add_metadata_type(
name=name,
definition=definition,
concurrently=not allow_table_lock
)
return self.get_by_name(name)
def example_metadata_type():
return MetadataType(dict(name='eo',
dataset=dict(
id=['id'],
label=['ga_label'],
creation_time=['creation_dt'],
measurements=['image', 'bands'],
grid_spatial=['grid_spatial', 'projection'],
sources=['lineage', 'source_datasets'])),
dataset_search_fields={})
def _make(self, definition, id_=None):
"""
:param dict definition:
:param int id_:
:rtype: datacube.model.MetadataType
"""
return MetadataType(
definition,
dataset_search_fields=self._db.get_dataset_fields(definition),
id_=id_)
def can_update(self, metadata_type, allow_unsafe_updates=False):
"""
Check if metadata type can be updated. Return bool,safe_changes,unsafe_changes
Safe updates currently allow new search fields to be added, description to be changed.
:param datacube.model.MetadataType metadata_type: updated MetadataType
:param bool allow_unsafe_updates: Allow unsafe changes. Use with caution.
:rtype: bool,list[change],list[change]
"""
MetadataType.validate(metadata_type.definition)
existing = self.get_by_name(metadata_type.name)
if not existing:
raise ValueError('Unknown metadata type %s, cannot update – '
'did you intend to add it?' % metadata_type.name)
updates_allowed = {
('description', ):
changes.allow_any,
# You can add new fields safely but not modify existing ones.
(
'dataset', ):
changes.allow_extension,
('dataset', 'search_fields'):
changes.allow_extension
}
doc_changes = get_doc_changes(
existing.definition, jsonify_document(metadata_type.definition))
good_changes, bad_changes = changes.classify_changes(
doc_changes, updates_allowed)
for offset, old_val, new_val in good_changes:
_LOG.info("Safe change in %s from %r to %r",
_readable_offset(offset), old_val, new_val)
for offset, old_val, new_val in bad_changes:
_LOG.info("Unsafe change in %s from %r to %r",
_readable_offset(offset), old_val, new_val)
return allow_unsafe_updates or not bad_changes, good_changes, bad_changes
def _make(self, query_row):
"""
:rtype list[datacube.model.MetadataType]
"""
definition = query_row['definition']
dataset_ = definition['dataset']
return MetadataType(
query_row['name'],
dataset_,
dataset_search_fields=self._db.get_dataset_fields(query_row),
id_=query_row['id'])
def test_metadata_type():
m = MetadataType(
{
'name':
'eo',
'dataset':
dict(id=['id'],
label=['ga_label'],
creation_time=['creation_dt'],
measurements=['image', 'bands'],
sources=['lineage', 'source_datasets'],
format=['format', 'name'])
},
dataset_search_fields={})
assert 'eo' in str(m)
assert 'eo' in repr(m)
assert m.name == 'eo'
assert m.description is None
assert m.dataset_reader({}) is not None
# again but without dataset_search_fields
m = MetadataType(m.definition)
assert 'eo' in str(m)
assert 'eo' in repr(m)
assert m.name == 'eo'
assert m.description is None
assert m.dataset_reader({}) is not None
def mk_sample_product(name,
description='Sample',
measurements=['red', 'green', 'blue']):
eo_type = MetadataType(
{
'name':
'eo',
'description':
'Sample',
'dataset':
dict(
id=['id'],
label=['ga_label'],
creation_time=['creation_dt'],
measurements=['image', 'bands'],
sources=['lineage', 'source_datasets'],
format=['format', 'name'],
)
},
dataset_search_fields={})
common = dict(dtype='int16', nodata=-999, units='1', aliases=[])
def mk_measurement(m):
if isinstance(m, str):
return dict(name=m, **common)
if isinstance(m, tuple):
name, dtype, nodata = m
m = common.copy()
m.update(name=name, dtype=dtype, nodata=nodata)
return m
if isinstance(m, dict):
m_merged = common.copy()
m_merged.update(m)
return m_merged
assert False and 'Only support str|dict|(name, dtype, nodata)'
measurements = [mk_measurement(m) for m in measurements]
return DatasetType(
eo_type,
dict(name=name,
description=description,
metadata_type='eo',
metadata={},
measurements=measurements))
def test_without_lineage_sources():
def mk_sample(v):
return dict(lineage={'source_datasets': v, 'a': 'a', 'b': 'b'},
aa='aa',
bb=dict(bb='bb'))
spec = mk_sample_product('tt')
x = {'a': 1}
assert without_lineage_sources(x, spec) == x
assert without_lineage_sources(x, spec, inplace=True) == x
x = {'a': 1, 'lineage': {}}
assert without_lineage_sources(x, spec) == x
assert without_lineage_sources(x, spec, inplace=True) == x
x = mk_sample(1)
assert without_lineage_sources(x, spec) != x
assert x['lineage']['source_datasets'] == 1
x = mk_sample(2)
assert without_lineage_sources(x, spec, inplace=True) == x
assert x['lineage']['source_datasets'] == {}
assert mk_sample(10) != mk_sample({})
assert without_lineage_sources(mk_sample(10), spec) == mk_sample({})
assert without_lineage_sources(mk_sample(10), spec, inplace=True) == mk_sample({})
# check behaviour when `sources` is not defined for the type
no_sources_type = MetadataType({
'name': 'eo',
'description': 'Sample',
'dataset': dict(
id=['id'],
label=['ga_label'],
creation_time=['creation_dt'],
measurements=['image', 'bands'],
format=['format', 'name'],
)
}, dataset_search_fields={})
assert without_lineage_sources(mk_sample(10), no_sources_type) == mk_sample(10)
assert without_lineage_sources(mk_sample(10), no_sources_type, inplace=True) == mk_sample(10)
def mk_sample_eo(name='eo'):
eo_yaml = f"""
name: {name}
description: Sample
dataset:
id: ['id']
label: ['ga_label']
creation_time: ['creation_dt']
measurements: ['image', 'bands']
sources: ['lineage', 'source_datasets']
format: ['format', 'name']
grid_spatial: ['grid_spatial', 'projection']
search_fields:
time:
type: 'datetime-range'
min_offset: [['time']]
max_offset: [['time']]
"""
return MetadataType(parse_yaml(eo_yaml))
**rio_args) as dst:
dst.write(sample_data, 1)
return sample_geotiff, geobox, sample_data
return internal_make_sample_geotiff
_EXAMPLE_METADATA_TYPE = MetadataType(
{
'name':
'eo',
'dataset':
dict(
id=['id'],
label=['ga_label'],
creation_time=['creation_dt'],
measurements=['image', 'bands'],
sources=['lineage', 'source_datasets'],
format=['format', 'name'],
)
},
dataset_search_fields={})
_EXAMPLE_DATASET_TYPE = DatasetType(
_EXAMPLE_METADATA_TYPE, {
'name':
'ls5_nbar_scene',
'description':
"Landsat 5 NBAR 25 metre",
'metadata_type':
def eo3_metadata(eo3_metadata_file):
(_, doc), *_ = read_documents(eo3_metadata_file)
return MetadataType(doc)
def mk_sample_product(name,
description='Sample',
measurements=('red', 'green', 'blue'),
with_grid_spec=False,
storage=None):
if storage is None and with_grid_spec is True:
storage = {
'crs': 'EPSG:3577',
'resolution': {
'x': 25,
'y': -25
},
'tile_size': {
'x': 100000.0,
'y': 100000.0
}
}
eo_type = MetadataType(
{
'name':
'eo',
'description':
'Sample',
'dataset':
dict(
id=['id'],
label=['ga_label'],
creation_time=['creation_dt'],
measurements=['image', 'bands'],
sources=['lineage', 'source_datasets'],
format=['format', 'name'],
grid_spatial=['grid_spatial', 'projection'],
)
},
dataset_search_fields={
'time':
parse_search_field({
'type': 'datetime-range',
'min_offset': [['time']],
'max_offset': [['time']],
}),
})
common = dict(dtype='int16', nodata=-999, units='1', aliases=[])
def mk_measurement(m):
if isinstance(m, str):
return dict(name=m, **common)
elif isinstance(m, tuple):
name, dtype, nodata = m
m = common.copy()
m.update(name=name, dtype=dtype, nodata=nodata)
return m
elif isinstance(m, dict):
m_merged = common.copy()
m_merged.update(m)
return m_merged
else:
raise ValueError('Only support str|dict|(name, dtype, nodata)')
measurements = [mk_measurement(m) for m in measurements]
definition = dict(name=name,
description=description,
metadata_type='eo',
metadata={},
measurements=measurements)
if storage is not None:
definition['storage'] = storage
return DatasetType(eo_type, definition)
},
'lineage': {
'source_datasets': {}
}
}
}
}
}
}
}
}
_EXAMPLE_METADATA_TYPE = MetadataType(
'eo',
dict(id=['id'],
label=['ga_label'],
creation_time=['creation_dt'],
measurements=['image', 'bands'],
sources=['lineage', 'source_datasets']),
dataset_search_fields={})
_EXAMPLE_DATASET_TYPE = DatasetType(_EXAMPLE_METADATA_TYPE, {
'name': 'eo',
'description': "",
'metadata_type': 'eo',
'metadata': {}
})
def _build_dataset(doc):
sources = {
name: _build_dataset(src)
