def get_metadata(
self, keys: Union[Sequence[str], Mapping[str,
str]]) -> Dict[str, Any]:
keys = tuple(self._key_dict_to_sequence(keys))
if len(keys) != len(self.key_names):
raise exceptions.InvalidKeyError(
f'Got wrong number of keys (available keys: {self.key_names})')
conn = self._connection
where_string = ' AND '.join([f'{key}=?' for key in self.key_names])
row = conn.execute(f'SELECT * FROM metadata WHERE {where_string}',
keys).fetchone()
if not row: # support lazy loading
filepath = self.get_datasets(dict(zip(self.key_names, keys)),
page=0,
limit=1)
if not filepath:
raise exceptions.DatasetNotFoundError(
f'No dataset found for given keys {keys}')
# compute metadata and try again
metadata = self.compute_metadata(
filepath[keys], max_shape=self.LAZY_LOADING_MAX_SHAPE)
self.insert(keys, filepath[keys], metadata=metadata)
row = conn.execute(f'SELECT * FROM metadata WHERE {where_string}',
keys).fetchone()
assert row
data_columns, _ = zip(*self._METADATA_COLUMNS)
encoded_data = {col: row[col] for col in self.key_names + data_columns}
return self._decode_data(encoded_data)
def get_metadata(
self, keys: Union[Sequence[str], Mapping[str,
str]]) -> Dict[str, Any]:
keys = tuple(self._key_dict_to_sequence(keys))
if len(keys) != len(self.key_names):
raise exceptions.InvalidKeyError('Got wrong number of keys')
cursor = self._cursor
where_string = ' AND '.join([f'{key}=%s' for key in self.key_names])
cursor.execute(f'SELECT * FROM metadata WHERE {where_string}', keys)
row = cursor.fetchone()
if not row: # support lazy loading
filepath = self.get_datasets(dict(zip(self.key_names, keys)))
if not filepath:
raise exceptions.DatasetNotFoundError(
f'No dataset found for given keys {keys}')
assert len(filepath) == 1
# compute metadata and try again
self.insert(keys, filepath[keys], skip_metadata=False)
cursor.execute(f'SELECT * FROM metadata WHERE {where_string}',
keys)
row = cursor.fetchone()
assert row
data_columns, _ = zip(*self._METADATA_COLUMNS)
encoded_data = {col: row[col] for col in self.key_names + data_columns}
return self._decode_data(encoded_data)
def get_metadata(self, keys: ExtendedKeysType) -> Dict[str, Any]:
"""Return all stored metadata for given keys.
Arguments:
keys: Keys of the requested dataset. Can either be given as a sequence of key values,
or as a mapping ``{key_name: key_value}``.
Returns:
A :class:`dict` with the values
- ``range``: global minimum and maximum value in dataset
- ``bounds``: physical bounds covered by dataset in latitude-longitude projection
- ``convex_hull``: GeoJSON shape specifying total data coverage in latitude-longitude
projection
- ``percentiles``: array of pre-computed percentiles from 1% through 99%
- ``mean``: global mean
- ``stdev``: global standard deviation
- ``metadata``: any additional client-relevant metadata
"""
keys = self._standardize_keys(keys)
metadata = self.meta_store.get_metadata(keys)
if metadata is None:
# metadata is not computed yet, trigger lazy loading
dataset = self.get_datasets(keys)
if not dataset:
raise exceptions.DatasetNotFoundError('No dataset found')
path = squeeze(dataset.values())
metadata = self.compute_metadata(
path, max_shape=self.LAZY_LOADING_MAX_SHAPE)
try:
self.insert(keys, path, metadata=metadata)
except exceptions.DatabaseNotWritableError as exc:
raise exceptions.DatabaseNotWritableError(
"Lazy loading requires a writable database") from exc
# ensure standardized/consistent output (types and floating point precision)
metadata = self.meta_store.get_metadata(keys)
assert metadata is not None
return metadata
def delete(self, keys: Union[Sequence[str], Mapping[str, str]]) -> None:
cursor = self._cursor
if len(keys) != len(self.key_names):
raise exceptions.InvalidKeyError(
f'Got wrong number of keys (available keys: {self.key_names})')
keys = self._key_dict_to_sequence(keys)
key_dict = dict(zip(self.key_names, keys))
if not self.get_datasets(key_dict):
raise exceptions.DatasetNotFoundError(
f'No dataset found with keys {keys}')
where_string = ' AND '.join([f'{key}=%s' for key in self.key_names])
cursor.execute(f'DELETE FROM datasets WHERE {where_string}', keys)
cursor.execute(f'DELETE FROM metadata WHERE {where_string}', keys)
def delete(self, keys: KeysType) -> None:
if not self.get_datasets(keys):
raise exceptions.DatasetNotFoundError(f'No dataset found with keys {keys}')
datasets_table = sqla.Table('datasets', self.sqla_metadata, autoload_with=self.sqla_engine)
metadata_table = sqla.Table('metadata', self.sqla_metadata, autoload_with=self.sqla_engine)
self.connection.execute(
datasets_table
.delete()
.where(*[datasets_table.c.get(column) == value for column, value in keys.items()])
)
self.connection.execute(
metadata_table
.delete()
.where(*[metadata_table.c.get(column) == value for column, value in keys.items()])
)