def _store(self, met_data: METData) -> None:
"""Store MET data within a Datasource
Here we do some retrieve on the request JSON to
make the metadata more easily searchable and of a similar
format to Datasources used in other modules of OpenGHG.
Args:
met_data: Dataset
Returns:
None
"""
from openghg.store.base import Datasource
metadata = met_data.metadata
datasource = Datasource()
datasource.add_data(metadata=metadata,
data=met_data.data,
data_type="met")
datasource.save()
date_str = f"{metadata['start_date']}_{metadata['end_date']}"
name = "_".join((metadata["site"], metadata["network"], date_str))
self._datasource_uuids[datasource.uuid()] = name
# Write this updated object back to the object store
self.save()
def test_exists():
d = Datasource()
d.save()
exists = Datasource.exists(datasource_id=d.uuid())
assert exists == True
def assign_data(
data_dict: Dict,
lookup_results: Dict,
overwrite: bool,
data_type: str = "timeseries",
) -> Dict[str, str]:
"""Assign data to a Datasource. This will either create a new Datasource
Create or get an existing Datasource for each gas in the file
Args:
data_dict: Dictionary containing data and metadata for species
lookup_results: Dictionary of lookup results]
overwrite: If True overwrite current data stored
Returns:
dict: Dictionary of UUIDs of Datasources data has been assigned to keyed by species name
"""
from openghg.store.base import Datasource
uuids = {}
for key in data_dict:
metadata = data_dict[key]["metadata"]
data = data_dict[key]["data"]
# Our lookup results and gas data have the same keys
uuid = lookup_results[key]
# TODO - Could this be done somewhere else? It doesn't feel quite right it
# being here
# Add the read metadata to the Dataset attributes being careful
# not to overwrite any attributes that are already there
to_add = {k: v for k, v in metadata.items() if k not in data.attrs}
data.attrs.update(to_add)
# If we have a UUID for this Datasource load the existing object
# from the object store
if uuid is False:
datasource = Datasource()
else:
datasource = Datasource.load(uuid=uuid)
# Add the dataframe to the datasource
datasource.add_data(metadata=metadata,
data=data,
overwrite=overwrite,
data_type=data_type)
# Save Datasource to object store
datasource.save()
uuids[key] = datasource.uuid()
return uuids
def test_shallow_then_load_data(data):
metadata = data["ch4"]["metadata"]
data = data["ch4"]["data"]
d = Datasource()
d.add_data(metadata=metadata, data=data, data_type="timeseries")
d.save()
new_d = Datasource.load(uuid=d.uuid(), shallow=True)
assert not new_d._data
ds_data = new_d.data()
assert ds_data
ch4_data = ds_data["2014-01-30-11:12:30+00:00_2014-11-30-11:23:30+00:00"]
assert ch4_data.time[0] == pd.Timestamp("2014-01-30-11:12:30")
