def _store(self, met_data: METData) -> None: """Store MET data within a Datasource Here we do some retrieve on the request JSON to make the metadata more easily searchable and of a similar format to Datasources used in other modules of OpenGHG. Args: met_data: Dataset Returns: None """ from openghg.store.base import Datasource metadata = met_data.metadata datasource = Datasource() datasource.add_data(metadata=metadata, data=met_data.data, data_type="met") datasource.save() date_str = f"{metadata['start_date']}_{metadata['end_date']}" name = "_".join((metadata["site"], metadata["network"], date_str)) self._datasource_uuids[datasource.uuid()] = name # Write this updated object back to the object store self.save()
def test_exists(): d = Datasource() d.save() exists = Datasource.exists(datasource_id=d.uuid()) assert exists == True
def assign_data( data_dict: Dict, lookup_results: Dict, overwrite: bool, data_type: str = "timeseries", ) -> Dict[str, str]: """Assign data to a Datasource. This will either create a new Datasource Create or get an existing Datasource for each gas in the file Args: data_dict: Dictionary containing data and metadata for species lookup_results: Dictionary of lookup results] overwrite: If True overwrite current data stored Returns: dict: Dictionary of UUIDs of Datasources data has been assigned to keyed by species name """ from openghg.store.base import Datasource uuids = {} for key in data_dict: metadata = data_dict[key]["metadata"] data = data_dict[key]["data"] # Our lookup results and gas data have the same keys uuid = lookup_results[key] # TODO - Could this be done somewhere else? It doesn't feel quite right it # being here # Add the read metadata to the Dataset attributes being careful # not to overwrite any attributes that are already there to_add = {k: v for k, v in metadata.items() if k not in data.attrs} data.attrs.update(to_add) # If we have a UUID for this Datasource load the existing object # from the object store if uuid is False: datasource = Datasource() else: datasource = Datasource.load(uuid=uuid) # Add the dataframe to the datasource datasource.add_data(metadata=metadata, data=data, overwrite=overwrite, data_type=data_type) # Save Datasource to object store datasource.save() uuids[key] = datasource.uuid() return uuids
def test_shallow_then_load_data(data): metadata = data["ch4"]["metadata"] data = data["ch4"]["data"] d = Datasource() d.add_data(metadata=metadata, data=data, data_type="timeseries") d.save() new_d = Datasource.load(uuid=d.uuid(), shallow=True) assert not new_d._data ds_data = new_d.data() assert ds_data ch4_data = ds_data["2014-01-30-11:12:30+00:00_2014-11-30-11:23:30+00:00"] assert ch4_data.time[0] == pd.Timestamp("2014-01-30-11:12:30")