def read_file(
filepath: Union[str, Path],
species: str,
source: str,
domain: str,
date: str,
high_time_resolution: Optional[bool] = False,
period: Optional[str] = None,
overwrite: bool = False,
) -> Dict:
"""Read emissions file
Args:
filepath: Path of emissions file
species: Species name
domain: Emissions domain
source: Emissions source
high_time_resolution: If this is a high resolution file
period: Period of measurements, if not passed this is inferred from the time coords
overwrite: Should this data overwrite currently stored data.
"""
from collections import defaultdict
from xarray import open_dataset
from openghg.store import assign_data
from openghg.util import (
clean_string,
hash_file,
timestamp_tzaware,
timestamp_now,
)
species = clean_string(species)
source = clean_string(source)
domain = clean_string(domain)
date = clean_string(date)
filepath = Path(filepath)
em_store = Emissions.load()
file_hash = hash_file(filepath=filepath)
if file_hash in em_store._file_hashes and not overwrite:
raise ValueError(
f"This file has been uploaded previously with the filename : {em_store._file_hashes[file_hash]}."
)
em_data = open_dataset(filepath)
# Some attributes are numpy types we can't serialise to JSON so convert them
# to their native types here
attrs = {}
for key, value in em_data.attrs.items():
try:
attrs[key] = value.item()
except AttributeError:
attrs[key] = value
author_name = "OpenGHG Cloud"
em_data.attrs["author"] = author_name
metadata = {}
metadata.update(attrs)
metadata["species"] = species
metadata["domain"] = domain
metadata["source"] = source
metadata["date"] = date
metadata["author"] = author_name
metadata["processed"] = str(timestamp_now())
metadata["start_date"] = str(timestamp_tzaware(em_data.time[0].values))
metadata["end_date"] = str(timestamp_tzaware(em_data.time[-1].values))
metadata["max_longitude"] = round(float(em_data["lon"].max()), 5)
metadata["min_longitude"] = round(float(em_data["lon"].min()), 5)
metadata["max_latitude"] = round(float(em_data["lat"].max()), 5)
metadata["min_latitude"] = round(float(em_data["lat"].min()), 5)
metadata["time_resolution"] = "high" if high_time_resolution else "standard"
if period is not None:
metadata["time_period"] = period
key = "_".join((species, source, domain, date))
emissions_data: DefaultDict[str, Dict[str, Union[Dict, Dataset]]] = defaultdict(dict)
emissions_data[key]["data"] = em_data
emissions_data[key]["metadata"] = metadata
keyed_metadata = {key: metadata}
lookup_results = em_store.datasource_lookup(metadata=keyed_metadata)
data_type = "emissions"
datasource_uuids = assign_data(
data_dict=emissions_data,
lookup_results=lookup_results,
overwrite=overwrite,
data_type=data_type,
)
em_store.add_datasources(datasource_uuids=datasource_uuids, metadata=keyed_metadata)
# Record the file hash in case we see this file again
em_store._file_hashes[file_hash] = filepath.name
em_store.save()
return datasource_uuids
def read_file(
filepath: Union[str, Path],
model: str,
species: str,
start_date: Optional[str] = None,
end_date: Optional[str] = None,
setup: Optional[str] = None,
overwrite: bool = False,
) -> Dict:
"""Read Eulerian model output
Args:
filepath: Path of Eulerian model species output
model: Eulerian model name
species: Species name
start_date: Start date (inclusive) associated with model run
end_date: End date (exclusive) associated with model run
setup: Additional setup details for run
overwrite: Should this data overwrite currently stored data.
"""
# TODO: As written, this currently includes some light assumptions that we're dealing with GEOSChem SpeciesConc format.
# May need to split out into multiple modules (like with ObsSurface) or into separate retrieve functions as needed.
from collections import defaultdict
from openghg.util import (
clean_string,
hash_file,
timestamp_now,
timestamp_tzaware,
)
from openghg.store import assign_data
from xarray import open_dataset
from pandas import Timestamp as pd_Timestamp
model = clean_string(model)
species = clean_string(species)
start_date = clean_string(start_date)
end_date = clean_string(end_date)
setup = clean_string(setup)
filepath = Path(filepath)
em_store = EulerianModel.load()
file_hash = hash_file(filepath=filepath)
if file_hash in em_store._file_hashes and not overwrite:
raise ValueError(
f"This file has been uploaded previously with the filename : {em_store._file_hashes[file_hash]}."
)
em_data = open_dataset(filepath)
# Check necessary 4D coordinates are present and rename if necessary (for consistency)
check_coords = {
"time": ["time"],
"lat": ["lat", "latitude"],
"lon": ["lon", "longitude"],
"lev": ["lev", "level", "layer", "sigma_level"],
}
for name, coord_options in check_coords.items():
for coord in coord_options:
if coord in em_data.coords:
break
else:
raise ValueError(
"Input data must contain one of '{coord_options}' co-ordinate"
)
if name != coord:
print("Renaming co-ordinate '{coord}' to '{name}'")
em_data = em_data.rename({coord: name})
attrs = em_data.attrs
# author_name = "OpenGHG Cloud"
# em_data.attrs["author"] = author_name
metadata = {}
metadata.update(attrs)
metadata["model"] = model
metadata["species"] = species
metadata["processed"] = str(timestamp_now())
if start_date is None:
if len(em_data["time"]) > 1:
start_date = str(timestamp_tzaware(em_data.time[0].values))
else:
try:
start_date = attrs["simulation_start_date_and_time"]
except KeyError:
raise Exception(
"Unable to derive start_date from data, please provide as an input."
)
else:
start_date = timestamp_tzaware(start_date)
start_date = str(start_date)
if end_date is None:
if len(em_data["time"]) > 1:
end_date = str(timestamp_tzaware(em_data.time[-1].values))
else:
try:
end_date = attrs["simulation_end_date_and_time"]
except KeyError:
raise Exception(
"Unable to derive `end_date` from data, please provide as an input."
)
else:
end_date = timestamp_tzaware(end_date)
end_date = str(end_date)
date = str(pd_Timestamp(start_date).date())
metadata["date"] = date
metadata["start_date"] = start_date
metadata["end_date"] = end_date
metadata["max_longitude"] = round(float(em_data["lon"].max()), 5)
metadata["min_longitude"] = round(float(em_data["lon"].min()), 5)
metadata["max_latitude"] = round(float(em_data["lat"].max()), 5)
metadata["min_latitude"] = round(float(em_data["lat"].min()), 5)
history = metadata.get("history")
if history is None:
history = ""
metadata[
"history"] = history + f" {str(timestamp_now())} Processed onto OpenGHG cloud"
key = "_".join((model, species, date))
model_data: DefaultDict[str, Dict[str,
Union[Dict,
Dataset]]] = defaultdict(dict)
model_data[key]["data"] = em_data
model_data[key]["metadata"] = metadata
keyed_metadata = {key: metadata}
lookup_results = em_store.datasource_lookup(metadata=keyed_metadata)
data_type = "eulerian_model"
datasource_uuids = assign_data(
data_dict=model_data,
lookup_results=lookup_results,
overwrite=overwrite,
data_type=data_type,
)
em_store.add_datasources(datasource_uuids=datasource_uuids,
metadata=keyed_metadata)
# Record the file hash in case we see this file again
em_store._file_hashes[file_hash] = filepath.name
em_store.save()
return datasource_uuids
def read_file(
filepath: Union[str, Path],
site: str,
height: str,
domain: str,
model: str,
metmodel: Optional[str] = None,
species: Optional[str] = None,
network: Optional[str] = None,
retrieve_met: bool = False,
overwrite: bool = False,
high_res: bool = False,
# model_params: Optional[Dict] = None,
) -> Dict[str, str]:
"""Reads footprints data files and returns the UUIDS of the Datasources
the processed data has been assigned to
Args:
filepath: Path of file to load
site: Site name
network: Network name
height: Height above ground level in metres
domain: Domain of footprints
model_params: Model run parameters
retrieve_met: Whether to also download meterological data for this footprints area
overwrite: Overwrite any currently stored data
Returns:
dict: UUIDs of Datasources data has been assigned to
"""
from collections import defaultdict
from xarray import open_dataset
from openghg.util import (
hash_file,
timestamp_tzaware,
timestamp_now,
clean_string,
)
from openghg.store import assign_data
filepath = Path(filepath)
site = clean_string(site)
network = clean_string(network)
height = clean_string(height)
domain = clean_string(domain)
fp = Footprints.load()
file_hash = hash_file(filepath=filepath)
if file_hash in fp._file_hashes and not overwrite:
raise ValueError(
f"This file has been uploaded previously with the filename : {fp._file_hashes[file_hash]}."
)
fp_data = open_dataset(filepath)
# Need to read the metadata from the footprints and then store it
# Do we need to chunk the footprints / will a Datasource store it correctly?
metadata: Dict[str, Union[str, float, List[float]]] = {}
metadata["data_type"] = "footprints"
metadata["site"] = site
metadata["height"] = height
metadata["domain"] = domain
metadata["model"] = model
if species is not None:
metadata["species"] = clean_string(species)
if network is not None:
metadata["network"] = clean_string(network)
if metmodel is not None:
metadata["metmodel"] = clean_string(metmodel)
metadata["start_date"] = str(timestamp_tzaware(fp_data.time[0].values))
metadata["end_date"] = str(timestamp_tzaware(fp_data.time[-1].values))
metadata["max_longitude"] = round(float(fp_data["lon"].max()), 5)
metadata["min_longitude"] = round(float(fp_data["lon"].min()), 5)
metadata["max_latitude"] = round(float(fp_data["lat"].max()), 5)
metadata["min_latitude"] = round(float(fp_data["lat"].min()), 5)
metadata["time_resolution"] = "standard_time_resolution"
# If it's a high resolution footprints file we'll have two sets of lat/long values
if high_res:
try:
metadata["max_longitude_high"] = round(
float(fp_data["lon_high"].max()), 5)
metadata["min_longitude_high"] = round(
float(fp_data["lon_high"].min()), 5)
metadata["max_latitude_high"] = round(
float(fp_data["lat_high"].max()), 5)
metadata["min_latitude_high"] = round(
float(fp_data["lat_high"].min()), 5)
metadata["time_resolution"] = "high_time_resolution"
except KeyError:
raise KeyError("Unable to find lat_high or lon_high data.")
metadata["heights"] = [float(h) for h in fp_data.height.values]
# Do we also need to save all the variables we have available in this footprints?
metadata["variables"] = list(fp_data.keys())
# if model_params is not None:
# metadata["model_parameters"] = model_params
# Set the attributes of this Dataset
fp_data.attrs = {
"author": "OpenGHG Cloud",
"processed": str(timestamp_now())
}
# This might seem longwinded now but will help when we want to read
# more than one footprints at a time
key = "_".join((site, domain, model, height))
footprint_data: DefaultDict[str,
Dict[str,
Union[Dict,
Dataset]]] = defaultdict(dict)
footprint_data[key]["data"] = fp_data
footprint_data[key]["metadata"] = metadata
# This will be removed when we process multiple files
keyed_metadata = {key: metadata}
lookup_results = fp.datasource_lookup(metadata=keyed_metadata)
data_type = "footprints"
datasource_uuids: Dict[str, str] = assign_data(
data_dict=footprint_data,
lookup_results=lookup_results,
overwrite=overwrite,
data_type=data_type,
)
fp.add_datasources(datasource_uuids=datasource_uuids,
metadata=keyed_metadata)
# Record the file hash in case we see this file again
fp._file_hashes[file_hash] = filepath.name
fp.save()
return datasource_uuids
def get_obs_surface(
site: str,
species: str,
inlet: str = None,
start_date: Optional[Union[str, Timestamp]] = None,
end_date: Optional[Union[str, Timestamp]] = None,
average: Optional[str] = None,
network: Optional[str] = None,
instrument: Optional[str] = None,
calibration_scale: Optional[str] = None,
keep_missing: Optional[bool] = False,
skip_ranking: Optional[bool] = False,
) -> ObsData:
"""Get measurements from one site.
Args:
site: Site of interest e.g. MHD for the Mace Head site.
species: Species identifier e.g. ch4 for methane.
start_date: Output start date in a format that Pandas can interpret
end_date: Output end date in a format that Pandas can interpret
inlet: Inlet label
average: Averaging period for each dataset. Each value should be a string of
the form e.g. "2H", "30min" (should match pandas offset aliases format).
keep_missing: Keep missing data points or drop them.
network: Network for the site/instrument (must match number of sites).
instrument: Specific instrument for the site (must match number of sites).
calibration_scale: Convert to this calibration scale
Returns:
ObsData: ObsData object
"""
from pandas import Timestamp, Timedelta
import numpy as np
from xarray import concat as xr_concat
from openghg.retrieve import search
from openghg.store import recombine_datasets
from openghg.util import clean_string, load_json, timestamp_tzaware
site_info = load_json(filename="acrg_site_info.json")
site = site.upper()
if site not in site_info:
raise ValueError(
f"No site called {site}, please enter a valid site name.")
# Find the correct synonym for the passed species
species = clean_string(_synonyms(species))
# Get the observation data
obs_results = search(
site=site,
species=species,
inlet=inlet,
start_date=start_date,
end_date=end_date,
instrument=instrument,
find_all=True,
skip_ranking=skip_ranking,
)
if not obs_results:
raise ValueError(f"Unable to find results for {species} at {site}")
# TODO - for some reason mypy doesn't pick up the ObsData being returned here, look into this
# GJ - 2021-07-19
retrieved_data: ObsData = obs_results.retrieve(site=site,
species=species,
inlet=inlet) # type: ignore
data = retrieved_data.data
if data.attrs["inlet"] == "multiple":
data.attrs["inlet_height_magl"] = "multiple"
retrieved_data.metadata["inlet"] = "multiple"
if start_date is not None and end_date is not None:
start_date_tzaware = timestamp_tzaware(start_date)
end_date_tzaware = timestamp_tzaware(end_date)
end_date_tzaware_exclusive = end_date_tzaware - Timedelta(
1, unit="nanosecond"
) # Deduct 1 ns to make the end day (date) exclusive.
# Slice the data to only cover the dates we're interested in
data = data.sel(
time=slice(start_date_tzaware, end_date_tzaware_exclusive))
try:
start_date_data = timestamp_tzaware(data.time[0].values)
end_date_data = timestamp_tzaware(data.time[-1].values)
except AttributeError:
raise AttributeError(
"This dataset does not have a time attribute, unable to read date range"
)
if average is not None:
# GJ - 2021-03-09
# TODO - check by RT
# # Average the Dataset over a given period
# if keep_missing is True:
# # Create a dataset with one element and NaNs to prepend or append
# ds_single_element = data[{"time": 0}]
# for v in ds_single_element.variables:
# if v != "time":
# ds_single_element[v].values = np.nan
# ds_concat = []
# # Pad with an empty entry at the start date
# if timestamp_tzaware(data.time.min()) > start_date:
# ds_single_element_start = ds_single_element.copy()
# ds_single_element_start.time.values = Timestamp(start_date)
# ds_concat.append(ds_single_element_start)
# ds_concat.append(data)
# # Pad with an empty entry at the end date
# if data.time.max() < Timestamp(end_date):
# ds_single_element_end = ds_single_element.copy()
# ds_single_element_end.time.values = Timestamp(end_date) - Timedelta("1ns")
# ds_concat.append(ds_single_element_end)
# data = xr_concat(ds_concat, dim="time")
# # Now sort to get everything in the right order
# data = data.sortby("time")
# First do a mean resample on all variables
ds_resampled = data.resample(time=average).mean(skipna=False,
keep_attrs=True)
# keep_attrs doesn't seem to work for some reason, so manually copy
ds_resampled.attrs = data.attrs.copy()
average_in_seconds = Timedelta(average).total_seconds()
ds_resampled.attrs["averaged_period"] = average_in_seconds
ds_resampled.attrs["averaged_period_str"] = average
# For some variables, need a different type of resampling
data_variables: List[str] = [str(v) for v in data.variables]
for var in data_variables:
if "repeatability" in var:
ds_resampled[var] = (np.sqrt(
(data[var]**2).resample(time=average).sum()) /
data[var].resample(time=average).count())
# Copy over some attributes
if "long_name" in data[var].attrs:
ds_resampled[var].attrs["long_name"] = data[var].attrs[
"long_name"]
if "units" in data[var].attrs:
ds_resampled[var].attrs["units"] = data[var].attrs["units"]
# Create a new variability variable, containing the standard deviation within the resampling period
ds_resampled[f"{species}_variability"] = (data[species].resample(
time=average).std(skipna=False, keep_attrs=True))
# If there are any periods where only one measurement was resampled, just use the median variability
ds_resampled[f"{species}_variability"][
ds_resampled[f"{species}_variability"] ==
0.0] = ds_resampled[f"{species}_variability"].median()
# Create attributes for variability variable
ds_resampled[f"{species}_variability"].attrs[
"long_name"] = f"{data.attrs['long_name']}_variability"
ds_resampled[f"{species}_variability"].attrs["units"] = data[
species].attrs["units"]
# Resampling may introduce NaNs, so remove, if not keep_missing
if keep_missing is False:
ds_resampled = ds_resampled.dropna(dim="time")
data = ds_resampled
# Rename variables
rename: Dict[str, str] = {}
data_variables = [str(v) for v in data.variables]
for var in data_variables:
if var.lower() == species.lower():
rename[var] = "mf"
if "repeatability" in var:
rename[var] = "mf_repeatability"
if "variability" in var:
rename[var] = "mf_variability"
if "number_of_observations" in var:
rename[var] = "mf_number_of_observations"
if "status_flag" in var:
rename[var] = "status_flag"
if "integration_flag" in var:
rename[var] = "integration_flag"
data = data.rename_vars(rename) # type: ignore
data.attrs["species"] = species
if "calibration_scale" in data.attrs:
data.attrs["scale"] = data.attrs.pop("calibration_scale")
if calibration_scale is not None:
data = _scale_convert(data, species, calibration_scale)
metadata = retrieved_data.metadata
metadata.update(data.attrs)
obs_data = ObsData(data=data, metadata=metadata)
# It doesn't make sense to do this now as we've only got a single Dataset
# # Now check if the units match for each of the observation Datasets
# units = set((f.data.mf.attrs["units"] for f in obs_files))
# scales = set((f.data.attrs["scale"] for f in obs_files))
# if len(units) > 1:
# raise ValueError(
# f"Units do not match for these observation Datasets {[(f.mf.attrs['station_long_name'],f.attrs['units']) for f in obs_files]}"
# )
# if len(scales) > 1:
# print(
# f"Scales do not match for these observation Datasets {[(f.mf.attrs['station_long_name'],f.attrs['units']) for f in obs_files]}"
# )
# print("Suggestion: set calibration_scale to convert scales")
return obs_data
def retrieve(
self, site: str = None, species: str = None, inlet: str = None
) -> Union[Dict[str, ObsData], ObsData]:
"""Retrieve some or all of the data found in the object store.
Args:
site: Three letter site code
species: Species name
Returns:
ObsData or dict
"""
site = clean_string(site)
species = clean_string(species)
inlet = clean_string(inlet)
# If inlet is not specified, check if this is unambiguous
# If so, set inlet to be the only value and continue.
if inlet is None:
try:
potential_inlets = self.results[site][species].keys()
except KeyError:
pass
else:
if len(potential_inlets) == 1:
inlet = list(potential_inlets)[0]
if self.ranked_data:
if all((site, species, inlet)):
# TODO - how to do this in a cleaner way?
site = str(site)
species = str(species)
inlet = str(inlet)
return self._create_obsdata(site=site, species=species, inlet=inlet)
results = {}
if site is not None and species is not None:
try:
_ = self.results[site][species]["keys"]
except KeyError:
raise KeyError(f"Unable to find data keys for {species} at {site}.")
return self._create_obsdata(site=site, species=species)
# Get the data for all the species at that site
if site is not None and species is None:
for sp in self.results[site]:
key = "_".join((site, sp))
results[key] = self._create_obsdata(site=site, species=sp)
return results
# Get the data for all the species at that site
if site is None and species is not None:
for a_site in self.results:
key = "_".join((a_site, species))
try:
results[key] = self._create_obsdata(site=a_site, species=species)
except KeyError:
pass
return results
for a_site, species_list in self.results.items():
for sp in species_list:
key = "_".join((a_site, sp))
results[key] = self._create_obsdata(site=a_site, species=sp)
return results
else:
# if len(self.results) == 1 and not all((species, inlet)):
# raise ValueError("Please pass species and inlet")
if not all((species, site, inlet)):
raise ValueError("Please pass site, species and inlet")
# TODO - how to do this in a cleaner way for mypy?
site = str(site)
species = str(species)
inlet = str(inlet)
return self._create_obsdata(site=site, species=species, inlet=inlet)
def _read_data(
data_filepath: Path,
site: str,
network: str,
inlet: Optional[str] = None,
instrument: Optional[str] = None,
sampling_period: Optional[str] = None,
measurement_type: Optional[str] = None,
) -> Dict:
"""Read the datafile passed in and extract the data we require.
Args:
data_filepath: Path to file
site: Three letter site code
network: Network name
inlet: Inlet height
instrument: Instrument name
sampling_period: Sampling period including the unit (using pandas frequency aliases like '1H' or '1min')
measurement_type: Measurement type e.g. insitu, flask
Returns:
dict: Dictionary of gas data
"""
from datetime import datetime
from pandas import RangeIndex, read_csv, NaT
import warnings
from openghg.util import clean_string
split_fname = data_filepath.stem.split(".")
site = site.lower()
try:
site_fname = clean_string(split_fname[0])
inlet_fname = clean_string(split_fname[3])
except IndexError:
raise ValueError(
"Error reading metadata from filename, we expect a form hfd.picarro.1minute.100m.dat"
)
if site_fname != site:
raise ValueError(
"Site mismatch between passed site code and that read from filename."
)
if "m" not in inlet_fname:
raise ValueError(
"No inlet found, we expect filenames such as: bsd.picarro.1minute.108m.dat"
)
if inlet is not None and inlet != inlet_fname:
raise ValueError(
"Inlet mismatch between passed inlet and that read from filename.")
else:
inlet = inlet_fname
# Function to parse the datetime format found in the datafile
def parse_date(date: str): # type: ignore
try:
return datetime.strptime(date, "%y%m%d %H%M%S")
except ValueError:
return NaT
# Catch dtype warnings
# TODO - look at setting dtypes - read header and data separately?
with warnings.catch_warnings():
warnings.simplefilter("ignore")
data = read_csv(
data_filepath,
header=None,
skiprows=1,
sep=r"\s+",
index_col=["0_1"],
parse_dates=[[0, 1]],
date_parser=parse_date,
)
data.index.name = "time"
# Drop any rows with NaNs
# This is now done before creating metadata
data = data.dropna(axis="rows", how="any")
# Get the number of gases in dataframe and number of columns of data present for each gas
n_gases, n_cols = _gas_info(data=data)
header = data.head(2)
skip_cols = sum([header[column][0] == "-" for column in header.columns])
metadata = _read_metadata(filepath=data_filepath, data=data)
if network is not None:
metadata["network"] = network
if sampling_period is not None:
# Compare against value extracted from the file name
file_sampling_period = Timedelta(seconds=metadata["sampling_period"])
comparison_seconds = abs(sampling_period -
file_sampling_period).total_seconds()
tolerance_seconds = 1
if comparison_seconds > tolerance_seconds:
raise ValueError(
f"Input sampling period {sampling_period} does not match to value "
f"extracted from the file name of {metadata['sampling_period']} seconds."
)
# Read the scale from JSON
# I'll leave this here for the possible future movement from class to functions
network_metadata = load_json(filename="process_gcwerks_parameters.json")
crds_metadata = network_metadata["CRDS"]
# This dictionary is used to store the gas data and its associated metadata
combined_data = {}
for n in range(n_gases):
# Slice the columns
gas_data = data.iloc[:, skip_cols + n * n_cols:skip_cols +
(n + 1) * n_cols]
# Reset the column numbers
gas_data.columns = RangeIndex(gas_data.columns.size)
species = gas_data[0][0]
species = species.lower()
column_labels = [
species,
f"{species}_variability",
f"{species}_number_of_observations",
]
# Name columns
gas_data = gas_data.set_axis(column_labels,
axis="columns",
inplace=False)
header_rows = 2
# Drop the first two rows now we have the name
gas_data = gas_data.drop(index=gas_data.head(header_rows).index,
inplace=False)
# Cast data to float64 / double
gas_data = gas_data.astype("float64")
# Here we can convert the Dataframe to a Dataset and then write the attributes
gas_data = gas_data.to_xarray()
site_attributes = _get_site_attributes(site=site,
inlet=inlet,
crds_metadata=crds_metadata)
scale = crds_metadata["default_scales"].get(species.upper(), "NA")
# Create a copy of the metadata dict
species_metadata = metadata.copy()
species_metadata["species"] = clean_string(species)
species_metadata["inlet"] = inlet
species_metadata["scale"] = scale
species_metadata["long_name"] = site_attributes["long_name"]
combined_data[species] = {
"metadata": species_metadata,
"data": gas_data,
"attributes": site_attributes,
}
return combined_data
def get_attributes(
ds: Dataset,
species: str,
site: str,
network: str = None,
global_attributes: Dict[str, str] = None,
units: str = None,
scale: str = None,
sampling_period: str = None,
date_range: List[str] = None,
) -> Dataset:
"""
This function writes attributes to an xarray.Dataset so that they conform with
the CF Convention v1.6
Attributes of the xarray DataSet are modified, and variable names are changed
If the species is a standard mole fraction then either:
- species name will used in lower case in the file and variable names
but with any hyphens taken out
- name will be changed according to the species_translator dictionary
If the species is isotopic data or a non-standard variable (e.g. APO):
- Isotopes species names should begin with a "D"
(Annoyingly, the code currently picks up "Desflurane" too. I've
fixed this for now, but if we get a lot of other "D" species, we
should make this better)
- I suggest naming for isotopologues should be d<species><isotope>, e.g.
dCH4C13, or dCO2C14
- Any non-standard variables should be listed in the species_translator
dictionary
Args:
ds: Should contain variables such as "ch4", "ch4 repeatability".
Must have a "time" dimension.
species: Species name. e.g. "CH4", "HFC-134a", "dCH4C13"
site: Three-letter site code
network: Network site is associated with
global_attribuates: Dictionary containing any info you want to
add to the file header (e.g. {"Contact": "Contact_Name"})
units: This routine will try to guess the units
unless this is specified. Options are in units_interpret
scale: Calibration scale for species.
sampling_period: Number of seconds for which air
sample is taken. Only for time variable attribute
date_range: Start and end date for output
If you only want an end date, just put a very early start date
(e.g. ["1900-01-01", "2010-01-01"])
"""
from pandas import Timestamp as pd_Timestamp
from openghg.util import clean_string, load_json, timestamp_now
# from numpy import unique as np_unique
if not isinstance(ds, Dataset):
raise TypeError("This function only accepts xarray Datasets")
# Current CF Conventions (v1.7) demand that valid variable names
# begin with a letter and be composed of letters, digits and underscores
# Here variable names are also made lowercase to enable easier matching below
# TODO - could I just cast ds.variables as as type for mypy instead of doing this?
# variable_names = [str(v) for v in ds.variables]
# Is this better?
variable_names = cast(Dict[str, Any], ds.variables)
to_underscores = {var: var.lower().replace(" ", "_") for var in variable_names}
ds = ds.rename(to_underscores) # type: ignore
species_attrs = load_json(filename="species_attributes.json")
attributes_data = load_json("attributes.json")
species_translator = attributes_data["species_translation"]
unit_species = attributes_data["unit_species"]
unit_species_long = attributes_data["unit_species_long"]
unit_interpret = attributes_data["unit_interpret"]
species_upper = species.upper()
species_lower = species.lower()
variable_names = cast(Dict[str, Any], ds.variables)
matched_keys = [var for var in variable_names if species_lower in var]
# If we don't have any variables to rename, raise an error
if not matched_keys:
raise NameError(f"Cannot find species {species} in Dataset variables")
species_rename = {}
for var in matched_keys:
try:
species_label = species_translator[species_upper]["chem"]
except KeyError:
species_label = clean_string(species_lower)
species_rename[var] = var.replace(species_lower, species_label)
ds = ds.rename(species_rename) # type: ignore
# Global attributes
global_attributes_default = {
"conditions_of_use": "Ensure that you contact the data owner at the outset of your project.",
"source": "In situ measurements of air",
"Conventions": "CF-1.6",
}
if global_attributes is not None:
# TODO - for some reason mypy doesn't see a Dict[str,str] as a valid Mapping[Hashable, Any] type
global_attributes.update(global_attributes_default) # type: ignore
else:
global_attributes = global_attributes_default
global_attributes["file_created"] = str(timestamp_now())
global_attributes["processed_by"] = "OpenGHG_Cloud"
global_attributes["species"] = species_label
if scale is None:
global_attributes["calibration_scale"] = "unknown"
else:
global_attributes["calibration_scale"] = scale
# Update the Dataset attributes
ds.attrs.update(global_attributes) # type: ignore
# Add some site attributes
site_attributes = _site_info_attributes(site.upper(), network)
ds.attrs.update(site_attributes)
# Species-specific attributes
# Long name
if species_upper.startswith("D") and species_upper != "DESFLURANE" or species_upper == "APD":
sp_long = species_translator[species_upper]["name"]
elif species_upper == "RN":
sp_long = "radioactivity_concentration_of_222Rn_in_air"
elif species_upper in species_translator:
name = species_translator[species_upper]["name"]
sp_long = f"mole_fraction_of_{name}_in_air"
else:
sp_long = f"mole_fraction_of_{species_label}_in_air"
ancillary_variables = []
variable_names = cast(Dict[str, Any], ds.variables)
matched_keys = [var for var in variable_names if species_lower in var.lower()]
# Write units as attributes to variables containing any of these
match_words = ["variability", "repeatability", "stdev", "count"]
for key in variable_names:
key = key.lower()
if species_label.lower() in key:
# Standard name attribute
# ds[key].attrs["standard_name"]=key.replace(species_label, sp_long)
ds[key].attrs["long_name"] = key.replace(species_label, sp_long)
# If units are required for variable, add attribute
if key == species_label or any(word in key for word in match_words):
if units is not None:
if units in unit_interpret:
ds[key].attrs["units"] = unit_interpret[units]
else:
ds[key].attrs["units"] = unit_interpret["else"]
else:
# TODO - merge these species attributes into a single simpler JSON
try:
ds[key].attrs["units"] = unit_species[species_upper]
except KeyError:
try:
ds[key].attrs["units"] = species_attrs[species_label.upper()]["units"]
except KeyError:
ds[key].attrs["units"] = "NA"
# If units are non-standard, add explanation
if species_upper in unit_species_long:
ds[key].attrs["units_description"] = unit_species_long[species_upper]
# Add to list of ancilliary variables
if key != species_label:
ancillary_variables.append(key)
# TODO - for the moment skip this step - check status of ancilliary variables in standard
# Write ancilliary variable list
# ds[species_label].attrs["ancilliary_variables"] = ", ".join(ancillary_variables)
# Add quality flag attributes
# NOTE - I've removed the whitespace before status_flag and integration_flag here
variable_names = cast(Dict[str, Any], ds.variables)
quality_flags = [key for key in variable_names if "status_flag" in key]
# Not getting long_name for c2f6
for key in quality_flags:
ds[key] = ds[key].astype(int)
try:
long_name = ds[species_label].attrs["long_name"]
except KeyError:
raise KeyError(key, quality_flags)
ds[key].attrs = {
"flag_meaning": "0 = unflagged, 1 = flagged",
"long_name": f"{long_name} status_flag",
}
variable_names = cast(Dict[str, Any], ds.variables)
# Add integration flag attributes
integration_flags = [key for key in variable_names if "integration_flag" in key]
for key in integration_flags:
ds[key] = ds[key].astype(int)
long_name = ds[species_label].attrs["long_name"]
ds[key].attrs = {
"flag_meaning": "0 = area, 1 = height",
"standard_name": f"{long_name} integration_flag",
"comment": "GC peak integration method (by height or by area). Does not indicate data quality",
}
# Set time encoding
# Check if there are duplicate time stamps
# I feel there should be a more pandas way of doing this
# but xarray doesn't currently have a duplicates method
# See this https://github.com/pydata/xarray/issues/2108
# if len(set(ds.time.values)) < len(ds.time.values):
# if len(np_unique(ds.time.values)) < len(ds.time.values):
# print("WARNING. Duplicate time stamps")
first_year = pd_Timestamp(str(ds.time[0].values)).year
ds.time.encoding = {"units": f"seconds since {str(first_year)}-01-01 00:00:00"}
time_attributes: Dict[str, str] = {}
time_attributes["label"] = "left"
time_attributes["standard_name"] = "time"
time_attributes["comment"] = (
"Time stamp corresponds to beginning of sampling period. "
+ "Time since midnight UTC of reference date. "
+ "Note that sampling periods are approximate."
)
if sampling_period is not None:
time_attributes["sampling_period_seconds"] = sampling_period
ds.time.attrs.update(time_attributes)
# If a date range is specified, slice dataset
if date_range:
ds = ds.loc[dict(time=slice(*date_range))]
return ds
def parse_gcwerks(
data_filepath: Union[str, Path],
precision_filepath: Union[str, Path],
site: str,
network: str,
inlet: Optional[str] = None,
instrument: Optional[str] = None,
sampling_period: Optional[str] = None,
measurement_type: Optional[str] = None,
) -> Dict:
"""Reads a GC data file by creating a GC object and associated datasources
Args:
data_filepath: Path of data file
precision_filepath: Path of precision file
site: Three letter code or name for site
instrument: Instrument name
network: Network name
Returns:
dict: Dictionary of source_name : UUIDs
"""
from pathlib import Path
from openghg.standardise.meta import assign_attributes
from openghg.util import clean_string, load_json
data_filepath = Path(data_filepath)
precision_filepath = Path(precision_filepath)
# Do some setup for processing
# Load site data
gcwerks_data = load_json(filename="process_gcwerks_parameters.json")
gc_params = gcwerks_data["GCWERKS"]
network = clean_string(network)
# We don't currently do anything with inlet here as it's always read from data
# or taken from process_gcwerks_parameters.json
if inlet is not None:
inlet = clean_string(inlet)
if instrument is not None:
instrument = clean_string(instrument)
# Check if the site code passed matches that read from the filename
site = _check_site(
filepath=data_filepath,
site_code=site,
gc_params=gc_params,
)
# If we're not passed the instrument name and we can't find it raise an error
if instrument is None:
instrument = _check_instrument(filepath=data_filepath,
gc_params=gc_params,
should_raise=True)
else:
fname_instrument = _check_instrument(filepath=data_filepath,
gc_params=gc_params,
should_raise=False)
if fname_instrument is not None and instrument != fname_instrument:
raise ValueError(
f"Mismatch between instrument passed as argument {instrument} and instrument read from filename {fname_instrument}"
)
instrument = str(instrument)
gas_data = _read_data(
data_filepath=data_filepath,
precision_filepath=precision_filepath,
site=site,
instrument=instrument,
network=network,
sampling_period=sampling_period,
gc_params=gc_params,
)
# Assign attributes to the data for CF compliant NetCDFs
gas_data = assign_attributes(data=gas_data, site=site)
return gas_data
def _read_raw_data(
data_filepath: Path,
species: str,
inlet: str,
sampling_period: str,
measurement_type: str = "flask",
) -> Dict:
"""Separates the gases stored in the dataframe in
separate dataframes and returns a dictionary of gases
with an assigned UUID as gas:UUID and a list of the processed
dataframes
Args:
data_filepath: Path of datafile
species: Species string such as CH4, CO
measurement_type: Type of measurements e.g. flask
Returns:
dict: Dictionary containing attributes, data and metadata keys
"""
from openghg.util import clean_string, read_header, load_json
from pandas import read_csv, Timestamp
header = read_header(filepath=data_filepath)
column_names = header[-1][14:].split()
def date_parser(year: str, month: str, day: str, hour: str, minute: str,
second: str) -> Timestamp:
return Timestamp(year, month, day, hour, minute, second)
date_parsing = {
"time": [
"sample_year",
"sample_month",
"sample_day",
"sample_hour",
"sample_minute",
"sample_seconds",
]
}
data_types = {
"sample_year": int,
"sample_month": int,
"sample_day": int,
"sample_hour": int,
"sample_minute": int,
"sample_seconds": int,
}
# Number of header lines to skip
n_skip = len(header)
data = read_csv(
data_filepath,
skiprows=n_skip,
names=column_names,
sep=r"\s+",
dtype=data_types,
parse_dates=date_parsing,
date_parser=date_parser,
index_col="time",
skipinitialspace=True,
)
# Drop duplicates
data = data.loc[~data.index.duplicated(keep="first")]
# Check if the index is sorted
if not data.index.is_monotonic_increasing:
data = data.sort_index()
# Read the site code from the Dataframe
site = str(data["sample_site_code"][0]).upper()
site_data = load_json("acrg_site_info.json")
# If this isn't a site we recognize try and read it from the filename
if site not in site_data:
site = str(data_filepath.name).split("_")[1].upper()
if site not in site_data:
raise ValueError(f"The site {site} is not recognized.")
if species is not None:
# If we're passed a species ensure that it is in fact the correct species
data_species = str(data["parameter_formula"].values[0]).lower()
passed_species = species.lower()
if data_species != passed_species:
raise ValueError(
f"Mismatch between passed species ({passed_species}) and species read from data ({data_species})"
)
species = species.upper()
flag = []
selection_flag = []
for flag_str in data.analysis_flag:
flag.append(flag_str[0] == ".")
selection_flag.append(int(flag_str[1] != "."))
combined_data = {}
data[species + "_status_flag"] = flag
data[species + "_selection_flag"] = selection_flag
data = data[data[species + "_status_flag"]]
data = data[[
"sample_latitude",
"sample_longitude",
"sample_altitude",
"analysis_value",
"analysis_uncertainty",
species + "_selection_flag",
]]
rename_dict = {
"analysis_value": species,
"analysis_uncertainty": species + "_repeatability",
"sample_longitude": "longitude",
"sample_latitude": "latitude",
"sample_altitude": "altitude",
}
data = data.rename(columns=rename_dict, inplace=False)
data = data.to_xarray()
# TODO - this could do with a better name
noaa_params = load_json("attributes.json")["NOAA"]
site_attributes = noaa_params["global_attributes"]
site_attributes["inlet_height_magl"] = "NA"
site_attributes["instrument"] = noaa_params["instrument"][species.upper()]
site_attributes["sampling_period"] = sampling_period
metadata = {}
metadata["species"] = clean_string(species)
metadata["site"] = site
metadata["measurement_type"] = measurement_type
metadata["network"] = "NOAA"
metadata["inlet"] = inlet
metadata["sampling_period"] = sampling_period
metadata["instrument"] = noaa_params["instrument"][species.upper()]
combined_data[species.lower()] = {
"metadata": metadata,
"data": data,
"attributes": site_attributes,
}
return combined_data
def _read_obspack(
data_filepath: Union[str, Path],
site: str,
inlet: str,
sampling_period: str,
measurement_type: str,
instrument: Optional[str] = None,
) -> Dict[str, Dict]:
"""Read NOAA ObsPack NetCDF files
Args:
data_filepath: Path to file
site: Three letter site code
inlet: Inlet height, if no height use measurement type e.g. flask
measurement_type: One of flask, insity or pfp
instrument: Instrument name
sampling_period: Sampling period
Returns:
dict: Dictionary of results
"""
import xarray as xr
from openghg.util import clean_string
from openghg.standardise.meta import assign_attributes
valid_types = ("flask", "insitu", "pfp")
if measurement_type not in valid_types:
raise ValueError(f"measurement_type must be one of {valid_types}")
obspack_ds = xr.open_dataset(data_filepath)
# orig_attrs = obspack_ds.attrs
# Want to find and drop any duplicate time values for the original dataset
# Using xarray directly we have to do in a slightly convoluted way as this is not well built
# into the xarray workflow yet - https://github.com/pydata/xarray/pull/5239
# - can use da.drop_duplicates() but only on one variable at a time and not on the whole Dataset
# This method keeps attributes for each of the variables including units
# The dimension within the original dataset is called "obs" and has no associated coordinates
# Extract time from original Dataset (dimension is "obs")
time = obspack_ds.time
# To keep associated "obs" dimension, need to assign coordinate values to this (just 0, len(obs))
time = time.assign_coords(obs=obspack_ds.obs)
# Make "time" the primary dimension (while retaining "obs") and add "time" values as coordinates
time = time.swap_dims(dims_dict={"obs": "time"})
time = time.assign_coords(time=time)
# Drop any duplicate time values and extract the associated "obs" values
# TODO: Work out what to do with duplicates - may be genuine multiple measurements
time_unique = time.drop_duplicates(dim="time", keep="first")
obs_unique = time_unique.obs
# Estimate sampling period using metadata and midpoint time
if sampling_period == "NOT_SET":
sampling_period_estimate = _estimate_sampling_period(obspack_ds)
else:
sampling_period_estimate = -1.0
species = clean_string(obspack_ds.attrs["dataset_parameter"])
network = "NOAA"
# Use these obs values to filter the original dataset to remove any repeated times
processed_ds = obspack_ds.sel(obs=obs_unique)
processed_ds = processed_ds.set_coords(["time"])
# Rename variables to match our internal standard
# "value_std_dev" --> f"{species}_variability"
# "value_unc" --> ??
# TODO: Clarify what "value_unc" should be renamed to
variable_names = {
"value": species,
"value_std_dev": f"{species}_variability",
"value_unc": f"{species}_variability", # May need to be updated
"nvalue": f"{species}_number_of_observations"
}
to_extract = [
name for name in variable_names.keys() if name in processed_ds
]
name_dict = {
name: key
for name, key in variable_names.items() if name in to_extract
}
if not to_extract:
wanted = variable_names.keys()
raise ValueError(
f"No valid data columns found in converted DataFrame. We expect the following data variables in the passed NetCDF: {wanted}"
)
processed_ds = processed_ds[to_extract]
processed_ds = processed_ds.rename(name_dict)
processed_ds = processed_ds.sortby("time")
try:
# Extract units attribute from value data variable
units = processed_ds[species].units
except (KeyError, AttributeError):
print("Unable to extract units from 'value' within input dataset")
else:
if units == "mol mol-1":
units = "1"
elif units == "millimol mol-1":
units = "1e-3"
elif units == "micromol mol-1":
units = "1e-6"
elif units == "nmol mol-1":
units = "1e-9"
elif units == "pmol mol-1":
units = "1e-12"
else:
print(f"Using unit {units} directly")
# raise ValueError(f"Did not recognise input units from file: {units}")
metadata = {}
metadata["site"] = site
metadata["inlet"] = inlet
metadata["network"] = network
metadata["measurement_type"] = measurement_type
metadata["species"] = species
metadata["units"] = units
metadata["sampling_period"] = sampling_period
if instrument is not None:
metadata["instrument"] = instrument
else:
try:
metadata["instrument"] = obspack_ds.attrs["instrument"]
except KeyError:
pass
if sampling_period_estimate >= 0.0:
metadata["sampling_period_estimate"] = str(
sampling_period_estimate
) # convert to string to keep consistent with "sampling_period"
# TODO: At the moment all attributes from the NOAA ObsPack are being copied
# plus any variables we're adding.
# - decide if we want to reduce this
attributes = obspack_ds.attrs
attributes["sampling_period"] = sampling_period
if sampling_period_estimate >= 0.0:
attributes["sampling_period_estimate"] = str(sampling_period_estimate)
gas_data = {
species: {
"data": processed_ds,
"metadata": metadata,
"attributes": attributes
}
}
gas_data = assign_attributes(data=gas_data, site=site, network=network)
return gas_data
def parse_cranfield(
data_filepath: Union[str, Path],
site: Optional[str] = None,
network: Optional[str] = None,
inlet: Optional[str] = None,
instrument: Optional[str] = None,
sampling_period: Optional[str] = None,
measurement_type: Optional[str] = None,
) -> Dict:
"""Creates a CRDS object holding data stored within Datasources
Args:
filepath: Path of file to load
data_filepath : Filepath of data to be read
site: Name of site
network: Name of network
Returns:
dict: Dictionary of gas data
"""
from pandas import read_csv
from openghg.util import clean_string
if sampling_period is None:
sampling_period = "NOT_SET"
data_filepath = Path(data_filepath)
data = read_csv(data_filepath, parse_dates=["Date"], index_col="Date")
data = data.rename(
columns={
"Methane/ppm": "ch4",
"Methane stdev/ppm": "ch4 variability",
"CO2/ppm": "co2",
"CO2 stdev/ppm": "co2 variability",
"CO/ppm": "co",
"CO stdev/ppm": "co variability",
}
)
data.index.name = "time"
# Convert CH4 and CO to ppb
data["ch4"] = data["ch4"] * 1e3
data["ch4 variability"] = data["ch4 variability"] * 1e3
data["co"] = data["co"] * 1e3
data["co variability"] = data["co variability"] * 1e3
metadata = {}
metadata["site"] = "THB"
metadata["instrument"] = "CRDS"
metadata["sampling_period"] = str(sampling_period)
metadata["height"] = "10magl"
metadata["inlet"] = "10magl"
metadata["network"] = "CRANFIELD"
# TODO - this feels fragile
species: List[str] = [col for col in data.columns if " " not in col]
combined_data = {}
# Number of columns of data for each species
n_cols = 2
for n, sp in enumerate(species):
# for sp in species:
# Create a copy of the metadata dict
species_metadata = metadata.copy()
species_metadata["species"] = str(clean_string(sp))
# Here we don't want to match the co in co2
# For now we'll just have 2 columns for each species
# cols = [col for col in data.columns if sp in col]
gas_data = data.iloc[:, n * n_cols : (n + 1) * n_cols]
# Convert from a pandas DataFrame to an xarray Dataset
gas_data = gas_data.to_xarray()
combined_data[sp] = {"metadata": species_metadata, "data": gas_data}
return combined_data
def parse_npl(
data_filepath: pathType,
site: str = "NPL",
network: str = "LGHG",
inlet: str = None,
instrument: str = None,
sampling_period: str = None,
measurement_type: str = None,
) -> Dict:
"""Reads NPL data files and returns the UUIDS of the Datasources
the processed data has been assigned to
Args:
data_filepath: Path of file to load
site: Site name
Returns:
list: UUIDs of Datasources data has been assigned to
"""
if sampling_period is None:
sampling_period = "NOT_SET"
data_filepath = Path(data_filepath)
site = "NPL"
attributes_data = load_json(filename="attributes.json")
npl_params = attributes_data["NPL"]
# mypy doesn't like NaT or NaNs - look into this
def parser(date: str): # type: ignore
try:
return datetime.strptime(str(date), "%d/%m/%Y %H:%M")
except ValueError:
return NaT
data = read_csv(data_filepath, index_col=0, date_parser=parser)
# Drop the NaT/NaNs
data = data.loc[data.index.dropna()]
# Rename columns
rename_dict = {"Cal_CO2_dry": "CO2", "Cal_CH4_dry": "CH4"}
data = data.rename(columns=rename_dict)
data.index.name = "time"
if inlet is None:
inlet = "NA"
gas_data = {}
for species in data.columns:
processed_data = data.loc[:, [species]].sort_index().to_xarray()
# Convert methane to ppb
if species == "CH4":
processed_data[species] *= 1000
# No averaging applied to raw obs, set variability to 0 to allow get_obs to calculate
# when averaging
processed_data["{} variability".format(
species)] = processed_data[species] * 0.0
site_attributes = npl_params["global_attributes"]
site_attributes["inlet_height_magl"] = npl_params["inlet"]
site_attributes["instrument"] = npl_params["instrument"]
metadata = {
"species": clean_string(species),
"sampling_period": str(sampling_period),
"site": "NPL",
"network": "LGHG",
"inlet": inlet,
}
# TODO - add in better metadata reading
gas_data[species] = {
"metadata": metadata,
"data": processed_data,
"attributes": site_attributes,
}
gas_data = assign_attributes(data=gas_data, site=site, network=network)
return gas_data
def read_file(
filepath: multiPathType,
data_type: str,
network: str,
site: str,
inlet: str = None,
instrument: str = None,
sampling_period: str = None,
measurement_type: str = "insitu",
overwrite: bool = False,
) -> Dict:
"""Process files and store in the object store. This function
utilises the process functions of the other classes in this submodule
to handle each data type.
Args:
filepath: Filepath(s)
data_type: Data type, for example CRDS, GCWERKS
site: Site code/name
network: Network name
inlet: Inlet height. If retrieve multiple files pass None, OpenGHG will attempt to
read inlets from data.
instrument: Instrument name
sampling_period: Sampling period in pandas style (e.g. 2H for 2 hour period, 2m for 2 minute period).
measurement_type: Type of measurement e.g. insitu, flask
overwrite: Overwrite previously uploaded data
Returns:
dict: Dictionary of Datasource UUIDs
"""
from collections import defaultdict
from pathlib import Path
from pandas import Timedelta
import sys
from tqdm import tqdm
from openghg.util import load_surface_parser, hash_file, clean_string, verify_site
from openghg.types import SurfaceTypes
from openghg.store import assign_data
if not isinstance(filepath, list):
filepath = [filepath]
try:
data_type = SurfaceTypes[data_type.upper()].value
except KeyError:
raise ValueError(f"Unknown data type {data_type} selected.")
# Test that the passed values are valid
# Check validity of site, instrument, inlet etc in acrg_site_info.json
# Clean the strings
site = verify_site(site=site)
network = clean_string(network)
inlet = clean_string(inlet)
instrument = clean_string(instrument)
sampling_period = clean_string(sampling_period)
sampling_period_seconds: Union[str, None] = None
# If we have a sampling period passed we want the number of seconds
if sampling_period is not None:
sampling_period_seconds = str(Timedelta(sampling_period).total_seconds())
# Load the data retrieve object
parser_fn = load_surface_parser(data_type=data_type)
obs = ObsSurface.load()
results: resultsType = defaultdict(dict)
# Create a progress bar object using the filepaths, iterate over this below
with tqdm(total=len(filepath), file=sys.stdout) as progress_bar:
for fp in filepath:
if data_type == "GCWERKS":
try:
data_filepath = Path(fp[0])
precision_filepath = Path(fp[1])
except ValueError:
raise ValueError("For GCWERKS data both data and precision filepaths must be given.")
else:
data_filepath = Path(fp)
# try:
file_hash = hash_file(filepath=data_filepath)
if file_hash in obs._file_hashes and overwrite is False:
raise ValueError(
f"This file has been uploaded previously with the filename : {obs._file_hashes[file_hash]}."
)
progress_bar.set_description(f"Processing: {data_filepath.name}")
if data_type == "GCWERKS":
data = parser_fn(
data_filepath=data_filepath,
precision_filepath=precision_filepath,
site=site,
network=network,
inlet=inlet,
instrument=instrument,
sampling_period=sampling_period_seconds,
measurement_type=measurement_type,
)
else:
data = parser_fn(
data_filepath=data_filepath,
site=site,
network=network,
inlet=inlet,
instrument=instrument,
sampling_period=sampling_period_seconds,
measurement_type=measurement_type,
)
# Extract the metadata for each set of measurements to perform a Datasource lookup
metadata = {key: data["metadata"] for key, data in data.items()}
lookup_results = obs.datasource_lookup(metadata=metadata)
# Create Datasources, save them to the object store and get their UUIDs
datasource_uuids = assign_data(
data_dict=data, lookup_results=lookup_results, overwrite=overwrite
)
results["processed"][data_filepath.name] = datasource_uuids
# Record the Datasources we've created / appended to
obs.add_datasources(datasource_uuids, metadata)
# Store the hash as the key for easy searching, store the filename as well for
# ease of checking by user
obs._file_hashes[file_hash] = data_filepath.name
# except Exception:
# results["error"][data_filepath.name] = traceback.format_exc()
progress_bar.update(1)
# Save this object back to the object store
obs.save()
return results
def parse_btt(
data_filepath: Union[str, Path],
site: Optional[str] = "BTT",
network: Optional[str] = "LGHG",
inlet: Optional[str] = None,
instrument: Optional[str] = None,
) -> Dict:
"""Reads NPL data files and returns the UUIDS of the Datasources
the processed data has been assigned to
Args:
data_filepath: Path of file to load
site: Site name
Returns:
dict: Dictionary of gas data
"""
from openghg.standardise.meta import assign_attributes
from pandas import read_csv, Timestamp, to_timedelta, isnull
from numpy import nan as np_nan
from openghg.util import clean_string, load_json
# TODO: Decide what to do about inputs which aren't use anywhere
# at present - inlet, instrument, sampling_period, measurement_type
data_filepath = Path(data_filepath)
site = "BTT"
# Rename these columns
rename_dict = {"co2.cal": "CO2", "ch4.cal.ppb": "CH4"}
# We only want these species
species_extract = ["CO2", "CH4"]
# Take std-dev measurements from these columns for these species
species_sd = {"CO2": "co2.sd.ppm", "CH4": "ch4.sd.ppb"}
param_data = load_json(filename="attributes.json")
network_params = param_data["BTT"]
sampling_period = int(network_params["sampling_period"])
sampling_period_seconds = str(sampling_period) + "s"
data = read_csv(data_filepath)
data["time"] = Timestamp("2019-01-01 00:00") + to_timedelta(
data["DOY"] - 1, unit="D")
data["time"] = data["time"].dt.round(sampling_period_seconds)
data = data[~isnull(data.time)]
data = data.rename(columns=rename_dict)
data = data.set_index("time")
gas_data = {}
for species in species_extract:
processed_data = data.loc[:, [species]].sort_index()
# Create a variability column
species_stddev_label = species_sd[species]
processed_data[species][f"{species} variability"] = data[
species_stddev_label]
# Replace any values below zero with NaNs
processed_data[processed_data < 0] = np_nan
# Drop NaNs
processed_data = processed_data.dropna()
# Convert to a Dataset
processed_data = processed_data.to_xarray()
site_attributes = network_params["global_attributes"]
site_attributes["inlet_height_magl"] = network_params["inlet"]
site_attributes["instrument"] = network_params["instrument"]
site_attributes["sampling_period"] = sampling_period
# TODO - add in better metadata reading
metadata = {
"species": clean_string(species),
"sampling_period": str(sampling_period),
"site": "BTT",
}
gas_data[species] = {
"metadata": metadata,
"data": processed_data,
"attributes": site_attributes,
}
gas_data = assign_attributes(data=gas_data, site=site, network=network)
return gas_data
def parse_beaco2n(
data_filepath: Union[str, Path],
site: str,
network: str,
inlet: str,
instrument: Optional[str] = "shinyei",
sampling_period: Optional[str] = None,
) -> Dict:
"""Read BEACO2N data files
Args:
data_filepath: Data filepath
site: Site name
network: Network name
inlet: Inlet height in metres
instrument: Instrument name
sampling_period: Measurement sampling period
Returns:
dict: Dictionary of data
"""
import pandas as pd
from openghg.util import load_json
from collections import defaultdict
from openghg.util import clean_string
if sampling_period is None:
sampling_period = "NOT_SET"
data_filepath = Path(data_filepath)
datetime_columns = {"time": ["datetime"]}
use_cols = [1, 5, 6, 7, 8, 9, 10]
na_values = [-999.0]
site = clean_string(site)
try:
data = pd.read_csv(
data_filepath,
index_col="time",
usecols=use_cols,
parse_dates=datetime_columns,
na_values=na_values,
)
except ValueError as e:
raise ValueError(
f"Unable to read data file, please make sure it is in the standard BEACO2N format.\nError: {e}"
)
beaco2n_site_data = load_json("beaco2n_site_data.json")
try:
site_metadata = beaco2n_site_data[site.upper()]
except KeyError:
raise ValueError(f"Site {site} not recognized.")
site_metadata["comment"] = "Retrieved from http://beacon.berkeley.edu/"
# Check which columns we have in the data and build the rename dict
possible_rename_cols = {
"PM_ug/m3": "pm",
"PM_ug/m3_QC_level": "pm_qc",
"co2_ppm": "co2",
"co2_ppm_QC_level": "co2_qc",
"co_ppm": "co",
"co_ppm_QC_level": "co_qc",
}
# Not all columns are in data from different sites, i.e. Glasgow has a CO column
rename_cols = {k: v for k, v in possible_rename_cols.items() if k in data}
# Set all values below zero to NaN
data = data.rename(columns=rename_cols)
# Read the columns available and make sure we have them to iterate over
possible_measurement_types = ["pm", "co", "co2"]
measurement_types = [c for c in possible_measurement_types if c in data]
units = {"pm": "ug/m3", "co2": "ppm", "co": "ppm"}
gas_data: DefaultDict[str, Dict[str, Union[DataFrame,
Dict]]] = defaultdict(dict)
for mt in measurement_types:
m_data = data[[mt, f"{mt}_qc"]]
m_data = m_data.dropna(axis="rows", subset=[mt])
# Some sites don't have data for each type, skip that type if all NaNs
if m_data.index.empty:
continue
m_data = m_data.to_xarray()
species_metadata = {
"units": units[mt],
"site": site,
"species": clean_string(mt),
"inlet": clean_string(inlet),
"network": "beaco2n",
"sampling_period": str(sampling_period),
"instrument": instrument,
}
gas_data[mt]["data"] = m_data
gas_data[mt]["metadata"] = species_metadata
gas_data[mt]["attributes"] = site_metadata
# TODO - add CF Compliant attributes?
return gas_data
def search(**kwargs): # type: ignore
"""Search for observations data. Any keyword arguments may be passed to the
the function and these keywords will be used to search the metadata associated
with each Datasource.
Example / commonly used arguments are given below.
Args:
species: Terms to search for in Datasources
locations: Where to search for the terms in species
inlet: Inlet height such as 100m
instrument: Instrument name such as picarro
find_all: Require all search terms to be satisfied
start_date: Start datetime for search.
If None a start datetime of UNIX epoch (1970-01-01) is set
end_date: End datetime for search.
If None an end datetime of the current datetime is set
skip_ranking: If True skip ranking system, defaults to False
Returns:
dict: List of keys of Datasources matching the search parameters
"""
from addict import Dict as aDict
from copy import deepcopy
from itertools import chain as iter_chain
from openghg.store import ObsSurface, Footprints, Emissions, EulerianModel
from openghg.store.base import Datasource
from openghg.util import (
timestamp_now,
timestamp_epoch,
timestamp_tzaware,
clean_string,
closest_daterange,
find_daterange_gaps,
split_daterange_str,
load_json,
)
from openghg.dataobjects import SearchResults
# Get a copy of kwargs as we make some modifications below
kwargs_copy = deepcopy(kwargs)
# Do this here otherwise we have to produce them for every datasource
start_date = kwargs.get("start_date")
end_date = kwargs.get("end_date")
if start_date is None:
start_date = timestamp_epoch()
else:
start_date = timestamp_tzaware(start_date)
if end_date is None:
end_date = timestamp_now()
else:
end_date = timestamp_tzaware(end_date)
kwargs_copy["start_date"] = start_date
kwargs_copy["end_date"] = end_date
skip_ranking = kwargs_copy.get("skip_ranking", False)
try:
del kwargs_copy["skip_ranking"]
except KeyError:
pass
# As we might have kwargs that are None we want to get rid of those
search_kwargs = {k: clean_string(v) for k, v in kwargs_copy.items() if v is not None}
# Speices translation
species = search_kwargs.get("species")
if species is not None:
if not isinstance(species, list):
species = [species]
translator = load_json("species_translator.json")
updated_species = []
for s in species:
updated_species.append(s)
try:
translated = translator[s]
except KeyError:
pass
else:
updated_species.extend(translated)
search_kwargs["species"] = updated_species
data_type = search_kwargs.get("data_type", "timeseries")
valid_data_types = ("timeseries", "footprints", "emissions", "eulerian_model")
if data_type not in valid_data_types:
raise ValueError(f"{data_type} is not a valid data type, please select one of {valid_data_types}")
# Assume we want timeseries data
obj: Union[ObsSurface, Footprints, Emissions, EulerianModel] = ObsSurface.load()
if data_type == "footprints":
obj = Footprints.load()
elif data_type == "emissions":
obj = Emissions.load()
elif data_type == "eulerian_model":
obj = EulerianModel.load()
datasource_uuids = obj.datasources()
# Shallow load the Datasources so we can search their metadata
datasources = (Datasource.load(uuid=uuid, shallow=True) for uuid in datasource_uuids)
# For the time being this will return a dict until we know how best to represent
# the footprints and emissions results in a SearchResult object
if data_type in {"emissions", "footprints", "eulerian_model"}:
sources: Dict = aDict()
for datasource in datasources:
if datasource.search_metadata(**search_kwargs):
uid = datasource.uuid()
sources[uid]["keys"] = datasource.keys_in_daterange(start_date=start_date, end_date=end_date)
sources[uid]["metadata"] = datasource.metadata()
return sources
# Find the Datasources that contain matching metadata
matching_sources = {d.uuid(): d for d in datasources if d.search_metadata(**search_kwargs)}
# TODO - Update this as it only uses the ACRG repo JSON at the moment
# Check if this site only has one inlet, if so skip ranking
# if "site" in search_kwargs:
# site = search_kwargs["site"]
# if not isinstance(site, list) and not multiple_inlets(site=site):
# skip_ranking = True
# If there isn't *any* ranking data at all, skip all the ranking functionality
if not obj._rank_data:
skip_ranking = True
# If only one datasource has been returned, skip all the ranking functionality
if len(matching_sources) == 1:
skip_ranking = True
# If we have the site, inlet and instrument then just return the data
# TODO - should instrument be added here
if {"site", "inlet", "species"} <= search_kwargs.keys() or skip_ranking is True:
specific_sources = aDict()
for datasource in matching_sources.values():
specific_keys = datasource.keys_in_daterange(start_date=start_date, end_date=end_date)
if not specific_keys:
continue
metadata = datasource.metadata()
site = metadata["site"]
species = metadata["species"]
inlet = metadata["inlet"]
specific_sources[site][species][inlet]["keys"] = specific_keys
specific_sources[site][species][inlet]["metadata"] = metadata
return SearchResults(results=specific_sources.to_dict(), ranked_data=False)
highest_ranked = aDict()
for uid, datasource in matching_sources.items():
# Find the site and then the ranking
metadata = datasource.metadata()
# Get the site inlet and species
site = metadata["site"]
species = metadata["species"]
rank_data = obj.get_rank(uuid=uid, start_date=start_date, end_date=end_date)
# If this Datasource doesn't have any ranking data skip it and move on
if not rank_data:
continue
# There will only be a single rank key
rank_value = next(iter(rank_data))
# Get the daterange this rank covers
rank_dateranges = rank_data[rank_value]
# Each match we store gives us the information we need
# to retrieve the data
match = {"uuid": uid, "dateranges": rank_dateranges}
# Need to ensure we get all the dates covered
if species in highest_ranked[site]:
species_rank_data = highest_ranked[site][species]
# If we have a higher (lower number) rank save it
if rank_value < species_rank_data["rank"]:
species_rank_data["rank"] = rank_value
species_rank_data["matching"] = [match]
# If another Datasource has the same rank for another daterange
# we want to save that as well
elif rank_value == species_rank_data["rank"]:
species_rank_data["matching"].append(match)
else:
highest_ranked[site][species]["rank"] = rank_value
highest_ranked[site][species]["matching"] = [match]
if not highest_ranked:
raise ValueError(
(
"No ranking data set for the given search parameters."
" Please refine your search to include a specific site, species and inlet."
)
)
# Now we have the highest ranked data the dateranges there are ranks for
# we want to fill in the gaps with (currently) the highest inlet from that site
# We just want some rank_metadata to go along with the final data scheme
# Can key a key of date - inlet
data_keys: Dict = aDict()
for site, species in highest_ranked.items():
for sp, data in species.items():
# data_keys[site][sp]["keys"] = []
species_keys = []
species_rank_data = {}
species_metadata = {}
for match_data in data["matching"]:
uuid = match_data["uuid"]
match_dateranges = match_data["dateranges"]
# Get the datasource as it's already in the dictionary
# we created earlier
datasource = matching_sources[uuid]
metadata = datasource.metadata()
inlet = metadata["inlet"]
keys = []
for dr in match_dateranges:
date_keys = datasource.keys_in_daterange_str(daterange=dr)
if date_keys:
keys.extend(date_keys)
# We'll add this to the metadata in the search results we return at the end
species_rank_data[dr] = inlet
species_keys.extend(keys)
species_metadata[inlet] = metadata
# Only create the dictionary keys if we have some data keys
if species_keys:
data_keys[site][sp]["keys"] = species_keys
data_keys[site][sp]["rank_metadata"] = species_rank_data
data_keys[site][sp]["metadata"] = species_metadata
else:
continue
# We now need to retrieve data for the dateranges for which we don't have ranking data
# To do this find the gaps in the daterange over which the user has requested data
# and the dates for which we have ranking information
# Get the dateranges that are covered by ranking information
daterange_strs = list(iter_chain.from_iterable([m["dateranges"] for m in data["matching"]]))
# Find the gaps in the ranking coverage
gap_dateranges = find_daterange_gaps(
start_search=start_date, end_search=end_date, dateranges=daterange_strs
)
# We want the dateranges and inlets for those dateranges
inlet_dateranges = data_keys[site][sp]["rank_metadata"]
# These are the dateranges for which we have ranking information for this site and species
ranked_dateranges = list(data_keys[site][sp]["rank_metadata"].keys())
for gap_daterange in gap_dateranges:
# We want to select the inlet that's ranked for dates closest to the ones we have here
closest_dr = closest_daterange(to_compare=gap_daterange, dateranges=ranked_dateranges)
gap_start, gap_end = split_daterange_str(gap_daterange)
# Find the closest ranked inlet by date
chosen_inlet = inlet_dateranges[closest_dr]
inlet_metadata = data_keys[site][sp]["metadata"][chosen_inlet]
inlet_instrument = inlet_metadata["instrument"]
inlet_sampling_period = inlet_metadata["sampling_period"]
# Then we want to retrieve the correct metadata for those inlets
results: SearchResults = search(
site=site,
species=sp,
inlet=chosen_inlet,
instrument=inlet_instrument,
sampling_period=inlet_sampling_period,
start_date=gap_start,
end_date=gap_end,
) # type: ignore
if not results:
continue
# Retrieve the data keys
inlet_data_keys = results.keys(site=site, species=sp, inlet=chosen_inlet)
data_keys[site][sp]["keys"].extend(inlet_data_keys)
# Remove any duplicate keys
data_keys[site][sp]["keys"] = list(set(data_keys[site][sp]["keys"]))
# TODO - create a stub for addict
dict_data_keys = data_keys.to_dict() # type: ignore
return SearchResults(results=dict_data_keys, ranked_data=True)
def _split_species(
data: DataFrame,
site: str,
instrument: str,
species: List,
metadata: Dict,
units: Dict,
scale: Dict,
gc_params: Dict,
) -> Dict:
"""Splits the species into separate dataframe into sections to be stored within individual Datasources
Args:
data: DataFrame of raw data
site: Name of site from which this data originates
instrument: Name of instrument
species: List of species contained in data
metadata: Dictionary of metadata
units: Dictionary of units for each species
scale: Dictionary of scales for each species
gc_params: GCWERKS parameter dictionary
Returns:
dict: Dataframe of gas data and metadata
"""
from addict import Dict as aDict
from fnmatch import fnmatch
from openghg.util import load_json, clean_string
# Load species translator so we can keep species names consistent
attributes_data = load_json("attributes.json")
species_translator = attributes_data["species_translation"]
# Read inlets from the parameters
expected_inlets = _get_inlets(site_code=site, gc_params=gc_params)
try:
data_inlets = data["Inlet"].unique().tolist()
except KeyError:
raise KeyError(
"Unable to read inlets from data, please ensure this data is of the GC type expected by this retrieve module"
)
combined_data = aDict()
for spec in species:
# Skip this species if the data is all NaNs
if data[spec].isnull().all():
continue
# Here inlet is the inlet in the data and inlet_label is the label we want to use as metadata
for inlet, inlet_label in expected_inlets.items():
# Create a copy of metadata for local modification
spec_metadata = metadata.copy()
spec_metadata["units"] = units[spec]
spec_metadata["scale"] = scale[spec]
# If we've only got a single inlet
if inlet == "any" or inlet == "air":
spec_data = data[[
spec,
spec + " repeatability",
spec + " status_flag",
spec + " integration_flag",
"Inlet",
]]
spec_data = spec_data.dropna(axis="index", how="any")
spec_metadata["inlet"] = inlet_label
elif "date" in inlet:
dates = inlet.split("_")[1:]
data_sliced = data.loc[dates[0]:dates[1]]
spec_data = data_sliced[[
spec,
spec + " repeatability",
spec + " status_flag",
spec + " integration_flag",
"Inlet",
]]
spec_data = spec_data.dropna(axis="index", how="any")
spec_metadata["inlet"] = inlet_label
else:
# Find the inlet
matching_inlets = [i for i in data_inlets if fnmatch(i, inlet)]
if not matching_inlets:
continue
# Only set the label in metadata when we have the correct label
spec_metadata["inlet"] = inlet_label
# There should only be one matching label
select_inlet = matching_inlets[0]
# Take only data for this inlet from the dataframe
inlet_data = data.loc[data["Inlet"] == select_inlet]
spec_data = inlet_data[[
spec,
spec + " repeatability",
spec + " status_flag",
spec + " integration_flag",
"Inlet",
]]
spec_data = spec_data.dropna(axis="index", how="any")
# Now we drop the inlet column
spec_data = spec_data.drop("Inlet", axis="columns")
# Check that the Dataframe has something in it
if spec_data.empty:
continue
attributes = _get_site_attributes(site=site,
inlet=inlet_label,
instrument=instrument,
gc_params=gc_params)
attributes = attributes.copy()
# We want an xarray Dataset
spec_data = spec_data.to_xarray()
# Create a standardised / cleaned species label
try:
comp_species = species_translator[spec.upper()]["chem"]
except KeyError:
comp_species = clean_string(spec.lower())
# Add the cleaned species name to the metadata and alternative name if present
spec_metadata["species"] = comp_species
if comp_species != spec.lower() and comp_species != spec.upper():
spec_metadata["species_alt"] = spec
# Rename variables so they have lowercase and alphanumeric names
to_rename = {}
for var in spec_data.variables:
if spec in var:
new_name = var.replace(spec, comp_species)
to_rename[var] = new_name
spec_data = spec_data.rename(to_rename)
# As a single species may have measurements from multiple inlets we
# use the species and inlet as a key
data_key = f"{comp_species}_{inlet_label}"
combined_data[data_key]["metadata"] = spec_metadata
combined_data[data_key]["data"] = spec_data
combined_data[data_key]["attributes"] = attributes
to_return: Dict = combined_data.to_dict()
return to_return