def read(stage, dset):
"""Read the GNSS RINEX data.
Args:
stage (str): Name of current stage.
dset (Dataset): A dataset containing the data.
"""
dset.vars.update(file_vars())
station = dset.vars["station"]
# Read GNSS observation data either from Android raw file or RINEX file
if config.tech.format.str == "android":
parser = parsers.parse("gnss_android_raw_data",
rundate=dset.analysis["rundate"],
station=station)
else:
version, filepath = gnss.get_rinex_file_version("gnss_rinex_obs")
log.info(f"RINEX file format {version}")
if version.startswith("2"):
parser = parsers.parse("rinex2_obs",
rundate=dset.analysis["rundate"],
station=station)
elif version.startswith("3"):
parser = parsers.parse("rinex3_obs",
rundate=dset.analysis["rundate"],
station=station)
else:
log.fatal(
f"Unknown RINEX format {version} is used in file {filepath}")
parser.write_to_dataset(dset)
dset.write_as(stage=stage)
dset.read() # TODO: workaround because caching does not work correctly
def get_handling_file(time):
"""Read station-dependent info
"""
handling = parsers.parse(file_key="slr_handling_file", time=time)
for station in handling:
if "V" in handling[station]:
log.warn(
"New station {}, unreliable coordinates according to ILRS Handling file",
station)
if "E" in handling[station]:
log.warn(
"Should esimate bias for station {}, according to ILRS Handling file",
station)
if "R" in handling[station]:
log.warn(
"Should apply bias for station {}, according to ILRS Handling file",
station)
if "U" in handling[station]:
log.warn(
"Should estimate time bias for station {}, according to ILRS Handling file",
station)
if "X" in handling[station]:
log.warn(
"Should delete data for station {}, according to ILRS Handling file",
station)
return handling
def get_gnss_sat_clk(rundate):
"""Get GNSS satellite clock data
The parsing is done by :mod:`where.parsers.rinex_clk_parser`.
Returns:
Dataset: Dataset with precise satellite clock values stored as follows
==================== =============== ======= ========================================================
Field Type Unit Description
==================== =============== ======= ========================================================
sat_clock_bias numpy.ndarray m Satellite clock offset from GPS time
satellite list Satellite PRN number
time TimeTable Observation epoch in GPS time
time.data Time TODO
time.utc Time TODO
==================== =============== ======= ========================================================
"""
gnss_sat_clk_data = data.Dataset(rundate,
tech=None,
stage=None,
dataset_name="gnss_sat_clk",
dataset_id=0,
empty=True)
parser = parsers.parse("rinex_clk", rundate=rundate)
parser.write_to_dataset(gnss_sat_clk_data)
return gnss_sat_clk_data
def get_gravity_coefficients(gravity_field, truncation_level, rundate):
"""Get coefficients for a gravity field
The coefficient files come with normalized gravity coefficients.
The coefficients are returned in a dict with keys `C` and `S`,
each containing a table of coefficients `C[i, j]` and `S[i, j]`.
Args:
gravity_field: Name of gravity field.
truncation_level: Level of degree and order where coeffs are truncated.
rundate: Start time of integration.
Returns:
A dictionary containing C and S coefficients.
"""
log.info("Reading gravity field {} up to degree and order {}",
gravity_field, truncation_level)
try:
gravity_coefficients = parsers.parse(file_key="gravity_coefficients",
gravity_field=gravity_field,
truncation_level=truncation_level)
except FileNotFoundError:
log.fatal("Unknown gravity field {}, exiting!", gravity_field)
if gravity_field == "egm2008": # TODO: Why do we treat this differently ... Should be done by properties not name
apply_rates(gravity_coefficients, truncation_level, rundate)
return gravity_coefficients
def get_center_of_mass(sat_name):
"""Read station-dependent center of mass corrections from file
Args:
sat_name (String): Name of satellite.
Returns:
Dict: Center of mass corrections per station for the given satellite.
"""
return parsers.parse("slr_com", sat_name=sat_name)
def get_eccentricity(rundate):
"""Get Eccentricities for a given date
Args:
rundate: The run date of the data analysis.
Returns:
A dictionary of eccentricities.
"""
return parsers.parse(file_key="eccentricity", rundate=rundate)
def _read(self, dset_raw):
"""Read SP3 orbit file data and save it in a Dataset
In addition to the given date, we read data for the day before and after. This is needed to carry out correct
orbit interpolation at the start and end of a day.
TODO:
How well fits the orbits from day to day? Is it necessary to align the orbits?
Args:
dset_raw (Dataset): Dataset representing raw data from apriori orbit files
"""
date_to_read = dset_raw.rundate - timedelta(days=self.day_offset)
file_paths = list()
# Loop over days to read
while date_to_read <= dset_raw.rundate + timedelta(
days=self.day_offset):
if self.file_path is None:
file_path = files.path(
self.file_key, file_vars=config.date_vars(date_to_read))
else:
file_path = self.file_path
log.debug(f"Parse precise orbit file {file_path}")
# Generate temporary Dataset with orbit file data
dset_temp = data.Dataset(
rundate=date_to_read,
tech=dset_raw.vars["tech"],
stage="temporary",
dataset_name="",
dataset_id=0,
empty=True,
)
parser = parsers.parse(parser_name="orbit_sp3",
file_path=file_path,
rundate=date_to_read)
parser.write_to_dataset(dset_temp)
file_paths.append(str(parser.file_path))
# Extend Dataset dset_raw with temporary Dataset
date = date_to_read.strftime("%Y-%m-%d")
dset_raw.copy_from(
dset_temp,
meta_key=date) if dset_raw.num_obs == 0 else dset_raw.extend(
dset_temp, meta_key=date)
dset_raw.add_to_meta("parser", "file_path", file_paths)
date_to_read += timedelta(days=1)
return dset_raw
def get_ocean_tides():
"""Get ocean tidal loading coefficients
Reads ocean tidal coefficients from file using OceanTidesFes2004Parser for satellitte
displacements.
Returns:
A dictionary with information about ocean tidal coefficients.
"""
model = config.tech.orbit_ocean_tides.str
file_key = "ocean_tides_{}".format(model) if model else "ocean_tides"
return parsers.parse(file_key=file_key)
def get_solar_flux(rundate):
"""Read time-dependent solar flux from file
"""
flux = parsers.parse(file_key="solar_flux")
arc_length = config.tech.arc_length.int
date_to_add = rundate - timedelta(days=5)
time_list, flux_list = list(), list()
while True:
if date_to_add in flux:
# Note that 72000 sec corresponds to local noontime in Penticton
time_list.append((date_to_add - rundate).total_seconds() + 72000)
flux_list.append(flux[date_to_add])
if date_to_add > rundate + timedelta(days=arc_length):
break
date_to_add += timedelta(days=1)
return interpolate.interp1d(time_list, flux_list)
def _read(self, dset_raw, provider, version):
"""Read SP3 orbit file data and save it in a Dataset
Naming convention correspond to end of arc, at midnight, hence we add day_offset,
which is usually arc_length - 1
Args:
dset_raw (Dataset): Dataset representing raw data from apriori orbit files
provider: Str: Orbit provider
version: Str: Orbit version
"""
rundate = dset_raw.rundate
date_to_read = rundate + timedelta(days=self.day_offset)
file_vars = config.date_vars(date_to_read)
file_vars["provider"] = provider
file_vars["version"] = version
if self.file_path is None:
file_path = config.files.path(self.file_key, file_vars)
else:
file_path = self.file_path
log.debug(f"Parse precise orbit file {file_path}")
# Generate temporary Dataset with orbit file data
dset_orbit = data.Dataset(rundate=date_to_read,
tech=dset_raw.vars["tech"],
stage="orbit",
dataset_name="",
dataset_id=0,
empty=True)
parser = parsers.parse(parser_name="orbit_sp3",
file_path=file_path,
rundate=date_to_read)
parser.write_to_dataset(dset_orbit)
dset_orbit.add_to_meta("parser", "file_path", file_path)
return dset_orbit
def get(datasource_name, **kwargs):
"""Read data from the given data source
Simple data sources that only return data directly from a parser does not need an explicit apriori-file. This is
handled by looking in the parser-directory if a data source is not found in the apriori directory.
The import of where.parsers is done locally to avoid circular imports.
Args:
datasource_name (String): Name of apriori data source
kwargs: Input arguments to the data source
Returns:
The data from the data source (data type depends on source)
"""
try:
return plugins.call_one(package_name=__name__,
plugin_name=datasource_name,
**kwargs)
except exceptions.UnknownPluginError as apriori_err:
from where import parsers
try:
data = parsers.parse_key(file_key=datasource_name,
**kwargs).as_dict()
log.dev(
f"Called parsers.parse_key({datasource_name}) in apriori.get()"
)
return data
except (AttributeError) as att:
try:
data = parsers.parse(datasource_name, **kwargs)
log.dev(
f"Called parsers.parse({datasource_name}) in apriori.get()"
)
return data
except exceptions.UnknownPluginError:
raise apriori_err from None
def _read(self, dset_raw):
"""Read RINEX navigation file data and save it in a Dataset
One RINEX navigation file is normally written for each GNSS. The navigation file extension depends on the GNSS
(GPS: *.n, Galileo: *.l, GLONASS: *.g, ...). Therefore we have defined for each GNSS the navigation file
name in the Where configuration file `files.conf`. In addition mixed navigation files exists, which includes
navigation messages of different GNSS. We use following file keys in `files.conf`:
========= ==================
System File key
========= ==================
Galileo gnss_rinex_nav_E
GLONASS gnss_rinex_nav_R
GPS gnss_rinex_nav_G
Mixed gnss_rinex_nav_M
========= ==================
Depending on the configuration options `systems` and `use_mixed_brdc_file` following navigation files are read:
====================== ================== =======================================
Option File key What kind of navigation file is read?
====================== ================== =======================================
systems = G gnss_rinex_nav_G Only the GPS navigation file
systems = G E gnss_rinex_nav_G GPS and Galileo navigation files
gnss_rinex_nav_E
use_mixed_brdc_file gnss_rinex_nav_M Mixed GNSS navigation file
====================== ================== =======================================
Args:
dset_raw (Dataset): Dataset representing raw data from RINEX navigation file
"""
use_mixed_brdc_file = config.tech.get("use_mixed_brdc_file",
default=False).bool
systems = {"M"} if use_mixed_brdc_file == True else set(self.system)
file_paths = list()
for sys in systems:
# Generate temporary Dataset with orbit file data
dset_temp = data.Dataset(dset_raw.rundate,
dset_raw.vars["tech"],
"temporary",
"",
0,
empty=True)
parser = parsers.parse(
file_key=self.file_key.format(system=sys),
rundate=dset_raw.rundate,
file_vars=dict(dset_raw.vars,
file_key=self.file_key.format(system=sys)),
)
parser.write_to_dataset(dset_temp)
file_paths.append(str(parser.file_path))
# Extend Dataset dset_raw with temporary Dataset
if dset_raw.num_obs:
# Merge meta data information
# TODO: Handle meta data information correctly. Meta data information based on different GNSS navigation
# message files has to be merged correctly together. What to do with 'sat_sys' and 'leap_seconds'?
for date in dset_temp.meta.keys():
for key in ["iono_para", "time_sys_corr"]:
dset_temp.meta[date][key].update(
dset_raw.meta[date][key])
dset_raw.extend(dset_temp)
else:
dset_raw.copy_from(dset_temp)
dset_raw.add_to_meta("parser", "file_path", file_paths)
return dset_raw
minute = 0
second = 0
rundate = datetime(year, month, day, hour, minute, second)
file_vars = config.date_vars(rundate)
# Define 15 min dataset
file_path = config.files.path(file_key="test_gnss_orbit_interpolation_15min",
file_vars=file_vars)
orb_15min = data.Dataset(rundate,
tech=None,
stage=None,
dataset_name="gnss_precise_orbit_15min",
dataset_id=0,
empty=True)
parser = parsers.parse(parser_name="orbit_sp3c",
file_path=file_path,
rundate=rundate)
parser.write_to_dataset(orb_15min)
# Define 5 min control dataset
file_path = config.files.path(file_key="test_gnss_orbit_interpolation_5min",
file_vars=file_vars)
orb_5min = data.Dataset(rundate,
tech=None,
stage=None,
dataset_name="gnss_precise_orbit_15min",
dataset_id=0,
empty=True)
parser = parsers.parse(parser_name="orbit_sp3c",
file_path=file_path,
rundate=rundate)
def satellite_clock_correction(self):
"""Determine satellite clock correction based on precise satellite clock product
The GNSS satellite clock bias is read from RINEX clock files. Afterwards the satellite clock bias is determined
via a cubic interpolation for the observation time.
TODO:
* Beware of the extrapolation (bounds_error=False in interpolate).
* Check for satellite clock interpolation in:
"Hesselbarth, A.: Statische und kinematische GNSS-Auswertung mittels PPP, 2011"
Returns:
numpy.ndarray: GNSS satellite clock corrections for each observation
"""
correction = np.zeros(self.dset.num_obs)
sat_transmission_time = self.dset.time.gps.gpssec
# Get precise GNSS satellite clock values
clock_product = config.tech.get("clock_product", default="clk").str
if clock_product == "sp3":
all_sat_clk = self.dset_edit
elif clock_product == "clk":
# TODO: File path information has to be improved, because 3 consecutive days are read.
log.info(
"Calculating satellite clock correction (precise) based on RINEX clock file {}.",
files.path(file_key="gnss_rinex_clk"),
)
all_sat_clk = data.Dataset(rundate=self.dset.rundate,
tech=None,
stage=None,
dataset_name="gnss_sat_clk",
dataset_id=0,
empty=True)
parser = parsers.parse("rinex_clk", rundate=self.dset.rundate)
parser.write_to_dataset(
all_sat_clk
) # TODO Read RINEX clock file, from day before and day after.
# Needed for interpolation. Add handling if these clk-files
# are not available. Remove first and last observations?
# If precise clock products are not available broadcast
# ephemeris should be used.
else:
log.fatal(
"Unknown clock product '{}'. Configuration option 'clock_product' can only be 'sp3' or 'clk'.",
clock_product,
)
# Loop over all satellites given in configuration file
for sat in self.dset.unique("satellite"):
# Skip satellites, which are not given in RINEX clock file
if sat not in all_sat_clk.unique("satellite"):
# TODO: Maybe satellite is available in SP3 file, which includes also
# satellite clock bias, but only for every 15 min instead of
# every 5 min (or 30 s by use of igs<wwwwd>.clk_30s).
continue
idx = self.dset.filter(satellite=sat)
clk_idx = all_sat_clk.filter(satellite=sat)
# Interpolation of GNSS precise satellite clock values
# TODO: Check if interpolation method is ok.
sat_clock_bias_ip = interpolate.interp1d(
all_sat_clk.time.gps.gpssec[clk_idx],
all_sat_clk.sat_clock_bias[clk_idx],
axis=0,
kind="cubic",
bounds_error=False,
fill_value=all_sat_clk.sat_clock_bias[clk_idx][-1],
)
correction[idx] = sat_clock_bias_ip(sat_transmission_time[idx])
return correction
def satellite_clock_correction(self,
dset: "Dataset",
time: str = "time") -> np.ndarray:
"""Determine satellite clock correction based on precise satellite clock product
The GNSS satellite clock bias is read from RINEX clock files. Afterwards the satellite clock bias is determined
via a cubic interpolation for the observation time.
TODO:
* Beware of the extrapolation (bounds_error=False in interpolate).
* Check for satellite clock interpolation in:
"Hesselbarth, A.: Statische und kinematische GNSS-Auswertung mittels PPP, 2011"
Args:
dset: A Dataset containing model data.
time: Define time fields to be used. It can be for example 'time' or 'sat_time'. 'time' is related to
observation time and 'sat_time' to satellite transmission time.
Returns:
GNSS satellite clock corrections for each observation
"""
correction = np.zeros(dset.num_obs)
sat_transmission_time = dset[time].gps.gps_ws.seconds
# Get precise GNSS satellite clock values
clock_product = config.tech.get("clock_product", default="clk").str
if clock_product == "sp3":
all_sat_clk = self.dset_edit
elif clock_product == "clk":
# TODO: File path information has to be improved, because 3 consecutive days are read.
log.info(
f"Calculating satellite clock correction (precise) based on RINEX clock file "
f"{config.files.path(file_key='gnss_rinex_clk')}")
all_sat_clk = dataset.Dataset(rundate=dset.analysis["rundate"])
parser = parsers.parse("rinex_clk",
rundate=dset.analysis["rundate"])
parser.write_to_dataset(
all_sat_clk
) # TODO Read RINEX clock file, from day before and day after.
# Needed for interpolation. Add handling if these clk-files
# are not available. Remove first and last observations?
# If precise clock products are not available broadcast
# ephemeris should be used.
else:
log.fatal(
f"Unknown clock product {clock_product!r}. "
"Configuration option 'clock_product' can only be 'sp3' or 'clk'"
)
# Loop over all satellites given in configuration file
for sat in dset.unique("satellite"):
# Skip satellites, which are not given in RINEX clock file
if sat not in all_sat_clk.unique("satellite"):
# TODO: Maybe satellite is available in SP3 file, which includes also
# satellite clock bias, but only for every 15 min instead of
# every 5 min (or 30 s by use of igs<wwwwd>.clk_30s).
continue
idx = dset.filter(satellite=sat)
clk_idx = all_sat_clk.filter(satellite=sat)
# Interpolation of GNSS precise satellite clock values
# TODO: Check if interpolation method is ok.
sat_clock_bias_ip = interpolate.interp1d(
all_sat_clk.time.gps.gps_ws.seconds[clk_idx],
all_sat_clk.sat_clock_bias[clk_idx],
axis=0,
kind="cubic",
bounds_error=False,
fill_value=all_sat_clk.sat_clock_bias[clk_idx][-1],
)
correction[idx] = sat_clock_bias_ip(sat_transmission_time[idx])
return correction
