def __init__( self, rundate: datetime.date, **kwargs: Dict[str, Any], ) -> None: """Set up a new AprioriOrbit object, does not parse any data TODO: Remove dependency on rundate, use time to read correct files. (What to do with dataset?) Args: rundate: Date of model run. """ # MURKS: Should it be done like that. The technique is normally not given for unittest routines (like # test_broadcast.py). try: pipeline = config.analysis.pipeline.str except (AttributeError, exceptions.MissingSectionError): pipeline = None self.rundate = rundate self.pipeline = pipeline self._dset_raw = dataset.Dataset(rundate=rundate, pipeline=pipeline, stage=self.name, label="raw") self._dset_edit = dataset.Dataset(rundate=rundate, pipeline=pipeline, stage=self.name, label="edit") self._dset = None
def _get_outliers_dataset(dset: "Dataset") -> "Dataset": """Get dataset with outliers Args: dset: A dataset containing the data. Returns: Dataset with outliers or status 1 if no data for "calculate" stage are available """ # Get Dataset where no outliers are rejected dset_vars = {**dset.vars, **dset.analysis} dset_vars["stage"] = "calculate" dset_complete = dataset.Dataset.read(**dset_vars) if dset_complete.num_obs == 0: # NOTE: This is the case for concatencated Datasets, where "calculate" stage data are not available. return 1 # Get relative complement, which corresponds to "outlier" dataset #dset_outliers = dset_complete.complement_with(dset, complement_by=["time", "satellite"]) dset_outliers = dataset.Dataset( num_obs=0 ) #MURKS: complement_with does not exists so far in Dataset v3. return dset_outliers
def _get_outliers_dataset(dset: "Dataset") -> "Dataset": """Get dataset with outliers Args: dset: A dataset containing the data. Returns: Dataset with outliers or status 1 if no data for "calculate" stage are available """ # Get Dataset where no outliers are rejected dset_complete = dataset.Dataset( rundate=dset.analysis["rundate"], pipeline=dset.vars["pipeline"], stage="calculate", station=dset.vars["station"], ) if dset_complete.num_obs == 0: # NOTE: This is the case for concatencated Datasets, where "calculate" stage data are not available. return 1 # Get relative complement, which corresponds to "outlier" dataset dset_outliers = dset_complete.complement_with(dset, complement_by=["time", "satellite"]) return dset_outliers
def calculate_orbit(self, dset: "Dataset", time: str = "time") -> None: """Set Dataset representing calculated apriori orbit Args: dset: A dataset containing the data. time: Define time fields to be used. It can be for example 'time' or 'sat_time'. 'time' is related to observation time and 'sat_time' to satellite transmission time. """ if not dset.num_obs: log.fatal( f"Dataset is empty. No observation epochs given for calculating orbits." ) # TODO: Getting of 'id' and 'profile' -> Should it be done like that? try: profile = config.analysis.profile.str except exceptions.MissingEntryError: profile = None try: id_ = config.analysis.id.str except exceptions.MissingEntryError: id_ = None self._dset = dataset.Dataset( rundate=self.rundate, pipeline=self.pipeline, stage=self.name, label="orbit", profile=profile, id=id_, ) self._calculate(self._dset, dset, time=time)
def _get_outliers_dataset(dset: "Dataset") -> Union["Dataset", Enum]: """Get dataset with outliers Args: dset: A dataset containing the data. Returns: Dataset with outliers or error exit status if no data for "calculate" stage are available """ # Get Dataset where no outliers are rejected file_vars = {**dset.vars, **dset.analysis} file_vars["stage"] = "calculate" try: dset_complete = dataset.Dataset.read(**file_vars) except OSError: log.warn(f"Could not read dataset {config.files.path('dataset', file_vars=file_vars)}.") return enums.ExitStatus.error # Get relative complement, which corresponds to "outlier" dataset # dset_outliers = dset_complete.complement_with(dset, complement_by=["time", "satellite"]) dset_outliers = dataset.Dataset(num_obs=0) # MURKS: complement_with does not exists so far in Dataset v3. return dset_outliers
def __init__(self, rundate: datetime.date, **kwargs: Dict[str, Any]) -> None: """Set up a new AprioriOrbit object, does not parse any data TODO: Remove dependency on rundate, use time to read correct files. (What to do with dataset?) Args: rundate: Date of model run. """ # MURKS: Should it be done like that. The technique is normally not given for unittest routines (like # test_broadcast.py). try: pipeline = config.analysis.pipeline.str except exceptions.MissingEntryError: pipeline = None # TODO: Getting of 'id' and 'profile' -> Should it be done like that? try: profile = config.analysis.profile.str except exceptions.MissingEntryError: profile = None try: id_ = config.analysis.id.str except exceptions.MissingEntryError: id_ = None self.rundate = rundate self.pipeline = pipeline self._dset_raw = dataset.Dataset( rundate=rundate, pipeline=pipeline, stage=self.name, label="raw", profile=profile, id=id_, ) self._dset_edit = dataset.Dataset( rundate=rundate, pipeline=pipeline, stage=self.name, label="edit", profile=profile, id=id_, ) self._dset = None
def read_dset(rundate): with Timer(f"Finish read of day {rundate} in", logger=log.time): try: log.info(f"Reading data for {rundate}") return dataset.Dataset.read(**dict(dset_vars, rundate=rundate)) except OSError as err: log.warn(f"Unable to read data for {rundate}: {err}") return dataset.Dataset()
def as_dataset(self) -> "Dataset": """Return the parsed data as a Dataset Returns: A dataset containing the data. """ # Initialize dataset dset = dataset.Dataset() if not self.data: log.warn("No data in {self.file_path}.") return dset dset.num_obs = len(self.data["year"]) # Add time epochs = list() for year, doy, seconds in zip(self.data["year"], self.data["doy"], self.data["seconds"]): epochs.append( datetime.strptime("{:.0f} {:.0f}".format(year, doy), "%Y %j") + timedelta(seconds=seconds)) dset.add_time("time", val=epochs, scale="gps", fmt="datetime", write_level="operational") # Add system field if "system" in self.data.keys(): systems = [] for system in self.data["system"]: systems.append(enums.gnss_name_to_id[system.lower()].value) dset.add_text("system", val=systems) # Add satellite field if "satellite" in self.data.keys(): satellites = [] for system, satellite in zip(dset.system, self.data["satellite"]): satellites.append(system + str(satellite).zfill(2)) dset.add_text("satellite", val=satellites) # Add text and float fields fields = set(self.data.keys()) - { "year", "doy", "seconds", "system", "satellite" } for field in fields: if self.data[field].dtype.kind in { "U", "S" }: # Check if numpy type is string dset.add_text(field, val=self.data[field]) continue dset.add_float(field, val=self.data[field]) return dset
def _concatenate_datasets(from_date: date, to_date: date, dset_vars: Dict[str, str], only_for_rundate: bool) -> np.ndarray: """Concatenate datasets Args: from_date: Start date for reading Dataset. to_date: End date for reading Dataset. dset_vars: Common Dataset variables. only_for_rundate: Concatenate only data for given rundate. """ merged_vars = dset_vars.copy() merged_vars["id"] += "_concatenated" dset_merged = dataset.Dataset( **dict(merged_vars, rundate=from_date, empty=True)) date_to_read = from_date while date_to_read <= to_date: dset = dataset.Dataset().read(**dict(dset_vars, rundate=date_to_read)) current_date = date_to_read date_to_read += timedelta(days=1) if dset.num_obs == 0: log.info(f"No data to read for {current_date}") continue if only_for_rundate: _keep_data_only_for_rundate(dset) if dset.num_obs == 0: log.info(f"No data to read for {current_date}") continue log.info(f"Reading data for {current_date}") if not dset_merged: dset_merged.update_from(dset) else: dset_merged.extend(dset) return dset_merged
def __init__( self, rundate: datetime, sampling_rate: Union[float, None] = None, systems: Union[List[str], None] = None, satellites: Union[List[str], None] = None, site_pos: Union[List[float], None] = None, station: Union[str, None] = None, ): """Set up a new GnssSimulate object TODO: Remove dependency on rundate, use time to read correct files. (What to do with dataset?) Args: rundate (datetime.date): Date of model run. time (TimeTable): Time epochs at the satellite for which to calculate the apriori orbit. satellite (list): Strings with names of satellites. """ # MURKS: Should it be done like that. The technique is normally not given for unittest routines (like # test_broadcast.py). try: pipeline = config.analysis.pipeline.str except: pipeline = "gnss" # +MURKS systems = ["G"] satellites = ["G01", "G02"] site_pos = [3348186.1150, 465040.8615, 5390738.0919] station = "krss" # -MURKS self.sampling_rate = sampling_rate if sampling_rate else config.tech.simulate.sampling_rate.float self.systems = systems if systems else config.tech.simulate.systems.list self.satellites = satellites if satellites else config.tech.simulate.satellites.list self.site_pos = np.array(site_pos) if site_pos else np.array(config.tech.simulate.site_pos.list) self.station = station if station else config.tech.simulate.station.str # TODO: Generate station position self._dset_raw = dataset.Dataset(rundate=rundate, pipeline=pipeline, stage="simulate", label="raw") self._dset_orbit = dataset.Dataset(rundate=rundate, pipeline=pipeline, stage="simulate", label="orbit") self._dset = dataset.Dataset(rundate=rundate, pipeline=pipeline, stage="simulate", label="simulate")
def dset(self) -> "Dataset": """Dataset representing calculated apriori orbit Calculates data from `dset_edit` if the data are not already present. """ if self._dset == None: self._dset = dataset.Dataset(rundate=self.rundate, pipeline=self.pipeline, stage=self.name, label="orbit") self._calculate(self._dset, self._dset_edit) return self._dset
def run_stage(rundate, pipeline, dset, stage, prev_stage, **kwargs): # Skip stages where no dependencies have changed dep_path = config.files.path("depends", file_vars={ **kwargs, "stage": stage }) if not (dependencies.changed(dep_path) or util.check_options("-F", "--force")): log.info( f"Not necessary to run {stage} for {pipeline.upper()} {rundate.strftime(config.FMT_date)}" ) return if dset is None: try: # Read dataset from disk if it exists dset = dataset.Dataset.read(rundate=rundate, pipeline=pipeline, stage=prev_stage, label="last", **kwargs) except (OSError, ValueError): # Create emtpy dataset dset = dataset.Dataset(rundate=rundate, pipeline=pipeline, **kwargs) # Set up dependencies. Add dependencies to previous stage and config file dependencies.init(dep_path) if prev_stage is not None: dependencies.add(config.files.path("depends", file_vars={ **kwargs, "stage": prev_stage }), label="depends") dependencies.add(*config.tech.sources, label="config") # Delete old datasets for this stage dset.delete_stage(stage, **kwargs) # Call the current stage. Skip rest of stages if current stage returns False (compare with is since by # default stages return None) plugins.call(package_name=__name__, plugin_name=pipeline, part=stage, stage=stage, dset=dset, plugin_logger=log.info) dependencies.write() return dset
def __init__( self, rundate: datetime.date, time: Union[None, "TimeTable"] = None, satellite: Union[None, List[str]] = None, **kwargs: Dict[str, Any], ) -> None: """Set up a new AprioriOrbit object, does not parse any data TODO: Remove dependency on rundate, use time to read correct files. (What to do with dataset?) Args: rundate: Date of model run. time: Time epochs at the satellite for which to calculate the apriori orbit. satellite: Strings with names of satellites. """ # MURKS: Should it be done like that. The technique is normally not given for unittest routines (like # test_broadcast.py). try: pipeline = config.analysis.pipeline.str except AttributeError: pipeline = None self.rundate = rundate self.pipeline = pipeline self.time = time self.satellite = satellite self._dset_raw = dataset.Dataset(rundate=rundate, pipeline=pipeline, stage=self.name, label="raw") self._dset_edit = dataset.Dataset(rundate=rundate, pipeline=pipeline, stage=self.name, label="edit") self._dset = None
def _read(self, dset_raw): """Read SP3 orbit file data and save it in a Dataset In addition to the given date, we read data for the day before and after. This is needed to carry out correct orbit interpolation at the start and end of a day. TODO: How well fits the orbits from day to day? Is it necessary to align the orbits? Args: dset_raw (Dataset): Dataset representing raw data from apriori orbit files """ date_to_read = dset_raw.analysis["rundate"] - timedelta( days=self.day_offset) file_paths = list() # Loop over days to read while date_to_read <= dset_raw.analysis["rundate"] + timedelta( days=self.day_offset): if self.file_path is None: file_path = config.files.path( self.file_key, file_vars=config.date_vars(date_to_read)) else: file_path = self.file_path log.debug(f"Parse precise orbit file {file_path}") # Generate temporary Dataset with orbit file data dset_temp = dataset.Dataset(rundate=date_to_read, pipeline=dset_raw.vars["pipeline"], stage="temporary") parser = parsers.parse(parser_name="orbit_sp3", file_path=file_path, rundate=date_to_read) parser.write_to_dataset(dset_temp) file_paths.append(str(parser.file_path)) dependencies.add(str(parser.file_path), label=self.file_key) # Used for output writing # Extend Dataset dset_raw with temporary Dataset date = date_to_read.strftime("%Y-%m-%d") dset_raw.update_from( dset_temp) if dset_raw.num_obs == 0 else dset_raw.extend( dset_temp, meta_key=date) dset_raw.meta.add("file_path", file_paths, section="parser") date_to_read += timedelta(days=1) return dset_raw
def calculate_orbit(self, dset: "Dataset", time: str = "time") -> None: """Set Dataset representing calculated apriori orbit Args: dset: A dataset containing the data. time: Define time fields to be used. It can be for example 'time' or 'sat_time'. 'time' is related to observation time and 'sat_time' to satellite transmission time. """ if not dset.num_obs: log.fatal( f"Dataset is empty. No observation epochs given for calculating orbits." ) self._dset = dataset.Dataset(rundate=self.rundate, pipeline=self.pipeline, stage=self.name, label="orbit") self._calculate(self._dset, dset, time=time)
def _read(self, dset_raw, rundate, provider, version): """Read SP3 orbit file data and save it in a Dataset Naming convention correspond to end of arc, at midnight, hence we add day_offset, which is usually arc_length - 1 Args: dset_raw (Dataset): Dataset representing raw data from apriori orbit files rundate: Date of run, datetime object provider: Str: Orbit provider version: Str: Orbit version """ date_to_read = rundate + timedelta(days=self.day_offset) file_vars = config.date_vars(date_to_read) file_vars["provider"] = provider file_vars["version"] = version file_vars["sat_name"] = self.sat_name if self.file_path is None: file_path = config.files.path(self.file_key, file_vars) else: file_path = self.file_path log.debug(f"Parse precise orbit file {file_path}") # Generate dataset with external orbit file data dset_orbit = dataset.Dataset(tech=dset_raw.meta["tech"], stage="orbit", dataset_name="", dataset_id=0, empty=True) parser = parsers.parse(parser_name="orbit_sp3", file_path=file_path, rundate=date_to_read) parser.write_to_dataset(dset_orbit) return dset_orbit
def add_to_full_timeseries(dset): """Write some key variables to the full timeseries Args: dset: Dataset, data for a model run. """ dset_id = config.tech.timeseries.dataset_id.str.format(**dset.vars) try: # Read existing dataset dset_ts = dataset.Dataset.read( rundate=date(1970, 1, 1), pipeline=dset.vars["pipeline"], stage="timeseries", label=dset_id, session="", use_options=False, id=dset.analysis["id"], ) except OSError: # Start new timeseries dataset dset_ts = dataset.Dataset( rundate=date(1970, 1, 1), pipeline=dset.vars["pipeline"], stage="timeseries", label=dset_id, session="", use_options=False, id=dset.analysis["id"], ) dset_session = dataset.Dataset() # Add data to dset_session idx_fields = config.tech[WRITER].index.list field_values = [["all"] + list(dset.unique(f)) for f in idx_fields] idx_values = dict(zip(idx_fields, zip(*itertools.product(*field_values)))) # TODO: Remove combinations where filter leaves 0 observations num_obs = len(idx_values[idx_fields[0]]) # Length of any (in this case the first) field mean_epoch = dset.time.mean.utc rundate_str = dset.analysis["rundate"].strftime(config.FMT_date) session = dset.vars.get("session", "") status = dset.meta.get("analysis_status", "unchecked") session_type = dset.meta.get("input", dict()).get("session_type", "") dset_session.num_obs = num_obs dset_session.add_time("time", val=[mean_epoch] * num_obs, scale=mean_epoch.scale, fmt=mean_epoch.fmt) dset_session.add_text("rundate", val=[rundate_str] * num_obs) dset_session.add_text("session", val=[session] * num_obs) dset_session.add_text("status", val=[status] * num_obs) dset_session.add_text("session_type", val=[session_type] * num_obs) for field, value in idx_values.items(): dset_session.add_text(field, val=value) default_dset_str = f"{dset.vars['stage']}/{dset.vars['label']}" dsets = {default_dset_str: dset} for method, cfg_entry in config.tech[WRITER].items(): try: method_func = getattr(sys.modules[__name__], f"method_{method}") except AttributeError: log.warn(f"Method {method!r} is unknown") continue for field_cfg in cfg_entry.as_list(split_re=", *"): field_out = re.sub("[ -/:]", "_", field_cfg) func, _, field_dset = field_cfg.rpartition(":") field_in, _, dset_str = field_dset.partition("-") func = func if func else field_in dset_str = dset_str if dset_str else default_dset_str if dset_str not in dsets: stage, _, dset_id = dset_str.partition("/") dset_id = int(dset_id) if dset_id else "last" dsets[dset_str] = dataset.Dataset.read( rundate=dset.analysis["rundate"], pipeline=dset.vars["pipeline"], stage=stage, session=dset.vars["session"], label=dset_id, id=dset.analysis["id"], ) val, adder, unit = method_func(dsets[dset_str], field_in, idx_values, func) if adder: add_func = getattr(dset_session, adder) add_func(field_out, val=val, unit=unit) # hack to get solved neq data into the time series: # TODO: unhack this :P Add as a method_neq instead? if "normal equation" in dset.meta: _add_solved_neq_fields(dset, dset_session, idx_values) # Filter timeseries dataset to remove any previous data for this rundate and session if dset_ts.num_obs > 0: keep_idx = np.logical_not(dset_ts.filter(rundate=rundate_str, session=session)) dset_ts.subset(keep_idx) # Extend timeseries dataset with dset_session and write to disk dset_ts.extend(dset_session) dset_ts.write() log.info(f"Updating timeseries dataset")
def as_dataset(self) -> "Dataset": """Return the parsed data as a Dataset Returns: A dataset containing the data. """ # Spring constellation definition system_def = { "0": "", # Unknown "1": "G", # GPS "2": "R", # GLONASS "3": "S", # SBAS "4": "E", # Galileo "5": "C", # BeiDou "6": "J", # QZSS } field_spring_to_where = { "3DSpeed": "site_vel_3d", "Clock": "delay.gnss_satellite_clock", "EastSpeed": "site_vel_east", "GroupDelay": "delay.gnss_total_group_delay", "HSpeed": "site_vel_h", "IODE": "used_iode", "NorthSpeed": "site_vel_north", "PseudoRange": "delay.gnss_range", "SatInView": "num_satellite_available", "TropoDelay": "troposphere_dT", "UISD": "delay.gnss_ionosphere", "UsedSat": "num_satellite_used", "EastvsRef": "site_pos_vs_ref_east", "NorthvsRef": "site_pos_vs_ref_north", "VerticalvsRef": "site_pos_vs_ref_up", "VerticalSpeed": "site_vel_up", "XSpeed": "site_vel_x", "YSpeed": "site_vel_y", "ZSpeed": "site_vel_z", } # Initialize dataset dset = dataset.Dataset() if not self.data: log.warn("No data in {self.file_path}.") return dset dset.num_obs = len(self.data["GPSEpoch"]) # Add time dset.add_time( "time", val=[ dateutil.parser.parse(v.replace("UTC", "")) for v in self.data["UTCDateTime"] ], scale="utc", fmt="datetime", write_level="operational", ) # Add system field based on Constellation column if "Constellation" in self.data.keys(): dset.add_text("system", val=[ system_def[str(value)] for value in self.data["Constellation"] ]) # Add satellite field based on PRN column if "PRN" in self.data.keys(): prn_data = [] for prn in self.data["PRN"]: if prn >= 1 and prn <= 32: # Handling of GPS satellites prn_data.append("G" + str(prn).zfill(2)) elif prn >= 38 and prn <= 70: # Handling of GLONASS satellites prn_data.append("R" + str(prn - 38).zfill(2)) elif prn >= 71 and prn <= 140: # Handling of Galileo satellites prn_data.append("E" + str(prn - 70).zfill(2)) elif prn >= 191 and prn <= 222: # Handling of BeiDou satellites prn_data.append("C" + str(prn - 191).zfill(2)) else: log.fatal(f"Spring PRN number '{prn}' is unknown.") dset.add_text("satellite", val=prn_data) dset.add_text("system", np.array(prn_data).astype("U1")) # Add position field based on Latitude, Longitude and Height column if "Latitude" in self.data.keys(): pos = Position( val=np.vstack((self.data["Latitude"] * Unit.deg2rad, self.data["Longitude"] * Unit.deg2rad, self.data["Height"])).T, system="llh", ) if "XPos" in self.data.keys(): dset.add_position("sat_pos", val=pos.trs, system="trs", time=dset.time) else: dset.add_position("site_pos", val=pos.trs, system="trs", time=dset.time) # Define fields to save in dataset remove_time_fields = { "Constellation", "GPSEpoch", "GPSWeek", "GPSSecond", "PRN", "", "UTCDateTime" } fields = set(self.data.keys()) - remove_time_fields # Add text and float fields for field in fields: where_fieldname = field_spring_to_where[ field] if field in field_spring_to_where.keys( ) else field.lower() if self.data[field].dtype.kind in { "U", "S" }: # Check if numpy type is string dset.add_text(where_fieldname, val=self.data[field]) continue dset.add_float(where_fieldname, val=self.data[field]) return dset
def compare(date: "datedoy", pipeline: "pipeline", items: "option", specifier: "option"): log.init(log_level="info") dsets = dict() # Additional options stage = util.read_option_value("--stage") writer_names = util.read_option_value("--writers").replace(",", " ").split() items_ = [s.strip() for s in items.split(",")] # Get optional options label = util.read_option_value("--label", default="None") # TODO label = "last" if label == "last" else label station = util.read_option_value("--station", default="") id_ = util.read_option_value("--id", default="") # Update configuration of Where analysis config.where.update_from_options(_clean_sys_argv(pipeline)) # Get dataset variables dset_vars = config.create_file_vars(rundate=date, pipeline=pipeline) # Read datasets for given specifier if specifier == "id": for id_ in items_: try: dset = dataset.Dataset().read(rundate=date, pipeline=pipeline, stage=stage, label=label, id=id_, station=station) except OSError: log.warn(f"No data to read for Dataset id '{id_}'.") continue dset.vars.update(dset_vars) dset.vars["id"] = id_ dsets.update({id_: dset}) elif specifier == "station": for station in items_: try: dset = dataset.Dataset().read(rundate=date, pipeline=pipeline, stage=stage, label=label, id=id_, station=station) except OSError: log.warn(f"No data to read for Dataset station '{station}'.") continue dset.vars.update(dset_vars) dset.vars["station"] = station dsets.update({station: dset}) elif specifier == "stage": for stage in items_: try: dset = dataset.Dataset().read(rundate=date, pipeline=pipeline, stage=stage, label=label, id=id_, station=station) except OSError: log.warn(f"No data to read for Dataset stage '{stage}'.") continue dset.vars.update(dset_vars) dset.vars["stage"] = stage dsets.update({stage: dset}) else: log.fatal( f"Specifier {specifier} is not defined. It should be either 'id', 'station' or 'stage'." ) if len(dsets) == 0: log.fatal(f"All given datasets are empty [{', '.join(dsets.keys())}].") elif len(dsets) == 1: log.warn( f"Nothing to compare. Only dataset '{list(dsets.keys())[0]}' is available." ) # Loop over writers for writer in writer_names: write(writer, dset=dsets)
def satellite_clock_correction(self, dset: "Dataset", time: str = "time") -> np.ndarray: """Determine satellite clock correction based on precise satellite clock product The GNSS satellite clock bias is read from RINEX clock files. Afterwards the satellite clock bias is determined via a cubic interpolation for the observation time. TODO: * Beware of the extrapolation (bounds_error=False in interpolate). * Check for satellite clock interpolation in: "Hesselbarth, A.: Statische und kinematische GNSS-Auswertung mittels PPP, 2011" Args: dset: A Dataset containing model data. time: Define time fields to be used. It can be for example 'time' or 'sat_time'. 'time' is related to observation time and 'sat_time' to satellite transmission time. Returns: GNSS satellite clock corrections for each observation """ correction = np.zeros(dset.num_obs) sat_transmission_time = dset[time].gps.gps_ws.seconds # Get precise GNSS satellite clock values clock_product = config.tech.get("clock_product", default="clk").str if clock_product == "sp3": all_sat_clk = self.dset_edit elif clock_product == "clk": # TODO: File path information has to be improved, because 3 consecutive days are read. log.info( f"Calculating satellite clock correction (precise) based on RINEX clock file " f"{config.files.path(file_key='gnss_rinex_clk')}") all_sat_clk = dataset.Dataset(rundate=dset.analysis["rundate"]) parser = parsers.parse("rinex_clk", rundate=dset.analysis["rundate"]) parser.write_to_dataset( all_sat_clk ) # TODO Read RINEX clock file, from day before and day after. # Needed for interpolation. Add handling if these clk-files # are not available. Remove first and last observations? # If precise clock products are not available broadcast # ephemeris should be used. else: log.fatal( f"Unknown clock product {clock_product!r}. " "Configuration option 'clock_product' can only be 'sp3' or 'clk'" ) # Loop over all satellites given in configuration file for sat in dset.unique("satellite"): # Skip satellites, which are not given in RINEX clock file if sat not in all_sat_clk.unique("satellite"): # TODO: Maybe satellite is available in SP3 file, which includes also # satellite clock bias, but only for every 15 min instead of # every 5 min (or 30 s by use of igs<wwwwd>.clk_30s). continue idx = dset.filter(satellite=sat) clk_idx = all_sat_clk.filter(satellite=sat) # Interpolation of GNSS precise satellite clock values # TODO: Check if interpolation method is ok. sat_clock_bias_ip = interpolate.interp1d( all_sat_clk.time.gps.gps_ws.seconds[clk_idx], all_sat_clk.sat_clock_bias[clk_idx], axis=0, kind="cubic", bounds_error=False, fill_value=all_sat_clk.sat_clock_bias[clk_idx][-1], ) correction[idx] = sat_clock_bias_ip(sat_transmission_time[idx]) return correction
def main(date: "datedoy", pipeline: "pipeline", items: "option", specifier: "option"): log.init(log_level="info") dsets = dict() # Additional options stage = util.read_option_value("--stage") writer_names = util.read_option_value("--writers").replace(",", " ").split() items_ = [s.strip() for s in items.split(",")] # Get optional options label = util.read_option_value("--label", default="None") # TODO label = "last" if label == "last" else label station = util.read_option_value("--station", default="") id_ = util.read_option_value("--id", default="") # Get dataset variables dset_vars = dict(pipeline=pipeline, stage=stage, station=station, label=label, id=id_) dset_vars = config.create_file_vars(rundate=date, **dset_vars) # Read datasets for given specifier if specifier == "id": for id_ in items_: dset = dataset.Dataset().read(rundate=date, pipeline=pipeline, stage=stage, label=label, id=id_, station=station) if dset.num_obs == 0: log.warn(f"Dataset '{id_}' is empty.") continue dset_vars[ "id"] = id_ #TODO: Better solution for handling of dataset variables? dset.vars.update( dset_vars ) # Necessary for example for getting correct file path in used writers. dsets.update({id_: dset}) elif specifier == "station": for station in items_: dset = dataset.Dataset().read(rundate=date, pipeline=pipeline, stage=stage, label=label, id=id_, station=station) if dset.num_obs == 0: log.warn(f"Dataset '{station}' is empty.") continue dset_vars[ "station"] = station #TODO: Better solution for handling of dataset variables? dset.vars.update( dset_vars ) # Necessary for example for getting correct file path in used writers. dsets.update({station: dset}) elif specifier == "stage": for stage in items_: dset = dataset.Dataset().read(rundate=date, pipeline=pipeline, stage=stage, label=label, id=id_, station=station) if dset.num_obs == 0: log.warn(f"Dataset '{stage}' is empty.") continue dset_vars[ "stage"] = stage #TODO: Better solution for handling of dataset variables? dset.vars.update( dset_vars ) # Necessary for example for getting correct file path in used writers. dsets.update({stage: dset}) else: log.fatal( f"Specifier {specifier} is not defined. It should be either 'id', 'station' or 'stage'." ) if len(dsets) == 0: log.fatal(f"All given datasets are empty [{', '.join(dsets.keys())}].") elif len(dsets) == 1: log.warn( f"Nothing to compare. Only dataset '{list(dsets.keys())[0]}' is available." ) # Loop over writers for writer in writer_names: write(writer, dset=dsets)
def setUp(self): """ The first test set up is based on the bc_velo.c program, which is published in :cite:`remondi2004` and following RINEX navigation file sample: /* Sample Broadcast Message in unit of radians, seconds, meters. 20 01 7 23 2 0 0.0 -.857324339449D-04 -.272848410532D-11 .000000000000D+00 .200000000000D+02 .886875000000D+02 .465376527657D-08 .105827953357D+01 .457651913166D-05 .223578442819D-02 .177137553692D-05 .515379589081D+04 .936000000000D+05 .651925802231D-07 .164046615454D+01 -.856816768646D-07 .961685061380D+00 .344968750000D+03 .206374037770D+01 -.856928551657D-08 .342514267094D-09 .000000000000D+00 .112400000000D+04 .000000000000D+00 .200000000000D+01 .000000000000D+00 -.651925802231D-08 .276000000000D+03 .865800000000D+05 .000000000000D+00 .000000000000D+00 .000000000000D+00 */ The second test set up compares results from Where against gLAB solution for satellite G20 and epoch 2016-03-01 00:00:00.0. /* Sample Broadcast Message in unit of radians, seconds, meters for satellite G20 and / epoch 2016-03-01 00:00:00.0 20 16 3 1 0 0 0.0 0.396233052015D-03 0.261479726760D-11 0.000000000000D+00 0.100000000000D+02-0.231562500000D+02 0.530236372187D-08 0.253477496869D+00 -0.111199915409D-05 0.483385741245D-02 0.810064375401D-05 0.515369705963D+04 0.172800000000D+06-0.141561031342D-06 0.304306271006D+00 0.372529029846D-08 0.926615731710D+00 0.207250000000D+03 0.133849764271D+01-0.843427989304D-08 -0.164292557730D-09 0.100000000000D+01 0.188600000000D+04 0.000000000000D+00 0.200000000000D+01 0.000000000000D+00-0.838190317154D-08 0.100000000000D+02 0.172770000000D+06 0.400000000000D+01 0.000000000000D+00 0.000000000000D+00 The third test set up compares results from Where against CNES solution for satellite E01 and epoch 2016-03-01 01:30:00.0. /* Sample Broadcast Message in unit of radians, seconds, meters for satellite E01 and / epoch 2019-07-01 00:00:00.0 E01 2019 07 01 00 00 00-6.374700460583D-04-8.085976332950D-12 0.000000000000D+00 1.600000000000D+01 2.106250000000D+02 2.413671967697D-09 5.641607228729D-01 9.929761290550D-06 1.870252890512D-04 7.383525371552D-06 5.440612319946D+03 8.640000000000D+04-3.725290298462D-09 2.424721177655D-01 1.657754182816D-07 9.878562635157D-01 1.958125000000D+02 3.073143357419D+00-5.291648989849D-09 2.003654888840D-10 2.580000000000D+02 2.060000000000D+03 0.000000000000D+00 3.120000000000D+00 0.000000000000D+00-1.862645149231D-09 0.000000000000D+00 8.714000000000D+04 0.000000000000D+00 0.000000000000D+00 0.000000000000D+00 """ # Initialize logging log.init(log_level="debug") # Get GNSS ephemeris data for testing if TEST == "test_1": file_name = "test2040.01n" year = 2001 month = 7 day = 23 hour = 2 minute = 0 second = 0 satellite = "G20" self.system = "G" # GNSS identifier # Satellite transmission time self.t_sat_gpsweek = 1124.0 self.t_sat_gpssec = 86400.00 elif TEST == "test_2": file_name = "test0610.16n" year = 2016 month = 3 day = 1 hour = 0 minute = 0 second = 0 satellite = "G20" self.system = "G" # GNSS identifier # Satellite transmission time self.t_sat_gpsweek = 1886.0 self.t_sat_gpssec = 172799.92312317 elif TEST == "test_3": file_name = "TEST00CNS_R_20191820000_01D_EN.rnx" year = 2019 month = 7 day = 1 hour = 1 minute = 30 second = 0 satellite = "E01" self.system = "E" # GNSS identifier # Satellite transmission time self.t_sat_gpsweek = 2060.0 self.t_sat_gpssec = 91800.0 rundate = datetime(year, month, day, hour, minute, second) # Initialize configuration config.init(rundate=rundate, pipeline="gnss") # Generate observation datast self.dset = dataset.Dataset(num_obs=1, rundate=rundate) self.dset.add_time(name="time", val=rundate, scale="gps", fmt="datetime") #self.dset.add_time( # name="time", val=Time(val=[self.t_sat_gpsweek], val2=[self.t_sat_gpssec], fmt="gps_ws", scale="gps") #) self.dset.add_text(name="satellite", val=[satellite]) # Get broadcast ephemeris self.brdc = apriori.get( "orbit", rundate=rundate, system=tuple({self.system}), station="test", apriori_orbit="broadcast", file_path=pathlib.Path(__file__).parent / "files" / file_name, ) self.idx = 0 # Broadcast ephemeris index