def compute_dops(az: np.ndarray, el: np.ndarray) -> Tuple[np.ndarray, ...]: """Compute dilution of precision (DOP) for an observation epoch It should be noted, that the weight of observations is not considered. The observation weight matrix is assumed to be an identity matrix. The cofactor matrix Q is related to a topocentric coordinate system (north, east, up): | q_nn q_ne q_nu q_nt | Q = | q_ne q_ee q_eu q_et | | q_nu q_eu q_nn q_nt | | q_nt q_et q_nt q_tt | Reference: Banerjee, P. and Bose, A. (1996): "Evaluation of GPS PDOP from elevation and azimuth of satellites", Indian Journal of Radio & Space Physics, Vol. 25, April 1996, pp. 110-113 Args: az: Satellite azimuth angle (radians) el: Satellite elevation angle (radians) Returns: Tuple with GDOP, PDOP, TDOP, HDOP and VDOP """ hdop = np.array([0.0]) vdop = np.array([0.0]) # Construct the design matrix H based on observed & valid satellites # # | -cos(e1) * cos(a1) -cos(e1) * sin(a1) -sin(e1) 1 | # | -cos(e2) * cos(a2) -cos(e2) * sin(a2) -sin(e2) 1 | # | -cos(e3) * cos(a3) -cos(e3) * sin(a3) -sin(a3) 1 | # H = | -cos(e4) * cos(a4) -cos(e4) * sin(a4) -sin(e4) 1 | # | .. .. .. .. | # | -cos(en) * cos(an) -cos(en) * sin(an) -sin(an) 1 | # H = np.stack((np.cos(el) * np.sin(az), np.cos(el) * np.cos(az), np.sin(el), np.ones(el.shape)), axis=1) H = np.stack( (-np.cos(el) * np.cos(az), -np.cos(el) * np.sin(az), -np.sin(el), np.ones(el.shape)), axis=1) Q = H.T @ H # H^t*H # User info log.debug("Q=H^t*H::") log.debug(Q) # Check if the inverse of Q exists by computing the conditional number (or computation of the detereminant) if not np.isfinite(np.linalg.cond(Q)): log.warn( "Error by computing the inverse of the co-factor matrix Q (DOP determination)." ) return None, None, None, None, None else: Q = np.linalg.inv(Q) # (H*H^t)^{-1} gdop = np.sqrt(np.trace(Q)) # GDOP pdop = np.sqrt(np.trace(Q[0:3])) # PDOP hdop = np.sqrt(np.trace(Q[0:2])) # HDOP vdop = np.sqrt(Q[2, 2]) # VDOP tdop = np.sqrt(Q[3, 3]) # TDOP return gdop, pdop, tdop, hdop, vdop
def parse_blocks(self, fid: Iterable[bytes]) -> None: """Parse contents of Sinex blocks Contents of Sinex blocks are stored as separate numpy-arrays in self._sinex Args: fid: Pointer to file being read. """ # Get set of interesting Sinex blocks, index them by marker sinex_blocks = {b.marker: b for b in self.sinex_blocks} # Iterate until all interesting Sinex blocks have been found or whole file is read try: while sinex_blocks: # Find next block (line that starts with +) fid = itertools.dropwhile(lambda ln: not ln.startswith(b"+"), fid) block_header = next(fid).decode(self.file_encoding or "utf-8") marker, *params = block_header[1:].strip().split() if marker not in sinex_blocks: continue # Find lines in block, remove comments and parse lines, store parameters for later lines = [ ln for ln in itertools.takewhile(lambda ln: not ln.startswith(b"-"), fid) if ln.startswith(b" ") ] self._sinex[marker] = self.parse_lines(lines, sinex_blocks[marker].fields) if params: self._sinex.setdefault("__params__", dict())[marker] = params del sinex_blocks[marker] except StopIteration: # File ended without reading all sinex_blocks missing = ", ".join(sinex_blocks) log.warn(f"SinexParser {self.parser_name!r} did not find Sinex blocks {missing} in file {self.file_path}")
def get_field_by_attrs(dset: "Dataset", attrs: Tuple[str], unit: str) -> np.ndarray: """Get field values of a Dataset specified by the field attributes If necessary the unit of the data fields are corrected to the defined 'output' unit. Args: dset: Dataset, a dataset containing the data. attrs: Field attributes (e.g. for Time object: (<scale>, <time format>)). unit: Unit used for output. Returns: Array with Dataset field values """ f = dset for attr in attrs: f = getattr(f, attr) # Convert 'unit' if necessary if unit: field = f"{'.'.join(attrs)}" if dset.unit(field): field_unit = dset.unit(field)[0] try: log.debug( f"Convert dataset field {field} from unit {field_unit} to {unit}." ) f = f * Unit(field_unit).to(unit).m except exceptions.UnitError: log.warn(f"Cannot convert from '{field_unit}' to '{unit}'.") return f
def glob_variable(self, file_key, variable, pattern, file_vars=None): """Find all possible values of variable """ # Find available paths file_vars = dict() if file_vars is None else dict(file_vars) file_vars[variable] = "*" search_paths = self.glob_paths(file_key, file_vars) # Set up the regular expression re_vars = {**file_vars, variable: f"(?P<{variable}>__pattern__)"} path_pattern = str(self.path(file_key, file_vars=re_vars, default=".*")).replace("\\", "\\\\") for i in itertools.count(): # Give unique names to each occurance of variable path_pattern = path_pattern.replace(f"<{variable}>", f"<{variable}__{i}>", 1) if f"<{variable}>" not in path_pattern: break re_pattern = re.compile(path_pattern.replace("__pattern__", pattern)) # Find each match values = set() for search_path in search_paths: match = re_pattern.search(str(search_path)) if match: matches = set(match.groupdict().values()) if len(matches) > 1: log.warn(f"Found multiple values for {variable!r} in {search_path}: {', '.join(matches)}") values |= matches return values
def _parse_raw(self, line: Dict[str, str], _: Dict[str, Any]) -> None: """Parse 'Raw' entries of GNSS raw data file to instance variable 'data'. """ # TODO: Make parsing depending of 'State' value. What do the 'State' numbers mean? constellationType = { "0": None, "1": "G", "2": "S", "3": "R", "4": "J", "5": "C", "6": "E" } for k, v in line.items(): if k == "dummy": continue if k == "Svid": system = constellationType[line["ConstellationType"]] if system == None: log.warn("GNSS is unknown.") continue self.data.setdefault("system", list()).append(system) self.data.setdefault("satellite", list()).append(system + str(v).zfill(2)) if v == "": v = float("nan") self.data.setdefault(k, list()).append(float(v))
def get_field(dset: "Dataset", field: str, attrs: Tuple[str], unit: str) -> np.ndarray: """Get field values of a Dataset specified by the field attributes If necessary the unit of the data fields are corrected to the defined 'output' unit. Args: dset: Dataset, a dataset containing the data. field: Field name. attrs: Field attributes (e.g. for Time object: (<scale>, <time format>)). unit: Unit used for output. Returns: Array with Dataset field values """ f = dset[field] for attr in attrs: f = getattr(f, attr) # Convert 'unit' if necessary if unit: field_attrs = field if len(attrs) == 0 else f"{field}.{'.'.join(attrs)}" try: field_unit = dset.unit(field_attrs)[0] except (exceptions.UnitError, TypeError) as e: log.debug(f"Skip unit conversion for field '{field_attrs}'.") return f # Skip unit conversion for text fields, which do not have a unit. try: log.debug(f"Convert dataset field {field} from unit {field_unit} to {unit}.") f = f * Unit(field_unit).to(unit).m except (exceptions.UnitError): log.warn(f"Cannot convert from '{field_unit}' to '{unit}' for field {field}.") return f
def as_dataset(self) -> "Dataset": """Return the parsed data as a Dataset Returns: A dataset containing the data. """ # Initialize dataset dset = dataset.Dataset() if not self.data: log.warn("No data in {self.file_path}.") return dset dset.num_obs = len(self.data["year"]) # Add time epochs = list() for year, doy, seconds in zip(self.data["year"], self.data["doy"], self.data["seconds"]): epochs.append( datetime.strptime("{:.0f} {:.0f}".format(year, doy), "%Y %j") + timedelta(seconds=seconds)) dset.add_time(name="time", val=epochs, scale="gps", fmt="datetime", write_level="operational") # Add system field if "system" in self.data.keys(): systems = [] for system in self.data["system"]: systems.append(enums.gnss_name_to_id[system.lower()].value) dset.add_text("system", val=systems) # Add system field if "satellite" in self.data.keys(): satellites = [] for system, satellite in zip(dset.system, self.data["satellite"]): satellites.append(system + str(satellite).zfill(2)) dset.add_text("satellite", val=satellites) # Add text and float fields fields = set(self.data.keys()) - { "year", "doy", "seconds", "system", "satellite" } for field in fields: if self.data[field].dtype.kind in { "U", "S" }: # Check if numpy type is string dset.add_text(field, val=self.data[field]) continue dset.add_float(field, val=self.data[field]) return dset
def parse_matrix_func(self: "SinexParser", data: np.array, lower_upper: str, type: str = "") -> Dict[str, Any]: """Parser for {marker} data Converts the input data to a symmetric matrix and adds it to self.data['{marker}']. The NEQ-Matrix Row/Column Number correspond to the Estimated Parameters Index in the {size_marker} block. Missing elements in the matrix are assumed to be zero (0); consequently, zero elements may be omitted to reduce the size of this block. Args: data: Input data, raw data for {marker} block. lower_upper: Either 'L' or 'U', indicating whether the matrix is given in lower or upper form. type: Information about the type of matrix, optional. Returns: Dictionary with symmetric matrix as a numpy array. """ # Size of matrix is given by {size_marker}-block, initialize to all zeros try: n = len(self._sinex[size_marker]) except KeyError: n = max(data["row_idx"]) log.warn( f"{size_marker!r}-block was not parsed. Guessing at size of normal equation matrix (n={n})." ) matrix = np.zeros((n, n)) # Loop through each line of values and put it in the correct place in the matrix (cannot simply reshape as # elements may have been omitted) values = np.stack((data["value_0"], data["value_1"], data["value_2"]), axis=1) for row, col, vals in zip(data["row_idx"], data["column_idx"], values): vals = vals[~np.isnan(vals)] idx = slice(row - 1, row), slice(col - 1, col - 1 + len(vals)) matrix[idx] = vals # Add symmetrical elements, depending on whether the matrix being represented in lower or upper form if lower_upper.upper() == "L": matrix = np.tril(matrix) + np.tril(matrix, k=-1).T elif lower_upper.upper() == "U": matrix = np.triu(matrix) + np.triu(matrix, k=1).T else: log.warn( f"'L' or 'U' not specified for {marker}. Trying to create a symmetric matrix anyway." ) matrix = matrix + matrix.T - np.diag(np.diag(matrix)) return {"matrix": matrix, "type": type}
def _parse_coord_comparison(self, line: Dict[str, str], cache: Dict[str, Any]) -> None: """Parse station coordinate comparison table """ if line["station"].strip().lower(): cache["station"] = line["station"].strip().lower() station = cache["station"] self.data.setdefault(station, dict()) coord_def = { "N": "north", "E": "east", "U": "up", } coord_key = coord_def[line['flag_coord'].strip()] self.data[station][f"coord_comp_rms_{coord_key}"] = float( line["rms"]) * Unit.millimeter2meter if not f"coord_comp_rms_{coord_key}" in self.fields: self.fields.append(f"coord_comp_rms_{coord_key}") # Parse values line #----+----1----+----2----+----3----+----4----+----- # 1.21 -1.85 -0.41 0.90 -1.27 0.67 1.39 0.56 # 5.03 -7.11 -1.71 -0.84 5.30 4.37 # 0.00 0.00 if not "num_coord_files" in self.meta: log.warn( "Number of coordinate files are unknown. Daily comparison values can not be read." ) return len_values = self.meta[ "num_coord_files"] * 6 # length of line depends on number of files line_values = line["values"].ljust(len_values) values = [line_values[i:i + 6] for i in range(0, len_values, 6)] for idx, value in enumerate(values): value = float( "inf") if value.strip() == "******" else value.strip() if value: values[idx] = float(value) * Unit.millimeter2meter else: values[idx] = float('nan') self.data[station][f"coord_comp_{coord_key}"] = values if not f"coord_comp_{coord_key}" in self.fields: self.fields.append(f"coord_comp_{coord_key}")
def parse(self) -> "SinexParser": """Parse data Override default parse() due to special handling of setup_parser for Sinex files """ if self.data_available: self.read_data() if not self.data_available: # May have been set to False by self.read_data() log.warn(f"No data found by {self.__class__.__name__} in {self.file_path}") return self self.postprocess_data() return self
def as_dataset(self, ref_pos: Union[np.ndarray, List[float]]) -> "Dataset": """Return the parsed data as a Dataset Args: ref_pos: Reference position given in terrestrial reference system and meters Returns: A dataset containing the data. """ # Initialize dataset dset = dataset.Dataset() if not self.data: log.warn("No data in {self.file_path}.") return dset dset.num_obs = len(self.data["date"]) # Add position ref_pos = position.Position(np.repeat(np.array([ref_pos]), dset.num_obs, axis=0), system="trs") dset.add_position_delta( name="pos", val=np.stack( (self.data["east"], self.data["north"], self.data["vertical"]), axis=1) * Unit.millimeter2meter, system="enu", ref_pos=ref_pos, ) # Add position sigma sigma = np.stack((self.data["east_sigma"], self.data["north_sigma"], self.data["vertical_sigma"]), axis=1) dset.add_sigma(name="pos_sigma", val=dset.pos.val, sigma=sigma * Unit.millimeter2meter, unit="meter") # Add time dset.add_time(name="time", val=self.data["year"], scale="utc", fmt="decimalyear", write_level="operational") return dset
def parse_header_line(self, header_line: bytes) -> None: """Parse header of Sinex file Header information is stored in `self.meta`. Args: header_line: First line of Sinex file. """ if not header_line.startswith(b"%=SNX"): log.warn( f"The file '{self.file_path}' does not contain a valid SINEX header: {header_line.decode().strip()!r}" ) return # Add header information to self.meta header_data = self.parse_lines([header_line], self.header_fields) self.meta.update({n: header_data[n][()] for n in header_data.dtype.names})
def _get_doc(obj_name, obj, module): """Get and format the documentation for an object""" list_headers = ["Args", "Attributes", "Returns", "Examples"] member_docs = list() if not callable(obj): return f"\n### {obj_name} ({obj.__class__.__name__})\n`{obj_name} = {obj!r}`\n" if not obj.__doc__: log.warn(f"No docstring found for {obj.__qualname__}") doc = textwrap.dedent((" " * 100) + (obj.__doc__ or "")).lstrip() paragraphs = doc.split("\n\n") doc_w_lists = list() for paragraph in paragraphs: if paragraph.split(":")[0] in list_headers: paragraph = re.sub(r"^(\w+:)", r"**\1**\n", paragraph) # Bold header paragraph = re.sub(r"\n (\w+):", r"\n- `\1`:", paragraph) paragraph = re.sub(r"\n (\w)", r"\n\1", paragraph) doc_w_lists.append(paragraph) doc = "\n\n".join(doc_w_lists) if inspect.isclass(obj) and not isinstance(obj, type): for member_name in dir(obj): member = getattr(obj, member_name) if not _do_doc(member_name, member, module): continue member_doc = _get_doc(member_name, member, module) member_doc = re.sub(r"\n### ", rf"\n#### {obj_name}.", member_doc) member_docs.append(member_doc) doc += "\n".join(member_docs) try: signature = inspect.signature(obj) except (ValueError, TypeError): signature = "()" headline = f"### **{obj_name}**{'' if inspect.isclass(obj) else '()'}" qualname = f"Full name: `{module.__name__}.{obj_name}`" signature_str = f"Signature: `{signature}`" return f"\n{headline}\n\n{qualname}\n\n{signature_str}\n\n{doc}"
def as_dataset(self) -> "Dataset": """Return the parsed data as a Dataset GipsyX summary results are added to Dataset 'meta' variable. Args: dset: Dataset. Returns: A dataset containing the data. """ dset = dataset.Dataset(num_obs=0) if not self.data: log.warn("No data in {self.file_path}.") return dset dset.meta["summary"] = self.data return dset
def _match_pattern(self, search_paths, path_pattern, variable, pattern): """Look for variable matching pattern in the given search paths.""" for i in itertools.count(): # Give unique names to each occurance of variable path_pattern = path_pattern.replace(f"<{variable}>", f"<{variable}__{i}>", 1) if f"<{variable}>" not in path_pattern: break re_pattern = re.compile(path_pattern.replace("__pattern__", pattern)) # Find each match values = set() for search_path in search_paths: match = re_pattern.search(str(search_path)) if match: matches = set(match.groupdict().values()) if len(matches) > 1: log.warn(f"Found multiple values for {variable!r} in {search_path}: {', '.join(matches)}") values |= matches return values
def main(): """Main program for testing solution validation implementation #TODO: This should be done via midgard/tests/gnss !!! """ log.init(log_level="info") # Upper bound for GDOP max_gdops = 30.0 # Read command line arguments parser = get_my_parser() results = parser.parse_args() no_print = lambda _: None # User info/test log.info(f" number of arguments ={len(sys.argv):d}") log.info(f" program name ={sys.argv}") # Testing the implemented function if len(sys.argv) == 1: i_res_cnt = 9 n_val_sats = i_res_cnt alpha_siglev = 0.01 n_params = 5 my_residuals = np.random.normal(0.0, 1, size=i_res_cnt) az, el = np.random.normal(60, 30, size=(2 * n_val_sats)).reshape(-1, 2).T my_result = sol_validation(my_residuals, alpha_siglev, n_params) dops_vals = compute_dops(az, el) if dops_vals[0] <= 0.0 or dops_vals[0] > max_gdops: log.warn( f"sol_validation():: compute_dops(): not valid solution, number of valid sats={n_val_sats:02d} and GDOP={dops_vals[0]:.2f}" ) log.info(f" DOPS results::") log.info(f" compute_dops(): GDOP={dops_vals[0]:.2f}") log.info(f" compute_dops(): PDOP={dops_vals[1]:.2f}") log.info(f" compute_dops(): HDOP={dops_vals[2]:.2f}") log.info(f" compute_dops(): PDOP={dops_vals[3]:.2f}")
def parse(self) -> "Parser": """Parse data This is a basic implementation that carries out the whole pipeline of reading and parsing datafiles including calculating secondary data. Subclasses should typically implement (at least) the `read_data`-method. """ self.setup_parser() if self.data_available: self.read_data() if not self.data_available: # May have been set to False by self.read_data() log.warn( f"No data found by {self.__class__.__name__} in {self.file_path}" ) return self self.postprocess_data() return self
def sol_validation(residuals: np.ndarray, alpha_siglev: float, n_params: int = 4) -> bool: """Validating the GNSS solution is carried out using Chi-square test Use Chi-square test for outlier detection and rejection. Args: residuals: Postfit residuals for all satellites in each epoch alpha_siglev: Alpha significance level n_params: Number of parameters (states), normally 4 parameters for station coordinates and receiver clock Returns: Array containing False for observations to throw away. """ # Regular checks num_obs = len(residuals) df = num_obs - n_params - 1 if df < 0: log.warn( f"sol_validattion():: degree of freedom < 0 (df = {df}) --> TEST NOT PASSED" ) return False # Chi-square validation of residuals vv = np.dot(residuals, residuals) # sum (v(i) * v(i)) chi_sqr = stats.chi2.ppf(1 - alpha_siglev, df=df) if vv > chi_sqr: log.debug( f"sol_validattion():: number of valid obs={num_obs:03} vv={vv:.2f} chi-square value={chi_sqr:.2f}--> TEST NOT PASSED" ) return False else: log.debug( f"sol_validation():: number of valid obs={num_obs:02} vv={vv:.2f} < chi-square value={chi_sqr:.2f} --> TEST PASSED for alpha significance level= {(1.0-alpha_siglev)*100:.2f} %" ) return True
def as_dataset( self, ref_pos: Union[np.ndarray, List[float]] = [0.0, 0.0, 0.0]) -> "Dataset": """Return the parsed data as a Dataset Args: ref_pos: Reference position given in terrestrial reference system and meters Returns: Midgard Dataset where timeseries data are stored with following fields: | Field | Type | Description | |---------------------|-------------------|----------------------------------------------------------------| | obs.dpos | PositionDelta | Position delta object referred to a reference position | | obs.dpos_sigma_east | numpy.array | Standard deviation of east position | | obs.dpos_sigma_north| numpy.array | Standard deviation of north position | | obs.dpos_sigma_up | numpy.array | Standard deviation of up position | | time | Time | Parameter time given as TimeTable object | """ # Initialize dataset dset = dataset.Dataset() if not self.data: log.warn("No data in {self.file_path}.") return dset dset.num_obs = len(self.data["decimalyear"]) dset.meta.update(self.meta) # Add position ref_pos = position.Position(np.repeat(np.array([ref_pos]), dset.num_obs, axis=0), system="trs") dset.add_position_delta( name="obs.dpos", val=np.stack( (self.data["east"], self.data["north"], self.data["vertical"]), axis=1), system="enu", ref_pos=ref_pos, ) # TODO: sigma functionality has to be improved: dpos_sigma.enu.east, dpos_sigma.trs.x ## Add position sigma # sigma = np.stack((self.data["east_sigma"], self.data["north_sigma"], self.data["vertical_sigma"]), axis=1) # dset.add_sigma(name="dpos_sigma", val=dset.dpos.val, sigma=sigma, unit="meter") dset.add_float(name="obs.dpos_sigma_east", val=self.data["east_sigma"], unit="meter") dset.add_float(name="obs.dpos_sigma_north", val=self.data["north_sigma"], unit="meter") dset.add_float(name="obs.dpos_sigma_up", val=self.data["vertical_sigma"], unit="meter") # Add time dset.add_time(name="time", val=self.data["decimalyear"], scale="utc", fmt="decimalyear", write_level="operational") return dset
def write_to_dataset(self, dset) -> "Dataset": """Return the parsed data as a Dataset Args: dset (Dataset): The Dataset. Depending on the Spring CSV following dataset fields can be available: | Field | Former name | Description | |-----------------------|--------------------------------------------------------------------------------------| | adjbgd-dcb_mean | | | | adjbgd-dcb_med | | | | clk_diff_dt_mean | dB_mean | MEAN clock offset determined in each epoch | | clk_diff_with_dt_mean | dH_mean | Satellite clock correction difference corrected for satellite bias and | | | | the MEAN constellation clock offset in each epoch | | dr | | Satellite coordinate difference between broadcast and precise ephemeris| | | | in radial direction in [m] | | dx | | Satellite coordinate difference between broadcast and precise ephemeris| | | | for x-coordinate | | dy | | Satellite coordinate difference between broadcast and precise ephemeris| | | | for y-coordinate | | dz | | Satellite coordinate difference between broadcast and precise ephemeris| | | | for z-coordinate | | dh_med | | Satellite clock correction difference corrected for satellite bias and | | | | the MEDIAN clock offset in each epoch | | db_med | | MEDIAN constellation clock offset determined in each epoch | | dbgd_mean | | | | dbgd_med | | | | orb_diff_3d | d3D | 3D orbit difference | | satellite | SVID | Satellite number | | sqrt_a2_c2 | dAC | sqrt(a^2 + c^2) | | system | | System identifier | | sisre | URE_Av_mean | Global average user range error (signal-in-space range error) with use | | | | of MEAN constellation clock offset | | ure_av_med | | Global average user range error (signal-in-space range error) with use | | | | of MEDIAN constellation clock offset | | ure_wul_mean | | Global average user range error for worst user location with use of | | | | MEAN constellation clock offset | | ure_wul_med | | Global average user range error for worst user location with use of | | | | MEDIAN constellation clock offset | """ field_ure_control_tool_to_where = { "dAC(m)": "sqrt_a2_c2", "dB_mean(m)": "clk_diff_dt_mean", "dH_mean(m)": "clk_diff_with_dt_mean", "dR(m)": "dradial", "d3D(m)": "orb_diff_3d", "SVID": "satellite", "URE_Av_mean(m)": "sisre", } # Initialize dataset if not self.data: log.warn("No data in {self.file_path}.") return dset dset.num_obs = len(self.data["dX(m)"]) # Add time dset.add_time( "time", val=[ dateutil.parser.parse(self.data["YYYY/MM/DD"][i] + " " + self.data["HH:MM:SS"][i]) for i in range(0, dset.num_obs) ], scale="gps", fmt="datetime", write_level="operational", ) # Add system field dset.add_text("system", val=[s[0:1] for s in self.data["SVID"]]) # Add position field dset.add_position("orb_diff", itrs=np.vstack( (self.data["dX(m)"], self.data["dY(m)"], self.data["dZ(m)"])).T, time="time") # Define fields to save in dataset remove_fields = {"YYYY/MM/DD", "HH:MM:SS", "dX(m)", "dY(m)", "dZ(m)"} fields = set(self.data.keys()) - remove_fields # Add text and float fields for field in fields: where_fieldname = (field_ure_control_tool_to_where[field] if field in field_ure_control_tool_to_where.keys() else field.lower()) where_fieldname = where_fieldname.replace( "(m)", "") # Remove unit (m) from field name if self.data[field].dtype.kind in { "U", "S" }: # Check if numpy type is string dset.add_text(where_fieldname, val=self.data[field]) continue dset.add_float(where_fieldname, val=self.data[field], unit="meter")
def as_dataset( self, ref_pos: Union[np.ndarray, List[float], None] = None) -> "Dataset": """Return the parsed data as a Dataset Args: ref_pos: Reference position given in terrestrial reference system and meters Returns: Midgard Dataset where GALAT result data are stored with following fields: | Field | Type | Description | |--------------------------|-------------------|-----------------------------------------------------------| | hpe | np.ndarray | Horizontal Position Error of site position vs. reference | | | | position | | num_satellite_available | np.ndarray | Number of available satellites | | num_satellite_used | np.ndarray | Number of used satellites | | pdop | np.ndarray | Position dilution of precision | | site_pos | Position | Site position | | site_pos_vs_ref | PositionDelta | Site position versus reference coordinate | | site_vel_3d | np.ndarray | 3D site velocity | | time | Time | Parameter time given as TimeTable object | | vpe | np.ndarray | Vertical Position Error of site position vs. reference | | | | position | """ fields = { #"hpe": "meter", # Recalculated based on site position and given reference coordinate #"vpe": "meter", # Recalculated based on site position and given reference coordinate "site_vel_3d": "meter/second", "pdop": "", "num_satellite_available": "", "num_satellite_used": "", } # Initialize dataset dset = dataset.Dataset() if not self.data: log.warn("No data in {self.file_path}.") return dset dset.num_obs = len(self.data["time"]) # Add time field dset.add_time( "time", val=self.data["time"], scale="gps", fmt="datetime", ) # Add float fields for field in fields.keys(): dset.add_float(name=field, val=self.data[field], unit=fields[field]) # Add site position field dset.add_position( "site_pos", val=np.stack(( self.data["latitude"] * Unit.deg2rad, self.data["longitude"] * Unit.deg2rad, self.data["height"], ), axis=1), system="llh", ) # Use either reference position from RINEX header or given argument as reference position if ref_pos is None: ref_pos = position.Position( np.repeat( np.array([[ self.meta["pos_x"], self.meta["pos_y"], self.meta["pos_z"] ]]), dset.num_obs, axis=0, ), system="trs", ) else: ref_pos = position.Position(np.repeat(np.array([ref_pos]), dset.num_obs, axis=0), system="trs") # Add relative position dset.add_position_delta( name="site_pos_vs_ref", val=(dset.site_pos.trs - ref_pos.trs).val, system="trs", ref_pos=ref_pos, ) # Add HPE and VPE to dataset dset.add_float( "hpe", val=np.sqrt(dset.site_pos_vs_ref.enu.east**2 + dset.site_pos_vs_ref.enu.north**2), unit="meter", ) dset.add_float("vpe", val=np.absolute(dset.site_pos_vs_ref.enu.up), unit="meter") return dset
def as_dataset(self) -> "Dataset": """Return the parsed data as a Dataset Returns: A dataset containing the data. """ # Spring constellation definition system_def = { "0": "", # Unknown "1": "G", # GPS "2": "R", # GLONASS "3": "S", # SBAS "4": "E", # Galileo "5": "C", # BeiDou "6": "J", # QZSS } field_spring_to_where = { "3DSpeed": "site_vel_3d", "Clock": "delay.gnss_satellite_clock", "EastSpeed": "site_vel_east", "GroupDelay": "delay.gnss_total_group_delay", "HSpeed": "site_vel_h", "IODE": "used_iode", "NorthSpeed": "site_vel_north", "PseudoRange": "delay.gnss_range", "SatInView": "num_satellite_available", "TropoDelay": "troposphere_dT", "UISD": "delay.gnss_ionosphere", "UsedSat": "num_satellite_used", "EastvsRef": "site_pos_vs_ref_east", "NorthvsRef": "site_pos_vs_ref_north", "VerticalvsRef": "site_pos_vs_ref_up", "VerticalSpeed": "site_vel_up", "XSpeed": "site_vel_x", "YSpeed": "site_vel_y", "ZSpeed": "site_vel_z", } # Initialize dataset dset = dataset.Dataset() if not self.data: log.warn("No data in {self.file_path}.") return dset dset.num_obs = len(self.data["GPSEpoch"]) # Add time dset.add_time( "time", val=[ dateutil.parser.parse(v.replace("UTC", "")) for v in self.data["UTCDateTime"] ], scale="utc", fmt="datetime", write_level="operational", ) # Add system field based on Constellation column if "Constellation" in self.data.keys(): dset.add_text("system", val=[ system_def[str(value)] for value in self.data["Constellation"] ]) # Add satellite field based on PRN column if "PRN" in self.data.keys(): prn_data = [] for prn in self.data["PRN"]: if prn >= 1 and prn <= 32: # Handling of GPS satellites prn_data.append("G" + str(prn).zfill(2)) elif prn >= 38 and prn <= 70: # Handling of GLONASS satellites prn_data.append("R" + str(prn - 38).zfill(2)) elif prn >= 71 and prn <= 140: # Handling of Galileo satellites prn_data.append("E" + str(prn - 70).zfill(2)) elif prn >= 191 and prn <= 222: # Handling of BeiDou satellites prn_data.append("C" + str(prn - 191).zfill(2)) else: log.fatal(f"Spring PRN number '{prn}' is unknown.") dset.add_text("satellite", val=prn_data) dset.add_text("system", np.array(prn_data).astype("U1")) # Add position field based on Latitude, Longitude and Height column if "Latitude" in self.data.keys(): pos = Position( val=np.vstack((self.data["Latitude"] * Unit.deg2rad, self.data["Longitude"] * Unit.deg2rad, self.data["Height"])).T, system="llh", ) if "XPos" in self.data.keys(): dset.add_position("sat_pos", val=pos.trs, system="trs", time=dset.time) else: dset.add_position("site_pos", val=pos.trs, system="trs", time=dset.time) # Define fields to save in dataset remove_time_fields = { "Constellation", "GPSEpoch", "GPSWeek", "GPSSecond", "PRN", "", "UTCDateTime" } fields = set(self.data.keys()) - remove_time_fields # Add text and float fields for field in fields: where_fieldname = field_spring_to_where[ field] if field in field_spring_to_where.keys( ) else field.lower() if self.data[field].dtype.kind in { "U", "S" }: # Check if numpy type is string dset.add_text(where_fieldname, val=self.data[field]) continue dset.add_float(where_fieldname, val=self.data[field]) return dset
def as_dataset(self) -> "Dataset": """Return the parsed data as a Dataset Returns: Midgard Dataset where timeseries data are stored with following fields: | Field | Type | Description | |-----------------------|-------------------|--------------------------------------------------------------| | amplitude | numpy.array | Amplitude | | azimuth | numpy.array | Azimuth in [rad] | | frequency | numpy.array | GNSS frequency identifier | | peak2noise | numpy.array | Peak to noise | | satellite | numpy.array | Satellite number | | reflection_height | numpy.array | Reflection height in [m] | | time | Time | Time | """ freq_def = { 1: "L1", # G 2: "L2", # G 5: "L5", # G 20: "L2C", # G 101: "L1", # R 102: "L2", # R 201: "E1", # E 205: "E5a", # E 206: "E6", # E 207: "E5b", # E 208: "E5", # E 302: "B1_2", # C 306: "B3", # C 307: "B2b", # C } float_fields = { "amplitude": None, "azimuth": "radian", "peak2noise": None, "reflection_height": "meter", } # Initialize dataset dset = dataset.Dataset() if not self.data: log.warn("No data in {self.file_path}.") return dset dset.num_obs = len(self.data["time"]) # Add text fields satellite = list() system = list() for sat in self.data["satellite"]: if sat >= 1 and sat < 100: # GPS satellites system.append("G") satellite.append("G" + str(int(sat)).zfill(2)) elif sat >= 101 and sat < 200: # GLONASS satellites system.append("R") satellite.append("R" + str(int(sat))[1:3]) elif sat >= 201 and sat < 300: # Galileo satellites system.append("E") satellite.append("E" + str(int(sat))[1:3]) elif sat >= 301 and sat < 400: # BeiDou satellites system.append("C") satellite.append("C" + str(int(sat))[1:3]) else: log.fatal( "GNSSREFL satellite number {sat} is not defined. Valid satellite numbers are between [1-399]." ) dset.add_text( name="system", val=system, write_level="operational", ) dset.add_text( name="satellite", val=satellite, write_level="operational", ) dset.add_text( name="frequency", val=[freq_def[v] for v in self.data["frequency"]], write_level="operational", ) # Add time field dset.add_time( name="time", val=self.data["time"], scale="utc", fmt="datetime", write_level="operational", ) # Add float fields for field in float_fields.keys(): if field not in self.data.keys(): log.warn( f"Field '{field}' does not exist in file {self.meta['__data_path__']}." ) continue value = np.deg2rad( self.data[field]) if field == "azimuth" else self.data[field] unit = "" if float_fields[field] is None else float_fields[field] dset.add_float(name=field, val=value, unit=unit, write_level="operational") return dset
def download_file( self, file_key: str, file_vars: Optional[Dict[str, str]] = None, file_path: Optional[pathlib.Path] = None, create_dirs: bool = True, **path_args: Any, ) -> Optional[pathlib.Path]: """Download a file from the web and save it to disk Use pycurl (libcurl) to do the actual downloading. Requests might be nicer for this, but turned out to be much slower (and in practice unusable for bigger files) and also not really supporting ftp-downloads. Args: file_key: File key that should be downloaded. file_vars: File variables used to find path from file_key. file_path: Path where file will be saved, default is to read from configuration. create_dirs: Create directories as necessary before downloading file. path_args: Arguments passed on to .path() to find file_path. Returns: Path to downloaded file, None if no file was downloaded. """ # Do not download anything if download_missing class variable is False if not self.download_missing: return None # Do not download anything if url is not given in configuration if "url" not in self[file_key] or not self[file_key].url.str: return None # Get file_path from configuration if it's not given explicitly file_url = self.url(file_key, file_vars=file_vars, **path_args) is_zipped = self.is_path_zipped(file_url) path_args.update(is_zipped=is_zipped) if file_path is None: file_path = self.path(file_key, file_vars=file_vars, download_missing=False, **path_args) file_path = file_path.with_name(file_url.name) if create_dirs: file_path.parent.mkdir(parents=True, exist_ok=True) log.info(f"Download {file_key} from '{file_url}' to '{file_path}'") with builtins.open(file_path, mode="wb") as fid: c = pycurl.Curl() c.setopt(c.URL, file_url) c.setopt(c.WRITEDATA, fid) try: c.perform() if not (200 <= c.getinfo(c.HTTP_CODE) <= 299): raise pycurl.error() except pycurl.error: log.error(f"Problem downloading file: {c.getinfo(c.EFFECTIVE_URL)} ({c.getinfo(c.HTTP_CODE)})") if file_path.exists(): # Print first 10 lines to console head_of_file = f"Contents of '{file_path}':\n" + "\n".join(file_path.read_text().split("\n")[:10]) log.info(console.indent(head_of_file, num_spaces=8)) file_path.unlink() log.warn(f"Try to download '{file_url}' manually and save it at '{file_path}'") else: log.info(f"Done downloading {file_key}") finally: c.close() return file_path