def list_datasets(rundate, tech, session, stage, **kwargs): """List datasets in a given dataset file Args: rundate: Datetime, the model run date. tech: String, the technique. stage: String, the stage. kwargs: Other arguments are passed to files.open. Returns: List of strings describing the datasets. """ file_vars = dict( config.program_vars(rundate, tech, session=session, stage=stage, **kwargs), **config.date_vars(rundate)) try: with files.open("dataset_json", file_vars=file_vars) as fid: json_data = json.load(fid) except FileNotFoundError: return list() log.fatal( f"No data found for {tech.upper()} {stage} {rundate.strftime(config.FMT_date)}" ) return sorted(k for k in json_data.keys() if not k.startswith("_") and "/" in k)
def _parameters(rundate, pipeline, stage, session, **options): """Set up parameters for running Where Args: rundate (date): The model run date. pipeline (str): The pipeline. stage (str): The stage to compare. session (str): The session to compare. options (dict): Command line options that will be passed to Where. Returns: dict: Command and file_vars representing the given Where analysis """ user = "******" params = dict( command=( f"{where.__executable__} {rundate:%Y %m %d} --{pipeline} --session={session} -D -N " f"--user={user} --output=system_test:{stage} " + " ".join(f"--{o}={v}" for o, v in options.items()) ), file_vars=dict( rundate=rundate.strftime(config.FMT_date), tech=pipeline, stage=stage, user=user, id="", session=session, **config.date_vars(rundate), ), ) return params
def _analysis_vars(rundate, user="", id=""): analysis_vars = dict() analysis_vars["rundate"] = rundate # Rundate as date analysis_vars["user"] = user if user else getpass.getuser().lower() analysis_vars["id"] = id analysis_vars.update(config.date_vars(rundate)) return analysis_vars
def parse_dataset_id(rundate, tech, stage, dataset_name, dataset_id, **kwargs): """Allow for some advanced handling of dataset_id In addition to using regular numbers as dataset_id, some text keywords can be used: + 'last': Use the last dataset_id written to file, default 0 if no file is previously written. + 'all': Return a list of all dataset_ids in the file. """ if isinstance(dataset_id, (float, int)): return dataset_id # Use the JSON-file to find information about the dataset ids file_vars = dict( config.program_vars(rundate, tech, session=dataset_name, stage=stage, **kwargs), **config.date_vars(rundate)) try: with files.open("dataset_json", file_vars=file_vars) as fid: json_data = json.load(fid) except FileNotFoundError: json_data = dict() if dataset_id == "last": # If _last_dataset_id is not given, use dataset_id=0 as default return json_data.get(dataset_name, dict()).get("_last_dataset_id", 0) if dataset_id == "all": return [ int(k.split("/")[-1]) for k in json_data.keys() if k.startswith("{}/".format(dataset_name)) ]
def copy_log_from_where(rundate, pipeline, args): kwargs = dict() for a in args.split(): if "=" in a: a = a.split("=", maxsplit=1) kwargs[a[0].lstrip("-")] = a[1] file_vars = dict( **config.program_vars(rundate, pipeline, use_options=False, **kwargs), **config.date_vars(rundate)) log_level = config.where.runner.log_level.str current_level = "none" try: with config.files.open("log", file_vars=file_vars) as fid: for line in fid: line_level, _, text = line.partition(" ") line_level = line_level.strip().lower() current_level = line_level if line_level else current_level text = text.strip() if getattr(LogLevel, current_level) >= getattr( LogLevel, log_level) and text: # strip the 20 date characters from the text log.log(text[20:], current_level) except FileNotFoundError as err: log.warn(f"'{err}'")
def __init__(self, rundate, file_key="gnss_sinex_bias"): """Set up a new GNSS bias object by parsing SINEX bias file The parsing is done by :mod:`where.parsers.gnss_sinex_bias`. """ parser = parsers.parse_key(file_key=file_key, file_vars=config.date_vars(rundate)) self.data = parser.as_dict()
def _concatenate_datasets(from_date: date, to_date: date, dset_vars: Dict[str, str], only_for_rundate: bool) -> np.ndarray: """Concatenate datasets Args: from_date: Start date for reading Dataset. to_date: End date for reading Dataset. dset_vars: Common Dataset variables. only_for_rundate: Concatenate only data for given rundate. """ dset_merged = None def read_dset(rundate): with Timer(f"Finish read of day {rundate} in", logger=log.time): try: log.info(f"Reading data for {rundate}") return dataset.Dataset.read(**dict(dset_vars, rundate=rundate)) except OSError as err: log.warn(f"Unable to read data for {rundate}: {err}") return dataset.Dataset() date_to_read = from_date while date_to_read <= to_date: dset = read_dset(date_to_read) if dset: # Skip extension if dataset is empty if only_for_rundate: _keep_data_only_for_rundate(dset) if dset.num_obs == 0: log.warn(f"No data to for {date_to_read} in dataset") # Initialize merged dataset if dset_merged is None: dset_merged = dset # Merged dataset should be related to start date if date_to_read != from_date: dset.vars["rundate"] = from_date.strftime("%Y-%m-%d") dset.analysis["rundate"] = from_date dset.analysis.update(config.date_vars(from_date)) date_to_read += timedelta(days=1) continue with Timer(f"Finish extend for day {date_to_read} in", logger=log.time): dset_merged.extend(dset) date_to_read += timedelta(days=1) dset_merged.analysis.update( id=f"{dset_merged.analysis['id']}_concatenated") return dset_merged
def __init__(self, rundate, file_path): """Set up the basic information needed by the parser Args: rundate (datetime.date): The model run date. file_path (str): Path to orbit-file to parse. """ super().__init__(rundate=rundate, file_path=file_path) self.vars = config.date_vars(rundate)
def rundate(self, value): """Set rundate of dataset and update corresponding variables Args: value (Date): The model run date. """ self._rundate = value self.vars["rundate"] = value.strftime(config.FMT_date) self.vars.update(**config.date_vars(value))
def get_vlbi_master_schedule(rundate=None): """Read master file If rundate is not specified, used file_vars that are already set. """ file_vars = None if rundate is None else config.date_vars(rundate) parser = parsers.parse_key("vlbi_master_file", file_vars=file_vars) return VlbiMasterSchedule(parser.as_dict(), file_vars["yyyy"])
def _read(self, dset_raw): """Read SP3 orbit file data and save it in a Dataset In addition to the given date, we read data for the day before and after. This is needed to carry out correct orbit interpolation at the start and end of a day. TODO: How well fits the orbits from day to day? Is it necessary to align the orbits? Args: dset_raw (Dataset): Dataset representing raw data from apriori orbit files """ date_to_read = dset_raw.rundate - timedelta(days=self.day_offset) file_paths = list() # Loop over days to read while date_to_read <= dset_raw.rundate + timedelta( days=self.day_offset): if self.file_path is None: file_path = files.path( self.file_key, file_vars=config.date_vars(date_to_read)) else: file_path = self.file_path log.debug(f"Parse precise orbit file {file_path}") # Generate temporary Dataset with orbit file data dset_temp = data.Dataset( rundate=date_to_read, tech=dset_raw.vars["tech"], stage="temporary", dataset_name="", dataset_id=0, empty=True, ) parser = parsers.parse(parser_name="orbit_sp3", file_path=file_path, rundate=date_to_read) parser.write_to_dataset(dset_temp) file_paths.append(str(parser.file_path)) # Extend Dataset dset_raw with temporary Dataset date = date_to_read.strftime("%Y-%m-%d") dset_raw.copy_from( dset_temp, meta_key=date) if dset_raw.num_obs == 0 else dset_raw.extend( dset_temp, meta_key=date) dset_raw.add_to_meta("parser", "file_path", file_paths) date_to_read += timedelta(days=1) return dset_raw
def file_vars(): """Get a list of file variables for the current pipeline The active analysis variables are also made available, but may be overridden by the pipeline. """ file_vars = dict(config.analysis.config.as_dict(), **config.date_vars(config.analysis.rundate.date)) pipeline_file_vars = plugins.call(package_name=__name__, plugin_name=config.analysis.tech.str, part="file_vars") file_vars.update(pipeline_file_vars) return file_vars
def changed(self): """Construct date variables and split out timestamp """ parts = (self.currentText() + "/").split("/") rundate = datetime.strptime(parts[0], "%Y%m%d") if parts[0] else None vars_ = config.date_vars(rundate) if rundate else dict() vars_["rundate"] = rundate vars_["date"] = parts[0] vars_["timestamp"] = parts[1].split("_")[0] try: # Test if timestamp is valid datetime.strptime(vars_["timestamp"], config.FMT_dt_file) parts[1] = "_".join(parts[1].split("_")[1:]) except ValueError: vars_["timestamp"] = "" vars_["id"] = "_" + parts[1] if parts[1] else "" return vars_
def copy_log_from_where(rundate, pipeline, session): file_vars = dict(**config.program_vars(rundate, pipeline, session), **config.date_vars(rundate)) log_level = config.where.runner.log_level.str current_level = "none" try: with files.open("log", file_vars=file_vars) as fid: for line in fid: line_level, _, text = line.partition(" ") line_level = line_level.strip().lower() current_level = line_level if line_level else current_level text = text.strip() if getattr(LogLevel, current_level) >= getattr( LogLevel, log_level) and text: log.log(text, current_level) except FileNotFoundError as err: log.warn(f"'{err}'")
def read_data(self): """Read the data from three monthly datafiles """ files_read = [] date_to_read = self.rundate - timedelta(days=7) while date_to_read < self.rundate + timedelta(days=self.arc_length + 8): self.vars.update(config.date_vars(date_to_read)) file_path = files.path(self.file_key, file_vars=self.vars) if file_path not in files_read: files_read.append(file_path) self.dependencies.append(file_path) with files.open(self.file_key, file_vars=self.vars, mode="rt", encoding="latin_1") as fid: self.parse_file(fid) date_to_read += timedelta(days=1)
def get_vmf1_grid(time): """Read VMF1 gridded data files relevant for the given time epochs Args: time (Time): observation epochs Returns: A dictionary of functions that can interpolate in the VMF1 dataset. """ data = dict() min_time = time.utc.datetime if len(time) == 1 else min(time.utc.datetime) max_time = time.utc.datetime if len(time) == 1 else max(time.utc.datetime) start_hour = 6 * (min_time.hour // 6) start = min_time.replace(hour=start_hour, minute=0, second=0, microsecond=0) end_hour = 6 * (max_time.hour // 6) end = max_time.replace(hour=end_hour, minute=0, second=0, microsecond=0) + timedelta(hours=6) for datatype, multiplier in DATATYPE.items(): dt_to_read = start vmf1_data = dict() while dt_to_read <= end: file_vars = dict(config.date_vars(dt_to_read), type=datatype) data_chunk = parsers.parse_key(file_key="vmf1_grid", file_vars=file_vars).as_dict() if data_chunk: vmf1_data[dt_to_read] = (data_chunk["lat"], data_chunk["lon"], data_chunk["values"] * multiplier) dt_to_read += timedelta(hours=6) data[datatype] = vmf1_data funcs = {k: vmf1_interpolator(v) for k, v in data.items()} data = parsers.parse_key(file_key="orography_ell").as_dict() funcs["ell"] = RectBivariateSpline(data["lon"], data["lat"], data["values"].T) return funcs
def get_non_tidal_atmospheric_loading(time): """Read gridded data files relevant for the given time epochs Args: time (Time): observation epochs Returns: A function that can interpolate """ min_time = min(time.utc).datetime max_time = max(time.utc).datetime start_hour = 6 * (min_time.hour // 6) start = min_time.replace(hour=start_hour, minute=0, second=0, microsecond=0) end_hour = 6 * (max_time.hour // 6) end = max_time.replace(hour=end_hour, minute=0, second=0, microsecond=0) + timedelta(hours=6) dt_to_read = start data = dict(up={}, east={}, north={}) while dt_to_read <= end: file_vars = dict(config.date_vars(dt_to_read)) data_chunk = parsers.parse_key( file_key="non_tidal_atmospheric_loading", file_vars=file_vars).as_dict() if data_chunk: data["up"][dt_to_read] = (data_chunk["lat"], data_chunk["lon"], data_chunk["up"]) data["east"][dt_to_read] = (data_chunk["lat"], data_chunk["lon"], data_chunk["east"]) data["north"][dt_to_read] = (data_chunk["lat"], data_chunk["lon"], data_chunk["north"]) dt_to_read += timedelta(hours=6) return {k: create_interpolator(v) for k, v in data.items()}
def get_vmf1_station(time): """Read VMF1 station data files relevant for the given time epochs Args: time (Time): observation epochs Returns: A dictionary of functions that can interpolate in the VMF1 dataset. """ start = time.utc.datetime.date() if len(time) == 1 else min( time.utc.datetime).date() end = time.utc.datetime.date() if len(time) == 1 else max( time.utc.datetime).date() date_to_read = start vmf1_data = dict() while date_to_read <= end: file_vars = dict(config.date_vars(date_to_read)) parser = parsers.parse_key(file_key="vmf1_station", file_vars=file_vars) data_chunk = parser.as_dict() for sta, sta_data in data_chunk.items(): vmf1_data.setdefault(sta, {}) for k in sta_data.keys(): vmf1_data[sta][k] = vmf1_data[sta].get(k, []) + data_chunk[sta][k] date_to_read += timedelta(days=1) funcs = dict() for sta, sta_data in vmf1_data.items(): funcs.setdefault(sta, {}) mjd = sta_data.pop("mjd") # Use linear interpolation between each datapoint funcs[sta] = { k: interp1d(mjd, v, fill_value="extrapolate") for k, v in sta_data.items() } return funcs
def _read(self, dset_raw, provider, version): """Read SP3 orbit file data and save it in a Dataset Naming convention correspond to end of arc, at midnight, hence we add day_offset, which is usually arc_length - 1 Args: dset_raw (Dataset): Dataset representing raw data from apriori orbit files provider: Str: Orbit provider version: Str: Orbit version """ rundate = dset_raw.rundate date_to_read = rundate + timedelta(days=self.day_offset) file_vars = config.date_vars(date_to_read) file_vars["provider"] = provider file_vars["version"] = version if self.file_path is None: file_path = config.files.path(self.file_key, file_vars) else: file_path = self.file_path log.debug(f"Parse precise orbit file {file_path}") # Generate temporary Dataset with orbit file data dset_orbit = data.Dataset(rundate=date_to_read, tech=dset_raw.vars["tech"], stage="orbit", dataset_name="", dataset_id=0, empty=True) parser = parsers.parse(parser_name="orbit_sp3", file_path=file_path, rundate=date_to_read) parser.write_to_dataset(dset_orbit) dset_orbit.add_to_meta("parser", "file_path", file_path) return dset_orbit
def write_one_day(dset, date): """Write RINEX navigation file for given date Args: dset: Dataset, a dataset containing the data. date: Current date """ brdc = apriori.get( "orbit", rundate=dset.analysis["rundate"], system=tuple(dset.unique("system")), station=dset.vars["station"], apriori_orbit="broadcast", ) meta = brdc.dset_edit.meta[date.strftime("%Y-%m-%d")] data = brdc.dset_edit # TODO: Another possibility: brdc.dset_raw file_vars = {**dset.vars, **dset.analysis} file_vars["doy"] = config.date_vars(date)[ "doy" ] # TODO: workaround, so that all files are written in the same working directory -> does not work if year is changed. with config.files.open("output_rinex2_nav", file_vars=file_vars, mode="wt") as fid: # # Write RINEX navigation header # if meta["file_type"] == "N": file_type = "NAVIGATION DATA" fid.write("{:>9s}{:11s}{:40s}RINEX VERSION / TYPE\n".format(meta["version"], "", file_type)) fid.write( "{:20s}{:20s}{:20s}PGM / RUN BY / DATE\n".format(meta["program"], meta["run_by"], meta["file_created"]) ) for line in meta["comment"]: fid.write("{:60s}COMMENT\n".format(line)) fid.write( "{:>14.4e}{:>12.4e}{:>12.4e}{:>12.4e}{:10s}ION ALPHA\n" "".format( meta["iono_para"]["GPSA"]["para"][0], meta["iono_para"]["GPSA"]["para"][1], meta["iono_para"]["GPSA"]["para"][2], meta["iono_para"]["GPSA"]["para"][3], "", ) ) fid.write( "{:>14.4e}{:>12.4e}{:>12.4e}{:>12.4e}{:10s}ION BETA\n" "".format( meta["iono_para"]["GPSB"]["para"][0], meta["iono_para"]["GPSB"]["para"][1], meta["iono_para"]["GPSB"]["para"][2], meta["iono_para"]["GPSB"]["para"][3], "", ) ) # TODO fid.write('{:>22.12e}{:>19.12e}{:>9d}{:>9d}{:1s}DELTA-UTC: A0,A1,T,W\n' # ''.format(meta['a0'], meta['a1'], int(meta['t']), int(meta['w']), '')) fid.write("{:>6d}{:54s}LEAP SECONDS\n".format(int(meta["leap_seconds"]["leap_seconds"]), "")) fid.write("{:60s}END OF HEADER\n".format("")) # # Write RINEX navigation data # # TODO: # for drow in data.get_rows(): # fid.write('{d.sat:2d} {d.time:%Y%m%d %H%M%S} {d.inc0:13.4f}'.format(d=drow)) # fid.write(' {d.hurra:14.10f} ...'.format(d=drow)) for idx in range(0, data.num_obs): sat = int(data.satellite[idx][1:3]) d = data.time.gps.datetime[idx] fid.write( "{:2d}{:>3s}{:>3d}{:>3d}{:>3d}{:>3d}{:>5.1f}{:>19.12e}{:>19.12e}{:>19.12e}\n" "".format( sat, str(d.year)[2:4], d.month, d.day, d.hour, d.minute, d.second, data.sat_clock_bias[idx], data.sat_clock_drift[idx], data.sat_clock_drift_rate[idx], ) ) fid.write( "{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n" "".format(data.iode[idx], data.crs[idx], data.delta_n[idx], data.m0[idx]) ) fid.write( "{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n" "".format(data.cuc[idx], data.e[idx], data.cus[idx], data.sqrt_a[idx]) ) # TODO: toe depends on GNSS system time -> for BeiDou it has to be changed fid.write( "{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n" "".format(data.toe.gps.gpssec[idx], data.cic[idx], data.Omega[idx], data.cis[idx]) ) fid.write( "{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n" "".format(data.i0[idx], data.crc[idx], data.omega[idx], data.Omega_dot[idx]) ) # TODO: gnss_week depends on GNSS -> for BeiDou it has to be changed # TODO: codes_l2 only valid for GPS and QZSS -> Galileo data_source; rest None # TODO: 'G': 'l2p_flag', 'J': 'l2p_flag' fid.write( "{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n" "".format(data.idot[idx], data.codes_l2[idx], data.gnss_week[idx], data.l2p_flag[idx]) ) # TODO: 'G': 'iodc', 'J': 'iodc', 'E': 'bgd_e1_e5b', 'C': 'tgd_b2_b3' # TODO: 'G': 'tgd', 'J': 'tgd', 'E': 'bgd_e1_e5a', 'C': 'tgd_b1_b3', 'I': 'tgd' fid.write( "{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n" "".format(data.sv_accuracy[idx], data.sv_health[idx], data.tgd[idx], data.iodc[idx]) ) # TODO: transmission_time depends on GNSS system time -> for BeiDou it has to be changed # TODO: fit_interval only valid for GPS and QZSS -> for BeiDou age_of_clock_corr; rest None fid.write( "{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n" "".format(data.transmission_time.gps.gpssec[idx], data.fit_interval[idx], 0.0, 0.0) )
# Interpolation settings moving_window = True interpolation_method = "lagrange" # Methods: 'lagrange', 'interp1d' or 'InterpolatedUnivariateSpline' window_size = 10 # Definition of dummy date year = 2002 month = 1 day = 1 hour = 0 minute = 0 second = 0 rundate = datetime(year, month, day, hour, minute, second) file_vars = config.date_vars(rundate) # Define 15 min dataset file_path = config.files.path(file_key="test_gnss_orbit_interpolation_15min", file_vars=file_vars) orb_15min = data.Dataset(rundate, tech=None, stage=None, dataset_name="gnss_precise_orbit_15min", dataset_id=0, empty=True) parser = parsers.parse(parser_name="orbit_sp3c", file_path=file_path, rundate=rundate) parser.write_to_dataset(orb_15min)