def list_datasets(rundate, tech, session, stage, **kwargs):
"""List datasets in a given dataset file
Args:
rundate: Datetime, the model run date.
tech: String, the technique.
stage: String, the stage.
kwargs: Other arguments are passed to files.open.
Returns:
List of strings describing the datasets.
"""
file_vars = dict(
config.program_vars(rundate,
tech,
session=session,
stage=stage,
**kwargs), **config.date_vars(rundate))
try:
with files.open("dataset_json", file_vars=file_vars) as fid:
json_data = json.load(fid)
except FileNotFoundError:
return list()
log.fatal(
f"No data found for {tech.upper()} {stage} {rundate.strftime(config.FMT_date)}"
)
return sorted(k for k in json_data.keys()
if not k.startswith("_") and "/" in k)
def _parameters(rundate, pipeline, stage, session, **options):
"""Set up parameters for running Where
Args:
rundate (date): The model run date.
pipeline (str): The pipeline.
stage (str): The stage to compare.
session (str): The session to compare.
options (dict): Command line options that will be passed to Where.
Returns:
dict: Command and file_vars representing the given Where analysis
"""
user = "******"
params = dict(
command=(
f"{where.__executable__} {rundate:%Y %m %d} --{pipeline} --session={session} -D -N "
f"--user={user} --output=system_test:{stage} " + " ".join(f"--{o}={v}" for o, v in options.items())
),
file_vars=dict(
rundate=rundate.strftime(config.FMT_date),
tech=pipeline,
stage=stage,
user=user,
id="",
session=session,
**config.date_vars(rundate),
),
)
return params
def _analysis_vars(rundate, user="", id=""):
analysis_vars = dict()
analysis_vars["rundate"] = rundate # Rundate as date
analysis_vars["user"] = user if user else getpass.getuser().lower()
analysis_vars["id"] = id
analysis_vars.update(config.date_vars(rundate))
return analysis_vars
def parse_dataset_id(rundate, tech, stage, dataset_name, dataset_id, **kwargs):
"""Allow for some advanced handling of dataset_id
In addition to using regular numbers as dataset_id, some text keywords can be used:
+ 'last': Use the last dataset_id written to file, default 0 if no file is previously written.
+ 'all': Return a list of all dataset_ids in the file.
"""
if isinstance(dataset_id, (float, int)):
return dataset_id
# Use the JSON-file to find information about the dataset ids
file_vars = dict(
config.program_vars(rundate,
tech,
session=dataset_name,
stage=stage,
**kwargs), **config.date_vars(rundate))
try:
with files.open("dataset_json", file_vars=file_vars) as fid:
json_data = json.load(fid)
except FileNotFoundError:
json_data = dict()
if dataset_id == "last":
# If _last_dataset_id is not given, use dataset_id=0 as default
return json_data.get(dataset_name, dict()).get("_last_dataset_id", 0)
if dataset_id == "all":
return [
int(k.split("/")[-1]) for k in json_data.keys()
if k.startswith("{}/".format(dataset_name))
]
def copy_log_from_where(rundate, pipeline, args):
kwargs = dict()
for a in args.split():
if "=" in a:
a = a.split("=", maxsplit=1)
kwargs[a[0].lstrip("-")] = a[1]
file_vars = dict(
**config.program_vars(rundate, pipeline, use_options=False, **kwargs),
**config.date_vars(rundate))
log_level = config.where.runner.log_level.str
current_level = "none"
try:
with config.files.open("log", file_vars=file_vars) as fid:
for line in fid:
line_level, _, text = line.partition(" ")
line_level = line_level.strip().lower()
current_level = line_level if line_level else current_level
text = text.strip()
if getattr(LogLevel, current_level) >= getattr(
LogLevel, log_level) and text:
# strip the 20 date characters from the text
log.log(text[20:], current_level)
except FileNotFoundError as err:
log.warn(f"'{err}'")
def __init__(self, rundate, file_key="gnss_sinex_bias"):
"""Set up a new GNSS bias object by parsing SINEX bias file
The parsing is done by :mod:`where.parsers.gnss_sinex_bias`.
"""
parser = parsers.parse_key(file_key=file_key,
file_vars=config.date_vars(rundate))
self.data = parser.as_dict()
def _concatenate_datasets(from_date: date, to_date: date, dset_vars: Dict[str,
str],
only_for_rundate: bool) -> np.ndarray:
"""Concatenate datasets
Args:
from_date: Start date for reading Dataset.
to_date: End date for reading Dataset.
dset_vars: Common Dataset variables.
only_for_rundate: Concatenate only data for given rundate.
"""
dset_merged = None
def read_dset(rundate):
with Timer(f"Finish read of day {rundate} in", logger=log.time):
try:
log.info(f"Reading data for {rundate}")
return dataset.Dataset.read(**dict(dset_vars, rundate=rundate))
except OSError as err:
log.warn(f"Unable to read data for {rundate}: {err}")
return dataset.Dataset()
date_to_read = from_date
while date_to_read <= to_date:
dset = read_dset(date_to_read)
if dset: # Skip extension if dataset is empty
if only_for_rundate:
_keep_data_only_for_rundate(dset)
if dset.num_obs == 0:
log.warn(f"No data to for {date_to_read} in dataset")
# Initialize merged dataset
if dset_merged is None:
dset_merged = dset
# Merged dataset should be related to start date
if date_to_read != from_date:
dset.vars["rundate"] = from_date.strftime("%Y-%m-%d")
dset.analysis["rundate"] = from_date
dset.analysis.update(config.date_vars(from_date))
date_to_read += timedelta(days=1)
continue
with Timer(f"Finish extend for day {date_to_read} in",
logger=log.time):
dset_merged.extend(dset)
date_to_read += timedelta(days=1)
dset_merged.analysis.update(
id=f"{dset_merged.analysis['id']}_concatenated")
return dset_merged
def __init__(self, rundate, file_path):
"""Set up the basic information needed by the parser
Args:
rundate (datetime.date): The model run date.
file_path (str): Path to orbit-file to parse.
"""
super().__init__(rundate=rundate, file_path=file_path)
self.vars = config.date_vars(rundate)
def rundate(self, value):
"""Set rundate of dataset and update corresponding variables
Args:
value (Date): The model run date.
"""
self._rundate = value
self.vars["rundate"] = value.strftime(config.FMT_date)
self.vars.update(**config.date_vars(value))
def get_vlbi_master_schedule(rundate=None):
"""Read master file
If rundate is not specified, used file_vars that are already set.
"""
file_vars = None if rundate is None else config.date_vars(rundate)
parser = parsers.parse_key("vlbi_master_file", file_vars=file_vars)
return VlbiMasterSchedule(parser.as_dict(), file_vars["yyyy"])
def _read(self, dset_raw):
"""Read SP3 orbit file data and save it in a Dataset
In addition to the given date, we read data for the day before and after. This is needed to carry out correct
orbit interpolation at the start and end of a day.
TODO:
How well fits the orbits from day to day? Is it necessary to align the orbits?
Args:
dset_raw (Dataset): Dataset representing raw data from apriori orbit files
"""
date_to_read = dset_raw.rundate - timedelta(days=self.day_offset)
file_paths = list()
# Loop over days to read
while date_to_read <= dset_raw.rundate + timedelta(
days=self.day_offset):
if self.file_path is None:
file_path = files.path(
self.file_key, file_vars=config.date_vars(date_to_read))
else:
file_path = self.file_path
log.debug(f"Parse precise orbit file {file_path}")
# Generate temporary Dataset with orbit file data
dset_temp = data.Dataset(
rundate=date_to_read,
tech=dset_raw.vars["tech"],
stage="temporary",
dataset_name="",
dataset_id=0,
empty=True,
)
parser = parsers.parse(parser_name="orbit_sp3",
file_path=file_path,
rundate=date_to_read)
parser.write_to_dataset(dset_temp)
file_paths.append(str(parser.file_path))
# Extend Dataset dset_raw with temporary Dataset
date = date_to_read.strftime("%Y-%m-%d")
dset_raw.copy_from(
dset_temp,
meta_key=date) if dset_raw.num_obs == 0 else dset_raw.extend(
dset_temp, meta_key=date)
dset_raw.add_to_meta("parser", "file_path", file_paths)
date_to_read += timedelta(days=1)
return dset_raw
def file_vars():
"""Get a list of file variables for the current pipeline
The active analysis variables are also made available, but may be overridden by the pipeline.
"""
file_vars = dict(config.analysis.config.as_dict(),
**config.date_vars(config.analysis.rundate.date))
pipeline_file_vars = plugins.call(package_name=__name__,
plugin_name=config.analysis.tech.str,
part="file_vars")
file_vars.update(pipeline_file_vars)
return file_vars
def changed(self):
"""Construct date variables and split out timestamp
"""
parts = (self.currentText() + "/").split("/")
rundate = datetime.strptime(parts[0], "%Y%m%d") if parts[0] else None
vars_ = config.date_vars(rundate) if rundate else dict()
vars_["rundate"] = rundate
vars_["date"] = parts[0]
vars_["timestamp"] = parts[1].split("_")[0]
try:
# Test if timestamp is valid
datetime.strptime(vars_["timestamp"], config.FMT_dt_file)
parts[1] = "_".join(parts[1].split("_")[1:])
except ValueError:
vars_["timestamp"] = ""
vars_["id"] = "_" + parts[1] if parts[1] else ""
return vars_
def copy_log_from_where(rundate, pipeline, session):
file_vars = dict(**config.program_vars(rundate, pipeline, session),
**config.date_vars(rundate))
log_level = config.where.runner.log_level.str
current_level = "none"
try:
with files.open("log", file_vars=file_vars) as fid:
for line in fid:
line_level, _, text = line.partition(" ")
line_level = line_level.strip().lower()
current_level = line_level if line_level else current_level
text = text.strip()
if getattr(LogLevel, current_level) >= getattr(
LogLevel, log_level) and text:
log.log(text, current_level)
except FileNotFoundError as err:
log.warn(f"'{err}'")
def read_data(self):
"""Read the data from three monthly datafiles
"""
files_read = []
date_to_read = self.rundate - timedelta(days=7)
while date_to_read < self.rundate + timedelta(days=self.arc_length +
8):
self.vars.update(config.date_vars(date_to_read))
file_path = files.path(self.file_key, file_vars=self.vars)
if file_path not in files_read:
files_read.append(file_path)
self.dependencies.append(file_path)
with files.open(self.file_key,
file_vars=self.vars,
mode="rt",
encoding="latin_1") as fid:
self.parse_file(fid)
date_to_read += timedelta(days=1)
def get_vmf1_grid(time):
"""Read VMF1 gridded data files relevant for the given time epochs
Args:
time (Time): observation epochs
Returns:
A dictionary of functions that can interpolate in the VMF1 dataset.
"""
data = dict()
min_time = time.utc.datetime if len(time) == 1 else min(time.utc.datetime)
max_time = time.utc.datetime if len(time) == 1 else max(time.utc.datetime)
start_hour = 6 * (min_time.hour // 6)
start = min_time.replace(hour=start_hour,
minute=0,
second=0,
microsecond=0)
end_hour = 6 * (max_time.hour // 6)
end = max_time.replace(hour=end_hour, minute=0, second=0,
microsecond=0) + timedelta(hours=6)
for datatype, multiplier in DATATYPE.items():
dt_to_read = start
vmf1_data = dict()
while dt_to_read <= end:
file_vars = dict(config.date_vars(dt_to_read), type=datatype)
data_chunk = parsers.parse_key(file_key="vmf1_grid",
file_vars=file_vars).as_dict()
if data_chunk:
vmf1_data[dt_to_read] = (data_chunk["lat"], data_chunk["lon"],
data_chunk["values"] * multiplier)
dt_to_read += timedelta(hours=6)
data[datatype] = vmf1_data
funcs = {k: vmf1_interpolator(v) for k, v in data.items()}
data = parsers.parse_key(file_key="orography_ell").as_dict()
funcs["ell"] = RectBivariateSpline(data["lon"], data["lat"],
data["values"].T)
return funcs
def get_non_tidal_atmospheric_loading(time):
"""Read gridded data files relevant for the given time epochs
Args:
time (Time): observation epochs
Returns:
A function that can interpolate
"""
min_time = min(time.utc).datetime
max_time = max(time.utc).datetime
start_hour = 6 * (min_time.hour // 6)
start = min_time.replace(hour=start_hour,
minute=0,
second=0,
microsecond=0)
end_hour = 6 * (max_time.hour // 6)
end = max_time.replace(hour=end_hour, minute=0, second=0,
microsecond=0) + timedelta(hours=6)
dt_to_read = start
data = dict(up={}, east={}, north={})
while dt_to_read <= end:
file_vars = dict(config.date_vars(dt_to_read))
data_chunk = parsers.parse_key(
file_key="non_tidal_atmospheric_loading",
file_vars=file_vars).as_dict()
if data_chunk:
data["up"][dt_to_read] = (data_chunk["lat"], data_chunk["lon"],
data_chunk["up"])
data["east"][dt_to_read] = (data_chunk["lat"], data_chunk["lon"],
data_chunk["east"])
data["north"][dt_to_read] = (data_chunk["lat"], data_chunk["lon"],
data_chunk["north"])
dt_to_read += timedelta(hours=6)
return {k: create_interpolator(v) for k, v in data.items()}
def get_vmf1_station(time):
"""Read VMF1 station data files relevant for the given time epochs
Args:
time (Time): observation epochs
Returns:
A dictionary of functions that can interpolate in the VMF1 dataset.
"""
start = time.utc.datetime.date() if len(time) == 1 else min(
time.utc.datetime).date()
end = time.utc.datetime.date() if len(time) == 1 else max(
time.utc.datetime).date()
date_to_read = start
vmf1_data = dict()
while date_to_read <= end:
file_vars = dict(config.date_vars(date_to_read))
parser = parsers.parse_key(file_key="vmf1_station",
file_vars=file_vars)
data_chunk = parser.as_dict()
for sta, sta_data in data_chunk.items():
vmf1_data.setdefault(sta, {})
for k in sta_data.keys():
vmf1_data[sta][k] = vmf1_data[sta].get(k,
[]) + data_chunk[sta][k]
date_to_read += timedelta(days=1)
funcs = dict()
for sta, sta_data in vmf1_data.items():
funcs.setdefault(sta, {})
mjd = sta_data.pop("mjd")
# Use linear interpolation between each datapoint
funcs[sta] = {
k: interp1d(mjd, v, fill_value="extrapolate")
for k, v in sta_data.items()
}
return funcs
def _read(self, dset_raw, provider, version):
"""Read SP3 orbit file data and save it in a Dataset
Naming convention correspond to end of arc, at midnight, hence we add day_offset,
which is usually arc_length - 1
Args:
dset_raw (Dataset): Dataset representing raw data from apriori orbit files
provider: Str: Orbit provider
version: Str: Orbit version
"""
rundate = dset_raw.rundate
date_to_read = rundate + timedelta(days=self.day_offset)
file_vars = config.date_vars(date_to_read)
file_vars["provider"] = provider
file_vars["version"] = version
if self.file_path is None:
file_path = config.files.path(self.file_key, file_vars)
else:
file_path = self.file_path
log.debug(f"Parse precise orbit file {file_path}")
# Generate temporary Dataset with orbit file data
dset_orbit = data.Dataset(rundate=date_to_read,
tech=dset_raw.vars["tech"],
stage="orbit",
dataset_name="",
dataset_id=0,
empty=True)
parser = parsers.parse(parser_name="orbit_sp3",
file_path=file_path,
rundate=date_to_read)
parser.write_to_dataset(dset_orbit)
dset_orbit.add_to_meta("parser", "file_path", file_path)
return dset_orbit
def write_one_day(dset, date):
"""Write RINEX navigation file for given date
Args:
dset: Dataset, a dataset containing the data.
date: Current date
"""
brdc = apriori.get(
"orbit",
rundate=dset.analysis["rundate"],
system=tuple(dset.unique("system")),
station=dset.vars["station"],
apriori_orbit="broadcast",
)
meta = brdc.dset_edit.meta[date.strftime("%Y-%m-%d")]
data = brdc.dset_edit # TODO: Another possibility: brdc.dset_raw
file_vars = {**dset.vars, **dset.analysis}
file_vars["doy"] = config.date_vars(date)[
"doy"
] # TODO: workaround, so that all files are written in the same working directory -> does not work if year is changed.
with config.files.open("output_rinex2_nav", file_vars=file_vars, mode="wt") as fid:
#
# Write RINEX navigation header
#
if meta["file_type"] == "N":
file_type = "NAVIGATION DATA"
fid.write("{:>9s}{:11s}{:40s}RINEX VERSION / TYPE\n".format(meta["version"], "", file_type))
fid.write(
"{:20s}{:20s}{:20s}PGM / RUN BY / DATE\n".format(meta["program"], meta["run_by"], meta["file_created"])
)
for line in meta["comment"]:
fid.write("{:60s}COMMENT\n".format(line))
fid.write(
"{:>14.4e}{:>12.4e}{:>12.4e}{:>12.4e}{:10s}ION ALPHA\n"
"".format(
meta["iono_para"]["GPSA"]["para"][0],
meta["iono_para"]["GPSA"]["para"][1],
meta["iono_para"]["GPSA"]["para"][2],
meta["iono_para"]["GPSA"]["para"][3],
"",
)
)
fid.write(
"{:>14.4e}{:>12.4e}{:>12.4e}{:>12.4e}{:10s}ION BETA\n"
"".format(
meta["iono_para"]["GPSB"]["para"][0],
meta["iono_para"]["GPSB"]["para"][1],
meta["iono_para"]["GPSB"]["para"][2],
meta["iono_para"]["GPSB"]["para"][3],
"",
)
)
# TODO fid.write('{:>22.12e}{:>19.12e}{:>9d}{:>9d}{:1s}DELTA-UTC: A0,A1,T,W\n'
# ''.format(meta['a0'], meta['a1'], int(meta['t']), int(meta['w']), ''))
fid.write("{:>6d}{:54s}LEAP SECONDS\n".format(int(meta["leap_seconds"]["leap_seconds"]), ""))
fid.write("{:60s}END OF HEADER\n".format(""))
#
# Write RINEX navigation data
#
# TODO:
# for drow in data.get_rows():
# fid.write('{d.sat:2d} {d.time:%Y%m%d %H%M%S} {d.inc0:13.4f}'.format(d=drow))
# fid.write(' {d.hurra:14.10f} ...'.format(d=drow))
for idx in range(0, data.num_obs):
sat = int(data.satellite[idx][1:3])
d = data.time.gps.datetime[idx]
fid.write(
"{:2d}{:>3s}{:>3d}{:>3d}{:>3d}{:>3d}{:>5.1f}{:>19.12e}{:>19.12e}{:>19.12e}\n"
"".format(
sat,
str(d.year)[2:4],
d.month,
d.day,
d.hour,
d.minute,
d.second,
data.sat_clock_bias[idx],
data.sat_clock_drift[idx],
data.sat_clock_drift_rate[idx],
)
)
fid.write(
"{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n"
"".format(data.iode[idx], data.crs[idx], data.delta_n[idx], data.m0[idx])
)
fid.write(
"{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n"
"".format(data.cuc[idx], data.e[idx], data.cus[idx], data.sqrt_a[idx])
)
# TODO: toe depends on GNSS system time -> for BeiDou it has to be changed
fid.write(
"{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n"
"".format(data.toe.gps.gpssec[idx], data.cic[idx], data.Omega[idx], data.cis[idx])
)
fid.write(
"{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n"
"".format(data.i0[idx], data.crc[idx], data.omega[idx], data.Omega_dot[idx])
)
# TODO: gnss_week depends on GNSS -> for BeiDou it has to be changed
# TODO: codes_l2 only valid for GPS and QZSS -> Galileo data_source; rest None
# TODO: 'G': 'l2p_flag', 'J': 'l2p_flag'
fid.write(
"{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n"
"".format(data.idot[idx], data.codes_l2[idx], data.gnss_week[idx], data.l2p_flag[idx])
)
# TODO: 'G': 'iodc', 'J': 'iodc', 'E': 'bgd_e1_e5b', 'C': 'tgd_b2_b3'
# TODO: 'G': 'tgd', 'J': 'tgd', 'E': 'bgd_e1_e5a', 'C': 'tgd_b1_b3', 'I': 'tgd'
fid.write(
"{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n"
"".format(data.sv_accuracy[idx], data.sv_health[idx], data.tgd[idx], data.iodc[idx])
)
# TODO: transmission_time depends on GNSS system time -> for BeiDou it has to be changed
# TODO: fit_interval only valid for GPS and QZSS -> for BeiDou age_of_clock_corr; rest None
fid.write(
"{:22.12e}{:>19.12e}{:>19.12e}{:>19.12e}\n"
"".format(data.transmission_time.gps.gpssec[idx], data.fit_interval[idx], 0.0, 0.0)
)
# Interpolation settings
moving_window = True
interpolation_method = "lagrange" # Methods: 'lagrange', 'interp1d' or 'InterpolatedUnivariateSpline'
window_size = 10
# Definition of dummy date
year = 2002
month = 1
day = 1
hour = 0
minute = 0
second = 0
rundate = datetime(year, month, day, hour, minute, second)
file_vars = config.date_vars(rundate)
# Define 15 min dataset
file_path = config.files.path(file_key="test_gnss_orbit_interpolation_15min",
file_vars=file_vars)
orb_15min = data.Dataset(rundate,
tech=None,
stage=None,
dataset_name="gnss_precise_orbit_15min",
dataset_id=0,
empty=True)
parser = parsers.parse(parser_name="orbit_sp3c",
file_path=file_path,
rundate=rundate)
parser.write_to_dataset(orb_15min)
