def vectorized_llh(pos, ref_ellipsoid=2):
"""Vectorized version of SOFA gc2gd-function.
Converts xyz coordinates to latitude, longitude and height
@todo ref_ellipsoid
Args:
pos: xyz coordinates
Returns:
np.array: llh coordinates
"""
if np.array(pos).ndim == 1:
lon, lat, h, _ = sofa.iau_gc2gd(ref_ellipsoid, pos)
return np.array([lat, lon, h])
else:
llh = np.empty(pos.shape)
for i, xyz in enumerate(pos):
lon, lat, h, _ = sofa.iau_gc2gd(ref_ellipsoid, xyz)
llh[i, :] = (lat, lon, h)
return llh
def llh(self):
"""Transform geocentric coordinates to geodetic using the specified reference ellipsoid
Returns:
numpy.ndarray: Geodetic coordinates (latitude in radians, longitude in radians, height in meters)
"""
llh = np.empty((self.num_obs, 3))
for obs, itrs in enumerate(self.itrs):
longitude, latitude, height, _ = sofa.iau_gc2gd(self._ref_ellipsoid, itrs[:3])
llh[obs, :] = (latitude, longitude, height)
return llh
def _write_to_dataset(parser, dset, rundate, session):
data = parser.as_dict()
units = data.get("meta", {}).get("units", {})
# Session meta
dset.meta.add("tech", "vlbi")
dset.meta.add("file", parser.file_path.stem, section="input")
dset.meta.add("type", config.tech.obs_format.str.upper(), section="input")
if "meta" not in data:
# Only read master file if session_code is not available in data["meta"]
# This is to avoid a dependency to the master file which changes frequently
master = apriori.get("vlbi_master_schedule", rundate=rundate)
master_data = master.get((rundate.timetuple().tm_yday, session), {})
session_code = master_data.get("session_code", "")
else:
master = apriori.get("vlbi_master_schedule")
session_code = data["meta"].get("session_code", "")
dset.meta.add("session_code", session_code, section="input")
dset.meta.add("session_type",
master.session_type(session_code),
section="input")
log.info(f"Session code: {session_code}")
# Convert source names to official IERS names
source_names = apriori.get("vlbi_source_names")
iers_source_names = [
source_names[src]["iers_name"] if src in source_names else src
for src in data["source"]
]
# Replace the characeter "." with the letters "dot" in source names because "." has a special meaning in where
data["source"] = iers_source_names
# Replace spaces in station names with underscores to match official IVS name
data["station_1"] = np.char.replace(data["station_1"], " ", "_")
data["station_2"] = np.char.replace(data["station_2"], " ", "_")
dset.num_obs = len(data["time"])
dset.add_time("time",
val=data.pop("time"),
scale="utc",
fmt="isot",
write_level="operational")
# Source directions
crf = apriori.get("crf", time=dset.time)
ra = np.array([
crf[s].pos.right_ascension if s in crf else 0 for s in data["source"]
])
dec = np.array(
[crf[s].pos.declination if s in crf else 0 for s in data["source"]])
dset.add_direction("src_dir",
ra=ra,
dec=dec,
time=dset.time,
write_level="operational")
# Replace the characeter "." with the letters "dot" in source names because "." has a special meaning in where
data["source"] = [s.replace(".", "dot") for s in iers_source_names]
for field, values in data.items():
values = np.array(values)
if values.dtype.kind in {"U", "S"}:
multiplier = -1 if field.endswith("_1") else 1
dset.add_text(field,
val=values,
multiplier=multiplier,
write_level="operational")
elif values.dtype.kind in {"f", "i"}:
multiplier = -1 if field.endswith("_1") else 1
unit = units.get(field, None)
dset.add_float(field,
val=values,
multiplier=multiplier,
write_level="operational",
unit=unit)
elif values.dtype.kind in {"O"}:
continue
else:
log.warn(f"Unknown datatype {values.dtype} for field {field}")
# Station information
log.info(f"Found stations: {', '.join(dset.unique('station'))}")
trf = apriori.get("trf", time=dset.time)
station_codes = apriori.get("vlbi_station_codes")
dset.add_text(
"baseline",
val=np.array([
f"{s1}/{s2}"
for s1, s2 in zip(data["station_1"], data["station_2"])
]),
write_level="operational",
)
for site in dset.unique("station"):
if site in station_codes:
cdp = station_codes[site]["cdp"]
trf_site = trf[cdp]
else:
named_site = trf.named_site(site)
trf_site = trf.closest(named_site.pos)
cdp = trf_site.key
ignore_stations = config.tech.ignore_station.stations.list
logger = log.info if site in ignore_stations else log.warn
logger(
f"Undefined station name {site}. Assuming station is {trf_site.name} to get a cdp number."
)
data["pos_" + site] = trf_site.pos.trs.val
_site_pos = np.mean(data[f"pos_{site}"], axis=0)
log.debug(
f"Using position {_site_pos} for {site} from {trf_site.source}")
ivsname = station_codes[cdp]["name"]
data["sta_" + site] = dict(site_id=cdp, cdp=cdp, ivsname=ivsname)
# Positions
itrs_pos_1 = np.array(
[data["pos_" + s][i, :] for i, s in enumerate(data["station_1"])])
itrs_vel_1 = np.zeros((dset.num_obs, 3))
dset.add_posvel(
"site_pos_1",
val=np.concatenate((itrs_pos_1, itrs_vel_1), axis=1),
ellipsoid=ellipsoid.get(config.tech.reference_ellipsoid.str.upper()),
system="trs",
time=dset.time,
# other=dset.src_dir,
write_level="operational",
)
itrs_pos_2 = np.array(
[data["pos_" + s][i, :] for i, s in enumerate(data["station_2"])])
itrs_vel_2 = np.zeros((dset.num_obs, 3))
dset.add_posvel(
"site_pos_2",
val=np.concatenate((itrs_pos_2, itrs_vel_2), axis=1),
ellipsoid=ellipsoid.get(config.tech.reference_ellipsoid.str.upper()),
system="trs",
time=dset.time,
# other=dset.src_dir,
write_level="operational",
)
# Compute aberrated source directions
def aberrated_src_dir(site_pos):
"""See IERS2010 Conventions, equation 11.15"""
site_vel_gcrs = site_pos.gcrs.vel.val
eph = apriori.get("ephemerides", time=dset.time)
vel = eph.vel_bcrs("earth") + site_vel_gcrs
return (
dset.src_dir.unit_vector + vel / constant.c -
dset.src_dir.unit_vector *
(dset.src_dir.unit_vector[:, None, :] @ vel[:, :, None])[:, :, 0] /
constant.c)
k_1 = aberrated_src_dir(dset.site_pos_1)
dset.add_direction("abr_src_dir_1", val=k_1, system="gcrs", time=dset.time)
dset.site_pos_1.other = dset.abr_src_dir_1
k_2 = aberrated_src_dir(dset.site_pos_2)
dset.add_direction("abr_src_dir_2", val=k_2, system="gcrs", time=dset.time)
dset.site_pos_2.other = dset.abr_src_dir_2
# Station data
sta_fields = set().union(
*[v.keys() for k, v in data.items() if k.startswith("sta_")])
for field in sta_fields:
dset.add_text(field + "_1",
val=[data["sta_" + s][field] for s in data["station_1"]],
multiplier=-1) # write_level='analysis')
dset.add_text(field + "_2",
val=[data["sta_" + s][field] for s in data["station_2"]],
multiplier=1) # write_level='analysis')
# Station meta
station_keys = sorted([k for k, v in data.items() if k.startswith("sta_")])
pos_keys = sorted([k for k, v in data.items() if k.startswith("pos_")])
for sta_key, pos_key in zip(station_keys, pos_keys):
sta_name = sta_key.replace("sta_", "")
cdp = data[sta_key]["cdp"]
ivsname = station_codes[cdp]["name"]
longitude, latitude, height, _ = sofa.iau_gc2gd(
2, data[pos_key][0, :]) # TODO: Reference ellipsoid
dset.meta.add("cdp", cdp, section=ivsname)
dset.meta.add("site_id", cdp, section=ivsname)
dset.meta.add("domes", station_codes[cdp]["domes"], section=ivsname)
dset.meta.add("marker", station_codes[cdp]["marker"], section=ivsname)
dset.meta.add("description",
station_codes[cdp]["description"],
section=ivsname)
dset.meta.add("longitude", longitude, section=ivsname)
dset.meta.add("latitude", latitude, section=ivsname)
dset.meta.add("height", height, section=ivsname)
if sta_name != ivsname:
dset.meta.add("cdp", cdp, section=sta_name)
dset.meta.add("site_id", cdp, section=sta_name)
dset.meta.add("domes",
station_codes[cdp]["domes"],
section=sta_name)
dset.meta.add("marker",
station_codes[cdp]["marker"],
section=sta_name)
dset.meta.add("description",
station_codes[cdp]["description"],
section=sta_name)
dset.meta.add("longitude", longitude, section=sta_name)
dset.meta.add("latitude", latitude, section=sta_name)
dset.meta.add("height", height, section=sta_name)
# Final cleanup
# If there are more than 300 sources in a NGS-file the source names are gibberish
bad_source_idx = ra == 0
bad_sources = np.array(dset.source)[bad_source_idx]
for s in np.unique(bad_sources):
log.warn(
f"Unknown source {s}. Observations with this source is discarded")
dset.subset(np.logical_not(bad_source_idx))
def _write_to_dataset(parser1, parser2, dset, rundate):
"""Store SLR data in a dataset"""
data_all1 = parser1.as_dict()
data_all2 = parser2.as_dict()
if parser1.file_path == parser2.file_path:
collection = [data_all1]
else:
collection = [data_all1, data_all2]
# Meta information
dset.meta["tech"] = "slr"
dset.meta.add("file", parser1.file_path.stem, section="input")
dset.meta.add("file", parser2.file_path.stem, section="input")
dset.meta.add("type", config.tech.obs_format.str.upper(), section="input")
# Make new dict "obs_data" containing only data in relevant time interval:
arc_length = config.tech.arc_length.float
rundate_datetime = datetime(rundate.year, rundate.month, rundate.day)
obs_data = dict()
for data_all in collection:
for i, x in enumerate(data_all["meta"]["obs_time"]):
if rundate_datetime <= x < rundate_datetime + timedelta(
days=arc_length):
for key in ("meta", "obs", "obs_str"):
for field, val in data_all[key].items():
obs_data.setdefault(key, dict()).setdefault(
field, list()).append(val[i])
data_all.pop("meta")
data_all.pop("obs")
data_all.pop("obs_str")
for key in data_all.keys():
if key.startswith("met_"):
for key2, val in data_all[key].items():
obs_data.setdefault(key,
dict()).setdefault(key2,
list()).append(val)
elif key.startswith("satellite_"):
# TODO: Use this information in the future?
continue
elif key.startswith("station_"):
# TODO: Use this information in the future?
continue
else:
log.fatal(f"Unknown data type{key}")
obs_date = obs_data["meta"]["obs_date"]
time = [
obs_date[i] + timedelta(seconds=obs_data["meta"]["obs_sec"][i])
for i in range(0, len(obs_date))
]
dset.num_obs = len(obs_data["meta"]["obs_time"])
dset.add_time("time", val=time, scale="utc", fmt="datetime")
dset.add_text(val=obs_data["meta"]["station"], name="station")
dset.add_text(val=obs_data["meta"]["satellite"], name="satellite")
dset.add_float(val=obs_data["meta"]["bin_rms"],
unit="picoseconds",
name="bin_rms")
# Positions
trf = apriori.get("trf", time=dset.time)
for station in dset.unique("station"):
trf_site = trf[station]
station_pos = trf_site.pos.trs.val
log.debug(
f"Station position for {station} ({trf_site.name}) is (x,y,z) = {station_pos.mean(axis=0)}"
)
domes = trf_site.meta["domes"]
obs_data["pos_" + station] = station_pos
obs_data["station-other_" + station] = dict(domes=domes,
cdp=station,
site_id=station)
dset.add_position(
"site_pos",
time=dset.time,
system="trs",
val=np.array(
[obs_data["pos_" + s][idx] for idx, s in enumerate(dset.station)]),
)
# Station data
sta_fields = set().union(
*[v.keys() for k, v in obs_data.items() if k.startswith("station_")])
for field in sta_fields:
dset.add_float(field,
val=np.array([
float(obs_data["station_" + s][field])
for s in dset.station
]))
sta_fields = set().union(*[
v.keys() for k, v in obs_data.items() if k.startswith("station-other_")
])
for field in sta_fields:
dset.add_text(
field,
val=[obs_data["station-other_" + s][field] for s in dset.station])
# Station meta
station_keys = sorted(
[k for k, v in obs_data.items() if k.startswith("station-other_")])
pos_keys = sorted([k for k, v in obs_data.items() if k.startswith("pos_")])
for sta_key, pos_key in zip(station_keys, pos_keys):
sta_name = sta_key.replace("station-other_", "")
cdp = obs_data[sta_key]["cdp"]
dset.meta.add(sta_name, "site_id", cdp)
longitude, latitude, height, _ = sofa.iau_gc2gd(
2, obs_data[pos_key][0, :]) # TODO: Reference ellipsoid
dset.meta.add("cdp", cdp, section=sta_name)
dset.meta.add("site_id", cdp, section=sta_name)
dset.meta.add("domes", obs_data[sta_key]["domes"], section=sta_name)
dset.meta.add("marker", " ", section=sta_name)
dset.meta.add("description", " ", section=sta_name)
dset.meta.add("longitude", longitude, section=sta_name)
dset.meta.add("latitude", latitude, section=sta_name)
dset.meta.add("height", height, section=sta_name)
# Satellite data
sat_fields = set().union(
*[v.keys() for k, v in obs_data.items() if k.startswith("satellite_")])
for field in sat_fields:
dset.add_float(field,
val=np.array([
float(obs_data["satellite_" + s][field])
for s in dset.satellite
]))
# Observations
# In the dataset, obs_time is seconds since rundate:
v = [(obs_data["meta"]["obs_date"][i] - rundate_datetime).total_seconds() +
obs_data["meta"]["obs_sec"][i] for i in range(0, dset.num_obs)]
obs_data["obs"].pop("obs_time")
dset.add_float("obs_time", val=v)
for field, values in obs_data["obs"].items():
dset.add_float(field, val=np.array(values))
for field, values in obs_data["obs_str"].items():
dset.add_text(field, val=values)
return obs_data
def _write_to_dataset(parser, dset, rundate, session):
data = parser.as_dict()
# TODO: units on fields
# Session meta
dset.meta["tech"] = "vlbi"
dset.add_to_meta("input", "file", parser.file_path.stem)
dset.add_to_meta("input", "type", config.tech.obs_format.str.upper())
if "meta" not in data:
# Only read master file if session_code is not available in data["meta"]
# This is to avoid a dependency to the master file which changes frequently
master = apriori.get("vlbi_master_schedule", rundate=rundate)
master_data = master.get((rundate.timetuple().tm_yday, session), {})
session_code = master_data.get("session_code", "")
dset.add_to_meta("input", "session_code", session_code)
else:
master = apriori.get("vlbi_master_schedule")
session_code = data["meta"].get("session_code", "")
dset.add_to_meta("input", "session_code", session_code)
dset.add_to_meta("input", "session_type",
master.session_type(session_code))
log.info(f"Session code: {session_code}")
# Convert source names to official IERS names
source_names = apriori.get("vlbi_source_names")
iers_source_names = [
source_names[src]["iers_name"] if src in source_names else src
for src in data["source"]
]
data["source"] = iers_source_names
# Replace spaces in station names with underscores to match official IVS name
data["station_1"] = np.char.replace(data["station_1"], " ", "_")
data["station_2"] = np.char.replace(data["station_2"], " ", "_")
dset.num_obs = len(data["time"])
dset.add_time("time",
val=data.pop("time"),
scale="utc",
format="isot",
write_level="operational")
for field, values in data.items():
values = np.array(values)
if values.dtype.kind in {"U", "S"}:
dset.add_text(field, val=values, write_level="operational")
elif values.dtype.kind in {"f", "i"}:
dset.add_float(field, val=values, write_level="operational")
elif values.dtype.kind in {"O"}:
continue
else:
log.warn(f"Unknown datatype {values.dtype} for field {field}")
# Source directions
crf = apriori.get("crf", time=dset.time.mean.utc)
ra = np.array(
[crf[s].pos.crs[0] if s in crf else 0 for s in data["source"]])
dec = np.array(
[crf[s].pos.crs[1] if s in crf else 0 for s in data["source"]])
dset.add_direction("src_dir", ra=ra, dec=dec, write_level="operational")
# Station information
log.info(f"Found stations: {', '.join(dset.unique('station'))}")
trf = apriori.get("trf", time=dset.time)
station_codes = apriori.get("vlbi_station_codes")
dset.add_text(
"baseline",
val=np.array([
f"{s1}/{s2}"
for s1, s2 in zip(data["station_1"], data["station_2"])
]),
write_level="operational",
)
for site in dset.unique("station"):
if site in station_codes:
cdp = station_codes[site]["cdp"]
trf_site = trf[cdp]
else:
named_site = trf.named_site(site)
trf_site = trf.closest(named_site.pos)
cdp = trf_site.key
ignore_stations = config.tech.ignore_station.stations.list
logger = log.info if site in ignore_stations else log.warn
logger(
f"Undefined station name {site}. Assuming station is {trf_site.name}."
)
data["pos_" + site] = trf_site.pos.itrs
_site_pos = np.mean(data[f"pos_{site}"], axis=0)
log.debug(
f"Using position {_site_pos} for {site} from {trf_site.source}")
ivsname = station_codes[cdp]["name"]
data["sta_" + site] = dict(site_id=cdp, cdp=cdp, ivsname=ivsname)
# Positions
itrs_pos_1 = np.array(
[data["pos_" + s][i, :] for i, s in enumerate(data["station_1"])])
itrs_vel_1 = np.zeros((dset.num_obs, 3))
dset.add_posvel(
"site_pos_1",
time="time",
other="src_dir",
itrs=np.concatenate((itrs_pos_1, itrs_vel_1), axis=1),
write_level="operational",
)
itrs_pos_2 = np.array(
[data["pos_" + s][i, :] for i, s in enumerate(data["station_2"])])
itrs_vel_2 = np.zeros((dset.num_obs, 3))
dset.add_posvel(
"site_pos_2",
time="time",
other="src_dir",
itrs=np.concatenate((itrs_pos_2, itrs_vel_2), axis=1),
write_level="operational",
)
# Station data
sta_fields = set().union(
*[v.keys() for k, v in data.items() if k.startswith("sta_")])
for field in sta_fields:
dset.add_text(field + "_1",
val=[data["sta_" + s][field] for s in data["station_1"]
]) # write_level='analysis')
dset.add_text(field + "_2",
val=[data["sta_" + s][field] for s in data["station_2"]
]) # write_level='analysis')
# Station meta
station_keys = sorted([k for k, v in data.items() if k.startswith("sta_")])
pos_keys = sorted([k for k, v in data.items() if k.startswith("pos_")])
for sta_key, pos_key in zip(station_keys, pos_keys):
sta_name = sta_key.replace("sta_", "")
cdp = data[sta_key]["cdp"]
ivsname = station_codes[cdp]["name"]
longitude, latitude, height, _ = sofa.iau_gc2gd(
2, data[pos_key][0, :]) # TODO: Reference ellipsoid
dset.add_to_meta(ivsname, "cdp", cdp)
dset.add_to_meta(ivsname, "site_id", cdp)
dset.add_to_meta(ivsname, "domes", station_codes[cdp]["domes"])
dset.add_to_meta(ivsname, "marker", station_codes[cdp]["marker"])
dset.add_to_meta(ivsname, "description",
station_codes[cdp]["description"])
dset.add_to_meta(ivsname, "longitude", longitude)
dset.add_to_meta(ivsname, "latitude", latitude)
dset.add_to_meta(ivsname, "height", height)
if sta_name != ivsname:
dset.add_to_meta(sta_name, "cdp", cdp)
dset.add_to_meta(sta_name, "site_id", cdp)
dset.add_to_meta(sta_name, "domes", station_codes[cdp]["domes"])
dset.add_to_meta(sta_name, "marker", station_codes[cdp]["marker"])
dset.add_to_meta(sta_name, "description",
station_codes[cdp]["description"])
dset.add_to_meta(sta_name, "longitude", longitude)
dset.add_to_meta(sta_name, "latitude", latitude)
dset.add_to_meta(sta_name, "height", height)
# Final cleanup
# If there are more than 300 sources in a NGS-file the source names are gibberish
bad_source_idx = ra == 0
bad_sources = np.array(dset.source)[bad_source_idx]
for s in np.unique(bad_sources):
log.warn(
f"Unknown source {s}. Observations with this source is discarded")
dset.subset(np.logical_not(bad_source_idx))
