def rotator_update_thread(self):
""" Rotator updater thread """
if self.rotator_homing_enabled:
# Move rotator to 'home' position on startup.
self.home_rotator()
while self.rotator_thread_running:
_telem = None
_telem_time = None
# Grab the latest telemetry data
self.telem_lock.acquire()
try:
if self.latest_telemetry != None:
_telem = self.latest_telemetry.copy()
_telem_time = self.latest_telemetry_time
finally:
self.telem_lock.release()
# Proceed if we have valid telemetry.
if _telem != None:
try:
# Check if the telemetry is very old.
_telem_age = time.time() - _telem_time
# If the telemetry is older than our homing delay, move to our home position.
if _telem_age > self.rotator_homing_delay * 60.0:
self.home_rotator()
else:
# Check that the station position is not 0,0
if (self.station_position[0]
== 0.0) and (self.station_position[1] == 0.0):
self.log_error(
"Station position is 0,0 - not moving rotator."
)
else:
# Otherwise, calculate the new azimuth/elevation.
_position = position_info(
self.station_position,
[_telem["lat"], _telem["lon"], _telem["alt"]],
)
# Move to the new position
self.move_rotator(_position["bearing"],
_position["elevation"])
except Exception as e:
self.log_error("Error handling new telemetry - %s" %
str(e))
# Wait until the next update time.
_i = 0
while (_i <
self.rotator_update_rate) and self.rotator_thread_running:
time.sleep(1)
_i += 1
def telemetry_filter(telemetry):
""" Filter incoming radiosonde telemetry based on various factors,
- Invalid Position
- Invalid Altitude
- Abnormal range from receiver.
- Invalid serial number.
This function is defined within this script to avoid passing around large amounts of configuration data.
"""
global config
# First Check: zero lat/lon
# Note : Be careful this position not occurring very often but it is possible
if (telemetry["lat"] == 0.0) and (telemetry["lon"] == 0.0):
logging.warning(
"Zero Lat/Lon. Sonde {} does not have GPS lock.".format(
telemetry["id"]))
return False
# Second check: Altitude cap.
if telemetry["alt"] > config["max_altitude"]:
_altitude_breach = telemetry["alt"] - config["max_altitude"]
logging.warning(
"Sonde {} position breached altitude cap by {} m.".format(
telemetry["id"], _altitude_breach))
return False
# Third check: Number of satellites visible.
if "sats" in telemetry:
if telemetry["sats"] < 4:
logging.warning(
"Sonde {} can only see {} satellites - discarding position as bad."
.format(telemetry["id"], telemetry["sats"]))
return False
# Fourth check - is the payload more than x km from our listening station.
# Only run this check if a station location has been provided.
if (config["station_lat"] != 0.0) and (config["station_lon"] != 0.0):
# Calculate the distance from the station to the payload.
_listener = (
config["station_lat"],
config["station_lon"],
config["station_alt"],
)
_payload = (telemetry["lat"], telemetry["lon"], telemetry["alt"])
# Calculate using positon_info function from rotator_utils.py
_info = position_info(_listener, _payload)
if _info["straight_distance"] > config["max_radius_km"] * 1000:
_radius_breach = (_info["straight_distance"] / 1000.0 -
config["max_radius_km"])
logging.warning(
"Sonde {0} position breached radius cap by {0.1f} km.".format(
telemetry["id"], _radius_breach))
if config["radius_temporary_block"]:
logging.warning("Blocking for {} minutes.".format(
config["temporary_block_time"]))
return "TempBlock"
else:
return False
if (_info["straight_distance"] < config["min_radius_km"] *
1000) and config["radius_temporary_block"]:
logging.warning(
"Sonde {0} within minimum radius limit ({0.1f} km). Blocking for {2} minutes."
.format(telemetry["id"], config["min_radius_km"],
config["temporary_block_time"]))
return "TempBlock"
# Payload Serial Number Checks
_serial = telemetry["id"]
# Run a Regex to match known Vaisala RS92/RS41 serial numbers (YWWDxxxx)
# RS92: https://www.vaisala.com/sites/default/files/documents/Vaisala%20Radiosonde%20RS92%20Serial%20Number.pdf
# RS41: https://www.vaisala.com/sites/default/files/documents/Vaisala%20Radiosonde%20RS41%20Serial%20Number.pdf
# This will need to be re-evaluated if we're still using this code in 2021!
# UPDATE: Had some confirmation that Vaisala will continue to use the alphanumeric numbering up until
# ~2025-2030, so have expanded the regex to match (and also support some older RS92s)
vaisala_callsign_valid = re.match(r"[E-Z][0-5][\d][1-7]\d{4}", _serial)
# Regex to check DFM callsigns are valid.
# DFM serial numbers have at least 6 numbers (newer sondes have 8)
dfm_callsign_valid = re.match(r"DFM-\d{6}", _serial)
# Check Meisei sonde callsigns for validity.
# meisei_ims returns a callsign of IMS100-xxxxxx until it receives the serial number, so we filter based on the x's being present or not.
if "MEISEI" in telemetry["type"]:
meisei_callsign_valid = "x" not in _serial.split("-")[1]
else:
meisei_callsign_valid = False
if "MRZ" in telemetry["type"]:
mrz_callsign_valid = "x" not in _serial.split("-")[1]
else:
mrz_callsign_valid = False
# If Vaisala or DFMs, check the callsigns are valid. If M10, iMet or LMS6, just pass it through - we get callsigns immediately and reliably from these.
if (vaisala_callsign_valid or dfm_callsign_valid or meisei_callsign_valid
or mrz_callsign_valid or ("M10" in telemetry["type"])
or ("M20" in telemetry["type"]) or ("LMS" in telemetry["type"])
or ("IMET" in telemetry["type"])):
return "OK"
else:
_id_msg = "Payload ID {} is invalid.".format(telemetry["id"])
# Add in a note about DFM sondes and their oddness...
if "DFM" in telemetry["id"]:
_id_msg += " Note: DFM sondes may take a while to get an ID."
if "MRZ" in telemetry["id"]:
_id_msg += " Note: MRZ sondes may take a while to get an ID."
logging.warning(_id_msg)
return False
def telemetry_filter(telemetry):
""" Filter incoming radiosonde telemetry based on various factors,
- Invalid Position
- Invalid Altitude
- Abnormal range from receiver.
- Invalid serial number.
This function is defined within this script to avoid passing around large amounts of configuration data.
"""
global config
# First Check: zero lat/lon
if (telemetry['lat'] == 0.0) and (telemetry['lon'] == 0.0):
logging.warning("Zero Lat/Lon. Sonde %s does not have GPS lock." %
telemetry['id'])
return False
# Second check: Altitude cap.
if telemetry['alt'] > config['max_altitude']:
_altitude_breach = telemetry['alt'] - config['max_altitude']
logging.warning("Sonde %s position breached altitude cap by %d m." %
(telemetry['id'], _altitude_breach))
return False
# Third check: Number of satellites visible.
if 'sats' in telemetry:
if telemetry['sats'] < 4:
logging.warning(
"Sonde %s can only see %d SVs - discarding position as bad." %
(telemetry['id'], telemetry['sats']))
return False
# Fourth check - is the payload more than x km from our listening station.
# Only run this check if a station location has been provided.
if (config['station_lat'] != 0.0) and (config['station_lon'] != 0.0):
# Calculate the distance from the station to the payload.
_listener = (config['station_lat'], config['station_lon'],
config['station_alt'])
_payload = (telemetry['lat'], telemetry['lon'], telemetry['alt'])
# Calculate using positon_info function from rotator_utils.py
_info = position_info(_listener, _payload)
if _info['straight_distance'] > config['max_radius_km'] * 1000:
_radius_breach = _info['straight_distance'] / 1000.0 - config[
'max_radius_km']
logging.warning(
"Sonde %s position breached radius cap by %.1f km." %
(telemetry['id'], _radius_breach))
return False
# Payload Serial Number Checks
_serial = telemetry['id']
# Run a Regex to match known Vaisala RS92/RS41 serial numbers (YWWDxxxx)
# RS92: https://www.vaisala.com/sites/default/files/documents/Vaisala%20Radiosonde%20RS92%20Serial%20Number.pdf
# RS41: https://www.vaisala.com/sites/default/files/documents/Vaisala%20Radiosonde%20RS41%20Serial%20Number.pdf
# This will need to be re-evaluated if we're still using this code in 2021!
# UPDATE: Had some confirmation that Vaisala will continue to use the alphanumeric numbering up until
# ~2025-2030, so have expanded the regex to match (and also support some older RS92s)
vaisala_callsign_valid = re.match(r'[E-Z][0-5][\d][1-7]\d{4}', _serial)
# Regex to check DFM06/09/15/17 callsigns.
dfm_callsign_valid = re.match(r'DFM[01][5679]-\d{6}', _serial)
if vaisala_callsign_valid or dfm_callsign_valid or 'M10' in telemetry[
'type']:
return True
else:
_id_msg = "Payload ID %s is invalid." % telemetry['id']
# Add in a note about DFM sondes and their oddness...
if 'DFM' in telemetry['id']:
_id_msg += " Note: DFM sondes may take a while to get an ID."
logging.warning(_id_msg)
return False
def telemetry_filter(telemetry):
""" Filter incoming radiosonde telemetry based on various factors,
- Invalid Position
- Invalid Altitude
- Abnormal range from receiver.
- Invalid serial number.
This function is defined within this script to avoid passing around large amounts of configuration data.
"""
global config
# First Check: zero lat/lon
if (telemetry['lat'] == 0.0) and (telemetry['lon'] == 0.0):
logging.warning("Zero Lat/Lon. Sonde %s does not have GPS lock." %
telemetry['id'])
return False
# Second check: Altitude cap.
if telemetry['alt'] > config['max_altitude']:
_altitude_breach = telemetry['alt'] - config['max_altitude']
logging.warning("Sonde %s position breached altitude cap by %d m." %
(telemetry['id'], _altitude_breach))
return False
# Third check: Number of satellites visible.
if 'sats' in telemetry:
if telemetry['sats'] < 4:
logging.warning(
"Sonde %s can only see %d SVs - discarding position as bad." %
(telemetry['id'], telemetry['sats']))
return False
# Fourth check - is the payload more than x km from our listening station.
# Only run this check if a station location has been provided.
if (config['station_lat'] != 0.0) and (config['station_lon'] != 0.0):
# Calculate the distance from the station to the payload.
_listener = (config['station_lat'], config['station_lon'],
config['station_alt'])
_payload = (telemetry['lat'], telemetry['lon'], telemetry['alt'])
# Calculate using positon_info function from rotator_utils.py
_info = position_info(_listener, _payload)
if _info['straight_distance'] > config['max_radius_km'] * 1000:
_radius_breach = _info['straight_distance'] / 1000.0 - config[
'max_radius_km']
logging.warning(
"Sonde %s position breached radius cap by %.1f km." %
(telemetry['id'], _radius_breach))
if config['radius_temporary_block']:
logging.warning("Blocking for %d minutes." %
config['temporary_block_time'])
return "TempBlock"
else:
return False
if (_info['straight_distance'] < config['min_radius_km'] *
1000) and config['radius_temporary_block']:
logging.warning(
"Sonde %s within minimum radius limit (%.1f km). Blocking for %d minutes."
% (telemetry['id'], config['min_radius_km'],
config['temporary_block_time']))
return "TempBlock"
# Payload Serial Number Checks
_serial = telemetry['id']
# Run a Regex to match known Vaisala RS92/RS41 serial numbers (YWWDxxxx)
# RS92: https://www.vaisala.com/sites/default/files/documents/Vaisala%20Radiosonde%20RS92%20Serial%20Number.pdf
# RS41: https://www.vaisala.com/sites/default/files/documents/Vaisala%20Radiosonde%20RS41%20Serial%20Number.pdf
# This will need to be re-evaluated if we're still using this code in 2021!
# UPDATE: Had some confirmation that Vaisala will continue to use the alphanumeric numbering up until
# ~2025-2030, so have expanded the regex to match (and also support some older RS92s)
vaisala_callsign_valid = re.match(r'[E-Z][0-5][\d][1-7]\d{4}', _serial)
# Regex to check DFM callsigns are valid.
# DFM serial numbers have at least 6 numbers (newer sondes have 8)
dfm_callsign_valid = re.match(r'DFM-\d{6}', _serial)
# Check Meisei sonde callsigns for validity.
# meisei_ims returns a callsign of IMS100-0 until it receives the serial number, so we filter based on the 0 being present or not.
if 'MEISEI' in telemetry['type']:
meisei_callsign_valid = 'x' not in _serial.split('-')[1]
else:
meisei_callsign_valid = False
# If Vaisala or DFMs, check the callsigns are valid. If M10, iMet or LMS6, just pass it through.
if vaisala_callsign_valid or dfm_callsign_valid or meisei_callsign_valid or (
'M10' in telemetry['type']) or ('M20' in telemetry['type']) or (
'LMS' in telemetry['type']) or ('IMET' in telemetry['type']):
return "OK"
else:
_id_msg = "Payload ID %s is invalid." % telemetry['id']
# Add in a note about DFM sondes and their oddness...
if 'DFM' in telemetry['id']:
_id_msg += " Note: DFM sondes may take a while to get an ID."
logging.warning(_id_msg)
return False
def telemetry_filter(telemetry):
""" Filter incoming radiosonde telemetry based on various factors,
- Invalid Position
- Invalid Altitude
- Abnormal range from receiver.
- Invalid serial number.
This function is defined within this script to avoid passing around large amounts of configuration data.
"""
global config
# First Check: zero lat/lon
if (telemetry['lat'] == 0.0) and (telemetry['lon'] == 0.0):
logging.warning("Zero Lat/Lon. Sonde %s does not have GPS lock." %
telemetry['id'])
return False
# Second check: Altitude cap.
if telemetry['alt'] > config['max_altitude']:
_altitude_breach = telemetry['alt'] - config['max_altitude']
logging.warning("Sonde %s position breached altitude cap by %d m." %
(telemetry['id'], _altitude_breach))
return False
# Third check - is the payload more than x km from our listening station.
# Only run this check if a station location has been provided.
if (config['station_lat'] != 0.0) and (config['station_lon'] != 0.0):
# Calculate the distance from the station to the payload.
_listener = (config['station_lat'], config['station_lon'],
config['station_alt'])
_payload = (telemetry['lat'], telemetry['lon'], telemetry['alt'])
# Calculate using positon_info function from rotator_utils.py
_info = position_info(_listener, _payload)
if _info['straight_distance'] > config['max_radius_km'] * 1000:
_radius_breach = _info['straight_distance'] / 1000.0 - config[
'max_radius_km']
logging.warning(
"Sonde %s position breached radius cap by %.1f km." %
(telemetry['id'], _radius_breach))
return False
# Payload Serial Number Checks
_serial = telemetry['id']
# Run a Regex to match known Vaisala RS92/RS41 serial numbers (YWWDxxxx)
# RS92: https://www.vaisala.com/sites/default/files/documents/Vaisala%20Radiosonde%20RS92%20Serial%20Number.pdf
# RS41: https://www.vaisala.com/sites/default/files/documents/Vaisala%20Radiosonde%20RS41%20Serial%20Number.pdf
# This will need to be re-evaluated if we're still using this code in 2021!
vaisala_callsign_valid = re.match(r'[J-T][0-5][\d][1-7]\d{4}', _serial)
# Regex to check DFM06/09 callsigns.
# TODO: Check if this valid for DFM06s, and find out what's up with the 8-digit DFM09 callsigns.
dfm_callsign_valid = re.match(r'DFM0[69]-\d{6}', _serial)
if vaisala_callsign_valid or dfm_callsign_valid:
return True
else:
logging.warning("Payload ID %s does not match regex. Discarding." %
telemetry['id'])
return False
def telemetry_filter(telemetry):
"""Filter incoming radiosonde telemetry based on various factors,
- Invalid Position
- Invalid Altitude
- Abnormal range from receiver.
- Invalid serial number.
- Abnormal date (more than 6 hours from utcnow)
This function is defined within this script to avoid passing around large amounts of configuration data.
"""
global config
# First Check: zero lat/lon
if (telemetry["lat"] == 0.0) and (telemetry["lon"] == 0.0):
logging.warning("Zero Lat/Lon. Sonde %s does not have GPS lock." %
telemetry["id"])
return False
# Second check: Altitude cap.
if telemetry["alt"] > config["max_altitude"]:
_altitude_breach = telemetry["alt"] - config["max_altitude"]
logging.warning("Sonde %s position breached altitude cap by %d m." %
(telemetry["id"], _altitude_breach))
return False
# Third check: Number of satellites visible.
if "sats" in telemetry:
if telemetry["sats"] < 4:
logging.warning(
"Sonde %s can only see %d SVs - discarding position as bad." %
(telemetry["id"], telemetry["sats"]))
return False
# Fourth check - is the payload more than x km from our listening station.
# Only run this check if a station location has been provided.
if (config["station_lat"] != 0.0) and (config["station_lon"] != 0.0):
# Calculate the distance from the station to the payload.
_listener = (
config["station_lat"],
config["station_lon"],
config["station_alt"],
)
_payload = (telemetry["lat"], telemetry["lon"], telemetry["alt"])
# Calculate using positon_info function from rotator_utils.py
_info = position_info(_listener, _payload)
if _info["straight_distance"] > config["max_radius_km"] * 1000:
_radius_breach = (_info["straight_distance"] / 1000.0 -
config["max_radius_km"])
logging.warning(
"Sonde %s position breached radius cap by %.1f km." %
(telemetry["id"], _radius_breach))
if config["radius_temporary_block"]:
logging.warning("Blocking for %d minutes." %
config["temporary_block_time"])
return "TempBlock"
else:
return False
if (_info["straight_distance"] < config["min_radius_km"] *
1000) and config["radius_temporary_block"]:
logging.warning(
"Sonde %s within minimum radius limit (%.1f km). Blocking for %d minutes."
% (
telemetry["id"],
config["min_radius_km"],
config["temporary_block_time"],
))
return "TempBlock"
# DateTime Check
_delta_time = (datetime.datetime.now(datetime.timezone.utc) -
parse(telemetry["datetime"])).total_seconds()
logging.debug("Delta time: %d" % _delta_time)
if abs(_delta_time) > (3600 * config["sonde_time_threshold"]):
logging.warning(
"Sonde reported time too far from current UTC time. Either sonde time or system time is invalid. (Threshold: %d hours)"
% config["sonde_time_threshold"])
return False
# Payload Serial Number Checks
_serial = telemetry["id"]
# Run a Regex to match known Vaisala RS92/RS41 serial numbers (YWWDxxxx)
# RS92: https://www.vaisala.com/sites/default/files/documents/Vaisala%20Radiosonde%20RS92%20Serial%20Number.pdf
# RS41: https://www.vaisala.com/sites/default/files/documents/Vaisala%20Radiosonde%20RS41%20Serial%20Number.pdf
# This will need to be re-evaluated if we're still using this code in 2021!
# UPDATE: Had some confirmation that Vaisala will continue to use the alphanumeric numbering up until
# ~2025-2030, so have expanded the regex to match (and also support some older RS92s)
# Modified 2021-06 to be more flexible and match older sondes, and reprogrammed sondes.
# Still needs a letter at the start, but the numbers don't need to match the format exactly.
vaisala_callsign_valid = re.match(r"[C-Z][\d][\d][\d]\d{4}", _serial)
# Just make sure we're not getting the 'xxxxxxxx' unknown serial from the DFM decoder.
if "DFM" in telemetry["type"]:
dfm_callsign_valid = "x" not in _serial.split("-")[1]
else:
dfm_callsign_valid = False
# Check Meisei sonde callsigns for validity.
# meisei_ims returns a callsign of IMS100-xxxxxx until it receives the serial number, so we filter based on the x's being present or not.
if "MEISEI" in telemetry["type"]:
meisei_callsign_valid = "x" not in _serial.split("-")[1]
else:
meisei_callsign_valid = False
if "MRZ" in telemetry["type"]:
mrz_callsign_valid = "x" not in _serial.split("-")[1]
else:
mrz_callsign_valid = False
# If Vaisala or DFMs, check the callsigns are valid. If M10, iMet or LMS6, just pass it through - we get callsigns immediately and reliably from these.
if (vaisala_callsign_valid or dfm_callsign_valid or meisei_callsign_valid
or mrz_callsign_valid or ("M10" in telemetry["type"])
or ("M20" in telemetry["type"]) or ("LMS" in telemetry["type"])
or ("IMET" in telemetry["type"])):
return "OK"
else:
_id_msg = "Payload ID %s is invalid." % telemetry["id"]
# Add in a note about DFM sondes and their oddness...
if "DFM" in telemetry["id"]:
_id_msg += " Note: DFM sondes may take a while to get an ID."
if "MRZ" in telemetry["id"]:
_id_msg += " Note: MRZ sondes may take a while to get an ID."
logging.warning(_id_msg)
return False
def calculate_skewt_data(
datetime,
latitude,
longitude,
altitude,
temperature,
humidity,
pressure=None,
decimation=5,
):
""" Work through a set of sonde data, and produce a dataset suitable for plotting in skewt-js """
# A few basic checks initially
# Don't bother to plot data with not enough data points.
if len(datetime) < 10:
return []
# Figure out if we have any ascent data at all.
_burst_idx = np.argmax(altitude)
if _burst_idx == 0:
# We only have descent data.
return []
if altitude[0] > 20000:
# No point plotting SkewT plots for data only gathered above 10km altitude...
return []
_skewt = []
# Make sure we start on index one.
i = -1*decimation + 1
while i < _burst_idx:
i += decimation
try:
if temperature[i] < -260.0:
# If we don't have any valid temp data, just skip this point
# to avoid doing un-necessary calculations
continue
_time_delta = (parse(datetime[i]) - parse(datetime[i - 1])).total_seconds()
if _time_delta == 0:
continue
_old_pos = (latitude[i - 1], longitude[i - 1], altitude[i - 1])
_new_pos = (latitude[i], longitude[i], altitude[i])
_pos_delta = position_info(_old_pos, _new_pos)
_speed = _pos_delta["great_circle_distance"] / _time_delta
_bearing = (_pos_delta["bearing"] + 180.0) % 360.0
if pressure is None:
_pressure = getDensity(altitude[i], get_pressure=True) / 100.0
elif pressure[i] < 0.0:
_pressure = getDensity(altitude[i], get_pressure=True) / 100.0
else:
_pressure = pressure[i]
_temp = temperature[i]
if humidity[i] >= 0.0:
_rh = humidity[i]
_dp = (
243.04
* (np.log(_rh / 100) + ((17.625 * _temp) / (243.04 + _temp)))
/ (
17.625
- np.log(_rh / 100)
- ((17.625 * _temp) / (243.04 + _temp))
)
)
else:
_dp = -999.0
if np.isnan(_dp):
continue
_skewt.append(
{
"press": _pressure,
"hght": altitude[i],
"temp": _temp,
"dwpt": _dp,
"wdir": _bearing,
"wspd": _speed,
}
)
# Only produce data up to 50hPa (~20km alt), which is the top of the skewt plot.
# We *could* go above this, but the data becomes less useful at those altitudes.
if _pressure < 50.0:
break
except Exception as e:
print(str(e))
# Continue through the data..
return _skewt
def read_log_file(filename, skewt_decimation=10):
""" Read in a log file """
logging.debug(f"Attempting to read file: {filename}")
# Open the file and get the header line
_file = open(filename, "r")
_header = _file.readline()
# Initially assume a new style log file (> ~1.4.0)
# timestamp,serial,frame,lat,lon,alt,vel_v,vel_h,heading,temp,humidity,pressure,type,freq_mhz,snr,f_error_hz,sats,batt_v,burst_timer,aux_data
fields = {
"datetime": "f0",
"serial": "f1",
"frame": "f2",
"latitude": "f3",
"longitude": "f4",
"altitude": "f5",
"vel_v": "f6",
"vel_h": "f7",
"heading": "f8",
"temp": "f9",
"humidity": "f10",
"pressure": "f11",
"type": "f12",
"frequency": "f13",
"snr": "f14",
"sats": "f16",
"batt": "f17",
}
if "other" in _header:
# Older style log file
# timestamp,serial,frame,lat,lon,alt,vel_v,vel_h,heading,temp,humidity,type,freq,other
# 2020-06-06T00:58:09.001Z,R3670268,7685,-31.21523,137.68126,33752.4,5.9,2.1,44.5,-273.0,-1.0,RS41,401.501,SNR 5.4,FERROR -187,SATS 9,BATT 2.7
fields = {
"datetime": "f0",
"serial": "f1",
"frame": "f2",
"latitude": "f3",
"longitude": "f4",
"altitude": "f5",
"vel_v": "f6",
"vel_h": "f7",
"heading": "f8",
"temp": "f9",
"humidity": "f10",
"type": "f11",
"frequency": "f12",
}
# Only use a subset of the columns, as the number of columns can vary in this old format
_data = np.genfromtxt(
_file,
dtype=None,
encoding="ascii",
delimiter=",",
usecols=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12),
)
else:
# Grab everything
_data = np.genfromtxt(_file, dtype=None, encoding="ascii", delimiter=",")
_file.close()
if _data.size == 1:
# Deal with log files with only one entry cleanly.
_data = np.array([_data])
# Now we need to rearrange some data for easier use in the client
_output = {"serial": strip_sonde_serial(_data[fields["serial"]][0])}
# Path to display on the map
_output["path"] = np.column_stack(
(
_data[fields["latitude"]],
_data[fields["longitude"]],
_data[fields["altitude"]],
)
).tolist()
_output["first"] = _output["path"][0]
_output["first_time"] = _data[fields["datetime"]][0]
_output["last"] = _output["path"][-1]
_output["last_time"] = _data[fields["datetime"]][-1]
_burst_idx = np.argmax(_data[fields["altitude"]])
_output["burst"] = _output["path"][_burst_idx]
_output["burst_time"] = _data[fields["datetime"]][_burst_idx]
# Calculate first position info
_pos_info = position_info(
(
autorx.config.global_config["station_lat"],
autorx.config.global_config["station_lon"],
autorx.config.global_config["station_alt"],
),
_output["first"],
)
_output["first_range_km"] = _pos_info["straight_distance"] / 1000.0
_output["first_bearing"] = _pos_info["bearing"]
# Calculate last position info
_pos_info = position_info(
(
autorx.config.global_config["station_lat"],
autorx.config.global_config["station_lon"],
autorx.config.global_config["station_alt"],
),
_output["last"],
)
_output["last_range_km"] = _pos_info["straight_distance"] / 1000.0
_output["last_bearing"] = _pos_info["bearing"]
# TODO: Calculate data necessary for Skew-T plots
if "pressure" in fields:
_press = _data[fields["pressure"]]
else:
_press = None
if "snr" in fields:
_output["snr"] = _data[fields["snr"]].tolist()
_output["skewt"] = calculate_skewt_data(
_data[fields["datetime"]],
_data[fields["latitude"]],
_data[fields["longitude"]],
_data[fields["altitude"]],
_data[fields["temp"]],
_data[fields["humidity"]],
_press,
decimation=skewt_decimation,
)
return _output
def log_quick_look(filename):
""" Attempt to read in the first and last line in a log file, and return the first/last position observed. """
_filesize = os.path.getsize(filename)
# Open the file and get the header line
_file = open(filename, "r")
_header = _file.readline()
# Discard anything
if "timestamp,serial,frame,lat,lon,alt" not in _header:
return None
_output = {}
if "snr" in _header:
_output["has_snr"] = True
else:
_output["has_snr"] = False
try:
# Naeive read of the first data line
_first = _file.readline()
_fields = _first.split(",")
_first_datetime = _fields[0]
_serial = _fields[1]
_first_lat = float(_fields[3])
_first_lon = float(_fields[4])
_first_alt = float(_fields[5])
_pos_info = position_info(
(
autorx.config.global_config["station_lat"],
autorx.config.global_config["station_lon"],
autorx.config.global_config["station_alt"],
),
(_first_lat, _first_lon, _first_alt),
)
_output["first"] = {
"datetime": _first_datetime,
"lat": _first_lat,
"lon": _first_lon,
"alt": _first_alt,
"range_km": _pos_info["straight_distance"] / 1000.0,
"bearing": _pos_info["bearing"],
"elevation": _pos_info["elevation"],
}
except Exception as e:
# Couldn't read the first line, so likely no data.
return None
# Now we try and seek to near the end of the file.
_seek_point = _filesize - 300
if _seek_point < 0:
# Don't bother trying to read the last line, it'll be the same as the first line.
_output["last"] = _output["first"]
return _output
# Read in the rest of the file
try:
_file.seek(_seek_point)
_remainder = _file.read()
# Get the last line
_last_line = _remainder.split("\n")[-2]
_fields = _last_line.split(",")
_last_datetime = _fields[0]
_last_lat = float(_fields[3])
_last_lon = float(_fields[4])
_last_alt = float(_fields[5])
_pos_info = position_info(
(
autorx.config.global_config["station_lat"],
autorx.config.global_config["station_lon"],
autorx.config.global_config["station_alt"],
),
(_last_lat, _last_lon, _last_alt),
)
_output["last"] = {
"datetime": _last_datetime,
"lat": _last_lat,
"lon": _last_lon,
"alt": _last_alt,
"range_km": _pos_info["straight_distance"] / 1000.0,
"bearing": _pos_info["bearing"],
"elevation": _pos_info["elevation"],
}
return _output
except Exception as e:
# Couldn't read in the last line for some reason.
# Return what we have
logging.error(f"Error reading last line of {filename}: {str(e)}")
_output["last"] = _output["first"]
return _output
def normalised_snr(log_files,
min_range_km=10,
max_range_km=1000,
maxsnr=False,
meansnr=True,
normalise=True):
""" Read in ALL log files and store snr data into a set of bins, normalised to 50km range. """
_norm_range = 50 # km
_snr_count = 0
_title = autorx.config.global_config[
'habitat_uploader_callsign'] + " SNR Map"
# Initialise output array
_map = np.ones((360, 90)) * -100.0
_station = (autorx.config.global_config['station_lat'],
autorx.config.global_config['station_lon'],
autorx.config.global_config['station_alt'])
for _log in log_files:
if 'has_snr' in _log:
if _log['has_snr'] == False:
continue
# Read in the file.
_data = read_log_by_serial(_log['serial'])
if 'snr' not in _data:
# No SNR information, move on.
continue
logging.debug(
f"Got SNR data ({len(_data['snr'])}) for {_log['serial']}")
for _i in range(len(_data['path'])):
_snr = _data['snr'][_i]
# Discard obviously sus SNR values
if _snr > 40.0 or _snr < 5.0:
continue
_balloon = _data['path'][_i]
_pos_info = position_info(_station, _balloon)
_range = _pos_info['straight_distance'] / 1000.0
_bearing = int(math.floor(_pos_info['bearing']))
_elevation = int(math.floor(_pos_info['elevation']))
if _range < min_range_km or _range > max_range_km:
continue
# Limit elevation data to 0-90
if _elevation < 0:
_elevation = 0
if normalise:
_snr = _snr + 20 * np.log10(_range / _norm_range)
#print(f"{_bearing},{_elevation}: {_range} km, {_snr} dB, {_norm_snr} dB")
if _map[_bearing, _elevation] < -10.0:
_map[_bearing, _elevation] = _snr
else:
if meansnr:
_map[_bearing, _elevation] = np.mean(
[_map[_bearing, _elevation], _snr])
elif maxsnr:
if _snr > _map[_bearing, _elevation]:
_map[_bearing, _elevation] = _snr
print(_map)
plt.figure(figsize=(12, 6))
plt.imshow(np.flipud(_map.T), vmin=0, vmax=40, extent=[0, 360, 0, 90])
plt.xlabel("Bearing (degrees true)")
plt.ylabel("Elevation (degrees)")
plt.title(_title)
if normalise:
plt.colorbar(label="Normalised SNR (dB)", shrink=0.5)
elif maxsnr:
plt.colorbar(label="Peak SNR (dB)", shrink=0.5)
