def process_observation(self, obs: Obs) -> Obs:
z = obs.get_response_value('z')
if not z:
return obs
d = obs.get_response_value('slopeDist')
if not d:
return obs
k = 0.13 # Refraction coefficient.
r = 6370000 # Earth radius.
k_e = (d * d) / (2 * r) # Correction of earth radius.
k_r = k * k_e # Correction of refraction.
r = k_e - k_r
self.logger.info('Updated height in observation "{}" of target "{}" '
'from {:3.4f} m to {:3.4f} m (refraction value: '
'{:3.5f} m)'.format(obs.get("name"),
obs.get("target"), z, z + r, r))
refraction = Obs.create_response_set('float', 'm', round(r, 6))
z_new = Obs.create_response_set('float', 'm', round(z + r, 5))
z_raw = Obs.create_response_set('float', 'm', z)
obs.data['responseSets']['refraction'] = refraction
obs.data['responseSets']['zRaw'] = z_raw
obs.data['responseSets']['z'] = z_new
return obs
def process_observation(self, obs: Obs) -> Obs:
if not self._is_valid_sensor_type(obs):
# Only total stations are supported.
return obs
hz = obs.get_response_value('hz')
v = obs.get_response_value('v')
dist = obs.get_response_value('slopeDist')
if None in [hz, v, dist]:
return obs
# Calculate the horizontal distance.
dist_hz = math.sin(v) * dist
if self._is_fixed_point(obs):
# Add measured Hz and calculated Hz to the fixed point.
self._update_fixed_point(obs)
self.logger.debug(f'Updated fixed point of target '
f'"{obs.get("target")}"')
self.logger.debug(
'Starting polar transformation of target "{}" (Hz = '
'{:3.5f} gon, V = {:3.5f} gon, dist = {:4.5f} m)'.format(
obs.get("target"), rad_to_gon(hz), rad_to_gon(v), dist_hz))
if self._is_adjustment_enabled:
# Add the adjustment value to the horizontal direction.
adj = self._get_adjustment_value()
self.logger.info('Calculated adjustment value for polar '
'transformation ({:3.5f} gon)'.format(
rad_to_gon(adj)))
hz += adj
# Calculate the coordinates of the observation.
x, y, z = self.transform(self._view_point.get('x'),
self._view_point.get('y'),
self._view_point.get('z'),
self._azimuth_point.get('x'),
self._azimuth_point.get('y'), hz, v, dist_hz)
self.logger.info('Transformed target "{}" (X = {:3.4f}, Y = {:3.4f}, '
'Z = {:3.4f})'.format(obs.get("target"), x, y, z))
# Add to observation data set.
response_sets = obs.get('responseSets')
response_sets['x'] = Obs.create_response_set('float', 'm', round(x, 5))
response_sets['y'] = Obs.create_response_set('float', 'm', round(y, 5))
response_sets['z'] = Obs.create_response_set('float', 'm', round(z, 5))
if self._is_adjustment_enabled:
response_sets['hzAdjusted'] = Obs.create_response_set(
'float', 'rad', round(hz, 16))
return obs
def _calculate_target_point(self, obs: Obs) -> Obs:
"""Calculates the coordinates of a target point and updates the
given `Observation` object.
Args:
obs: `Observation` object.
Returns:
The `Observation` object with calculated coordinates.
"""
hz = obs.get_response_value('hz')
v = obs.get_response_value('v')
dist = obs.get_response_value('slopeDist')
if None in [hz, v, dist]:
self.logger.warning(f'Hz, V, or distance missing in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return obs
# Calculate the coordinates in the global system (X, Y, Z).
x, y, z = self.calculate_point_coordinates(hz, v, dist,
self._view_point.get('x'),
self._view_point.get('y'),
self._view_point.get('z'),
self._a, self._o)
self.logger.info('Calculated coordinates of target point "{}" '
'(X = {:4.5f}, Y = {:4.5f}, Z = {:4.5f})'.format(
obs.get("target"), x, y, z))
# Do residual mismatch transformation.
if self._is_residual:
vx, vy = self._calculate_residual_mismatches(x, y)
self.logger.debug('Calculated improvements for target point "{}" '
'(dX = {:4.5f} m, dY = {:4.5f} m)'.format(
obs.get("target"), vx, vy))
x += vx
y += vy
self.logger.debug('Updated coordinates of target point "{}" '
'(X = {:4.5f}, Y = {:4.5f})'.format(
obs.get("target"), x, y))
# Add response set.
response_sets = obs.get('responseSets')
response_sets['x'] = Obs.create_response_set('float', 'm', x)
response_sets['y'] = Obs.create_response_set('float', 'm', y)
response_sets['z'] = Obs.create_response_set('float', 'm', z)
return obs
def process_observation(self, obs: Observation) -> Observation:
for response_set in self._response_sets:
return_code = obs.get_value('responseSets', response_set, 'value')
if return_code is None:
continue
if return_code == 0:
if obs.get('corrupted') is True:
obs.set('corrupted', False)
continue
# Get error message of the return code.
error_values = ReturnCodes.codes.get(return_code)
attempts = obs.get('attempts', 0)
# Retry measurement.
if error_values and attempts < self._retries:
obs.set('attempts', attempts + 1)
obs.set('corrupted', False)
obs.set('nextReceiver', 0)
self.logger.info(
f'Retrying observation "{obs.get("name")}" of '
f'target "{obs.get("target")}" due to return '
f'code {return_code} of response '
f'"{response_set}" (attempt {attempts + 1,} '
f'of {self._retries})')
else:
obs.set('corrupted', True)
if error_values:
# Return code related log message.
lvl, retry, msg = error_values
self.logger.log(
lvl * 10, f'Observation "{obs.get("name")}" of '
f'target "{obs.get("target")}": {msg} '
f'(code {return_code} in response '
f'"{response_set}")')
else:
# Generic log message.
self.logger.error(f'Error occurred on observation '
f'"{obs.get("name")}" (unknown code '
f'{return_code} in response '
f'"{response_set}")')
return obs
return obs
def _update_fixed_point(self, obs: Obs) -> None:
"""Adds horizontal direction, vertical angle, and slope distance
of the observation to a fixed point.
Args:
obs: `Observation` object.
"""
hz = obs.get_response_value('hz')
v = obs.get_response_value('v')
dist = obs.get_response_value('slopeDist')
if None in [hz, v, dist]:
return
# Calculate the coordinates of the fixed point if the Helmert
# transformation has been done already. Otherwise, use the datum from
# the configuration.
fixed_point = self._fixed_points.get(obs.get('target'))
if self._is_ready():
# Calculate fixed point coordinates.
x, y, z = self.calculate_point_coordinates(
hz, v, dist, self._view_point.get('x'),
self._view_point.get('y'), self._view_point.get('z'), self._a,
self._o)
self.logger.info('Calculated coordinates of fixed point "{}" '
'(X = {:3.5f}, Y = {:3.5f}, Z = {:3.5f})'.format(
obs.get("target"), x, y, z))
else:
# Get the coordinates of the fixed point from the configuration.
x = fixed_point.get('x')
y = fixed_point.get('y')
z = fixed_point.get('z')
# Update the values.
fixed_point['hz'] = hz
fixed_point['v'] = v
fixed_point['dist'] = dist
fixed_point['lastUpdate'] = time.time()
self.logger.debug(f'Updated fixed point of target '
f'"{obs.get("target")}"')
# Add global Cartesian coordinates of the fixed point to the
# observation.
response_sets = obs.get('responseSets')
response_sets['x'] = Obs.create_response_set('float', 'm', x)
response_sets['y'] = Obs.create_response_set('float', 'm', y)
response_sets['z'] = Obs.create_response_set('float', 'm', z)
def process_observation(self, obs: Observation) -> Observation:
"""Checks, if responses are inside defined bounds.
Args:
obs: The observation object.
Returns:
The untouched observation object.
"""
if not obs.get('name') in self._observations:
self.logger.warning(f'Undefined observation "{obs.get("name")}" '
f'of target "{obs.get("target")}"')
return obs
response_sets = self._observations.get(obs.get('name'))
for response_name, limits in response_sets.items():
response_value = obs.get_response_value(response_name)
if response_value is None or not self.is_number(response_value):
self.logger.warning(f'Response value "{response_name}" in '
f'observation "{obs.get("name")}" of '
f'target "{obs.get("target")}" is not a '
f'number')
continue
min_value = limits.get('min')
max_value = limits.get('max')
if min_value <= response_value <= max_value:
self.logger.debug(f'Response value "{response_name}" in '
f'observation "{obs.get("name")}" of target '
f'"{obs.get("target")}" is within limits')
elif response_value < min_value:
self.logger.critical(f'Response value "{response_name}" in '
f'observation "{obs.get("name")}" of '
f'target "{obs.get("target")}" is less '
f'than minimum ({response_value} < '
f'{min_value})')
elif response_value > max_value:
self.logger.critical(
f'Response value "{response_name}" in '
f'observation "{obs.get("name")}" of '
f'target "{obs.get("target")}" is greater '
f'than maximum ({response_value} > '
f'{max_value})')
return obs
def _is_fixed_point(self, obs: Obs) -> bool:
"""Checks if the given observation equals one of the defined fixed
points."""
if self._fixed_points.get(obs.get('target')):
return True
else:
return False
def process_observation(self, obs: Observation) -> Observation:
request_sets = obs.get('requestSets')
for set_name, request_set in request_sets.items():
request = request_set.get('request')
timeout = request_set.get('timeout')
sleep_time = request_set.get('sleepTime')
response = ''
self.logger.verbose(f'Sending request "{set_name}" to sensor '
f'"{obs.get("sensorName")}" on virtual port '
f'"{self.name}"')
for pattern in self.patterns:
reg_exp = re.compile(pattern)
parsed = reg_exp.match(request)
if not parsed:
continue
response = self.patterns[pattern](request)
self.logger.verbose(f'Received response '
f'"{self.sanitize(response)}" from sensor '
f'"{obs.get("sensorName")}" on virtual '
f'port "{self.name}"')
break
request_set['response'] = response
time.sleep((timeout * 0.25) + sleep_time)
obs.set('portName', self._name)
obs.set('timestamp', str(arrow.utcnow()))
return obs
def _update_fixed_point(self, obs: Obs) -> None:
"""Updates given fixed point.
Args:
obs: `Observation` object.
"""
fixed_point = self._fixed_points.get(obs.get('target'))
hz = obs.get_response_value('hz')
azimuth = self.get_azimuth_angle(self._azimuth_angle,
self._view_point.get('x'),
self._view_point.get('y'),
fixed_point.get('x'),
fixed_point.get('y'))
fixed_point['hz'] = hz
fixed_point['azimuth'] = azimuth
fixed_point['lastUpdate'] = time.time()
# Calculate the orientation.
delta_hz = azimuth - hz
if delta_hz < 0:
# Add 400 gon.
delta_hz += 2 * math.pi
fixed_point['deltaHz'] = delta_hz
def publish_observation(self, obs: Observation) -> None:
"""Prepares the observation for publishing and forwards it to the
messenger.
Args:
obs: Observation object.
"""
receivers = obs.get('receivers')
index = obs.get('nextReceiver')
# No receivers defined.
if not receivers:
logging.debug(f'No receivers defined in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return
# No index defined.
if (index is None) or (index < 0):
self.logger.warning(f'Undefined receiver in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return
# Receivers list has been processed and observation is finished.
if index >= len(receivers):
self.logger.info(f'Observation "{obs.get("name")}" of target '
f'"{obs.get("target")}" has been finished')
return
# Name of the sending module.
sender = receivers[index - 1]
# Increase the receivers index.
next_receiver = receivers[index]
obs.set('nextReceiver', index + 1)
# Create header and payload.
header = {'from': sender, 'type': 'observation'}
payload = obs.data
# Send the observation to the next module.
self.publish(next_receiver, header, payload)
def process_observation(self, obs: Observation) -> Observation:
for name, properties in self._config.items():
response_set = obs.get('responseSets').get(name)
if not response_set:
continue
source_value = response_set.get('value')
source_unit = response_set.get('unit')
if not source_value or not source_unit:
continue
if source_unit != properties.get('sourceUnit'):
self.logger.warning(f'Unit "{source_unit}" of response '
f'"{name}" in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}" does not match '
f'"{properties.get("sourceUnit")}"')
continue
if properties.get('conversionType') == 'scale':
target_value = self.scale(float(source_value),
properties.get('scalingValue'))
target_unit = properties.get('targetUnit')
self.logger.info(
'Converted response "{}" in observation "{}" '
'of target "{}" from {:.4f} {} to {:.4f} {}'.format(
name, obs.get("name"), obs.get("target"), source_value,
source_unit, target_value, target_unit))
response_set = Observation.create_response_set(
'float', target_unit, round(target_value, 5))
obs.data['responseSets'][name] = response_set
return obs
def _is_fixed_point(self, obs: Obs) -> bool:
"""Checks if the given observation equals one of the defined fixed
points.
Args:
obs: `Observation` object.
Returns:
True if observation is fixed point, False if not.
"""
if self._fixed_points.get(obs.get('target')):
return True
else:
return False
def _is_valid_sensor_type(self, obs: Obs) -> bool:
"""Returns whether or not the sensor is supported.
Args:
obs: `Observation` object.
Returns:
True if sensor is supported, False if not.
"""
sensor_type = obs.get('sensorType')
if not SensorType.is_total_station(sensor_type):
self.logger.error(f'Sensor type "{sensor_type}" not supported')
return False
return True
def process_observation(self, obs: Obs) -> Obs:
# Calculate the serial measurement of an observation in two faces.
hz_0 = obs.get_response_value('hz0')
hz_1 = obs.get_response_value('hz1')
v_0 = obs.get_response_value('v0')
v_1 = obs.get_response_value('v1')
dist_0 = obs.get_response_value('slopeDist0')
dist_1 = obs.get_response_value('slopeDist1')
if None in [hz_0, hz_1, v_0, v_1, dist_0, dist_1]:
return obs
# Calculate new Hz, V, and slope distance.
hz = hz_0 + hz_1
if hz_0 > hz_1:
hz += math.pi
else:
hz -= math.pi
hz /= 2
v = ((2 * math.pi) + (v_0 - v_1)) / 2
dist = (dist_0 + dist_1) / 2
# Save the calculated values.
response_sets = obs.get('responseSets')
response_sets['hz'] = Obs.create_response_set('float', 'rad', hz)
response_sets['v'] = Obs.create_response_set('float', 'rad', v)
response_sets['slopeDist'] = Obs.create_response_set('float' 'm', dist)
self.logger.debug(f'Calculated serial measurement with two faces for '
f'observation "{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return obs
def process_observation(self, obs: Observation) -> Observation:
"""Appends data to a flat file in CSV format.
Args:
obs: Observation object.
Returns:
The observation object.
"""
ts = arrow.get(obs.get('timestamp', 0))
file_date = {
# No file rotation, i.e., all data is stored in a single file.
FileRotation.NONE:
None,
# Every day a new file is created.
FileRotation.DAILY:
ts.format('YYYY-MM-DD'),
# Every month a new file is created.
FileRotation.MONTHLY:
ts.format('YYYY-MM'),
# Every year a new file is created.
FileRotation.YEARLY:
ts.format('YYYY')
}[self._file_rotation]
fn = self._file_name
fn = fn.replace('{{port}}', obs.get("portName"))
fn = fn.replace('{{date}}', f'{file_date}' if file_date else '')
fn = fn.replace(
'{{target}}',
f'{obs.get("target")}' if obs.get('target') is not None else '')
fn = fn.replace(
'{{name}}',
f'{obs.get("name")}' if obs.get('name') is not None else '')
fn += self._file_extension
for path in self._paths:
if not Path(path).exists():
self.logger.critical(f'Path "{path}" does not exist')
continue
file_path = Path(path, fn)
# Create a header if a new file has to be touched.
header = None
if not Path(file_path).is_file():
header = (f'# Target "{obs.get("target")}" of '
f'"{obs.get("sensorName")}" on '
f'"{obs.get("portName")}"\n')
# Open a file for each path.
with open(str(file_path), 'a') as fh:
# Add the header if necessary.
if header:
fh.write(header)
# Format the time stamp. For more information, see:
# http://arrow.readthedocs.io/en/latest/#tokens
date_time = ts.format(self._date_time_format)
# Create the CSV line starting with date and time.
line = date_time
if self._save_observation_id:
line += self._separator + obs.get('id')
if obs.get('target') is not None:
line += self._separator + obs.get('target')
response_sets = obs.get('responseSets')
for response_set_id in sorted(response_sets.keys()):
response_set = response_sets.get(response_set_id)
v = response_set.get('value')
u = response_set.get('unit')
line += self._separator + format(response_set_id)
line += self._separator + format(v)
line += self._separator + format(u)
# Write line to file.
fh.write(line + '\n')
self.logger.info(f'Saved observation "{obs.get("name")}" of '
f'target "{obs.get("target")}" from port '
f'"{obs.get("portName")}" to file '
f'"{str(file_path)}"')
return obs
def _calculate_view_point(self, obs: Obs) -> Union[Obs, None]:
"""Calculates the view point by doing a 2D Helmert transformation.
Args:
obs: `Observation` object. Needed for port and sensor information.
Returns:
New `Observation` object with view point coordinates.
"""
sum_local_x = sum_local_y = sum_local_z = 0 # [x], [y], [z].
sum_global_x = sum_global_y = sum_global_z = 0 # [X], [Y], [Z].
num_fixed_points = len(self._fixed_points) # n.
# Calculate the centroid coordinates of the view point.
for name, fixed_point in self._fixed_points.items():
hz = fixed_point.get('hz') # Horizontal direction.
v = fixed_point.get('v') # Vertical angle.
dist = fixed_point.get('dist') # Distance (slope or reduced).
if None in [hz, v, dist]:
self.logger.warning(f'Hz, V, or distance missing in fixed '
f'point "{name}"')
return
# Calculate Cartesian coordinates out of polar coordinates.
local_x, local_y, local_z = self.get_cartesian_coordinates(
hz, v, dist)
# Store local coordinates in the fixed point dictionary.
fixed_point['localX'] = local_x
fixed_point['localY'] = local_y
fixed_point['localZ'] = local_z
# Coordinates in the global system (X, Y, Z).
global_x = fixed_point.get('x')
global_y = fixed_point.get('y')
global_z = fixed_point.get('z')
if None in [global_x, global_y, global_z]:
self.logger.error(f'Undefined fixed point "{name}"')
# Sums of the coordinates.
sum_local_x += local_x
sum_local_y += local_y
sum_local_z += local_z
sum_global_x += global_x
sum_global_y += global_y
sum_global_z += global_z
# Coordinates of the centroids.
local_centroid_x = sum_local_x / num_fixed_points # x_s.
local_centroid_y = sum_local_y / num_fixed_points # y_s.
global_centroid_x = sum_global_x / num_fixed_points # X_s.
global_centroid_y = sum_global_y / num_fixed_points # Y_s.
# Calculate transformation parameters.
o_1 = o_2 = 0
a_1 = a_2 = 0
for name, fixed_point in self._fixed_points.items():
local_x = fixed_point.get('localX')
local_y = fixed_point.get('localY')
global_x = fixed_point.get('x')
global_y = fixed_point.get('y')
# Reduced coordinates of the centroids.
r_local_centroid_x = local_x - local_centroid_x
r_local_centroid_y = local_y - local_centroid_y
r_global_centroid_x = global_x - global_centroid_x
r_global_centroid_y = global_y - global_centroid_y
# o = [ x_i * Y_i - y_i * X_i ] * [ x_i^2 + y_i^2 ]^-1
o_1 += (r_local_centroid_x * r_global_centroid_y) -\
(r_local_centroid_y * r_global_centroid_x)
o_2 += math.pow(r_local_centroid_x, 2) +\
math.pow(r_local_centroid_y, 2)
# a = [ x_i * X_i + y_i * Y_i ] * [ x_i^2 + y_i^2 ]^-1
a_1 += (r_local_centroid_x * r_global_centroid_x) +\
(r_local_centroid_y * r_global_centroid_y)
a_2 += math.pow(r_local_centroid_x, 2) +\
math.pow(r_local_centroid_y, 2)
self._o = o_1 / o_2 if o_2 != 0 else 0 # Parameter o.
self._a = a_1 / a_2 if a_2 != 0 else 0 # Parameter a.
# Calculate the coordinates of the view point:
# Y_0 = Y_s - a * y_s - o * x_s
# X_0 = X_s - a * x_s + o * y_s
# Z_0 = ([Z] - [z]) / n
self._view_point['x'] = (global_centroid_x -
(self._a * local_centroid_x) +
(self._o * local_centroid_y))
self._view_point['y'] = (global_centroid_y -
(self._a * local_centroid_y) -
(self._o * local_centroid_x))
self._view_point['z'] = (sum_global_z - sum_local_z) / num_fixed_points
self.logger.info('Calculated coordinates of view point "{}" '
'(X = {:4.5f}, Y = {:4.5f}, Z = {:4.5f})'.format(
self._view_point.get('target'),
self._view_point.get('x'),
self._view_point.get('y'),
self._view_point.get('z')))
# Calculate the standard deviations.
sum_wx = sum_wy = 0 # [W_x], [W_y].
sum_wx_wx = sum_wy_wy = sum_wz_wz = 0 # [W_x^2], [W_y^2], [W_z^2].
for name, fixed_point in self._fixed_points.items():
local_x = fixed_point.get('localX')
local_y = fixed_point.get('localY')
local_z = fixed_point.get('localZ')
global_x = fixed_point.get('x')
global_y = fixed_point.get('y')
global_z = fixed_point.get('z')
view_point_x = self._view_point.get('x')
view_point_y = self._view_point.get('y')
view_point_z = self._view_point.get('z')
wx_i = ((-1 * view_point_x) - (self._a * local_x) +
(self._o * local_y) + global_x)
wy_i = ((-1 * view_point_y) - (self._a * local_y) -
(self._o * local_x) + global_y)
sum_wx += wx_i
sum_wy += wy_i
sum_wx_wx += wx_i * wx_i
sum_wy_wy += wy_i * wy_i
sum_wz_wz += math.pow(view_point_z - (global_z - local_z), 2)
# Sum of discrepancies should be 0, i.e., [W_x] = [W_y] = 0.
r_sum_wx = abs(round(sum_wx, 5))
r_sum_wy = abs(round(sum_wy, 5))
if r_sum_wx != 0 or r_sum_wy != 0:
self.logger.warning(f'Calculated coordinates of view point '
f'"{self._view_point.get("target")}" '
f'are inaccurate ([Wx] = {r_sum_wx}, '
f'[Wy] = {r_sum_wy})')
# Standard deviations.
sx = math.sqrt((sum_wx_wx + sum_wy_wy) / ((2 * num_fixed_points) - 4))
sy = sx
sz = math.sqrt(sum_wz_wz / (num_fixed_points - 1))
self.logger.debug(
'Calculated standard deviations '
'(sX = {:1.5f} m, sY = {:1.5f} m, sZ = {:1.5f} m)'.format(
sx, sy, sz))
# Scale factor.
m = math.sqrt((self._a * self._a) + (self._o * self._o))
self.logger.debug('Calculated scale factor (m = {:1.5f})'.format(m))
# Create response sets for the view point.
response_sets = {
'x': Obs.create_response_set('float', 'm', self._view_point['x']),
'y': Obs.create_response_set('float', 'm', self._view_point['y']),
'z': Obs.create_response_set('float', 'm', self._view_point['z']),
'stdDevX': Obs.create_response_set('float', 'm', sx),
'stdDevY': Obs.create_response_set('float', 'm', sy),
'stdDevZ': Obs.create_response_set('float', 'm', sz),
'scaleFactor': Obs.create_response_set('float', 'm', m)
}
# Create `Observation` instance for the view point.
view_point = Obs()
view_point.set('name', 'getViewPoint')
view_point.set('nextReceiver', 0)
view_point.set('nid', self._node_manager.node.id)
view_point.set('portName', obs.get('portName'))
view_point.set('pid', self._project_manager.project.id)
view_point.set('receivers', self._view_point.get('receivers'))
view_point.set('responseSets', response_sets)
view_point.set('sensorName', obs.get('sensorName'))
view_point.set('sensorType', obs.get('sensorType'))
view_point.set('target', self._view_point.get('target'))
view_point.set('timestamp', str(arrow.utcnow()))
# Return the `Observation` object of the view point.
return view_point
def process_observation(self, obs: Observation) -> Observation:
"""Extracts the values from the raw responses of the observation
using regular expressions.
Args:
obs: The observation object.
Returns:
The observation object with extracted and converted responses.
"""
for set_name, request_set in obs.get('requestSets').items():
if not request_set.get('enabled'):
# Request is disabled.
continue
if set_name not in obs.get('requestsOrder'):
# Request should be ignored.
continue
response = request_set.get('response')
response_pattern = request_set.get('responsePattern')
if response is None or len(response) == 0:
self.logger.warning(f'No response "{set_name}" in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}" from sensor '
f'"{obs.get("sensorName")}" on port '
f'"{obs.get("portName")}"')
continue
try:
pattern = re.compile(response_pattern)
match = pattern.search(response)
except Exception:
self.logger.error(f'Invalid regular expression for response '
f'"{set_name}" in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}" from sensor '
f'"{obs.get("sensorName")}" on port '
f'"{obs.get("portName")}"')
return obs
if not match:
self.logger.error(f'Response "{self.sanitize(response)}" of '
f'request "{set_name}" in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}" from sensor '
f'"{obs.get("sensorName")}" on port '
f'"{obs.get("portName")}" does not match '
f'extraction pattern')
return obs
# The regular expression pattern needs at least one named group
# defined. Otherwise, the extraction of the values fails.
#
# Right: "(?P<id>.*)"
# Wrong: ".*"
if pattern.groups == 0:
self.logger.error(
f'No group(s) defined in regular expression '
f'pattern in observation "{obs.get("name")}" '
f'of target "{obs.get("target")}"')
return obs
# Convert the type of the parsed raw values from string to the
# actual data type (float, int).
response_sets = obs.get('responseSets')
for group_name, raw_value in match.groupdict("").items():
if raw_value is None or len(raw_value) == 0:
self.logger.error(f'Undefined raw value in response set '
f'"{group_name}" in observation '
f'"{obs.get("name")}" of target '
f'"{ obs.get("target")}"')
continue
response_set = response_sets.get(group_name)
if not response_set:
self.logger.error(f'Undefined response set "{group_name}" '
f'in observation "{obs.get("name")}" '
f'of target "{obs.get("target")}"')
continue
response_type = response_set.get('type').lower()
if response_type == 'float':
# Convert raw value to float. Replace comma by dot.
response_value = self.to_float(raw_value)
elif response_type == 'integer':
# Convert raw value to int.
response_value = self.to_int(raw_value)
else:
response_value = raw_value
if response_value is not None:
self.logger.debug(
f'Extracted "{response_value}" from raw '
f'response "{group_name}" in observation '
f'"{ obs.get("name")}" of target '
f'"{obs.get("target")}"')
response_set['value'] = response_value
return obs
def process_observation(self, obs: Obs) -> Obs:
sensor_type = obs.get('sensorType')
# Update atmospheric data if sensor is a weather station.
if SensorType.is_weather_station(sensor_type):
self._update_meteorological_data(obs)
return obs
# Check if sensor is of type "total station".
if not SensorType.is_total_station(sensor_type):
self.logger.warning(f'Sensor type "{sensor_type}" not supported')
return obs
# Check if atmospheric data has been set.
if None in [self.temperature, self.pressure, not self.humidity]:
self.logger.warning('Missing temperature, air pressure, or '
'humidity')
return obs
# Check the age of the atmospheric data.
if self.last_update - time.time() > self._max_age:
self.logger.warning(f'Atmospheric data is older than '
f'{int(self._max_age / 3600)} hour(s)')
# Reduce the slope distance of the EDM measurement.
dist = obs.get_response_value(self._distance_name)
if dist is None:
self.logger.error(f'No distance set in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return obs
d_dist_1 = 0
d_dist_2 = 0
response_sets = obs.get('responseSets')
# Calculate the atmospheric reduction of the distance.
if self._is_atmospheric_correction:
c = self.get_atmospheric_correction(self._temperature,
self._pressure, self._humidity)
d_dist_1 = dist * c * math.pow(10, -6)
rs = Obs.create_response_set('float', 'none', round(c, 5))
response_sets['atmosphericPpm'] = rs
# Calculate the sea level reduction of the distance.
if self._is_sea_level_correction:
d_dist_2 = self.get_sea_level_correction(self._sensor_height)
rs = Obs.create_response_set('float', 'm', round(d_dist_2, 5))
response_sets['seaLevelDelta'] = rs
# Add corrected distance to the observation set.
if d_dist_1 != 0 or d_dist_2 != 0:
r_dist = dist + d_dist_1 + d_dist_2
self.logger.info('Reduced distance from {:0.5f} m to {:0.5f} m '
'(correction value: {:0.5f} m)'.format(
dist, r_dist, d_dist_1 + d_dist_2))
rs = Obs.create_response_set('float', 'm', round(r_dist, 5))
response_sets[self._distance_name + 'Raw'] =\
response_sets.get(self._distance_name)
response_sets[self._distance_name] = rs
return obs
def process_observation(self,
obs: Observation) -> Union[Observation, None]:
"""Sends a request to the attached sensor and forwards the response.
Args:
obs: Observation object.
Returns:
The extended observation object or None if connection failed.
"""
if not self._sock:
# Open socket connection.
if not self._open():
return
# Add the name of the Bluetooth port to the observation.
obs.set('portName', self.name)
requests_order = obs.get('requestsOrder', [])
request_sets = obs.get('requestSets')
if not requests_order:
self.logger.notice(f'No requests order defined in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}"')
# Send requests sequentially to the sensor.
for request_name in requests_order:
request_set = request_sets.get(request_name)
if not request_set:
self.logger.error(f'Request set "{request_name}" not found in '
f'observation "{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return
# The response of the sensor.
response = ''
response_delimiter = request_set.get('responseDelimiter')
# Data of the request set.
request = request_set.get('request')
sleep_time = request_set.get('sleepTime') or 1.0
timeout = request_set.get('timeout') or 1.0
# Send the request of the observation to the attached sensor.
self.logger.verbose(f'Sending request "{request_name}" of '
f'observation "{obs.get("name")}" to sensor '
f'"{obs.get("sensorName")}" ...')
# Write to Bluetooth port.
self._send(request)
# Get the response of the sensor.
response = self._receive(response_delimiter, timeout)
self.logger.verbose(f'Received response '
f'"{self.sanitize(response)}" for request '
f'"{request_name}" of observation '
f'"{obs.get("name")}" from sensor '
f'"{obs.get("sensorName")}"')
# Add the raw response of the sensor to the observation set.
request_set['response'] = response
# Add the timestamp to the observation.
obs.set('timestamp', str(arrow.utcnow()))
# Sleep until the next request.
time.sleep(sleep_time)
return obs
def process_observation(self,
obs: Observation) -> Union[Observation, None]:
"""Processes an observation object. Sends request to sensor and stores
response.
Args:
obs: The observation object.
Returns:
The processed observation.
"""
# Turn on passive mode.
if obs.get('passiveMode'):
if self._is_passive:
# Restart passive listener.
self._is_passive = False
self._passive_listener.join()
self._is_passive = True
self._passive_listener = Thread(target=self.listen, args=(obs, ))
self._passive_listener.daemon = True
self._passive_listener.start()
self.logger.debug(
f'Started passive listener of port "{self.name}"')
return
# Turn off passive mode.
if self._is_passive and not obs.get('passiveMode'):
self._is_passive = False
self._passive_listener.join()
self.logger.debug(
f'Stopped passive listener of port "{self.name}"')
# Create new serial connection.
if not self._serial:
self._create()
if self._serial is None:
self.logger.error(f'Could not access port '
f'"{self._serial_port_config.port}"')
return
if not self._serial.is_open:
self.logger.verbose(f'Re-opening port '
f'"{self._serial_port_config.port}" ...')
self._serial.open()
self._serial.reset_output_buffer()
self._serial.reset_input_buffer()
# Add the name of this serial port module to the observation.
obs.set('portName', self._name)
requests_order = obs.get('requestsOrder', [])
request_sets = obs.get('requestSets')
if not requests_order:
self.logger.notice(f'No requests order defined in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}"')
# Send requests sequentially to the sensor.
for request_name in requests_order:
request_set = request_sets.get(request_name)
if not request_set:
self.logger.error(f'Request set "{request_name}" not found in '
f'observation "{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return
# The response of the sensor.
response = ''
response_delimiter = request_set.get('responseDelimiter')
# Data of the request set.
request = request_set.get('request')
sleep_time = request_set.get('sleepTime') or 0.0
timeout = request_set.get('timeout') or 0.0
# Send the request of the observation to the attached sensor.
self.logger.verbose(f'Sending request "{request_name}" of '
f'observation "{obs.get("name")}" to sensor '
f'"{obs.get("sensorName")}" ...')
for attempt in range(self._max_attempts):
if attempt > 0:
self.logger.info(f'Request attempt {attempt + 1} of '
f'{self._max_attempts}')
time.sleep(1)
# Write to the serial port.
self._write(request)
# Get the response of the sensor.
response = self._read(eol=response_delimiter,
length=0,
timeout=timeout)
self._serial.reset_output_buffer()
self._serial.reset_input_buffer()
if response:
self.logger.verbose(
f'Received response '
f'"{self.sanitize(response)}" for '
f'request "{request_name}" of '
f'observation "{obs.get("name")}" from '
f'sensor "{obs.get("sensorName")}"')
break
# Try next attempt if response is empty.
self.logger.warning(f'No response from sensor '
f'"{obs.get("sensorName")}" for '
f'observation "{obs.get("name")}" of '
f'target "{obs.get("target")}"')
# Add the raw response of the sensor to the observation set.
request_set['response'] = response
# Add the timestamp to the observation.
obs.set('timestamp', str(arrow.utcnow()))
# Sleep until the next request.
time.sleep(sleep_time)
return obs