def create_observation(self, data: Dict[str, Any]) -> Observation:
"""Creates an observation object.
Args:
data: The observation data.
"""
data['id'] = Observation.get_new_id()
data['sensorName'] = self._name
data['sensorType'] = self._type
# Character '\' is escaped in the JSON configuration file. Encoded
# bytes have to be decoded.
for set_name, request_set in data.get('requestSets').items():
if request_set.get('request'):
request_set['request'] = codecs.decode(
request_set.get('request'), 'unicode_escape')
if request_set.get('responseDelimiter'):
request_set['responseDelimiter'] = codecs.decode(
request_set.get('responseDelimiter'), 'unicode_escape')
if request_set.get('responsePattern'):
request_set['responsePattern'] = codecs.decode(
request_set.get('responsePattern'), 'unicode_escape')
return Observation(data)
def process_observation(self, obs: Observation) -> Observation:
request_sets = obs.get('requestSets')
for set_name, request_set in request_sets.items():
request = request_set.get('request')
timeout = request_set.get('timeout')
sleep_time = request_set.get('sleepTime')
response = ''
self.logger.verbose(f'Sending request "{set_name}" to sensor '
f'"{obs.get("sensorName")}" on virtual port '
f'"{self.name}"')
for pattern in self.patterns:
reg_exp = re.compile(pattern)
parsed = reg_exp.match(request)
if not parsed:
continue
response = self.patterns[pattern](request)
self.logger.verbose(f'Received response '
f'"{self.sanitize(response)}" from sensor '
f'"{obs.get("sensorName")}" on virtual '
f'port "{self.name}"')
break
request_set['response'] = response
time.sleep((timeout * 0.25) + sleep_time)
obs.set('portName', self._name)
obs.set('timestamp', str(arrow.utcnow()))
return obs
def process_observation(self, obs: Obs) -> Obs:
z = obs.get_response_value('z')
if not z:
return obs
d = obs.get_response_value('slopeDist')
if not d:
return obs
k = 0.13 # Refraction coefficient.
r = 6370000 # Earth radius.
k_e = (d * d) / (2 * r) # Correction of earth radius.
k_r = k * k_e # Correction of refraction.
r = k_e - k_r
self.logger.info('Updated height in observation "{}" of target "{}" '
'from {:3.4f} m to {:3.4f} m (refraction value: '
'{:3.5f} m)'.format(obs.get("name"),
obs.get("target"), z, z + r, r))
refraction = Obs.create_response_set('float', 'm', round(r, 6))
z_new = Obs.create_response_set('float', 'm', round(z + r, 5))
z_raw = Obs.create_response_set('float', 'm', z)
obs.data['responseSets']['refraction'] = refraction
obs.data['responseSets']['zRaw'] = z_raw
obs.data['responseSets']['z'] = z_new
return obs
def _update_fixed_point(self, obs: Obs) -> None:
"""Updates given fixed point.
Args:
obs: `Observation` object.
"""
fixed_point = self._fixed_points.get(obs.get('target'))
hz = obs.get_response_value('hz')
azimuth = self.get_azimuth_angle(self._azimuth_angle,
self._view_point.get('x'),
self._view_point.get('y'),
fixed_point.get('x'),
fixed_point.get('y'))
fixed_point['hz'] = hz
fixed_point['azimuth'] = azimuth
fixed_point['lastUpdate'] = time.time()
# Calculate the orientation.
delta_hz = azimuth - hz
if delta_hz < 0:
# Add 400 gon.
delta_hz += 2 * math.pi
fixed_point['deltaHz'] = delta_hz
def process_observation(self, obs: Obs) -> Obs:
if not self._is_valid_sensor_type(obs):
# Only total stations are supported.
return obs
hz = obs.get_response_value('hz')
v = obs.get_response_value('v')
dist = obs.get_response_value('slopeDist')
if None in [hz, v, dist]:
return obs
# Calculate the horizontal distance.
dist_hz = math.sin(v) * dist
if self._is_fixed_point(obs):
# Add measured Hz and calculated Hz to the fixed point.
self._update_fixed_point(obs)
self.logger.debug(f'Updated fixed point of target '
f'"{obs.get("target")}"')
self.logger.debug(
'Starting polar transformation of target "{}" (Hz = '
'{:3.5f} gon, V = {:3.5f} gon, dist = {:4.5f} m)'.format(
obs.get("target"), rad_to_gon(hz), rad_to_gon(v), dist_hz))
if self._is_adjustment_enabled:
# Add the adjustment value to the horizontal direction.
adj = self._get_adjustment_value()
self.logger.info('Calculated adjustment value for polar '
'transformation ({:3.5f} gon)'.format(
rad_to_gon(adj)))
hz += adj
# Calculate the coordinates of the observation.
x, y, z = self.transform(self._view_point.get('x'),
self._view_point.get('y'),
self._view_point.get('z'),
self._azimuth_point.get('x'),
self._azimuth_point.get('y'), hz, v, dist_hz)
self.logger.info('Transformed target "{}" (X = {:3.4f}, Y = {:3.4f}, '
'Z = {:3.4f})'.format(obs.get("target"), x, y, z))
# Add to observation data set.
response_sets = obs.get('responseSets')
response_sets['x'] = Obs.create_response_set('float', 'm', round(x, 5))
response_sets['y'] = Obs.create_response_set('float', 'm', round(y, 5))
response_sets['z'] = Obs.create_response_set('float', 'm', round(z, 5))
if self._is_adjustment_enabled:
response_sets['hzAdjusted'] = Obs.create_response_set(
'float', 'rad', round(hz, 16))
return obs
def _calculate_target_point(self, obs: Obs) -> Obs:
"""Calculates the coordinates of a target point and updates the
given `Observation` object.
Args:
obs: `Observation` object.
Returns:
The `Observation` object with calculated coordinates.
"""
hz = obs.get_response_value('hz')
v = obs.get_response_value('v')
dist = obs.get_response_value('slopeDist')
if None in [hz, v, dist]:
self.logger.warning(f'Hz, V, or distance missing in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return obs
# Calculate the coordinates in the global system (X, Y, Z).
x, y, z = self.calculate_point_coordinates(hz, v, dist,
self._view_point.get('x'),
self._view_point.get('y'),
self._view_point.get('z'),
self._a, self._o)
self.logger.info('Calculated coordinates of target point "{}" '
'(X = {:4.5f}, Y = {:4.5f}, Z = {:4.5f})'.format(
obs.get("target"), x, y, z))
# Do residual mismatch transformation.
if self._is_residual:
vx, vy = self._calculate_residual_mismatches(x, y)
self.logger.debug('Calculated improvements for target point "{}" '
'(dX = {:4.5f} m, dY = {:4.5f} m)'.format(
obs.get("target"), vx, vy))
x += vx
y += vy
self.logger.debug('Updated coordinates of target point "{}" '
'(X = {:4.5f}, Y = {:4.5f})'.format(
obs.get("target"), x, y))
# Add response set.
response_sets = obs.get('responseSets')
response_sets['x'] = Obs.create_response_set('float', 'm', x)
response_sets['y'] = Obs.create_response_set('float', 'm', y)
response_sets['z'] = Obs.create_response_set('float', 'm', z)
return obs
def do_handle_observation(self, header: Dict, payload: Dict) -> None:
"""Handles an observation by forwarding it to the processing method and
prepares the result for publishing.
Args:
header: Message header.
payload: Message payload.
"""
obs = Observation(payload)
if self._is_running:
obs = self.process_observation(Observation(payload))
if obs:
self.publish_observation(obs)
def process_observation(self, obs: Observation) -> Observation:
"""Checks, if responses are inside defined bounds.
Args:
obs: The observation object.
Returns:
The untouched observation object.
"""
if not obs.get('name') in self._observations:
self.logger.warning(f'Undefined observation "{obs.get("name")}" '
f'of target "{obs.get("target")}"')
return obs
response_sets = self._observations.get(obs.get('name'))
for response_name, limits in response_sets.items():
response_value = obs.get_response_value(response_name)
if response_value is None or not self.is_number(response_value):
self.logger.warning(f'Response value "{response_name}" in '
f'observation "{obs.get("name")}" of '
f'target "{obs.get("target")}" is not a '
f'number')
continue
min_value = limits.get('min')
max_value = limits.get('max')
if min_value <= response_value <= max_value:
self.logger.debug(f'Response value "{response_name}" in '
f'observation "{obs.get("name")}" of target '
f'"{obs.get("target")}" is within limits')
elif response_value < min_value:
self.logger.critical(f'Response value "{response_name}" in '
f'observation "{obs.get("name")}" of '
f'target "{obs.get("target")}" is less '
f'than minimum ({response_value} < '
f'{min_value})')
elif response_value > max_value:
self.logger.critical(
f'Response value "{response_name}" in '
f'observation "{obs.get("name")}" of '
f'target "{obs.get("target")}" is greater '
f'than maximum ({response_value} > '
f'{max_value})')
return obs
def _is_fixed_point(self, obs: Obs) -> bool:
"""Checks if the given observation equals one of the defined fixed
points."""
if self._fixed_points.get(obs.get('target')):
return True
else:
return False
def run(self) -> None:
"""Iterates trough the observation set and sends observations to an
external callback function."""
# Return if observation is disabled.
if not self._obs.get('enabled'):
return
# Disable the observation if it should run one time only.
if self._obs.get('onetime'):
self._obs.set('enabled', False)
# Make a deep copy, since we don't want to do any changes to the
# observation in our observation set.
obs_copy = copy.deepcopy(self._obs)
# Set IDs.
obs_copy.set('id', Observation.get_new_id())
obs_copy.set('pid', self._project_id)
obs_copy.set('nid', self._node_id)
# Insert the name of the port module or the virtual sensor at the
# beginning of the receivers list.
receivers = obs_copy.get('receivers')
receivers.insert(0, self._port_name)
obs_copy.set('receivers', receivers)
# Set the next receiver to the module following the port.
obs_copy.set('nextReceiver', 1)
self.logger.info(f'Starting job "{self._obs.get("name")}" for port '
f'"{self._port_name}" ...')
# Get the sleep time of the whole observation.
sleep_time = obs_copy.get('sleepTime', 0)
# Create target, header, and payload in order to send the observation.
target = self._port_name
header = Observation.get_header()
header['from'] = 'job'
payload = obs_copy.data
# Fire and forget the observation.
self._uplink(target, header, payload)
# Sleep until the next observation.
self.logger.debug(f'Next observation starts in {sleep_time} s')
time.sleep(sleep_time)
def _update_fixed_point(self, obs: Obs) -> None:
"""Adds horizontal direction, vertical angle, and slope distance
of the observation to a fixed point.
Args:
obs: `Observation` object.
"""
hz = obs.get_response_value('hz')
v = obs.get_response_value('v')
dist = obs.get_response_value('slopeDist')
if None in [hz, v, dist]:
return
# Calculate the coordinates of the fixed point if the Helmert
# transformation has been done already. Otherwise, use the datum from
# the configuration.
fixed_point = self._fixed_points.get(obs.get('target'))
if self._is_ready():
# Calculate fixed point coordinates.
x, y, z = self.calculate_point_coordinates(
hz, v, dist, self._view_point.get('x'),
self._view_point.get('y'), self._view_point.get('z'), self._a,
self._o)
self.logger.info('Calculated coordinates of fixed point "{}" '
'(X = {:3.5f}, Y = {:3.5f}, Z = {:3.5f})'.format(
obs.get("target"), x, y, z))
else:
# Get the coordinates of the fixed point from the configuration.
x = fixed_point.get('x')
y = fixed_point.get('y')
z = fixed_point.get('z')
# Update the values.
fixed_point['hz'] = hz
fixed_point['v'] = v
fixed_point['dist'] = dist
fixed_point['lastUpdate'] = time.time()
self.logger.debug(f'Updated fixed point of target '
f'"{obs.get("target")}"')
# Add global Cartesian coordinates of the fixed point to the
# observation.
response_sets = obs.get('responseSets')
response_sets['x'] = Obs.create_response_set('float', 'm', x)
response_sets['y'] = Obs.create_response_set('float', 'm', y)
response_sets['z'] = Obs.create_response_set('float', 'm', z)
def listen(self, obs_draft: Observation) -> None:
"""Threaded method for passive mode. Reads incoming data from serial
port. Used for sensors which start streaming data without prior
request.
Args:
obs_draft: The observation draft with the response pattern.
"""
while self._is_running and self._is_passive:
if not obs_draft:
self.logger.warning(f'No draft observation set for passive '
f'listener on port "{self.name}"')
break
if not self._serial:
self._create()
if self._serial is None:
self.logger.error(f'Can\'t access port '
f'"{self._serial_port_config.port}" for '
f'passive listening')
return
if not self._serial.is_open:
self.logger.info(f'Re-opening port '
f'"{self._serial_port_config.port}" for '
f'passive listening ...')
self._serial.open()
self._serial.reset_input_buffer()
obs = copy.deepcopy(obs_draft)
obs.set('id', Observation.get_new_id())
obs.set('portName', self.name)
draft = obs.get('requestSets').get('draft')
timeout = draft.get('timeout', 1.0)
response_delimiter = draft.get('responseDelimiter', '')
length = draft.get('responseLength', 0)
request = draft.get('request')
if request:
self._write(request)
response = self._read(eol=response_delimiter,
length=length,
timeout=timeout)
if response:
self.logger.verbose(
f'Received "{self.sanitize(response)}" '
f'from sensor "{obs.get("sensorName")}" on '
f'port "{self._name}"')
draft['response'] = response
obs.set('timestamp', str(arrow.utcnow()))
self.publish_observation(obs)
def publish_observation(self, obs: Observation) -> None:
"""Prepares the observation for publishing and forwards it to the
messenger.
Args:
obs: Observation object.
"""
receivers = obs.get('receivers')
index = obs.get('nextReceiver')
# No receivers defined.
if not receivers:
logging.debug(f'No receivers defined in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return
# No index defined.
if (index is None) or (index < 0):
self.logger.warning(f'Undefined receiver in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return
# Receivers list has been processed and observation is finished.
if index >= len(receivers):
self.logger.info(f'Observation "{obs.get("name")}" of target '
f'"{obs.get("target")}" has been finished')
return
# Name of the sending module.
sender = receivers[index - 1]
# Increase the receivers index.
next_receiver = receivers[index]
obs.set('nextReceiver', index + 1)
# Create header and payload.
header = {'from': sender, 'type': 'observation'}
payload = obs.data
# Send the observation to the next module.
self.publish(next_receiver, header, payload)
def rerun_pending(obs, working_dir, args):
pd_log = cst.pending_log
# Read the current pending log
lines = np.array(read_pending_log())
folder_date = working_dir.replace(cst.base_path, '').split(os.sep)[1]
folder_datetime = datetime.strptime(folder_date, '%y%m%d')
if len(lines) == 0:
ut.Print("Pending log empty!", args, True)
return
# Overwrite the pending log
with open(pd_log, "wb") as f:
# Re-append the header
np.savetxt(f, lines[0].reshape(1, lines[0].shape[0]), delimiter=",", fmt="%s")
# Loop over every old line and check if it needs to be re-run
for line in lines[1:]:
line = np.array([entry.strip() for entry in line])
# Unpack data and rerun the reduction process
date, frame_type, binning, fltr, bias_off, dark_off, flat_off, expiry, file_path = line
# Reinitialise the observation object and working dir
line_obs = Observation(file_path)
line_working_dir = (os.path.split(file_path)[0]).replace(cst.tele_path, cst.base_path)
tele_data_dir = line_working_dir.replace(cst.base_path, cst.tele_path)
# Check its expiry date if is has one
try:
expiry_date = datetime.strptime(line[-2], "%d-%m-%Y")
if expiry_date.date() < folder_datetime.date():
# We cannot hope to find a better version, ever. So, we can
# safely continue with new actions/plugins.
run_plugins_single(line_obs, line_working_dir, args, file, "pending")
except ValueError: pass
# Re-reduce a light file
if frame_type == "Light file":
max_days_off = 365
# Cut on calculation time if possible
try:
max_days_off = abs(max(int(bias_off), int(dark_off), int(flat_off)))
except ValueError: pass
# Re-reduce light file. Will add a new entry in the pending log, but hopefully with a
# lower max_days_off. When the time's there, it will expire.
red.reduce_img(line_obs, line_working_dir, args, file_path, max_days_off)
# Re-reduce dark frame
elif frame_type == "Dark file":
line_working_dir = (os.path.split(os.path.split(file_path)[0])[0]).replace(cst.tele_path, cst.base_path)
cor.recreate_mdark(line_obs, line_working_dir, args, file_path, binning, fltr)
# Re-reduce flat field
elif frame_type == "Flat file":
line_working_dir = (os.path.split(os.path.split(file_path)[0])[0]).replace(cst.tele_path, cst.base_path)
cor.recreate_mflat(line_obs, line_working_dir, args, file_path, binning, fltr)
def create_observation(self):
""" Function that creates the observation object for the
passed target. Also performs some small checkups to
be sure that we have some proper data.
"""
target = self.target
args = self.args
ps.running(f"Looking for files in {self.target}")
self.obs = Observation(target)
self.check_obs()
ps.done("Observation Object initialized")
def observations() -> List[Observation]:
"""Returns a List with examples observation objects.
Returns:
List of examples observations.
"""
file_path = 'tests/data/observations.json'
with open(file_path) as fh:
data = json.loads(fh.read())
return [Observation(n) for n in data]
def process_observation(self, obs: Observation) -> Observation:
for name, properties in self._config.items():
response_set = obs.get('responseSets').get(name)
if not response_set:
continue
source_value = response_set.get('value')
source_unit = response_set.get('unit')
if not source_value or not source_unit:
continue
if source_unit != properties.get('sourceUnit'):
self.logger.warning(f'Unit "{source_unit}" of response '
f'"{name}" in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}" does not match '
f'"{properties.get("sourceUnit")}"')
continue
if properties.get('conversionType') == 'scale':
target_value = self.scale(float(source_value),
properties.get('scalingValue'))
target_unit = properties.get('targetUnit')
self.logger.info(
'Converted response "{}" in observation "{}" '
'of target "{}" from {:.4f} {} to {:.4f} {}'.format(
name, obs.get("name"), obs.get("target"), source_value,
source_unit, target_value, target_unit))
response_set = Observation.create_response_set(
'float', target_unit, round(target_value, 5))
obs.data['responseSets'][name] = response_set
return obs
def _is_fixed_point(self, obs: Obs) -> bool:
"""Checks if the given observation equals one of the defined fixed
points.
Args:
obs: `Observation` object.
Returns:
True if observation is fixed point, False if not.
"""
if self._fixed_points.get(obs.get('target')):
return True
else:
return False
def _is_valid_sensor_type(self, obs: Obs) -> bool:
"""Returns whether or not the sensor is supported.
Args:
obs: `Observation` object.
Returns:
True if sensor is supported, False if not.
"""
sensor_type = obs.get('sensorType')
if not SensorType.is_total_station(sensor_type):
self.logger.error(f'Sensor type "{sensor_type}" not supported')
return False
return True
def process_observation(self, obs: Observation) -> Observation:
if not self._is_enabled:
return obs
for receiver in self._receivers:
obs_copy = copy.deepcopy(obs)
target = f'{receiver}/{obs_copy.get("target")}'
obs_copy.set('nextReceiver', 0)
obs_copy.set('receivers', [target])
self.logger.debug(f'Publishing observation '
f'"{obs_copy.get("name")}" of target '
f'"{obs_copy.get("target")}" to "{target}"')
header = Observation.get_header()
payload = obs_copy.data
self.publish(target, header, payload)
return obs
def process_observation(self, obs: Obs) -> Obs:
# Calculate the serial measurement of an observation in two faces.
hz_0 = obs.get_response_value('hz0')
hz_1 = obs.get_response_value('hz1')
v_0 = obs.get_response_value('v0')
v_1 = obs.get_response_value('v1')
dist_0 = obs.get_response_value('slopeDist0')
dist_1 = obs.get_response_value('slopeDist1')
if None in [hz_0, hz_1, v_0, v_1, dist_0, dist_1]:
return obs
# Calculate new Hz, V, and slope distance.
hz = hz_0 + hz_1
if hz_0 > hz_1:
hz += math.pi
else:
hz -= math.pi
hz /= 2
v = ((2 * math.pi) + (v_0 - v_1)) / 2
dist = (dist_0 + dist_1) / 2
# Save the calculated values.
response_sets = obs.get('responseSets')
response_sets['hz'] = Obs.create_response_set('float', 'rad', hz)
response_sets['v'] = Obs.create_response_set('float', 'rad', v)
response_sets['slopeDist'] = Obs.create_response_set('float' 'm', dist)
self.logger.debug(f'Calculated serial measurement with two faces for '
f'observation "{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return obs
def _update_meteorological_data(self, obs: Obs) -> None:
"""Updates the temperature, air pressure, and humidity attributes by
using the measured data of a weather station."""
# Update temperature.
t = obs.get_response_value('temperature')
if t is not None:
self.temperature = t
# Update pressure.
p = obs.get_response_value('pressure')
if p is not None:
self.pressure = p
# Update humidity.
if (obs.has_response_value('humidity')
and obs.has_response_type('humidity')):
h = obs.get_response_value('humidity')
u = obs.get_response_unit('humidity')
self.humidity = h / 100 if u == '%' else h
def process_observation(self, obs: Observation) -> Observation:
"""Appends data to a flat file in CSV format.
Args:
obs: Observation object.
Returns:
The observation object.
"""
ts = arrow.get(obs.get('timestamp', 0))
file_date = {
# No file rotation, i.e., all data is stored in a single file.
FileRotation.NONE:
None,
# Every day a new file is created.
FileRotation.DAILY:
ts.format('YYYY-MM-DD'),
# Every month a new file is created.
FileRotation.MONTHLY:
ts.format('YYYY-MM'),
# Every year a new file is created.
FileRotation.YEARLY:
ts.format('YYYY')
}[self._file_rotation]
fn = self._file_name
fn = fn.replace('{{port}}', obs.get("portName"))
fn = fn.replace('{{date}}', f'{file_date}' if file_date else '')
fn = fn.replace(
'{{target}}',
f'{obs.get("target")}' if obs.get('target') is not None else '')
fn = fn.replace(
'{{name}}',
f'{obs.get("name")}' if obs.get('name') is not None else '')
fn += self._file_extension
for path in self._paths:
if not Path(path).exists():
self.logger.critical(f'Path "{path}" does not exist')
continue
file_path = Path(path, fn)
# Create a header if a new file has to be touched.
header = None
if not Path(file_path).is_file():
header = (f'# Target "{obs.get("target")}" of '
f'"{obs.get("sensorName")}" on '
f'"{obs.get("portName")}"\n')
# Open a file for each path.
with open(str(file_path), 'a') as fh:
# Add the header if necessary.
if header:
fh.write(header)
# Format the time stamp. For more information, see:
# http://arrow.readthedocs.io/en/latest/#tokens
date_time = ts.format(self._date_time_format)
# Create the CSV line starting with date and time.
line = date_time
if self._save_observation_id:
line += self._separator + obs.get('id')
if obs.get('target') is not None:
line += self._separator + obs.get('target')
response_sets = obs.get('responseSets')
for response_set_id in sorted(response_sets.keys()):
response_set = response_sets.get(response_set_id)
v = response_set.get('value')
u = response_set.get('unit')
line += self._separator + format(response_set_id)
line += self._separator + format(v)
line += self._separator + format(u)
# Write line to file.
fh.write(line + '\n')
self.logger.info(f'Saved observation "{obs.get("name")}" of '
f'target "{obs.get("target")}" from port '
f'"{obs.get("portName")}" to file '
f'"{str(file_path)}"')
return obs
def process_observation(self, obs: Observation) -> Observation:
for response_set in self._response_sets:
return_code = obs.get_value('responseSets', response_set, 'value')
if return_code is None:
continue
if return_code == 0:
if obs.get('corrupted') is True:
obs.set('corrupted', False)
continue
# Get error message of the return code.
error_values = ReturnCodes.codes.get(return_code)
attempts = obs.get('attempts', 0)
# Retry measurement.
if error_values and attempts < self._retries:
obs.set('attempts', attempts + 1)
obs.set('corrupted', False)
obs.set('nextReceiver', 0)
self.logger.info(
f'Retrying observation "{obs.get("name")}" of '
f'target "{obs.get("target")}" due to return '
f'code {return_code} of response '
f'"{response_set}" (attempt {attempts + 1,} '
f'of {self._retries})')
else:
obs.set('corrupted', True)
if error_values:
# Return code related log message.
lvl, retry, msg = error_values
self.logger.log(
lvl * 10, f'Observation "{obs.get("name")}" of '
f'target "{obs.get("target")}": {msg} '
f'(code {return_code} in response '
f'"{response_set}")')
else:
# Generic log message.
self.logger.error(f'Error occurred on observation '
f'"{obs.get("name")}" (unknown code '
f'{return_code} in response '
f'"{response_set}")')
return obs
return obs
def process_observation(self, obs: Observation) -> Observation:
"""Extracts the values from the raw responses of the observation
using regular expressions.
Args:
obs: The observation object.
Returns:
The observation object with extracted and converted responses.
"""
for set_name, request_set in obs.get('requestSets').items():
if not request_set.get('enabled'):
# Request is disabled.
continue
if set_name not in obs.get('requestsOrder'):
# Request should be ignored.
continue
response = request_set.get('response')
response_pattern = request_set.get('responsePattern')
if response is None or len(response) == 0:
self.logger.warning(f'No response "{set_name}" in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}" from sensor '
f'"{obs.get("sensorName")}" on port '
f'"{obs.get("portName")}"')
continue
try:
pattern = re.compile(response_pattern)
match = pattern.search(response)
except Exception:
self.logger.error(f'Invalid regular expression for response '
f'"{set_name}" in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}" from sensor '
f'"{obs.get("sensorName")}" on port '
f'"{obs.get("portName")}"')
return obs
if not match:
self.logger.error(f'Response "{self.sanitize(response)}" of '
f'request "{set_name}" in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}" from sensor '
f'"{obs.get("sensorName")}" on port '
f'"{obs.get("portName")}" does not match '
f'extraction pattern')
return obs
# The regular expression pattern needs at least one named group
# defined. Otherwise, the extraction of the values fails.
#
# Right: "(?P<id>.*)"
# Wrong: ".*"
if pattern.groups == 0:
self.logger.error(
f'No group(s) defined in regular expression '
f'pattern in observation "{obs.get("name")}" '
f'of target "{obs.get("target")}"')
return obs
# Convert the type of the parsed raw values from string to the
# actual data type (float, int).
response_sets = obs.get('responseSets')
for group_name, raw_value in match.groupdict("").items():
if raw_value is None or len(raw_value) == 0:
self.logger.error(f'Undefined raw value in response set '
f'"{group_name}" in observation '
f'"{obs.get("name")}" of target '
f'"{ obs.get("target")}"')
continue
response_set = response_sets.get(group_name)
if not response_set:
self.logger.error(f'Undefined response set "{group_name}" '
f'in observation "{obs.get("name")}" '
f'of target "{obs.get("target")}"')
continue
response_type = response_set.get('type').lower()
if response_type == 'float':
# Convert raw value to float. Replace comma by dot.
response_value = self.to_float(raw_value)
elif response_type == 'integer':
# Convert raw value to int.
response_value = self.to_int(raw_value)
else:
response_value = raw_value
if response_value is not None:
self.logger.debug(
f'Extracted "{response_value}" from raw '
f'response "{group_name}" in observation '
f'"{ obs.get("name")}" of target '
f'"{obs.get("target")}"')
response_set['value'] = response_value
return obs
def process_observation(self,
obs: Observation) -> Union[Observation, None]:
"""Sends a request to the attached sensor and forwards the response.
Args:
obs: Observation object.
Returns:
The extended observation object or None if connection failed.
"""
if not self._sock:
# Open socket connection.
if not self._open():
return
# Add the name of the Bluetooth port to the observation.
obs.set('portName', self.name)
requests_order = obs.get('requestsOrder', [])
request_sets = obs.get('requestSets')
if not requests_order:
self.logger.notice(f'No requests order defined in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}"')
# Send requests sequentially to the sensor.
for request_name in requests_order:
request_set = request_sets.get(request_name)
if not request_set:
self.logger.error(f'Request set "{request_name}" not found in '
f'observation "{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return
# The response of the sensor.
response = ''
response_delimiter = request_set.get('responseDelimiter')
# Data of the request set.
request = request_set.get('request')
sleep_time = request_set.get('sleepTime') or 1.0
timeout = request_set.get('timeout') or 1.0
# Send the request of the observation to the attached sensor.
self.logger.verbose(f'Sending request "{request_name}" of '
f'observation "{obs.get("name")}" to sensor '
f'"{obs.get("sensorName")}" ...')
# Write to Bluetooth port.
self._send(request)
# Get the response of the sensor.
response = self._receive(response_delimiter, timeout)
self.logger.verbose(f'Received response '
f'"{self.sanitize(response)}" for request '
f'"{request_name}" of observation '
f'"{obs.get("name")}" from sensor '
f'"{obs.get("sensorName")}"')
# Add the raw response of the sensor to the observation set.
request_set['response'] = response
# Add the timestamp to the observation.
obs.set('timestamp', str(arrow.utcnow()))
# Sleep until the next request.
time.sleep(sleep_time)
return obs
def process_observation(self, obs: Obs) -> Obs:
sensor_type = obs.get('sensorType')
# Update atmospheric data if sensor is a weather station.
if SensorType.is_weather_station(sensor_type):
self._update_meteorological_data(obs)
return obs
# Check if sensor is of type "total station".
if not SensorType.is_total_station(sensor_type):
self.logger.warning(f'Sensor type "{sensor_type}" not supported')
return obs
# Check if atmospheric data has been set.
if None in [self.temperature, self.pressure, not self.humidity]:
self.logger.warning('Missing temperature, air pressure, or '
'humidity')
return obs
# Check the age of the atmospheric data.
if self.last_update - time.time() > self._max_age:
self.logger.warning(f'Atmospheric data is older than '
f'{int(self._max_age / 3600)} hour(s)')
# Reduce the slope distance of the EDM measurement.
dist = obs.get_response_value(self._distance_name)
if dist is None:
self.logger.error(f'No distance set in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return obs
d_dist_1 = 0
d_dist_2 = 0
response_sets = obs.get('responseSets')
# Calculate the atmospheric reduction of the distance.
if self._is_atmospheric_correction:
c = self.get_atmospheric_correction(self._temperature,
self._pressure, self._humidity)
d_dist_1 = dist * c * math.pow(10, -6)
rs = Obs.create_response_set('float', 'none', round(c, 5))
response_sets['atmosphericPpm'] = rs
# Calculate the sea level reduction of the distance.
if self._is_sea_level_correction:
d_dist_2 = self.get_sea_level_correction(self._sensor_height)
rs = Obs.create_response_set('float', 'm', round(d_dist_2, 5))
response_sets['seaLevelDelta'] = rs
# Add corrected distance to the observation set.
if d_dist_1 != 0 or d_dist_2 != 0:
r_dist = dist + d_dist_1 + d_dist_2
self.logger.info('Reduced distance from {:0.5f} m to {:0.5f} m '
'(correction value: {:0.5f} m)'.format(
dist, r_dist, d_dist_1 + d_dist_2))
rs = Obs.create_response_set('float', 'm', round(r_dist, 5))
response_sets[self._distance_name + 'Raw'] =\
response_sets.get(self._distance_name)
response_sets[self._distance_name] = rs
return obs
def process_observation(self,
obs: Observation) -> Union[Observation, None]:
"""Processes an observation object. Sends request to sensor and stores
response.
Args:
obs: The observation object.
Returns:
The processed observation.
"""
# Turn on passive mode.
if obs.get('passiveMode'):
if self._is_passive:
# Restart passive listener.
self._is_passive = False
self._passive_listener.join()
self._is_passive = True
self._passive_listener = Thread(target=self.listen, args=(obs, ))
self._passive_listener.daemon = True
self._passive_listener.start()
self.logger.debug(
f'Started passive listener of port "{self.name}"')
return
# Turn off passive mode.
if self._is_passive and not obs.get('passiveMode'):
self._is_passive = False
self._passive_listener.join()
self.logger.debug(
f'Stopped passive listener of port "{self.name}"')
# Create new serial connection.
if not self._serial:
self._create()
if self._serial is None:
self.logger.error(f'Could not access port '
f'"{self._serial_port_config.port}"')
return
if not self._serial.is_open:
self.logger.verbose(f'Re-opening port '
f'"{self._serial_port_config.port}" ...')
self._serial.open()
self._serial.reset_output_buffer()
self._serial.reset_input_buffer()
# Add the name of this serial port module to the observation.
obs.set('portName', self._name)
requests_order = obs.get('requestsOrder', [])
request_sets = obs.get('requestSets')
if not requests_order:
self.logger.notice(f'No requests order defined in observation '
f'"{obs.get("name")}" of target '
f'"{obs.get("target")}"')
# Send requests sequentially to the sensor.
for request_name in requests_order:
request_set = request_sets.get(request_name)
if not request_set:
self.logger.error(f'Request set "{request_name}" not found in '
f'observation "{obs.get("name")}" of target '
f'"{obs.get("target")}"')
return
# The response of the sensor.
response = ''
response_delimiter = request_set.get('responseDelimiter')
# Data of the request set.
request = request_set.get('request')
sleep_time = request_set.get('sleepTime') or 0.0
timeout = request_set.get('timeout') or 0.0
# Send the request of the observation to the attached sensor.
self.logger.verbose(f'Sending request "{request_name}" of '
f'observation "{obs.get("name")}" to sensor '
f'"{obs.get("sensorName")}" ...')
for attempt in range(self._max_attempts):
if attempt > 0:
self.logger.info(f'Request attempt {attempt + 1} of '
f'{self._max_attempts}')
time.sleep(1)
# Write to the serial port.
self._write(request)
# Get the response of the sensor.
response = self._read(eol=response_delimiter,
length=0,
timeout=timeout)
self._serial.reset_output_buffer()
self._serial.reset_input_buffer()
if response:
self.logger.verbose(
f'Received response '
f'"{self.sanitize(response)}" for '
f'request "{request_name}" of '
f'observation "{obs.get("name")}" from '
f'sensor "{obs.get("sensorName")}"')
break
# Try next attempt if response is empty.
self.logger.warning(f'No response from sensor '
f'"{obs.get("sensorName")}" for '
f'observation "{obs.get("name")}" of '
f'target "{obs.get("target")}"')
# Add the raw response of the sensor to the observation set.
request_set['response'] = response
# Add the timestamp to the observation.
obs.set('timestamp', str(arrow.utcnow()))
# Sleep until the next request.
time.sleep(sleep_time)
return obs
def observation() -> Observation:
return Observation()
def test_create_response_test(self, observation: Observation) -> None:
response_set = observation.create_response_set('test', 'none', 0.0)
assert response_set == {'type': 'test', 'unit': 'none', 'value': 0.0}
