def create_observation(self, data: Dict[str, Any]) -> Observation: """Creates an observation object. Args: data: The observation data. """ data['id'] = Observation.get_new_id() data['sensorName'] = self._name data['sensorType'] = self._type # Character '\' is escaped in the JSON configuration file. Encoded # bytes have to be decoded. for set_name, request_set in data.get('requestSets').items(): if request_set.get('request'): request_set['request'] = codecs.decode( request_set.get('request'), 'unicode_escape') if request_set.get('responseDelimiter'): request_set['responseDelimiter'] = codecs.decode( request_set.get('responseDelimiter'), 'unicode_escape') if request_set.get('responsePattern'): request_set['responsePattern'] = codecs.decode( request_set.get('responsePattern'), 'unicode_escape') return Observation(data)
def process_observation(self, obs: Observation) -> Observation: request_sets = obs.get('requestSets') for set_name, request_set in request_sets.items(): request = request_set.get('request') timeout = request_set.get('timeout') sleep_time = request_set.get('sleepTime') response = '' self.logger.verbose(f'Sending request "{set_name}" to sensor ' f'"{obs.get("sensorName")}" on virtual port ' f'"{self.name}"') for pattern in self.patterns: reg_exp = re.compile(pattern) parsed = reg_exp.match(request) if not parsed: continue response = self.patterns[pattern](request) self.logger.verbose(f'Received response ' f'"{self.sanitize(response)}" from sensor ' f'"{obs.get("sensorName")}" on virtual ' f'port "{self.name}"') break request_set['response'] = response time.sleep((timeout * 0.25) + sleep_time) obs.set('portName', self._name) obs.set('timestamp', str(arrow.utcnow())) return obs
def process_observation(self, obs: Obs) -> Obs: z = obs.get_response_value('z') if not z: return obs d = obs.get_response_value('slopeDist') if not d: return obs k = 0.13 # Refraction coefficient. r = 6370000 # Earth radius. k_e = (d * d) / (2 * r) # Correction of earth radius. k_r = k * k_e # Correction of refraction. r = k_e - k_r self.logger.info('Updated height in observation "{}" of target "{}" ' 'from {:3.4f} m to {:3.4f} m (refraction value: ' '{:3.5f} m)'.format(obs.get("name"), obs.get("target"), z, z + r, r)) refraction = Obs.create_response_set('float', 'm', round(r, 6)) z_new = Obs.create_response_set('float', 'm', round(z + r, 5)) z_raw = Obs.create_response_set('float', 'm', z) obs.data['responseSets']['refraction'] = refraction obs.data['responseSets']['zRaw'] = z_raw obs.data['responseSets']['z'] = z_new return obs
def _update_fixed_point(self, obs: Obs) -> None: """Updates given fixed point. Args: obs: `Observation` object. """ fixed_point = self._fixed_points.get(obs.get('target')) hz = obs.get_response_value('hz') azimuth = self.get_azimuth_angle(self._azimuth_angle, self._view_point.get('x'), self._view_point.get('y'), fixed_point.get('x'), fixed_point.get('y')) fixed_point['hz'] = hz fixed_point['azimuth'] = azimuth fixed_point['lastUpdate'] = time.time() # Calculate the orientation. delta_hz = azimuth - hz if delta_hz < 0: # Add 400 gon. delta_hz += 2 * math.pi fixed_point['deltaHz'] = delta_hz
def process_observation(self, obs: Obs) -> Obs: if not self._is_valid_sensor_type(obs): # Only total stations are supported. return obs hz = obs.get_response_value('hz') v = obs.get_response_value('v') dist = obs.get_response_value('slopeDist') if None in [hz, v, dist]: return obs # Calculate the horizontal distance. dist_hz = math.sin(v) * dist if self._is_fixed_point(obs): # Add measured Hz and calculated Hz to the fixed point. self._update_fixed_point(obs) self.logger.debug(f'Updated fixed point of target ' f'"{obs.get("target")}"') self.logger.debug( 'Starting polar transformation of target "{}" (Hz = ' '{:3.5f} gon, V = {:3.5f} gon, dist = {:4.5f} m)'.format( obs.get("target"), rad_to_gon(hz), rad_to_gon(v), dist_hz)) if self._is_adjustment_enabled: # Add the adjustment value to the horizontal direction. adj = self._get_adjustment_value() self.logger.info('Calculated adjustment value for polar ' 'transformation ({:3.5f} gon)'.format( rad_to_gon(adj))) hz += adj # Calculate the coordinates of the observation. x, y, z = self.transform(self._view_point.get('x'), self._view_point.get('y'), self._view_point.get('z'), self._azimuth_point.get('x'), self._azimuth_point.get('y'), hz, v, dist_hz) self.logger.info('Transformed target "{}" (X = {:3.4f}, Y = {:3.4f}, ' 'Z = {:3.4f})'.format(obs.get("target"), x, y, z)) # Add to observation data set. response_sets = obs.get('responseSets') response_sets['x'] = Obs.create_response_set('float', 'm', round(x, 5)) response_sets['y'] = Obs.create_response_set('float', 'm', round(y, 5)) response_sets['z'] = Obs.create_response_set('float', 'm', round(z, 5)) if self._is_adjustment_enabled: response_sets['hzAdjusted'] = Obs.create_response_set( 'float', 'rad', round(hz, 16)) return obs
def _calculate_target_point(self, obs: Obs) -> Obs: """Calculates the coordinates of a target point and updates the given `Observation` object. Args: obs: `Observation` object. Returns: The `Observation` object with calculated coordinates. """ hz = obs.get_response_value('hz') v = obs.get_response_value('v') dist = obs.get_response_value('slopeDist') if None in [hz, v, dist]: self.logger.warning(f'Hz, V, or distance missing in observation ' f'"{obs.get("name")}" of target ' f'"{obs.get("target")}"') return obs # Calculate the coordinates in the global system (X, Y, Z). x, y, z = self.calculate_point_coordinates(hz, v, dist, self._view_point.get('x'), self._view_point.get('y'), self._view_point.get('z'), self._a, self._o) self.logger.info('Calculated coordinates of target point "{}" ' '(X = {:4.5f}, Y = {:4.5f}, Z = {:4.5f})'.format( obs.get("target"), x, y, z)) # Do residual mismatch transformation. if self._is_residual: vx, vy = self._calculate_residual_mismatches(x, y) self.logger.debug('Calculated improvements for target point "{}" ' '(dX = {:4.5f} m, dY = {:4.5f} m)'.format( obs.get("target"), vx, vy)) x += vx y += vy self.logger.debug('Updated coordinates of target point "{}" ' '(X = {:4.5f}, Y = {:4.5f})'.format( obs.get("target"), x, y)) # Add response set. response_sets = obs.get('responseSets') response_sets['x'] = Obs.create_response_set('float', 'm', x) response_sets['y'] = Obs.create_response_set('float', 'm', y) response_sets['z'] = Obs.create_response_set('float', 'm', z) return obs
def do_handle_observation(self, header: Dict, payload: Dict) -> None: """Handles an observation by forwarding it to the processing method and prepares the result for publishing. Args: header: Message header. payload: Message payload. """ obs = Observation(payload) if self._is_running: obs = self.process_observation(Observation(payload)) if obs: self.publish_observation(obs)
def process_observation(self, obs: Observation) -> Observation: """Checks, if responses are inside defined bounds. Args: obs: The observation object. Returns: The untouched observation object. """ if not obs.get('name') in self._observations: self.logger.warning(f'Undefined observation "{obs.get("name")}" ' f'of target "{obs.get("target")}"') return obs response_sets = self._observations.get(obs.get('name')) for response_name, limits in response_sets.items(): response_value = obs.get_response_value(response_name) if response_value is None or not self.is_number(response_value): self.logger.warning(f'Response value "{response_name}" in ' f'observation "{obs.get("name")}" of ' f'target "{obs.get("target")}" is not a ' f'number') continue min_value = limits.get('min') max_value = limits.get('max') if min_value <= response_value <= max_value: self.logger.debug(f'Response value "{response_name}" in ' f'observation "{obs.get("name")}" of target ' f'"{obs.get("target")}" is within limits') elif response_value < min_value: self.logger.critical(f'Response value "{response_name}" in ' f'observation "{obs.get("name")}" of ' f'target "{obs.get("target")}" is less ' f'than minimum ({response_value} < ' f'{min_value})') elif response_value > max_value: self.logger.critical( f'Response value "{response_name}" in ' f'observation "{obs.get("name")}" of ' f'target "{obs.get("target")}" is greater ' f'than maximum ({response_value} > ' f'{max_value})') return obs
def _is_fixed_point(self, obs: Obs) -> bool: """Checks if the given observation equals one of the defined fixed points.""" if self._fixed_points.get(obs.get('target')): return True else: return False
def run(self) -> None: """Iterates trough the observation set and sends observations to an external callback function.""" # Return if observation is disabled. if not self._obs.get('enabled'): return # Disable the observation if it should run one time only. if self._obs.get('onetime'): self._obs.set('enabled', False) # Make a deep copy, since we don't want to do any changes to the # observation in our observation set. obs_copy = copy.deepcopy(self._obs) # Set IDs. obs_copy.set('id', Observation.get_new_id()) obs_copy.set('pid', self._project_id) obs_copy.set('nid', self._node_id) # Insert the name of the port module or the virtual sensor at the # beginning of the receivers list. receivers = obs_copy.get('receivers') receivers.insert(0, self._port_name) obs_copy.set('receivers', receivers) # Set the next receiver to the module following the port. obs_copy.set('nextReceiver', 1) self.logger.info(f'Starting job "{self._obs.get("name")}" for port ' f'"{self._port_name}" ...') # Get the sleep time of the whole observation. sleep_time = obs_copy.get('sleepTime', 0) # Create target, header, and payload in order to send the observation. target = self._port_name header = Observation.get_header() header['from'] = 'job' payload = obs_copy.data # Fire and forget the observation. self._uplink(target, header, payload) # Sleep until the next observation. self.logger.debug(f'Next observation starts in {sleep_time} s') time.sleep(sleep_time)
def _update_fixed_point(self, obs: Obs) -> None: """Adds horizontal direction, vertical angle, and slope distance of the observation to a fixed point. Args: obs: `Observation` object. """ hz = obs.get_response_value('hz') v = obs.get_response_value('v') dist = obs.get_response_value('slopeDist') if None in [hz, v, dist]: return # Calculate the coordinates of the fixed point if the Helmert # transformation has been done already. Otherwise, use the datum from # the configuration. fixed_point = self._fixed_points.get(obs.get('target')) if self._is_ready(): # Calculate fixed point coordinates. x, y, z = self.calculate_point_coordinates( hz, v, dist, self._view_point.get('x'), self._view_point.get('y'), self._view_point.get('z'), self._a, self._o) self.logger.info('Calculated coordinates of fixed point "{}" ' '(X = {:3.5f}, Y = {:3.5f}, Z = {:3.5f})'.format( obs.get("target"), x, y, z)) else: # Get the coordinates of the fixed point from the configuration. x = fixed_point.get('x') y = fixed_point.get('y') z = fixed_point.get('z') # Update the values. fixed_point['hz'] = hz fixed_point['v'] = v fixed_point['dist'] = dist fixed_point['lastUpdate'] = time.time() self.logger.debug(f'Updated fixed point of target ' f'"{obs.get("target")}"') # Add global Cartesian coordinates of the fixed point to the # observation. response_sets = obs.get('responseSets') response_sets['x'] = Obs.create_response_set('float', 'm', x) response_sets['y'] = Obs.create_response_set('float', 'm', y) response_sets['z'] = Obs.create_response_set('float', 'm', z)
def listen(self, obs_draft: Observation) -> None: """Threaded method for passive mode. Reads incoming data from serial port. Used for sensors which start streaming data without prior request. Args: obs_draft: The observation draft with the response pattern. """ while self._is_running and self._is_passive: if not obs_draft: self.logger.warning(f'No draft observation set for passive ' f'listener on port "{self.name}"') break if not self._serial: self._create() if self._serial is None: self.logger.error(f'Can\'t access port ' f'"{self._serial_port_config.port}" for ' f'passive listening') return if not self._serial.is_open: self.logger.info(f'Re-opening port ' f'"{self._serial_port_config.port}" for ' f'passive listening ...') self._serial.open() self._serial.reset_input_buffer() obs = copy.deepcopy(obs_draft) obs.set('id', Observation.get_new_id()) obs.set('portName', self.name) draft = obs.get('requestSets').get('draft') timeout = draft.get('timeout', 1.0) response_delimiter = draft.get('responseDelimiter', '') length = draft.get('responseLength', 0) request = draft.get('request') if request: self._write(request) response = self._read(eol=response_delimiter, length=length, timeout=timeout) if response: self.logger.verbose( f'Received "{self.sanitize(response)}" ' f'from sensor "{obs.get("sensorName")}" on ' f'port "{self._name}"') draft['response'] = response obs.set('timestamp', str(arrow.utcnow())) self.publish_observation(obs)
def publish_observation(self, obs: Observation) -> None: """Prepares the observation for publishing and forwards it to the messenger. Args: obs: Observation object. """ receivers = obs.get('receivers') index = obs.get('nextReceiver') # No receivers defined. if not receivers: logging.debug(f'No receivers defined in observation ' f'"{obs.get("name")}" of target ' f'"{obs.get("target")}"') return # No index defined. if (index is None) or (index < 0): self.logger.warning(f'Undefined receiver in observation ' f'"{obs.get("name")}" of target ' f'"{obs.get("target")}"') return # Receivers list has been processed and observation is finished. if index >= len(receivers): self.logger.info(f'Observation "{obs.get("name")}" of target ' f'"{obs.get("target")}" has been finished') return # Name of the sending module. sender = receivers[index - 1] # Increase the receivers index. next_receiver = receivers[index] obs.set('nextReceiver', index + 1) # Create header and payload. header = {'from': sender, 'type': 'observation'} payload = obs.data # Send the observation to the next module. self.publish(next_receiver, header, payload)
def rerun_pending(obs, working_dir, args): pd_log = cst.pending_log # Read the current pending log lines = np.array(read_pending_log()) folder_date = working_dir.replace(cst.base_path, '').split(os.sep)[1] folder_datetime = datetime.strptime(folder_date, '%y%m%d') if len(lines) == 0: ut.Print("Pending log empty!", args, True) return # Overwrite the pending log with open(pd_log, "wb") as f: # Re-append the header np.savetxt(f, lines[0].reshape(1, lines[0].shape[0]), delimiter=",", fmt="%s") # Loop over every old line and check if it needs to be re-run for line in lines[1:]: line = np.array([entry.strip() for entry in line]) # Unpack data and rerun the reduction process date, frame_type, binning, fltr, bias_off, dark_off, flat_off, expiry, file_path = line # Reinitialise the observation object and working dir line_obs = Observation(file_path) line_working_dir = (os.path.split(file_path)[0]).replace(cst.tele_path, cst.base_path) tele_data_dir = line_working_dir.replace(cst.base_path, cst.tele_path) # Check its expiry date if is has one try: expiry_date = datetime.strptime(line[-2], "%d-%m-%Y") if expiry_date.date() < folder_datetime.date(): # We cannot hope to find a better version, ever. So, we can # safely continue with new actions/plugins. run_plugins_single(line_obs, line_working_dir, args, file, "pending") except ValueError: pass # Re-reduce a light file if frame_type == "Light file": max_days_off = 365 # Cut on calculation time if possible try: max_days_off = abs(max(int(bias_off), int(dark_off), int(flat_off))) except ValueError: pass # Re-reduce light file. Will add a new entry in the pending log, but hopefully with a # lower max_days_off. When the time's there, it will expire. red.reduce_img(line_obs, line_working_dir, args, file_path, max_days_off) # Re-reduce dark frame elif frame_type == "Dark file": line_working_dir = (os.path.split(os.path.split(file_path)[0])[0]).replace(cst.tele_path, cst.base_path) cor.recreate_mdark(line_obs, line_working_dir, args, file_path, binning, fltr) # Re-reduce flat field elif frame_type == "Flat file": line_working_dir = (os.path.split(os.path.split(file_path)[0])[0]).replace(cst.tele_path, cst.base_path) cor.recreate_mflat(line_obs, line_working_dir, args, file_path, binning, fltr)
def create_observation(self): """ Function that creates the observation object for the passed target. Also performs some small checkups to be sure that we have some proper data. """ target = self.target args = self.args ps.running(f"Looking for files in {self.target}") self.obs = Observation(target) self.check_obs() ps.done("Observation Object initialized")
def observations() -> List[Observation]: """Returns a List with examples observation objects. Returns: List of examples observations. """ file_path = 'tests/data/observations.json' with open(file_path) as fh: data = json.loads(fh.read()) return [Observation(n) for n in data]
def process_observation(self, obs: Observation) -> Observation: for name, properties in self._config.items(): response_set = obs.get('responseSets').get(name) if not response_set: continue source_value = response_set.get('value') source_unit = response_set.get('unit') if not source_value or not source_unit: continue if source_unit != properties.get('sourceUnit'): self.logger.warning(f'Unit "{source_unit}" of response ' f'"{name}" in observation ' f'"{obs.get("name")}" of target ' f'"{obs.get("target")}" does not match ' f'"{properties.get("sourceUnit")}"') continue if properties.get('conversionType') == 'scale': target_value = self.scale(float(source_value), properties.get('scalingValue')) target_unit = properties.get('targetUnit') self.logger.info( 'Converted response "{}" in observation "{}" ' 'of target "{}" from {:.4f} {} to {:.4f} {}'.format( name, obs.get("name"), obs.get("target"), source_value, source_unit, target_value, target_unit)) response_set = Observation.create_response_set( 'float', target_unit, round(target_value, 5)) obs.data['responseSets'][name] = response_set return obs
def _is_fixed_point(self, obs: Obs) -> bool: """Checks if the given observation equals one of the defined fixed points. Args: obs: `Observation` object. Returns: True if observation is fixed point, False if not. """ if self._fixed_points.get(obs.get('target')): return True else: return False
def _is_valid_sensor_type(self, obs: Obs) -> bool: """Returns whether or not the sensor is supported. Args: obs: `Observation` object. Returns: True if sensor is supported, False if not. """ sensor_type = obs.get('sensorType') if not SensorType.is_total_station(sensor_type): self.logger.error(f'Sensor type "{sensor_type}" not supported') return False return True
def process_observation(self, obs: Observation) -> Observation: if not self._is_enabled: return obs for receiver in self._receivers: obs_copy = copy.deepcopy(obs) target = f'{receiver}/{obs_copy.get("target")}' obs_copy.set('nextReceiver', 0) obs_copy.set('receivers', [target]) self.logger.debug(f'Publishing observation ' f'"{obs_copy.get("name")}" of target ' f'"{obs_copy.get("target")}" to "{target}"') header = Observation.get_header() payload = obs_copy.data self.publish(target, header, payload) return obs
def process_observation(self, obs: Obs) -> Obs: # Calculate the serial measurement of an observation in two faces. hz_0 = obs.get_response_value('hz0') hz_1 = obs.get_response_value('hz1') v_0 = obs.get_response_value('v0') v_1 = obs.get_response_value('v1') dist_0 = obs.get_response_value('slopeDist0') dist_1 = obs.get_response_value('slopeDist1') if None in [hz_0, hz_1, v_0, v_1, dist_0, dist_1]: return obs # Calculate new Hz, V, and slope distance. hz = hz_0 + hz_1 if hz_0 > hz_1: hz += math.pi else: hz -= math.pi hz /= 2 v = ((2 * math.pi) + (v_0 - v_1)) / 2 dist = (dist_0 + dist_1) / 2 # Save the calculated values. response_sets = obs.get('responseSets') response_sets['hz'] = Obs.create_response_set('float', 'rad', hz) response_sets['v'] = Obs.create_response_set('float', 'rad', v) response_sets['slopeDist'] = Obs.create_response_set('float' 'm', dist) self.logger.debug(f'Calculated serial measurement with two faces for ' f'observation "{obs.get("name")}" of target ' f'"{obs.get("target")}"') return obs
def _update_meteorological_data(self, obs: Obs) -> None: """Updates the temperature, air pressure, and humidity attributes by using the measured data of a weather station.""" # Update temperature. t = obs.get_response_value('temperature') if t is not None: self.temperature = t # Update pressure. p = obs.get_response_value('pressure') if p is not None: self.pressure = p # Update humidity. if (obs.has_response_value('humidity') and obs.has_response_type('humidity')): h = obs.get_response_value('humidity') u = obs.get_response_unit('humidity') self.humidity = h / 100 if u == '%' else h
def process_observation(self, obs: Observation) -> Observation: """Appends data to a flat file in CSV format. Args: obs: Observation object. Returns: The observation object. """ ts = arrow.get(obs.get('timestamp', 0)) file_date = { # No file rotation, i.e., all data is stored in a single file. FileRotation.NONE: None, # Every day a new file is created. FileRotation.DAILY: ts.format('YYYY-MM-DD'), # Every month a new file is created. FileRotation.MONTHLY: ts.format('YYYY-MM'), # Every year a new file is created. FileRotation.YEARLY: ts.format('YYYY') }[self._file_rotation] fn = self._file_name fn = fn.replace('{{port}}', obs.get("portName")) fn = fn.replace('{{date}}', f'{file_date}' if file_date else '') fn = fn.replace( '{{target}}', f'{obs.get("target")}' if obs.get('target') is not None else '') fn = fn.replace( '{{name}}', f'{obs.get("name")}' if obs.get('name') is not None else '') fn += self._file_extension for path in self._paths: if not Path(path).exists(): self.logger.critical(f'Path "{path}" does not exist') continue file_path = Path(path, fn) # Create a header if a new file has to be touched. header = None if not Path(file_path).is_file(): header = (f'# Target "{obs.get("target")}" of ' f'"{obs.get("sensorName")}" on ' f'"{obs.get("portName")}"\n') # Open a file for each path. with open(str(file_path), 'a') as fh: # Add the header if necessary. if header: fh.write(header) # Format the time stamp. For more information, see: # http://arrow.readthedocs.io/en/latest/#tokens date_time = ts.format(self._date_time_format) # Create the CSV line starting with date and time. line = date_time if self._save_observation_id: line += self._separator + obs.get('id') if obs.get('target') is not None: line += self._separator + obs.get('target') response_sets = obs.get('responseSets') for response_set_id in sorted(response_sets.keys()): response_set = response_sets.get(response_set_id) v = response_set.get('value') u = response_set.get('unit') line += self._separator + format(response_set_id) line += self._separator + format(v) line += self._separator + format(u) # Write line to file. fh.write(line + '\n') self.logger.info(f'Saved observation "{obs.get("name")}" of ' f'target "{obs.get("target")}" from port ' f'"{obs.get("portName")}" to file ' f'"{str(file_path)}"') return obs
def process_observation(self, obs: Observation) -> Observation: for response_set in self._response_sets: return_code = obs.get_value('responseSets', response_set, 'value') if return_code is None: continue if return_code == 0: if obs.get('corrupted') is True: obs.set('corrupted', False) continue # Get error message of the return code. error_values = ReturnCodes.codes.get(return_code) attempts = obs.get('attempts', 0) # Retry measurement. if error_values and attempts < self._retries: obs.set('attempts', attempts + 1) obs.set('corrupted', False) obs.set('nextReceiver', 0) self.logger.info( f'Retrying observation "{obs.get("name")}" of ' f'target "{obs.get("target")}" due to return ' f'code {return_code} of response ' f'"{response_set}" (attempt {attempts + 1,} ' f'of {self._retries})') else: obs.set('corrupted', True) if error_values: # Return code related log message. lvl, retry, msg = error_values self.logger.log( lvl * 10, f'Observation "{obs.get("name")}" of ' f'target "{obs.get("target")}": {msg} ' f'(code {return_code} in response ' f'"{response_set}")') else: # Generic log message. self.logger.error(f'Error occurred on observation ' f'"{obs.get("name")}" (unknown code ' f'{return_code} in response ' f'"{response_set}")') return obs return obs
def process_observation(self, obs: Observation) -> Observation: """Extracts the values from the raw responses of the observation using regular expressions. Args: obs: The observation object. Returns: The observation object with extracted and converted responses. """ for set_name, request_set in obs.get('requestSets').items(): if not request_set.get('enabled'): # Request is disabled. continue if set_name not in obs.get('requestsOrder'): # Request should be ignored. continue response = request_set.get('response') response_pattern = request_set.get('responsePattern') if response is None or len(response) == 0: self.logger.warning(f'No response "{set_name}" in observation ' f'"{obs.get("name")}" of target ' f'"{obs.get("target")}" from sensor ' f'"{obs.get("sensorName")}" on port ' f'"{obs.get("portName")}"') continue try: pattern = re.compile(response_pattern) match = pattern.search(response) except Exception: self.logger.error(f'Invalid regular expression for response ' f'"{set_name}" in observation ' f'"{obs.get("name")}" of target ' f'"{obs.get("target")}" from sensor ' f'"{obs.get("sensorName")}" on port ' f'"{obs.get("portName")}"') return obs if not match: self.logger.error(f'Response "{self.sanitize(response)}" of ' f'request "{set_name}" in observation ' f'"{obs.get("name")}" of target ' f'"{obs.get("target")}" from sensor ' f'"{obs.get("sensorName")}" on port ' f'"{obs.get("portName")}" does not match ' f'extraction pattern') return obs # The regular expression pattern needs at least one named group # defined. Otherwise, the extraction of the values fails. # # Right: "(?P<id>.*)" # Wrong: ".*" if pattern.groups == 0: self.logger.error( f'No group(s) defined in regular expression ' f'pattern in observation "{obs.get("name")}" ' f'of target "{obs.get("target")}"') return obs # Convert the type of the parsed raw values from string to the # actual data type (float, int). response_sets = obs.get('responseSets') for group_name, raw_value in match.groupdict("").items(): if raw_value is None or len(raw_value) == 0: self.logger.error(f'Undefined raw value in response set ' f'"{group_name}" in observation ' f'"{obs.get("name")}" of target ' f'"{ obs.get("target")}"') continue response_set = response_sets.get(group_name) if not response_set: self.logger.error(f'Undefined response set "{group_name}" ' f'in observation "{obs.get("name")}" ' f'of target "{obs.get("target")}"') continue response_type = response_set.get('type').lower() if response_type == 'float': # Convert raw value to float. Replace comma by dot. response_value = self.to_float(raw_value) elif response_type == 'integer': # Convert raw value to int. response_value = self.to_int(raw_value) else: response_value = raw_value if response_value is not None: self.logger.debug( f'Extracted "{response_value}" from raw ' f'response "{group_name}" in observation ' f'"{ obs.get("name")}" of target ' f'"{obs.get("target")}"') response_set['value'] = response_value return obs
def process_observation(self, obs: Observation) -> Union[Observation, None]: """Sends a request to the attached sensor and forwards the response. Args: obs: Observation object. Returns: The extended observation object or None if connection failed. """ if not self._sock: # Open socket connection. if not self._open(): return # Add the name of the Bluetooth port to the observation. obs.set('portName', self.name) requests_order = obs.get('requestsOrder', []) request_sets = obs.get('requestSets') if not requests_order: self.logger.notice(f'No requests order defined in observation ' f'"{obs.get("name")}" of target ' f'"{obs.get("target")}"') # Send requests sequentially to the sensor. for request_name in requests_order: request_set = request_sets.get(request_name) if not request_set: self.logger.error(f'Request set "{request_name}" not found in ' f'observation "{obs.get("name")}" of target ' f'"{obs.get("target")}"') return # The response of the sensor. response = '' response_delimiter = request_set.get('responseDelimiter') # Data of the request set. request = request_set.get('request') sleep_time = request_set.get('sleepTime') or 1.0 timeout = request_set.get('timeout') or 1.0 # Send the request of the observation to the attached sensor. self.logger.verbose(f'Sending request "{request_name}" of ' f'observation "{obs.get("name")}" to sensor ' f'"{obs.get("sensorName")}" ...') # Write to Bluetooth port. self._send(request) # Get the response of the sensor. response = self._receive(response_delimiter, timeout) self.logger.verbose(f'Received response ' f'"{self.sanitize(response)}" for request ' f'"{request_name}" of observation ' f'"{obs.get("name")}" from sensor ' f'"{obs.get("sensorName")}"') # Add the raw response of the sensor to the observation set. request_set['response'] = response # Add the timestamp to the observation. obs.set('timestamp', str(arrow.utcnow())) # Sleep until the next request. time.sleep(sleep_time) return obs
def process_observation(self, obs: Obs) -> Obs: sensor_type = obs.get('sensorType') # Update atmospheric data if sensor is a weather station. if SensorType.is_weather_station(sensor_type): self._update_meteorological_data(obs) return obs # Check if sensor is of type "total station". if not SensorType.is_total_station(sensor_type): self.logger.warning(f'Sensor type "{sensor_type}" not supported') return obs # Check if atmospheric data has been set. if None in [self.temperature, self.pressure, not self.humidity]: self.logger.warning('Missing temperature, air pressure, or ' 'humidity') return obs # Check the age of the atmospheric data. if self.last_update - time.time() > self._max_age: self.logger.warning(f'Atmospheric data is older than ' f'{int(self._max_age / 3600)} hour(s)') # Reduce the slope distance of the EDM measurement. dist = obs.get_response_value(self._distance_name) if dist is None: self.logger.error(f'No distance set in observation ' f'"{obs.get("name")}" of target ' f'"{obs.get("target")}"') return obs d_dist_1 = 0 d_dist_2 = 0 response_sets = obs.get('responseSets') # Calculate the atmospheric reduction of the distance. if self._is_atmospheric_correction: c = self.get_atmospheric_correction(self._temperature, self._pressure, self._humidity) d_dist_1 = dist * c * math.pow(10, -6) rs = Obs.create_response_set('float', 'none', round(c, 5)) response_sets['atmosphericPpm'] = rs # Calculate the sea level reduction of the distance. if self._is_sea_level_correction: d_dist_2 = self.get_sea_level_correction(self._sensor_height) rs = Obs.create_response_set('float', 'm', round(d_dist_2, 5)) response_sets['seaLevelDelta'] = rs # Add corrected distance to the observation set. if d_dist_1 != 0 or d_dist_2 != 0: r_dist = dist + d_dist_1 + d_dist_2 self.logger.info('Reduced distance from {:0.5f} m to {:0.5f} m ' '(correction value: {:0.5f} m)'.format( dist, r_dist, d_dist_1 + d_dist_2)) rs = Obs.create_response_set('float', 'm', round(r_dist, 5)) response_sets[self._distance_name + 'Raw'] =\ response_sets.get(self._distance_name) response_sets[self._distance_name] = rs return obs
def process_observation(self, obs: Observation) -> Union[Observation, None]: """Processes an observation object. Sends request to sensor and stores response. Args: obs: The observation object. Returns: The processed observation. """ # Turn on passive mode. if obs.get('passiveMode'): if self._is_passive: # Restart passive listener. self._is_passive = False self._passive_listener.join() self._is_passive = True self._passive_listener = Thread(target=self.listen, args=(obs, )) self._passive_listener.daemon = True self._passive_listener.start() self.logger.debug( f'Started passive listener of port "{self.name}"') return # Turn off passive mode. if self._is_passive and not obs.get('passiveMode'): self._is_passive = False self._passive_listener.join() self.logger.debug( f'Stopped passive listener of port "{self.name}"') # Create new serial connection. if not self._serial: self._create() if self._serial is None: self.logger.error(f'Could not access port ' f'"{self._serial_port_config.port}"') return if not self._serial.is_open: self.logger.verbose(f'Re-opening port ' f'"{self._serial_port_config.port}" ...') self._serial.open() self._serial.reset_output_buffer() self._serial.reset_input_buffer() # Add the name of this serial port module to the observation. obs.set('portName', self._name) requests_order = obs.get('requestsOrder', []) request_sets = obs.get('requestSets') if not requests_order: self.logger.notice(f'No requests order defined in observation ' f'"{obs.get("name")}" of target ' f'"{obs.get("target")}"') # Send requests sequentially to the sensor. for request_name in requests_order: request_set = request_sets.get(request_name) if not request_set: self.logger.error(f'Request set "{request_name}" not found in ' f'observation "{obs.get("name")}" of target ' f'"{obs.get("target")}"') return # The response of the sensor. response = '' response_delimiter = request_set.get('responseDelimiter') # Data of the request set. request = request_set.get('request') sleep_time = request_set.get('sleepTime') or 0.0 timeout = request_set.get('timeout') or 0.0 # Send the request of the observation to the attached sensor. self.logger.verbose(f'Sending request "{request_name}" of ' f'observation "{obs.get("name")}" to sensor ' f'"{obs.get("sensorName")}" ...') for attempt in range(self._max_attempts): if attempt > 0: self.logger.info(f'Request attempt {attempt + 1} of ' f'{self._max_attempts}') time.sleep(1) # Write to the serial port. self._write(request) # Get the response of the sensor. response = self._read(eol=response_delimiter, length=0, timeout=timeout) self._serial.reset_output_buffer() self._serial.reset_input_buffer() if response: self.logger.verbose( f'Received response ' f'"{self.sanitize(response)}" for ' f'request "{request_name}" of ' f'observation "{obs.get("name")}" from ' f'sensor "{obs.get("sensorName")}"') break # Try next attempt if response is empty. self.logger.warning(f'No response from sensor ' f'"{obs.get("sensorName")}" for ' f'observation "{obs.get("name")}" of ' f'target "{obs.get("target")}"') # Add the raw response of the sensor to the observation set. request_set['response'] = response # Add the timestamp to the observation. obs.set('timestamp', str(arrow.utcnow())) # Sleep until the next request. time.sleep(sleep_time) return obs
def observation() -> Observation: return Observation()
def test_create_response_test(self, observation: Observation) -> None: response_set = observation.create_response_set('test', 'none', 0.0) assert response_set == {'type': 'test', 'unit': 'none', 'value': 0.0}