def insert_current(self, collection, obj, store_permanently=True): self.validate_collection(collection) obj_id = self._make_id() obj = create_storage_obj(collection, obj, obj_id=obj_id) current_fn = self._get_file(collection, permanent=False) result = obj_id try: # Overwrite current collection file with obj. to_json(obj, filename=current_fn, append=False) except Exception as e: self._warn( f"Problem serializing object for insertion: {e} {current_fn} {obj!r}" ) result = None if not store_permanently: return result collection_fn = self._get_file(collection) try: # Append obj to collection file. to_json(obj, filename=collection_fn, append=True) return obj_id except Exception as e: self._warn( "Problem inserting object into collection: {}, {!r}".format( e, obj)) return None
def send_message(self, topic, message): """ Responsible for actually sending message across a topic Args: topic(str): Name of topic to send on. The name must match topic_name_re. message: Message to be sent (a string or a dict). """ if not isinstance(topic, str): raise ValueError('Topic name must be a string') elif not self.topic_name_re.fullmatch(topic): raise ValueError('Topic name ("{}") is not valid'.format(topic)) if topic == 'PANCHAT': self.logger.info(f"{topic} {message}") if isinstance(message, str): message = to_json({ "message": message, "timestamp": current_time(pretty=True), }) elif isinstance(message, dict): message = to_json(message) else: raise ValueError('Message value must be a string or dict') # Build the full message with topic full_message = f'{topic} {message}' # Send the message # self.socket.send_string(full_message, flags=zmq.NOBLOCK) self.socket.send_string(full_message)
def insert(self, collection, obj): obj_id = self._make_id() obj = create_storage_obj(collection, obj, obj_id) collection_fn = self._get_file(collection) try: # Insert record into file to_json(obj, filename=collection_fn) return obj_id except Exception as e: raise error.InvalidSerialization( f"Problem inserting object into collection: {e}, {obj!r}")
def insert(self, collection, obj): self.validate_collection(collection) obj_id = self._make_id() obj = create_storage_obj(collection, obj, obj_id=obj_id) collection_fn = self._get_file(collection) try: # Insert record into file to_json(obj, filename=collection_fn) return obj_id except Exception as e: self._warn( "Problem inserting object into collection: {}, {!r}".format( e, obj)) return None
def reset_conf(config_host, config_port): url = f'http://{config_host}:{config_port}/reset-config' response = requests.post(url, data=to_json({'reset': True}), headers={'Content-Type': 'application/json'} ) assert response.ok
def test_quantity(): json_str = serializers.to_json(dict(foo='42 deg', bar='foo deg')) assert json_str == '{"foo": "42 deg", "bar": "foo deg"}' json_obj = serializers.from_json(json_str) # Make sure we made a quantity when we could. assert json_obj['foo'] == 42 * u.deg # And not when we can't. assert json_obj['bar'] == 'foo deg'
def insert_current(self, collection, obj, store_permanently=True): obj_id = self._make_id() obj = create_storage_obj(collection, obj, obj_id) current_fn = self._get_file(collection, permanent=False) result = obj_id try: # Overwrite current collection file with obj. to_json(obj, filename=current_fn, append=False) except Exception as e: raise error.InvalidSerialization( f"Problem serializing object for insertion: {e} {current_fn} {obj!r}" ) if not store_permanently: return result else: return self.insert(collection, obj)
def insert(self, collection, obj): obj_id = self._make_id() obj = create_storage_obj(collection, obj, obj_id) try: obj = to_json(obj) except Exception as e: raise error.InvalidSerialization( f"Problem inserting object into collection: {e}, {obj!r}") with self.lock: self.collections.setdefault(collection, {})[obj_id] = obj return obj_id
def insert(self, collection, obj): self.validate_collection(collection) obj_id = self._make_id() obj = create_storage_obj(collection, obj, obj_id=obj_id) try: obj = to_json(obj) except Exception as e: self._warn("Problem inserting object into collection: {}, {!r}".format(e, obj)) return None with self.lock: self.collections.setdefault(collection, {})[obj_id] = obj return obj_id
def set_config(key, new_value, host='localhost', port='6563', parse=True): """Set config item in config server. Given a `key` entry, update the config to match. The `key` is a dot accessible string, as given by [scalpl](https://pypi.org/project/scalpl/). See Examples in `get_config` for details. Examples: >>> from astropy import units as u >>> set_config('location.horizon', 35 * u.degree) {'location.horizon': <Quantity 35. deg>} >>> get_config(key='location.horizon') <Quantity 35. deg> >>> set_config('location.horizon', 30 * u.degree) {'location.horizon': <Quantity 30. deg>} Args: key (str): The key to update, see Examples in `get_config` for details. new_value (scalar|object): The new value for the key, can be any serializable object. host (str, optional): The config server host, defaults to '127.0.0.1'. port (str, optional): The config server port, defaults to 6563. parse (bool, optional): If response should be parsed by `~panoptes.utils.serializers.from_json`, default True. Returns: dict: The updated config entry. Raises: Exception: Raised if the config server is not available. """ url = f'http://{host}:{port}/set-config' json_str = serializers.to_json({key: new_value}) config_entry = None try: # We use our own serializer so pass as `data` instead of `json`. response = requests.post(url, data=json_str, headers={'Content-Type': 'application/json'}) if not response.ok: raise Exception(f'Cannot access config server: {response.text}') except Exception as e: get_root_logger().info(f'Problem with set_config: {e!r}') else: if parse: config_entry = serializers.from_json( response.content.decode('utf8')) else: config_entry = response.json() return config_entry
def insert_current(self, collection, obj, store_permanently=True): obj_id = self._make_id() obj = create_storage_obj(collection, obj, obj_id) try: obj = to_json(obj) except Exception as e: raise error.InvalidSerialization( f"Problem serializing object for insertion: {e} {obj!r}") with self.lock: self.current[collection] = obj if store_permanently: self.collections.setdefault(collection, {})[obj_id] = obj return obj_id
def generate_next_message_bytes(self, now): """Generate the next message (report) from the simulated Arduino.""" # Not worrying here about emulating the 32-bit nature of millis (wraps in 49 days) elapsed = int((now - self.start_time).total_seconds() * 1000) self.report_num += 1 self.message['millis'] = elapsed self.message['report_num'] = self.report_num if self.command_lines: self.message['commands'] = self.command_lines self.command_lines = [] s = to_json(self.message) + '\r\n' if 'commands' in self.message: del self.message['commands'] self.logger.debug('generate_next_message -> {!r}', s) b = s.encode(encoding='ascii') return b
def test_config_reset(dynamic_config_server, config_port, config_host): # Check we are at default. assert get_config('location.horizon', port=config_port) == 30 * u.degree # Set to new value. set_config_return = set_config('location.horizon', 47 * u.degree, port=config_port) assert set_config_return == {'location.horizon': 47 * u.degree} # Check we have changed. assert get_config('location.horizon', port=config_port) == 47 * u.degree # Reset config url = f'http://{config_host}:{config_port}/reset-config' response = requests.post(url, data=serializers.to_json({'reset': True}), headers={'Content-Type': 'application/json'}) assert response.ok # Check we are at default again. assert get_config('location.horizon', port=config_port) == 30 * u.degree
def set_config(key, new_value, host=None, port=None, parse=True): """Set config item in config server. Given a `key` entry, update the config to match. The `key` is a dot accessible string, as given by `scalpl <https://pypi.org/project/scalpl/>`_. See Examples in :func:`get_config` for details. Examples: .. doctest:: >>> from astropy import units as u >>> # Can use astropy units. >>> set_config('location.horizon', 35 * u.degree) {'location.horizon': <Quantity 35. deg>} >>> get_config(key='location.horizon') <Quantity 35. deg> >>> # String equivalent works for 'deg', 'm', 's'. >>> set_config('location.horizon', '30 deg') {'location.horizon': <Quantity 30. deg>} Args: key (str): The key to update, see Examples in :func:`get_config` for details. new_value (scalar|object): The new value for the key, can be any serializable object. host (str, optional): The config server host. First checks for PANOPTES_CONFIG_HOST env var, defaults to 'localhost'. port (str or int, optional): The config server port. First checks for PANOPTES_CONFIG_HOST env var, defaults to 6563. parse (bool, optional): If response should be parsed by :func:`panoptes.utils.serializers.from_json`, default True. Returns: dict: The updated config entry. Raises: Exception: Raised if the config server is not available. """ host = host or os.getenv('PANOPTES_CONFIG_HOST', 'localhost') port = port or os.getenv('PANOPTES_CONFIG_PORT', 6563) url = f'http://{host}:{port}/set-config' json_str = to_json({key: new_value}) config_entry = None try: # We use our own serializer so pass as `data` instead of `json`. logger.info(f'Calling set_config on url={url!r}') response = requests.post(url, data=json_str, headers={'Content-Type': 'application/json'}) if not response.ok: # pragma: no cover raise Exception(f'Cannot access config server: {response.text}') except Exception as e: logger.warning(f'Problem with set_config: {e!r}') else: if parse: config_entry = from_json(response.content.decode('utf8')) else: config_entry = response.json() return config_entry
def test_bad_serialization(): with pytest.raises(error.InvalidSerialization): serializers.to_json(pytest)
def test_roundtrip_json(obj): config_str = serializers.to_json(obj) config = serializers.from_json(config_str) assert config['name'] == obj['name'] assert config['location']['latitude'] == obj['location']['latitude']
def get_root_logger(profile='panoptes', log_config=None): """Creates a root logger for PANOPTES used by the PanBase object. Args: profile (str, optional): The name of the logger to use, defaults to 'panoptes'. log_config (dict|None, optional): Configuration options for the logger. See https://docs.python.org/3/library/logging.config.html for available options. Default is `None`, which then looks up the values in the `log.yaml` config file. Returns: logger(logging.logger): A configured instance of the logger """ # Get log info from config log_config = log_config if log_config else load_default() # If we already created a logger for this profile and log_config, return that. logger_key = (profile, to_json(log_config, sort_keys=True)) try: return all_loggers[logger_key] except KeyError: pass # Alter the log_config to use UTC times if log_config.get('use_utc', True): # TODO(jamessynge): Figure out why 'formatters' is sometimes # missing from the log_config. It is hard to understand how # this could occur given that none of the callers of # get_root_logger pass in their own log_config. if 'formatters' not in log_config: # pragma: no cover # TODO(jamessynge): Raise a custom exception in this case instead # of issuing a warning; after all, a standard dict will throw a # KeyError in the for loop below if 'formatters' is missing. warn('formatters is missing from log_config!') warn(f'log_config: {log_config!r}') log_fname_datetime = datetime.datetime.utcnow().strftime( '%Y%m%dT%H%M%SZ') # Make the log use UTC logging.Formatter.converter = time.gmtime else: log_fname_datetime = datetime.datetime.now().strftime('%Y%m%dT%H%M%S') # Setup log file names invoked_script = os.path.basename(sys.argv[0]) log_dir = os.getenv('PANLOG', '') if not log_dir: log_dir = os.path.join(os.getenv('PANDIR', gettempdir()), 'logs') per_run_dir = os.path.join(log_dir, 'per-run', invoked_script) log_fname = '{}-{}-{}'.format(invoked_script, log_fname_datetime, os.getpid()) # Create the directory for the per-run files. os.makedirs(per_run_dir, exist_ok=True) # Set log filename and rotation for handler in log_config.get('handlers', []): # Set the filename partial_fname = '{}-{}.log'.format(log_fname, handler) full_log_fname = os.path.join(per_run_dir, partial_fname) log_config['handlers'][handler].setdefault('filename', full_log_fname) # Setup the TimedRotatingFileHandler for middle of day log_config['handlers'][handler].setdefault( 'atTime', datetime.time(hour=11, minute=30)) # Create a symlink to the log file with just the name of the script and the handler # (level), as this makes it easier to find the latest file. log_symlink = os.path.join(log_dir, '{}-{}.log'.format(invoked_script, handler)) log_symlink_target = os.path.abspath(full_log_fname) with suppress(FileNotFoundError): os.unlink(log_symlink) os.symlink(log_symlink_target, log_symlink) # Configure the logger logging.config.dictConfig(log_config) # Get the logger and set as attribute to class logger = logging.getLogger(profile) # Set custom LogRecord logging.setLogRecordFactory(StrFormatLogRecord) logger.info('{:*^80}'.format(' Starting PanLogger ')) # TODO(jamessynge) Output name of script, cmdline args, etc. And do son # when the log rotates too! all_loggers[logger_key] = logger return logger
def reset_conf(): response = requests.post(url, data=serializers.to_json({'reset': True}), headers={'Content-Type': 'application/json'}) assert response.ok