def insert_current(self, collection, obj, store_permanently=True):
self.validate_collection(collection)
obj_id = self._make_id()
obj = create_storage_obj(collection, obj, obj_id=obj_id)
current_fn = self._get_file(collection, permanent=False)
result = obj_id
try:
# Overwrite current collection file with obj.
to_json(obj, filename=current_fn, append=False)
except Exception as e:
self._warn(
f"Problem serializing object for insertion: {e} {current_fn} {obj!r}"
)
result = None
if not store_permanently:
return result
collection_fn = self._get_file(collection)
try:
# Append obj to collection file.
to_json(obj, filename=collection_fn, append=True)
return obj_id
except Exception as e:
self._warn(
"Problem inserting object into collection: {}, {!r}".format(
e, obj))
return None
def send_message(self, topic, message):
""" Responsible for actually sending message across a topic
Args:
topic(str): Name of topic to send on. The name must
match topic_name_re.
message: Message to be sent (a string or a dict).
"""
if not isinstance(topic, str):
raise ValueError('Topic name must be a string')
elif not self.topic_name_re.fullmatch(topic):
raise ValueError('Topic name ("{}") is not valid'.format(topic))
if topic == 'PANCHAT':
self.logger.info(f"{topic} {message}")
if isinstance(message, str):
message = to_json({
"message": message,
"timestamp": current_time(pretty=True),
})
elif isinstance(message, dict):
message = to_json(message)
else:
raise ValueError('Message value must be a string or dict')
# Build the full message with topic
full_message = f'{topic} {message}'
# Send the message
# self.socket.send_string(full_message, flags=zmq.NOBLOCK)
self.socket.send_string(full_message)
def insert(self, collection, obj):
obj_id = self._make_id()
obj = create_storage_obj(collection, obj, obj_id)
collection_fn = self._get_file(collection)
try:
# Insert record into file
to_json(obj, filename=collection_fn)
return obj_id
except Exception as e:
raise error.InvalidSerialization(
f"Problem inserting object into collection: {e}, {obj!r}")
def insert(self, collection, obj):
self.validate_collection(collection)
obj_id = self._make_id()
obj = create_storage_obj(collection, obj, obj_id=obj_id)
collection_fn = self._get_file(collection)
try:
# Insert record into file
to_json(obj, filename=collection_fn)
return obj_id
except Exception as e:
self._warn(
"Problem inserting object into collection: {}, {!r}".format(
e, obj))
return None
def reset_conf(config_host, config_port):
url = f'http://{config_host}:{config_port}/reset-config'
response = requests.post(url,
data=to_json({'reset': True}),
headers={'Content-Type': 'application/json'}
)
assert response.ok
def test_quantity():
json_str = serializers.to_json(dict(foo='42 deg', bar='foo deg'))
assert json_str == '{"foo": "42 deg", "bar": "foo deg"}'
json_obj = serializers.from_json(json_str)
# Make sure we made a quantity when we could.
assert json_obj['foo'] == 42 * u.deg
# And not when we can't.
assert json_obj['bar'] == 'foo deg'
def insert_current(self, collection, obj, store_permanently=True):
obj_id = self._make_id()
obj = create_storage_obj(collection, obj, obj_id)
current_fn = self._get_file(collection, permanent=False)
result = obj_id
try:
# Overwrite current collection file with obj.
to_json(obj, filename=current_fn, append=False)
except Exception as e:
raise error.InvalidSerialization(
f"Problem serializing object for insertion: {e} {current_fn} {obj!r}"
)
if not store_permanently:
return result
else:
return self.insert(collection, obj)
def insert(self, collection, obj):
obj_id = self._make_id()
obj = create_storage_obj(collection, obj, obj_id)
try:
obj = to_json(obj)
except Exception as e:
raise error.InvalidSerialization(
f"Problem inserting object into collection: {e}, {obj!r}")
with self.lock:
self.collections.setdefault(collection, {})[obj_id] = obj
return obj_id
def insert(self, collection, obj):
self.validate_collection(collection)
obj_id = self._make_id()
obj = create_storage_obj(collection, obj, obj_id=obj_id)
try:
obj = to_json(obj)
except Exception as e:
self._warn("Problem inserting object into collection: {}, {!r}".format(e, obj))
return None
with self.lock:
self.collections.setdefault(collection, {})[obj_id] = obj
return obj_id
def set_config(key, new_value, host='localhost', port='6563', parse=True):
"""Set config item in config server.
Given a `key` entry, update the config to match. The `key` is a dot accessible
string, as given by [scalpl](https://pypi.org/project/scalpl/). See Examples in `get_config`
for details.
Examples:
>>> from astropy import units as u
>>> set_config('location.horizon', 35 * u.degree)
{'location.horizon': <Quantity 35. deg>}
>>> get_config(key='location.horizon')
<Quantity 35. deg>
>>> set_config('location.horizon', 30 * u.degree)
{'location.horizon': <Quantity 30. deg>}
Args:
key (str): The key to update, see Examples in `get_config` for details.
new_value (scalar|object): The new value for the key, can be any serializable object.
host (str, optional): The config server host, defaults to '127.0.0.1'.
port (str, optional): The config server port, defaults to 6563.
parse (bool, optional): If response should be parsed by
`~panoptes.utils.serializers.from_json`, default True.
Returns:
dict: The updated config entry.
Raises:
Exception: Raised if the config server is not available.
"""
url = f'http://{host}:{port}/set-config'
json_str = serializers.to_json({key: new_value})
config_entry = None
try:
# We use our own serializer so pass as `data` instead of `json`.
response = requests.post(url,
data=json_str,
headers={'Content-Type': 'application/json'})
if not response.ok:
raise Exception(f'Cannot access config server: {response.text}')
except Exception as e:
get_root_logger().info(f'Problem with set_config: {e!r}')
else:
if parse:
config_entry = serializers.from_json(
response.content.decode('utf8'))
else:
config_entry = response.json()
return config_entry
def insert_current(self, collection, obj, store_permanently=True):
obj_id = self._make_id()
obj = create_storage_obj(collection, obj, obj_id)
try:
obj = to_json(obj)
except Exception as e:
raise error.InvalidSerialization(
f"Problem serializing object for insertion: {e} {obj!r}")
with self.lock:
self.current[collection] = obj
if store_permanently:
self.collections.setdefault(collection, {})[obj_id] = obj
return obj_id
def generate_next_message_bytes(self, now):
"""Generate the next message (report) from the simulated Arduino."""
# Not worrying here about emulating the 32-bit nature of millis (wraps in 49 days)
elapsed = int((now - self.start_time).total_seconds() * 1000)
self.report_num += 1
self.message['millis'] = elapsed
self.message['report_num'] = self.report_num
if self.command_lines:
self.message['commands'] = self.command_lines
self.command_lines = []
s = to_json(self.message) + '\r\n'
if 'commands' in self.message:
del self.message['commands']
self.logger.debug('generate_next_message -> {!r}', s)
b = s.encode(encoding='ascii')
return b
def test_config_reset(dynamic_config_server, config_port, config_host):
# Check we are at default.
assert get_config('location.horizon', port=config_port) == 30 * u.degree
# Set to new value.
set_config_return = set_config('location.horizon',
47 * u.degree,
port=config_port)
assert set_config_return == {'location.horizon': 47 * u.degree}
# Check we have changed.
assert get_config('location.horizon', port=config_port) == 47 * u.degree
# Reset config
url = f'http://{config_host}:{config_port}/reset-config'
response = requests.post(url,
data=serializers.to_json({'reset': True}),
headers={'Content-Type': 'application/json'})
assert response.ok
# Check we are at default again.
assert get_config('location.horizon', port=config_port) == 30 * u.degree
def set_config(key, new_value, host=None, port=None, parse=True):
"""Set config item in config server.
Given a `key` entry, update the config to match. The `key` is a dot accessible
string, as given by `scalpl <https://pypi.org/project/scalpl/>`_. See Examples in
:func:`get_config` for details.
Examples:
.. doctest::
>>> from astropy import units as u
>>> # Can use astropy units.
>>> set_config('location.horizon', 35 * u.degree)
{'location.horizon': <Quantity 35. deg>}
>>> get_config(key='location.horizon')
<Quantity 35. deg>
>>> # String equivalent works for 'deg', 'm', 's'.
>>> set_config('location.horizon', '30 deg')
{'location.horizon': <Quantity 30. deg>}
Args:
key (str): The key to update, see Examples in :func:`get_config` for details.
new_value (scalar|object): The new value for the key, can be any serializable object.
host (str, optional): The config server host. First checks for PANOPTES_CONFIG_HOST
env var, defaults to 'localhost'.
port (str or int, optional): The config server port. First checks for PANOPTES_CONFIG_HOST
env var, defaults to 6563.
parse (bool, optional): If response should be parsed by
:func:`panoptes.utils.serializers.from_json`, default True.
Returns:
dict: The updated config entry.
Raises:
Exception: Raised if the config server is not available.
"""
host = host or os.getenv('PANOPTES_CONFIG_HOST', 'localhost')
port = port or os.getenv('PANOPTES_CONFIG_PORT', 6563)
url = f'http://{host}:{port}/set-config'
json_str = to_json({key: new_value})
config_entry = None
try:
# We use our own serializer so pass as `data` instead of `json`.
logger.info(f'Calling set_config on url={url!r}')
response = requests.post(url,
data=json_str,
headers={'Content-Type': 'application/json'})
if not response.ok: # pragma: no cover
raise Exception(f'Cannot access config server: {response.text}')
except Exception as e:
logger.warning(f'Problem with set_config: {e!r}')
else:
if parse:
config_entry = from_json(response.content.decode('utf8'))
else:
config_entry = response.json()
return config_entry
def test_bad_serialization():
with pytest.raises(error.InvalidSerialization):
serializers.to_json(pytest)
def test_roundtrip_json(obj):
config_str = serializers.to_json(obj)
config = serializers.from_json(config_str)
assert config['name'] == obj['name']
assert config['location']['latitude'] == obj['location']['latitude']
def get_root_logger(profile='panoptes', log_config=None):
"""Creates a root logger for PANOPTES used by the PanBase object.
Args:
profile (str, optional): The name of the logger to use, defaults
to 'panoptes'.
log_config (dict|None, optional): Configuration options for the logger.
See https://docs.python.org/3/library/logging.config.html for
available options. Default is `None`, which then looks up the
values in the `log.yaml` config file.
Returns:
logger(logging.logger): A configured instance of the logger
"""
# Get log info from config
log_config = log_config if log_config else load_default()
# If we already created a logger for this profile and log_config, return that.
logger_key = (profile, to_json(log_config, sort_keys=True))
try:
return all_loggers[logger_key]
except KeyError:
pass
# Alter the log_config to use UTC times
if log_config.get('use_utc', True):
# TODO(jamessynge): Figure out why 'formatters' is sometimes
# missing from the log_config. It is hard to understand how
# this could occur given that none of the callers of
# get_root_logger pass in their own log_config.
if 'formatters' not in log_config: # pragma: no cover
# TODO(jamessynge): Raise a custom exception in this case instead
# of issuing a warning; after all, a standard dict will throw a
# KeyError in the for loop below if 'formatters' is missing.
warn('formatters is missing from log_config!')
warn(f'log_config: {log_config!r}')
log_fname_datetime = datetime.datetime.utcnow().strftime(
'%Y%m%dT%H%M%SZ')
# Make the log use UTC
logging.Formatter.converter = time.gmtime
else:
log_fname_datetime = datetime.datetime.now().strftime('%Y%m%dT%H%M%S')
# Setup log file names
invoked_script = os.path.basename(sys.argv[0])
log_dir = os.getenv('PANLOG', '')
if not log_dir:
log_dir = os.path.join(os.getenv('PANDIR', gettempdir()), 'logs')
per_run_dir = os.path.join(log_dir, 'per-run', invoked_script)
log_fname = '{}-{}-{}'.format(invoked_script, log_fname_datetime,
os.getpid())
# Create the directory for the per-run files.
os.makedirs(per_run_dir, exist_ok=True)
# Set log filename and rotation
for handler in log_config.get('handlers', []):
# Set the filename
partial_fname = '{}-{}.log'.format(log_fname, handler)
full_log_fname = os.path.join(per_run_dir, partial_fname)
log_config['handlers'][handler].setdefault('filename', full_log_fname)
# Setup the TimedRotatingFileHandler for middle of day
log_config['handlers'][handler].setdefault(
'atTime', datetime.time(hour=11, minute=30))
# Create a symlink to the log file with just the name of the script and the handler
# (level), as this makes it easier to find the latest file.
log_symlink = os.path.join(log_dir,
'{}-{}.log'.format(invoked_script, handler))
log_symlink_target = os.path.abspath(full_log_fname)
with suppress(FileNotFoundError):
os.unlink(log_symlink)
os.symlink(log_symlink_target, log_symlink)
# Configure the logger
logging.config.dictConfig(log_config)
# Get the logger and set as attribute to class
logger = logging.getLogger(profile)
# Set custom LogRecord
logging.setLogRecordFactory(StrFormatLogRecord)
logger.info('{:*^80}'.format(' Starting PanLogger '))
# TODO(jamessynge) Output name of script, cmdline args, etc. And do son
# when the log rotates too!
all_loggers[logger_key] = logger
return logger
def reset_conf():
response = requests.post(url,
data=serializers.to_json({'reset': True}),
headers={'Content-Type': 'application/json'})
assert response.ok
