def set_config(key, new_value, host='localhost', port='6563', parse=True):
"""Set config item in config server.
Given a `key` entry, update the config to match. The `key` is a dot accessible
string, as given by [scalpl](https://pypi.org/project/scalpl/). See Examples in `get_config`
for details.
Examples:
>>> from astropy import units as u
>>> set_config('location.horizon', 35 * u.degree)
{'location.horizon': <Quantity 35. deg>}
>>> get_config(key='location.horizon')
<Quantity 35. deg>
>>> set_config('location.horizon', 30 * u.degree)
{'location.horizon': <Quantity 30. deg>}
Args:
key (str): The key to update, see Examples in `get_config` for details.
new_value (scalar|object): The new value for the key, can be any serializable object.
host (str, optional): The config server host, defaults to '127.0.0.1'.
port (str, optional): The config server port, defaults to 6563.
parse (bool, optional): If response should be parsed by
`~panoptes.utils.serializers.from_json`, default True.
Returns:
dict: The updated config entry.
Raises:
Exception: Raised if the config server is not available.
"""
url = f'http://{host}:{port}/set-config'
json_str = serializers.to_json({key: new_value})
config_entry = None
try:
# We use our own serializer so pass as `data` instead of `json`.
response = requests.post(url,
data=json_str,
headers={'Content-Type': 'application/json'})
if not response.ok:
raise Exception(f'Cannot access config server: {response.text}')
except Exception as e:
get_root_logger().info(f'Problem with set_config: {e!r}')
else:
if parse:
config_entry = serializers.from_json(
response.content.decode('utf8'))
else:
config_entry = response.json()
return config_entry
def test_load_c_library():
# Called without a `path` this will use find_library to locate libc.
libc = load_c_library('c')
assert libc._name[:4] == 'libc'
libc = load_c_library('c', mode=None, logger=get_root_logger())
assert libc._name[:4] == 'libc'
def reset_config():
if request.is_json:
get_root_logger().warning(f'Resetting config server')
req_data = request.get_json()
if req_data['reset']:
# Reload the config
app.config['POCS'] = load_config(
config_files=app.config['config_file'],
ignore_local=app.config['ignore_local'])
app.config['POCS_cut'] = Cut(app.config['POCS'])
return jsonify(req_data)
return jsonify({
'success': False,
'msg': "Invalid. Need json request: {'reset': True}"
})
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.logger = get_root_logger()
self.simulator_thread = None
self.relay_queue = queue.Queue(maxsize=1)
self.json_queue = queue.Queue(maxsize=1)
self.json_bytes = bytearray()
self.stop = threading.Event()
self.stop.set()
self.device_simulator = None
def message_forwarder(messaging_ports):
cmd = shutil.which('panoptes-messaging-hub')
assert cmd is not None
args = [cmd]
# Note that the other programs using these port pairs consider
# them to be pub and sub, in that order, but the forwarder sees things
# in reverse: it subscribes to the port that others publish to,
# and it publishes to the port that others subscribe to.
for _, (sub, pub) in messaging_ports.items():
args.append('--pair')
args.append(str(sub))
args.append(str(pub))
get_root_logger().info('message_forwarder fixture starting: {}', args)
proc = subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
# It takes a while for the forwarder to start, so allow for that.
# TODO(jamessynge): Come up with a way to speed up these fixtures.
time.sleep(3)
yield messaging_ports
proc.terminate()
def can_connect_to_mongo():
global _can_connect_to_mongo
if _can_connect_to_mongo is None:
logger = get_root_logger()
try:
PanDB(db_type='mongo', db_name='panoptes_testing', logger=logger, connect=True)
_can_connect_to_mongo = True
except Exception:
_can_connect_to_mongo = False
logger.info('can_connect_to_mongo = {}', _can_connect_to_mongo)
return _can_connect_to_mongo
def __init__(self, host='localhost', port=3040, connect=True, *args, **kwargs):
self.logger = get_root_logger()
self._host = host
self._port = port
self._socket = None
self._is_connected = False
if connect:
self.connect()
def __init__(self, message, relay_queue, json_queue, chunk_size, stop,
logger):
"""
Args:
message: The message to be sent (millis and report_num will be added).
relay_queue: The queue.Queue instance from which relay command
bytes are read and acted upon. Elements are of type bytes.
json_queue: The queue.Queue instance to which json messages
(serialized to bytes) are written at ~9600 baud. Elements
are of type bytes (i.e. each element is a sequence of bytes of
length up to chunk_size).
chunk_size: The number of bytes to write to json_queue at a time.
stop: a threading.Event which is checked to see if run should stop executing.
logger: the Python logger to use for reporting messages.
"""
self.message = copy.deepcopy(message)
get_root_logger().critical(f'message: {message}')
self.relay_queue = relay_queue
self.json_queue = json_queue
self.stop = stop
self.logger = logger
# Time between producing messages.
self.message_delta = datetime.timedelta(seconds=2)
self.next_message_time = None
# Size of a chunk of bytes.
self.chunk_size = chunk_size
# Interval between outputing chunks of bytes.
chunks_per_second = 1000.0 / self.chunk_size
chunk_interval = 1.0 / chunks_per_second
self.logger.debug('chunks_per_second={} chunk_interval={}',
chunks_per_second, chunk_interval)
self.chunk_delta = datetime.timedelta(seconds=chunk_interval)
self.next_chunk_time = None
self.pending_json_bytes = bytearray()
self.pending_relay_bytes = bytearray()
self.command_lines = []
self.start_time = datetime.datetime.now()
self.report_num = 0
self.logger.info('ArduinoSimulator created')
def dynamic_config_server(config_host, config_port, config_server_args, images_dir, db_name):
"""If a test requires changing the configuration we use a function-scoped testing
server. We only do this on tests that require it so we are not constantly starting and stopping
the config server unless necessary. To use this, each test that requires it must use the
`dynamic_config_server` and `config_port` fixtures and must pass the `config_port` to all
instances that are created (propogated through PanBase).
"""
logger = get_root_logger()
logger.critical(f'Starting config_server for testing function')
def start_config_server():
# Load the config items into the app config.
for k, v in config_server_args.items():
app.config[k] = v
# Start the actual flask server.
app.run(host=config_host, port=config_port)
proc = Process(target=start_config_server)
proc.start()
logger.info(f'config_server started with PID={proc.pid}')
# Give server time to start
time.sleep(1)
# Adjust various config items for testing
unit_name = 'Generic PANOPTES Unit'
unit_id = 'PAN000'
logger.info(f'Setting testing name and unit_id to {unit_id}')
set_config('name', unit_name, port=config_port)
set_config('pan_id', unit_id, port=config_port)
logger.info(f'Setting testing database to {db_name}')
set_config('db.name', db_name, port=config_port)
fields_file = 'simulator.yaml'
logger.info(f'Setting testing scheduler fields_file to {fields_file}')
set_config('scheduler.fields_file', fields_file, port=config_port)
# TODO(wtgee): determine if we need separate directories for each module.
logger.info(f'Setting temporary image directory for testing')
set_config('directories.images', images_dir, port=config_port)
yield
logger.critical(f'Killing config_server started with PID={proc.pid}')
proc.terminate()
class SocialTwitter(object):
"""Social Messaging sink to output to Twitter."""
logger = get_root_logger()
def __init__(self, **kwargs):
consumer_key = kwargs.get('consumer_key', '')
if consumer_key == '':
raise ValueError('consumer_key parameter is not defined.')
consumer_secret = kwargs.get('consumer_secret', '')
if consumer_secret == '':
raise ValueError('consumer_secret parameter is not defined.')
access_token = kwargs.get('access_token', '')
if access_token == '':
raise ValueError('access_token parameter is not defined.')
access_token_secret = kwargs.get('access_token_secret', '')
if access_token_secret == '':
raise ValueError('access_token_secret parameter is not defined.')
# Output timestamp should always be True by default otherwise Twitter will reject duplicate statuses.
self.output_timestamp = kwargs.get("output_timestamp", True)
# Create a new twitter api object
try:
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
self.api = tweepy.API(auth)
except tweepy.TweepError:
msg = 'Error authenicating with Twitter. Please check your Twitter configuration.'
self.logger.warning(msg)
raise ValueError(msg)
def send_message(self, msg, timestamp):
try:
# update_status returns a tweepy Status instance, but we
# drop it on the floor because we don't have anything we
# can do with it.
if self.output_timestamp:
self.api.update_status('{} - {}'.format(msg, timestamp))
else:
self.api.update_status(msg)
except tweepy.TweepError:
self.logger.debug(
'Error tweeting message. Please check your Twitter configuration.'
)
def pytest_runtest_logfinish(nodeid, location):
"""Signal the complete finish of running a single test item.
This hook will be called after pytest_runtest_setup(),
pytest_runtest_call() and pytest_runtest_teardown() hooks.
Args:
nodeid (str) – full id of the item
location – a triple of (filename, linenum, testname)
"""
try:
logger = get_root_logger()
logger.critical('')
logger.critical(' END TEST {}', nodeid)
logger.critical('##########' * 8)
except Exception:
pass
def pytest_runtest_logreport(report):
"""Adds the failure info that pytest prints to stdout into the log."""
if report.skipped or report.outcome != 'failed':
return
try:
logger = get_root_logger()
logger.critical('')
logger.critical(' TEST {} FAILED during {}\n\n{}\n', report.nodeid, report.when,
report.longreprtext)
cnt = 15
if report.capstdout:
logger.critical('{}Captured stdout during {}{}\n{}\n', '= ' * cnt, report.when,
' =' * cnt, report.capstdout)
if report.capstderr:
logger.critical('{}Captured stderr during {}{}\n{}\n', '* ' * cnt, report.when,
' *' * cnt, report.capstderr)
except Exception:
pass
def message_forwarder(messaging_ports):
cmd = shutil.which('panoptes-messaging-hub')
assert cmd is not None
args = [cmd]
# Note that the other programs using these port pairs consider
# them to be pub and sub, in that order, but the forwarder sees things
# in reverse: it subscribes to the port that others publish to,
# and it publishes to the port that others subscribe to.
for _, (sub, pub) in messaging_ports.items():
args.append('--pair')
args.append(str(sub))
args.append(str(pub))
logger = get_root_logger()
logger.info('message_forwarder fixture starting: {}', args)
proc = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# It takes a while for the forwarder to start, so allow for that.
# TODO(jamessynge): Come up with a way to speed up these fixtures.
time.sleep(3)
# If message forwarder doesn't start, tell us why.
if proc.poll() is not None:
outs, errs = proc.communicate(timeout=5)
logger.info(f'outs: {outs!r}')
logger.info(f'errs: {errs!r}')
assert False
yield messaging_ports
# Make sure messager forwarder is still running at end.
assert proc.poll() is None
# Try to terminate, then communicate, then kill.
try:
proc.terminate()
outs, errs = proc.communicate(timeout=0.5)
except subprocess.TimeoutExpired:
proc.kill()
outs, errs = proc.communicate()
# Make sure message forwarder was killed.
assert proc.poll() is not None
def static_config_server(config_host, static_config_port, config_server_args, images_dir, db_name):
logger = get_root_logger()
logger.critical(f'Starting config_server for testing session')
def start_config_server():
# Load the config items into the app config.
for k, v in config_server_args.items():
app.config[k] = v
# Start the actual flask server.
app.run(host=config_host, port=static_config_port)
proc = Process(target=start_config_server)
proc.start()
logger.info(f'config_server started with PID={proc.pid}')
# Give server time to start
time.sleep(1)
# Adjust various config items for testing
unit_name = 'Generic PANOPTES Unit'
unit_id = 'PAN000'
logger.info(f'Setting testing name and unit_id to {unit_id}')
set_config('name', unit_name, port=static_config_port)
set_config('pan_id', unit_id, port=static_config_port)
logger.info(f'Setting testing database to {db_name}')
set_config('db.name', db_name, port=static_config_port)
fields_file = 'simulator.yaml'
logger.info(f'Setting testing scheduler fields_file to {fields_file}')
set_config('scheduler.fields_file', fields_file, port=static_config_port)
# TODO(wtgee): determine if we need separate directories for each module.
logger.info(f'Setting temporary image directory for testing')
set_config('directories.images', images_dir, port=static_config_port)
yield
logger.critical(f'Killing config_server started with PID={proc.pid}')
proc.terminate()
def __init__(self, bucket_name, project_id='panoptes-survey'):
"""Create an object that can interact easily with storage buckets.
Note:
This assumes that you have authenticated to the Google Cloud network
using your provided auth_key.
See the README:
https://github.com/panoptes/POCS/tree/develop/pocs/utils/google
Args:
bucket_name (str): Name of bucket to use.
project_id (str, optional): Project id hosting the bucket. Default 'panoptes-survey'
Raises:
error.GoogleCloudError: Error raised if valid connection cannot be formed for
given project, bucket, and authorization.
"""
self.logger = get_root_logger()
super(PanStorage, self).__init__()
try:
self.unit_id = load_config()['pan_id']
except KeyError:
raise error.GoogleCloudError("Missing pan_id in config "
"Cannot connect to Google services.")
assert re.match(r'PAN\d\d\d', self.unit_id) is not None
self.project_id = project_id
self.bucket_name = bucket_name
try:
self.client = storage.Client(project=self.project_id)
self.bucket = self.client.get_bucket(bucket_name)
except exceptions.Forbidden:
raise error.GoogleCloudError(
"Storage bucket does not exist or no permissions. "
"Ensure that the auth_key has valid permissions to the bucket. "
"or that you have executed 'gcloud auth'")
self.logger.info("Connected to storage bucket {}", self.bucket_name)
def static_config_server(config_host, static_config_port, config_path,
images_dir, db_name):
logger = get_root_logger()
logger.critical(f'Starting config_server for testing session')
proc = config_server(
host=config_host,
port=static_config_port,
config_file=config_path,
ignore_local=True,
)
logger.info(f'config_server started with PID={proc.pid}')
# Give server time to start
time.sleep(1)
# Adjust various config items for testing
unit_name = 'Generic PANOPTES Unit'
unit_id = 'PAN000'
logger.info(f'Setting testing name and unit_id to {unit_id}')
set_config('name', unit_name, port=static_config_port)
set_config('pan_id', unit_id, port=static_config_port)
logger.info(f'Setting testing database to {db_name}')
set_config('db.name', db_name, port=static_config_port)
fields_file = 'simulator.yaml'
logger.info(f'Setting testing scheduler fields_file to {fields_file}')
set_config('scheduler.fields_file', fields_file, port=static_config_port)
# TODO(wtgee): determine if we need separate directories for each module.
logger.info(f'Setting temporary image directory for testing')
set_config('directories.images', images_dir, port=static_config_port)
yield
logger.critical(f'Killing config_server started with PID={proc.pid}')
proc.terminate()
class SocialSlack(object):
"""Social Messaging sink to output to Slack."""
logger = get_root_logger()
def __init__(self, **kwargs):
self.web_hook = kwargs.get('webhook_url', '')
if self.web_hook == '':
raise ValueError('webhook_url parameter is not defined.')
else:
self.output_timestamp = kwargs.get('output_timestamp', False)
def send_message(self, msg, timestamp):
try:
if self.output_timestamp:
post_msg = '{} - {}'.format(msg, timestamp)
else:
post_msg = msg
# We ignore the response body and headers of a successful post.
requests.post(self.web_hook, json={'text': post_msg})
except Exception as e:
self.logger.debug('Error posting to slack: {}'.format(e))
def __init__(
self,
port=None,
baudrate=115200,
name=None,
timeout=2.0,
open_delay=0.0,
retry_limit=5,
retry_delay=0.5,
logger=None,
):
"""Create a SerialData instance and attempt to open a connection.
The device need not exist at the time this is called, in which case is_connected will
be false.
Args:
port: The port (e.g. /dev/tty123 or socket://host:port) to which to
open a connection.
baudrate: For true serial lines (e.g. RS-232), sets the baud rate of
the device.
name: Name of this object. Defaults to the name of the port.
timeout (float, optional): Timeout in seconds for both read and write.
Defaults to 2.0.
open_delay: Seconds to wait after opening the port.
retry_limit: Number of times to try readline() calls in read().
retry_delay: Delay between readline() calls in read().
logger (`logging.logger` or None, optional): A logger instance. If left as None
then `panoptes.utils.logger.get_root_logger` will be called.
Raises:
ValueError: If the serial parameters are invalid (e.g. a negative baudrate).
"""
if not logger:
logger = get_root_logger()
self.logger = logger
if not port:
raise ValueError('Must specify port for SerialData')
self.name = name or port
self.retry_limit = retry_limit
self.retry_delay = retry_delay
self.ser = serial.serial_for_url(port, do_not_open=True)
# Configure the PySerial class.
self.ser.baudrate = baudrate
self.ser.bytesize = serial.EIGHTBITS
self.ser.parity = serial.PARITY_NONE
self.ser.stopbits = serial.STOPBITS_ONE
self.ser.timeout = timeout
self.ser.write_timeout = timeout
self.ser.xonxoff = False
self.ser.rtscts = False
self.ser.dsrdtr = False
self.logger.debug('SerialData for {} created', self.name)
# Properties have been set to reasonable values, ready to open the port.
try:
self.ser.open()
except serial.serialutil.SerialException as err:
self.logger.debug('Unable to open {}. Error: {}', self.name, err)
return
open_delay = max(0.0, float(open_delay))
if open_delay > 0.0:
self.logger.debug('Opened {}, sleeping for {} seconds', self.name,
open_delay)
time.sleep(open_delay)
else:
self.logger.debug('Opened {}', self.name)
def db(db_type):
return PanDB(
db_type=db_type, db_name='panoptes_testing', logger=get_root_logger(), connect=True)
class PanMessaging(object):
"""Provides messaging services within a PANOPTES robotic telescope.
Supports broadcasting messages from publishers (e.g. a POCS or
ArduinoIO class instance) to subscribers (also typically class
instances). The publishers and subscribers may be in the same
process, or in separate processes. The messages all go through
a message forwarder; this is a process which listens for messages
from all publishers on one TCP port and forwards each message to
all subscribers that are connected to a second TCP port.
Do not create PanMessaging instances directly. Publishers should
call PanMessaging.create_publisher to create an instance of
PanMessaging, on which they can then call send_message.
Subscribers should call PanMessaging.create_subscriber to create
an instance of PanMessaging, on which they can then call
receive_message.
Messages are sent to topics, a name that can be used to allow
a high-level partitioning of messages. A topic name may not
include whitespace. Among the currently used topic names are:
* PANCHAT (sent from POCS.say)
* PAWS-CMD (sent from PAWS websockets.py)
* POCS (sent by class POCS)
* POCS-CMD (sent by class POCS)
* STATUS (sent by class POCS)
* weather (from peas/sensors.py)
* environment (from peas/sensors.py)
* telemetry:commands (in ArduinoIO... new)
* camera:commands (in ArduinoIO... new)
And some other topics are used in tests:
* Test-Topic (test_messaging.py)
* RUNNING (test_pocs.py)
* POCS-CMD (test_pocs.py)
The method receive_message will return messages from all topics;
the caller must check the returned topic name to determine if
the message value is of interest.
Note: PAWS doesn't use PanMessaging, which will likely result in
problems as we evolve PanMessaging and the set of topics.
TODO: Figure out how to share PanMessaging with PAWS.
The value of a message being sent may be a string (in which case it
is wrapped in a dict(message=<value>, timestamp=<now>) or a dict,
in which case it will be "scrubbed", i.e. the dict entries will be
modified as necessary to so that the dict can be serialized using
json.dumps.
TODO Pick an encoding of strings (e.g. UTF-8) so that non-ASCII
strings may be sent and received without corruption of the data
or exceptions being thrown.
ZeroMQ is used to provide the underlying pub-sub support. ZeroMQ
supports only a very basic message format: an array of bytes.
PanMessaging converts the provided message topic and value into
a byte array of this format:
<topic-name><space><serialized-value>
"""
logger = get_root_logger()
# Topic names must consist of the characters.
topic_name_re = re.compile('[a-zA-Z][-a-zA-Z0-9_.:]*')
def __init__(self, **kwargs):
"""Do not call this directly."""
# Create a new context
self.context = zmq.Context()
self.socket = None
@classmethod
def create_forwarder(cls, sub_port, pub_port, ready_fn=None, done_fn=None):
subscriber, publisher = PanMessaging.create_forwarder_sockets(sub_port, pub_port)
PanMessaging.run_forwarder(subscriber, publisher, ready_fn=ready_fn, done_fn=done_fn)
@classmethod
def create_forwarder_sockets(cls, sub_port, pub_port):
cls.logger.info('Creating forwarder sockets for {} -> {}', sub_port, pub_port)
subscriber = PanMessaging.create_subscriber(sub_port, bind=True, connect=False)
publisher = PanMessaging.create_publisher(pub_port, bind=True, connect=False)
return subscriber, publisher
@classmethod
def run_forwarder(cls, subscriber, publisher, ready_fn=None, done_fn=None):
publisher.logger.info('run_forwarder')
try:
if ready_fn:
ready_fn()
publisher.logger.info('run_forwarder calling zmq.device')
zmq.device(zmq.FORWARDER, subscriber.socket, publisher.socket)
except KeyboardInterrupt:
pass
except Exception as e:
publisher.logger.warning(e)
publisher.logger.warning("bringing down zmq device")
finally:
publisher.logger.info('run_forwarder closing publisher and subscriber')
publisher.close()
subscriber.close()
if done_fn:
done_fn()
@classmethod
def create_publisher(cls, port, bind=False, host='localhost', connect=True):
""" Create a publisher
Args:
port (int): The port (on localhost) to bind to.
Returns:
A ZMQ PUB socket
"""
obj = cls()
obj.logger.debug('Creating zmq message publisher.')
socket = obj.context.socket(zmq.PUB)
if bind:
obj.logger.debug(f'Binding publisher to port {port}')
socket.bind(f'tcp://*:{port}')
elif connect:
obj.logger.debug(f'Binding publisher to tcp://{host}:{port}')
socket.connect(f'tcp://{host}:{port}')
obj.socket = socket
return obj
@classmethod
def create_subscriber(cls, port, topic='', host='localhost', bind=False, connect=True):
""" Create a listener
Args:
port (int): The port (on localhost) to bind to.
topic (str): Which topic or topic prefix to subscribe to.
"""
obj = cls()
obj.logger.debug("Creating subscriber. Port: {} \tTopic: {}".format(port, topic))
socket = obj.context.socket(zmq.SUB)
if bind:
try:
socket.bind(f'tcp://*:{port}')
except zmq.error.ZMQError:
obj.logger.debug('Problem binding port {}'.format(port))
elif connect:
obj.logger.debug(f'Connecting subscriber to tcp://{host}:{port}')
socket.connect(f'tcp://{host}:{port}')
socket.setsockopt_string(zmq.SUBSCRIBE, topic)
obj.socket = socket
return obj
def send_message(self, topic, message):
""" Responsible for actually sending message across a topic
Args:
topic(str): Name of topic to send on. The name must
match topic_name_re.
message: Message to be sent (a string or a dict).
"""
if not isinstance(topic, str):
raise ValueError('Topic name must be a string')
elif not self.topic_name_re.fullmatch(topic):
raise ValueError('Topic name ("{}") is not valid'.format(topic))
if topic == 'PANCHAT':
self.logger.info(f"{topic} {message}")
if isinstance(message, str):
message = to_json({
"message": message,
"timestamp": current_time(pretty=True),
})
elif isinstance(message, dict):
message = to_json(message)
else:
raise ValueError('Message value must be a string or dict')
# Build the full message with topic
full_message = f'{topic} {message}'
# Send the message
# self.socket.send_string(full_message, flags=zmq.NOBLOCK)
self.socket.send_string(full_message)
def receive_message(self, blocking=True, flags=0, timeout_ms=0):
"""Receive a message
Receives a message for the current subscriber. Blocks by default, pass
`flags=zmq.NOBLOCK` for non-blocking.
Args:
blocking (bool, optional): If True, blocks until message
received or timeout__ms elapsed (if timeout_ms > 0).
flag (int, optional): Any valid recv flag, e.g. zmq.NOBLOCK
timeout_ms (int, optional): Time in milliseconds to wait for
a message to arrive. Only applies if blocking is True.
Returns:
tuple(str, dict): Tuple containing the topic and a dict
"""
topic = None
msg_obj = None
if not blocking:
flags = flags | zmq.NOBLOCK
elif timeout_ms > 0:
# Wait until a message is available or the timeout expires.
# TODO(jamessynge): Remove flags=..., confirm that works with
# the default flags value of zmq.POLLIN.
self.socket.poll(timeout=timeout_ms, flags=(zmq.POLLIN | zmq.POLLOUT))
# Don't block at this point, because we will have waited as long
# as necessary.
flags = flags | zmq.NOBLOCK
try:
# Ugh. So ugly with the strings.
message = self.socket.recv_string(flags=flags)
except Exception as e:
self.logger.warning(f'error in receive_message: {e!r}')
else:
topic, msg = message.split(' ', maxsplit=1)
try:
msg_obj = from_json(msg)
except Exception as e:
self.logger.warning(f'Error parsing message: {msg} {e!r}')
return topic, msg_obj
def close(self):
"""Close the socket """
self.socket.close()
self.context.term()
import numpy as np
from astropy.time import Time
from astropy.wcs import WCS
from astropy.stats import sigma_clipped_stats, sigma_clip
from panoptes.utils.images import fits as fits_utils
from panoptes.utils.logger import get_root_logger
from panoptes.utils.bayer import get_rgb_data
import logging
logger = get_root_logger()
logger.setLevel(logging.DEBUG)
def moving_average(data_set, periods=3):
"""Moving average.
Args:
data_set (`numpy.array`): An array of values over which to perform the moving average.
periods (int, optional): Number of periods.
Returns:
`numpy.array`: An array of the computed averages.
"""
weights = np.ones(periods) / periods
return np.convolve(data_set, weights, mode='same')
def get_pixel_drift(coords, files):
"""Get the pixel drift for a given set of coordinates.
def get_config(key=None,
host='localhost',
port='6563',
parse=True,
default=None):
"""Get a config item from the config server.
Return the config entry for the given `key`. If `key=None` (default), return
the entire config.
Nested keys can be specified as a string, as per [scalpl](https://pypi.org/project/scalpl/).
Examples:
>>> get_config(key='name')
'Generic PANOPTES Unit'
>>> get_config(key='location.horizon')
<Quantity 30. deg>
>>> get_config(key='location.horizon', parse=False)
'30.0 deg'
>>> get_config(key='cameras.devices[1].model')
'canon_gphoto2'
>>> # Returns `None` if key is not found.
>>> foobar = get_config(key='foobar')
>>> foobar is None
True
>>> # But you can supply a default.
>>> get_config(key='foobar', default='baz')
'baz'
>>> # Can use Quantities as well
>>> from astropy import units as u
>>> get_config(key='foobar', default=42 * u.meter)
<Quantity 42. m>
Args:
key (str): The key to update, see Examples in `get_config` for details.
host (str, optional): The config server host, defaults to '127.0.0.1'.
port (str, optional): The config server port, defaults to 6563.
parse (bool, optional): If response should be parsed by
`~panoptes.utils.serializers.from_json`, default True.
default (str, optional): The config server port, defaults to 6563.
Returns:
dict: The corresponding config entry.
Raises:
Exception: Raised if the config server is not available.
"""
url = f'http://{host}:{port}/get-config'
config_entry = default
try:
response = requests.post(url, json={'key': key})
except Exception as e:
get_root_logger().info(f'Problem with get_config: {e!r}')
else:
if not response.ok:
get_root_logger().info(
f'Problem with get_config: {response.content!r}')
else:
if response.text != 'null\n':
if parse:
config_entry = serializers.from_json(
response.content.decode('utf8'))
else:
config_entry = response.json()
if config_entry is None:
config_entry = default
return config_entry
from glob import glob
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from matplotlib.figure import Figure
from matplotlib import pyplot as plt
from tqdm import tqdm
from astropy.stats import sigma_clip
from panoptes.utils import current_time
from panoptes.utils.logger import get_root_logger
from panoptes.piaa.utils import plot
from panoptes.piaa.utils import pipeline
import logging
logger = get_root_logger()
logger.setLevel(logging.INFO)
plt.style.use('bmh')
def build_ref(build_params):
""" Build a reference PSC for the given PSC. """
psc_fn = build_params[0]
params = build_params[1]
# Load params
base_dir = params['base_dir']
processed_dir = params['processed_dir']
force = params['force']
