def __init__(self, **kwargs):
consumer_key = kwargs.get('consumer_key', '')
if consumer_key == '':
raise ValueError('consumer_key parameter is not defined.')
consumer_secret = kwargs.get('consumer_secret', '')
if consumer_secret == '':
raise ValueError('consumer_secret parameter is not defined.')
access_token = kwargs.get('access_token', '')
if access_token == '':
raise ValueError('access_token parameter is not defined.')
access_token_secret = kwargs.get('access_token_secret', '')
if access_token_secret == '':
raise ValueError('access_token_secret parameter is not defined.')
# Output timestamp should always be True by default otherwise Twitter will reject duplicate statuses.
self.output_timestamp = kwargs.get("output_timestamp", True)
# Create a new twitter api object
try:
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
self.api = tweepy.API(auth)
except tweepy.TweepError: # pragma: no cover
msg = 'Error authenicating with Twitter. Please check your Twitter configuration.'
logger.warning(msg)
raise ValueError(msg)
def server_is_running():
"""Thin-wrapper to check server."""
try:
return get_config(endpoint='heartbeat', verbose=False)
except Exception as e:
logger.warning(
f'server_is_running error (ignore if just starting server): {e!r}')
return False
def parse_config_directories(directories, must_exist=False):
"""Parse the config dictionary for common objects.
Given a `base` entry that corresponds to the absolute path of a directory,
prepend the `base` to all other relative directory entries.
If `must_exist=True`, then only update entry if the corresponding
directory exists on the filesystem.
.. doctest::
>>> dirs_config = dict(base='/var/panoptes', foo='bar', baz='bam')
>>> # If the relative dir doesn't exist but is required, return as is.
>>> parse_config_directories(dirs_config, must_exist=True)
{'base': '/var/panoptes', 'foo': 'bar', 'baz': 'bam'}
>>> # Default is to return anyway.
>>> parse_config_directories(dirs_config)
{'base': '/var/panoptes', 'foo': '/var/panoptes/bar', 'baz': '/var/panoptes/bam'}
>>> # If 'base' is not a valid absolute directory, return all as is.
>>> dirs_config = dict(base='panoptes', foo='bar', baz='bam')
>>> parse_config_directories(dirs_config, must_exist=False)
{'base': 'panoptes', 'foo': 'bar', 'baz': 'bam'}
Args:
directories (dict): The dictionary of directory information. Usually comes
from the "directories" entry in the config.
must_exist (bool): Only parse directory if it exists on the filesystem,
default False.
Returns:
dict: The same directory but with relative directories resolved.
"""
# Try to get the base directory first.
base_dir = directories.get('base', os.environ['PANDIR'])
if os.path.isdir(base_dir):
logger.trace(f'Using base_dir={base_dir!r} for setting config directories')
# Add the base directory to any relative dir.
for dir_name, rel_dir in directories.items():
# Only want relative directories.
if rel_dir.startswith('/') is False:
abs_dir = os.path.join(base_dir, rel_dir)
logger.trace(
f'base_dir={base_dir!r} rel_dir={rel_dir!r} abs_dir={abs_dir!r} must_exist={must_exist!r}')
if must_exist and not os.path.exists(abs_dir):
logger.warning(f'must_exist={must_exist!r} but abs_dir={abs_dir!r} does not exist, skipping')
else:
logger.trace(f'Setting {dir_name} to {abs_dir}')
directories[dir_name] = abs_dir
return directories
def start_server(host='localhost', port=6563):
try:
logger.info(f'Starting panoptes config server with {host}:{port}')
http_server = WSGIServer((host, int(port)), app, log=access_logs, error_log=error_logs)
http_server.serve_forever()
except OSError:
logger.warning(f'Problem starting config server, is another config server already running?')
return None
except Exception as e:
logger.warning(f'Problem starting config server: {e!r}')
return None
def cr2_to_pgm(cr2_fname,
pgm_fname=None,
overwrite=True,
*args,
**kwargs): # pragma: no cover
""" Convert CR2 file to PGM
Converts a raw Canon CR2 file to a netpbm PGM file via `dcraw`. Assumes
`dcraw` is installed on the system
Note:
This is a blocking call
Arguments:
cr2_fname {str} -- Name of CR2 file to convert
**kwargs {dict} -- Additional keywords to pass to script
Keyword Arguments:
pgm_fname {str} -- Name of PGM file to output, if None (default) then
use same name as CR2 (default: {None})
dcraw {str} -- Path to installed `dcraw` (default: {'dcraw'})
overwrite {bool} -- A bool indicating if existing PGM should be overwritten
(default: {True})
Returns:
str -- Filename of PGM that was created
"""
dcraw = shutil.which('dcraw')
if dcraw is None:
raise error.InvalidCommand('dcraw not found')
if pgm_fname is None:
pgm_fname = cr2_fname.replace('.cr2', '.pgm')
if os.path.exists(pgm_fname) and not overwrite:
logger.warning(
f"PGM file exists, returning existing file: {pgm_fname}")
else:
try:
# Build the command for this file
command = '{} -t 0 -D -4 {}'.format(dcraw, cr2_fname)
cmd_list = command.split()
logger.debug("PGM Conversion command: \n {}".format(cmd_list))
# Run the command
if subprocess.check_call(cmd_list) == 0:
logger.debug("PGM Conversion command successful")
except subprocess.CalledProcessError as err:
raise error.InvalidSystemCommand(
msg="File: {} \n err: {}".format(cr2_fname, err))
return pgm_fname
def get_current(self, collection):
current_fn = self._get_file(collection, permanent=False)
try:
with open(current_fn) as f:
msg = from_json(f.read())
return msg
except FileNotFoundError:
logger.warning(f"No record found for {collection}")
return None
def send_message(self, msg, timestamp):
try:
if self.output_timestamp:
post_msg = '{} - {}'.format(msg, timestamp)
else:
post_msg = msg
# We ignore the response body and headers of a successful post.
requests.post(self.web_hook, json={'text': post_msg})
except Exception as e: # pragma: no cover
logger.warning('Error posting to slack: {}'.format(e))
def reset_config():
"""Reset the configuration.
An endpoint that accepts a POST method. The json request object
must contain the key ``reset`` (with any value).
The method will reset the configuration to the original configuration files that were
used, skipping the local (and saved file).
.. note::
If the server was originally started with a local version of the file, those will
be skipped upon reload. This is not ideal but hopefully this method is not used too
much.
Returns:
str: A json string object containing the keys ``success`` and ``msg`` that indicate
success or failure.
"""
params = dict()
if request.method == 'GET':
params = request.args
elif request.method == 'POST':
params = request.get_json()
logger.warning(f'Resetting config server')
if params['reset']:
# Reload the config
config = load_config(config_files=app.config['config_file'],
load_local=app.config['load_local'])
# Add an entry to control running of the server.
config['config_server'] = dict(running=True)
app.config['POCS'] = config
app.config['POCS_cut'] = Cut(config)
else:
return jsonify({
'success': False,
'msg': "Invalid. Need json request: {'reset': True}"
})
return jsonify({
'success': True,
'msg': f'Configuration reset'
})
def wait_for_events(
events,
timeout=600,
sleep_delay=5 * u.second,
callback=None,
):
"""Wait for event(s) to be set.
This method will wait for a maximum of `timeout` seconds for all of the `events`
to complete.
Checks every `sleep_delay` seconds for the events to be set.
If provided, the `callback` will be called every `sleep_delay` seconds.
The callback should return `True` to continue waiting otherwise `False`
to interrupt the loop and return from the function.
.. doctest::
>>> import time
>>> import threading
>>> from panoptes.utils.time import wait_for_events
>>> # Create some events, normally something like taking an image.
>>> event0 = threading.Event()
>>> event1 = threading.Event()
>>> # Wait for 30 seconds but interrupt after 1 second by returning False from callback.
>>> def interrupt_cb(): time.sleep(1); return False
>>> # The function will return False if events are not set.
>>> wait_for_events([event0, event1], timeout=30, callback=interrupt_cb)
False
>>> # Timeout will raise an exception.
>>> wait_for_events([event0, event1], timeout=1)
Traceback (most recent call last):
File "<input>", line 1, in <module>
File ".../panoptes-utils/src/panoptes/utils/time.py", line 254, in wait_for_events
panoptes.utils.error.Timeout: Timeout: Timeout waiting for generic event
>>> # Set the events in another thread for normal usage.
>>> def set_events(): time.sleep(1); event0.set(); event1.set()
>>> threading.Thread(target=set_events).start()
>>> wait_for_events([event0, event1], timeout=30)
True
Args:
events (list(`threading.Event`)): An Event or list of Events to wait on.
timeout (float|`astropy.units.Quantity`): Timeout in seconds to wait for events,
default 600 seconds.
sleep_delay (float, optional): Time in seconds between event checks.
callback (callable): A periodic callback that should return `True` to continue
waiting or `False` to interrupt the loop. Can also be used for e.g. custom logging.
Returns:
bool: True if events were set, False otherwise.
Raises:
error.Timeout: Raised if events have not all been set before `timeout` seconds.
"""
with suppress(AttributeError):
sleep_delay = sleep_delay.to_value('second')
event_timer = CountdownTimer(timeout)
if not isinstance(events, list):
events = [events]
start_time = current_time()
while not all([event.is_set() for event in events]):
elapsed_secs = round((current_time() - start_time).to_value('second'),
2)
if event_timer.expired():
raise error.Timeout(
f"Timeout waiting for {len(events)} events after {elapsed_secs} seconds"
)
if callable(callback) and callback() is False:
logger.warning(
f"Waiting for {len(events)} events has been interrupted after {elapsed_secs} seconds"
)
break
# Sleep for a little bit.
event_timer.sleep(max_sleep=sleep_delay)
return all([event.is_set() for event in events])
def load_config(config_files=None, parse=True, load_local=True):
"""Load configuration information.
.. note::
This function is used by the config server and normal config usage should
be via a running config server.
This function supports loading of a number of different files. If no options
are passed to ``config_files`` then the default ``$PANDIR/conf_files/pocs.yaml``
will be loaded.
``config_files`` is a list and loaded in order, so the second entry will overwrite
any values specified by similarly named keys in the first entry.
``config_files`` should be specified by an absolute path, which can exist anywhere
on the filesystem.
Local versions of files can override built-in versions and are automatically loaded if
they exist alongside the specified config path. Local files have a ``<>_local.yaml`` name, where
``<>`` is the built-in file.
Given the following path:
::
/path/to/dir
|- my_conf.yaml
|- my_conf_local.yaml
You can do a ``load_config('/path/to/dir/my_conf.yaml')`` and both versions of the file will
be loaded, with the values in the local file overriding the non-local. Typically the local
file would also be ignored by ``git``, etc.
For example, the ``panoptes.utils.config.server.config_server`` will always save values to
a local version of the file so the default settings can always be recovered if necessary.
Local files can be ignored (mostly for testing purposes or for recovering default values)
with the ``load_local=False`` parameter.
Args:
config_files (list, optional): A list of files to load as config,
see Notes for details of how to specify files.
parse (bool, optional): If the config file should attempt to create
objects such as dates, astropy units, etc.
load_local (bool, optional): If local files should be used, see
Notes for details.
Returns:
dict: A dictionary of config items.
"""
config = dict()
config_files = listify(config_files)
logger.debug(f'Loading config files: config_files={config_files!r}')
for config_file in config_files:
try:
logger.debug(f'Adding config_file={config_file!r} to config dict')
_add_to_conf(config, config_file, parse=parse)
except Exception as e: # pragma: no cover
logger.warning(f"Problem with config_file={config_file!r}, skipping. {e!r}")
# Load local version of config
if load_local:
local_version = config_file.replace('.', '_local.')
if os.path.exists(local_version):
try:
_add_to_conf(config, local_version, parse=parse)
except Exception as e: # pragma: no cover
logger.warning(f"Problem with local_version={local_version!r}, skipping: {e!r}")
# parse_config_directories currently only corrects directory names.
if parse:
logger.trace(f'Parsing config={config!r}')
with suppress(KeyError):
config['directories'] = parse_config_directories(config['directories'])
logger.trace(f'Config directories parsed: config={config!r}')
return config
def get_solve_field(fname, replace=True, overwrite=True, timeout=30, **kwargs):
"""Convenience function to wait for `solve_field` to finish.
This function merely passes the `fname` of the image to be solved along to `solve_field`,
which returns a subprocess.Popen object. This function then waits for that command
to complete, populates a dictonary with the EXIF informaiton and returns. This is often
more useful than the raw `solve_field` function.
Example:
>>> from panoptes.utils.images import fits as fits_utils
>>> # Get our fits filename.
>>> fits_fn = getfixture('unsolved_fits_file')
>>> # Perform the solve.
>>> solve_info = fits_utils.get_solve_field(fits_fn)
>>> # Show solved filename.
>>> solve_info['solved_fits_file']
'.../unsolved.fits'
>>> # Pass a suggested location.
>>> ra = 15.23
>>> dec = 90
>>> radius = 5 # deg
>>> solve_info = fits_utils.solve_field(fits_fn, ra=ra, dec=dec, radius=radius)
>>> # Pass kwargs to `solve-field` program.
>>> solve_kwargs = {'--pnm': '/tmp/awesome.bmp', '--overwrite': True}
>>> solve_info = fits_utils.get_solve_field(fits_fn, **solve_kwargs, skip_solved=False)
>>> assert os.path.exists('/tmp/awesome.bmp')
Args:
fname ({str}): Name of FITS file to be solved.
replace (bool, optional): Saves the WCS back to the original file,
otherwise output base filename with `.new` extension. Default True.
overwrite (bool, optional): Clobber file, default True. Required if `replace=True`.
timeout (int, optional): The timeout for solving, default 30 seconds.
**kwargs ({dict}): Options to pass to `solve_field` should start with `--`.
Returns:
dict: Keyword information from the solved field.
"""
skip_solved = kwargs.get('skip_solved', True)
out_dict = {}
output = None
errs = None
header = getheader(fname)
wcs = WCS(header)
# Check for solved file
if skip_solved and wcs.is_celestial:
logger.info(
f"Skipping solved file (use skip_solved=False to solve again): {fname}"
)
out_dict.update(header)
out_dict['solved_fits_file'] = fname
return out_dict
# Set a default radius of 15
if overwrite:
kwargs['--overwrite'] = True
# Use unpacked version of file.
was_compressed = False
if fname.endswith('.fz'):
logger.debug(f'Uncompressing {fname}')
fname = funpack(fname)
logger.debug(f'Using {fname} for solving')
was_compressed = True
logger.debug(f'Solving with: {kwargs!r}')
proc = solve_field(fname, **kwargs)
try:
output, errs = proc.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
proc.kill()
output, errs = proc.communicate()
raise error.Timeout(f'Timeout while solving: {output!r} {errs!r}')
else:
if proc.returncode != 0:
logger.debug(f'Returncode: {proc.returncode}')
for log in [output, errs]:
if log and log > '':
logger.debug(f'Output on {fname}: {log}')
if proc.returncode == 3:
raise error.SolveError(f'solve-field not found: {output}')
new_fname = fname.replace('.fits', '.new')
if replace:
logger.debug(f'Overwriting original {fname}')
os.replace(new_fname, fname)
else:
fname = new_fname
try:
header = getheader(fname)
header.remove('COMMENT', ignore_missing=True, remove_all=True)
header.remove('HISTORY', ignore_missing=True, remove_all=True)
out_dict.update(header)
except OSError:
logger.warning(f"Can't read fits header for: {fname}")
# Check it was solved.
if WCS(header).is_celestial is False:
raise error.SolveError(
'File not properly solved, no WCS header present.')
# Remove WCS file.
os.remove(fname.replace('.fits', '.wcs'))
if was_compressed and replace:
logger.debug(f'Compressing plate-solved {fname}')
fname = fpack(fname)
out_dict['solved_fits_file'] = fname
return out_dict
def get_config(key=None,
host=None,
port=None,
endpoint='get-config',
parse=True,
default=None,
verbose=True):
"""Get a config item from the config server.
Return the config entry for the given ``key``. If ``key=None`` (default), return
the entire config.
Nested keys can be specified as a string, as per `scalpl <https://pypi.org/project/scalpl/>`_.
Examples:
.. doctest::
>>> get_config(key='name')
'Testing PANOPTES Unit'
>>> get_config(key='location.horizon')
<Quantity 30. deg>
>>> # With no parsing, the raw string (including quotes) is returned.
>>> get_config(key='location.horizon', parse=False)
'"30.0 deg"'
>>> get_config(key='cameras.devices[1].model')
'canon_gphoto2'
>>> # Returns `None` if key is not found.
>>> foobar = get_config(key='foobar')
>>> foobar is None
True
>>> # But you can supply a default.
>>> get_config(key='foobar', default='baz')
'baz'
>>> # Can use Quantities as well
>>> from astropy import units as u
>>> get_config(key='foobar', default=42 * u.meter)
<Quantity 42. m>
Notes:
By default all calls to this function will log at the `trace` level because
there are some calls (e.g. during POCS operation) that will be quite noisy.
Setting `verbose=True` changes those to `debug` log levels for an individual
call.
Args:
key (str): The key to update, see Examples in :func:`get_config` for details.
host (str, optional): The config server host. First checks for PANOPTES_CONFIG_HOST
env var, defaults to 'localhost'.
port (str or int, optional): The config server port. First checks for PANOPTES_CONFIG_HOST
env var, defaults to 6563.
endpoint (str, optional): The relative url endpoint to use for getting
the config items, default 'get-config'. See `server_is_running()`
for example of usage.
parse (bool, optional): If response should be parsed by
:func:`panoptes.utils.serializers.from_json`, default True.
default (str, optional): The config server port, defaults to 6563.
verbose (bool, optional): Determines the output log level, defaults to
True (i.e. `debug` log level). See notes for details.
Returns:
dict: The corresponding config entry.
Raises:
Exception: Raised if the config server is not available.
"""
log_level = 'DEBUG' if verbose else 'TRACE'
host = host or os.getenv('PANOPTES_CONFIG_HOST', 'localhost')
port = port or os.getenv('PANOPTES_CONFIG_PORT', 6563)
url = f'http://{host}:{port}/{endpoint}'
config_entry = default
try:
logger.log(log_level,
f'Calling get_config on url={url!r} with key={key!r}')
response = requests.post(url, json={'key': key, 'verbose': verbose})
if not response.ok: # pragma: no cover
raise InvalidConfig(
f'Config server returned invalid JSON: response.content={response.content!r}'
)
except Exception as e:
logger.warning(f'Problem with get_config: {e!r}')
else:
response_text = response.text.strip()
logger.log(log_level, f'Decoded response_text={response_text!r}')
if response_text != 'null':
logger.log(
log_level,
f'Received config key={key!r} response_text={response_text!r}'
)
if parse:
logger.log(
log_level,
f'Parsing config results: response_text={response_text!r}'
)
config_entry = from_json(response_text)
else:
config_entry = response_text
if config_entry is None:
logger.log(log_level,
f'No config entry found, returning default={default!r}')
config_entry = default
logger.log(log_level,
f'Config key={key!r}: config_entry={config_entry!r}')
return config_entry
def set_config(key, new_value, host=None, port=None, parse=True):
"""Set config item in config server.
Given a `key` entry, update the config to match. The `key` is a dot accessible
string, as given by `scalpl <https://pypi.org/project/scalpl/>`_. See Examples in
:func:`get_config` for details.
Examples:
.. doctest::
>>> from astropy import units as u
>>> # Can use astropy units.
>>> set_config('location.horizon', 35 * u.degree)
{'location.horizon': <Quantity 35. deg>}
>>> get_config(key='location.horizon')
<Quantity 35. deg>
>>> # String equivalent works for 'deg', 'm', 's'.
>>> set_config('location.horizon', '30 deg')
{'location.horizon': <Quantity 30. deg>}
Args:
key (str): The key to update, see Examples in :func:`get_config` for details.
new_value (scalar|object): The new value for the key, can be any serializable object.
host (str, optional): The config server host. First checks for PANOPTES_CONFIG_HOST
env var, defaults to 'localhost'.
port (str or int, optional): The config server port. First checks for PANOPTES_CONFIG_HOST
env var, defaults to 6563.
parse (bool, optional): If response should be parsed by
:func:`panoptes.utils.serializers.from_json`, default True.
Returns:
dict: The updated config entry.
Raises:
Exception: Raised if the config server is not available.
"""
host = host or os.getenv('PANOPTES_CONFIG_HOST', 'localhost')
port = port or os.getenv('PANOPTES_CONFIG_PORT', 6563)
url = f'http://{host}:{port}/set-config'
json_str = to_json({key: new_value})
config_entry = None
try:
# We use our own serializer so pass as `data` instead of `json`.
logger.info(f'Calling set_config on url={url!r}')
response = requests.post(url,
data=json_str,
headers={'Content-Type': 'application/json'})
if not response.ok: # pragma: no cover
raise Exception(f'Cannot access config server: {response.text}')
except Exception as e:
logger.warning(f'Problem with set_config: {e!r}')
else:
if parse:
config_entry = from_json(response.content.decode('utf8'))
else:
config_entry = response.json()
return config_entry