def __init__(self, db_name=None, **kwargs):
"""
Init base class for db instances.
Args:
db_name: Name of the database, typically 'panoptes' or 'panoptes_testing'.
"""
self.db_name = db_name
logger.info(f'Creating PanDB {self.db_name}')
def start_server(host='localhost', port=6563):
try:
logger.info(f'Starting panoptes config server with {host}:{port}')
http_server = WSGIServer((host, int(port)), app, log=access_logs, error_log=error_logs)
http_server.serve_forever()
except OSError:
logger.warning(f'Problem starting config server, is another config server already running?')
return None
except Exception as e:
logger.warning(f'Problem starting config server: {e!r}')
return None
def save_config(path, config, overwrite=True):
"""Save config to local yaml file.
Args:
path (str): Path to save, can be relative or absolute. See Notes in ``load_config``.
config (dict): Config to save.
overwrite (bool, optional): True if file should be updated, False
to generate a warning for existing config. Defaults to True
for updates.
Returns:
bool: If the save was successful.
Raises:
FileExistsError: If the local path already exists and ``overwrite=False``.
"""
# Make sure ends with '_local.yaml'
base, ext = os.path.splitext(path)
# Always want .yaml (although not actually used).
ext = '.yaml'
# Check for _local name.
if not base.endswith('_local'):
base = f'{base}_local'
full_path = f'{base}{ext}'
if os.path.exists(full_path) and overwrite is False:
raise FileExistsError(f"Path exists and overwrite=False: {full_path}")
else:
# Create directory if does not exist
os.makedirs(os.path.dirname(full_path), exist_ok=True)
logger.info(f'Saving config to {full_path}')
with open(full_path, 'w') as f:
to_yaml(config, stream=f)
logger.success(f'Config info saved to {full_path}')
return True
def run(context,
config_file=None,
save_local=True,
load_local=False,
heartbeat=True):
"""Runs the config server with command line options.
This function is installed as an entry_point for the module, accessible
at `panoptes-config-server`.
"""
host = context.obj.get('host')
port = context.obj.get('port')
server_process = server.config_server(config_file,
host=host,
port=port,
load_local=load_local,
save_local=save_local,
auto_start=False)
try:
print(f'Starting config server. Ctrl-c to stop')
server_process.start()
print(
f'Config server started on server_process.pid={server_process.pid!r}. '
f'Set "config_server.running=False" or Ctrl-c to stop')
# Loop until config told to stop.
while server_is_running():
time.sleep(heartbeat)
server_process.terminate()
server_process.join(30)
except KeyboardInterrupt:
logger.info(
f'Config server interrupted, shutting down {server_process.pid}')
server_process.terminate()
except Exception as e: # pragma: no cover
logger.error(f'Unable to start config server {e!r}')
def config_server(config_file,
host=None,
port=None,
load_local=True,
save_local=False,
auto_start=True,
access_logs=None,
error_logs='logger',
):
"""Start the config server in a separate process.
A convenience function to start the config server.
Args:
config_file (str or None): The absolute path to the config file to load. Checks for
PANOPTES_CONFIG_FILE env var and fails if not provided.
host (str, optional): The config server host. First checks for PANOPTES_CONFIG_HOST
env var, defaults to 'localhost'.
port (str or int, optional): The config server port. First checks for PANOPTES_CONFIG_HOST
env var, defaults to 6563.
load_local (bool, optional): If local config files should be used when loading, default True.
save_local (bool, optional): If setting new values should auto-save to local file, default False.
auto_start (bool, optional): If server process should be started automatically, default True.
access_logs ('default' or `logger` or `File`-like or None, optional): Controls access logs for
the gevent WSGIServer. The `default` string will cause access logs to go to stderr. The
string `logger` will use the panoptes logger. A File-like will write to file. The default
`None` will turn off all access logs.
error_logs ('default' or 'logger' or `File`-like or None, optional): Same as `access_logs` except we use
our `logger` as the default.
Returns:
multiprocessing.Process: The process running the config server.
"""
config_file = config_file or os.environ['PANOPTES_CONFIG_FILE']
logger.info(f'Starting panoptes-config-server with config_file={config_file!r}')
config = load_config(config_files=config_file, load_local=load_local)
logger.success(f'Config server Loaded {len(config)} top-level items')
# Add an entry to control running of the server.
config['config_server'] = dict(running=True)
logger.success(f'{config!r}')
cut_config = Cut(config)
app.config['config_file'] = config_file
app.config['save_local'] = save_local
app.config['load_local'] = load_local
app.config['POCS'] = config
app.config['POCS_cut'] = cut_config
logger.info(f'Config items saved to flask config-server')
# Set up access and error logs for server.
access_logs = logger if access_logs == 'logger' else access_logs
error_logs = logger if error_logs == 'logger' else error_logs
def start_server(host='localhost', port=6563):
try:
logger.info(f'Starting panoptes config server with {host}:{port}')
http_server = WSGIServer((host, int(port)), app, log=access_logs, error_log=error_logs)
http_server.serve_forever()
except OSError:
logger.warning(f'Problem starting config server, is another config server already running?')
return None
except Exception as e:
logger.warning(f'Problem starting config server: {e!r}')
return None
host = host or os.getenv('PANOPTES_CONFIG_HOST', 'localhost')
port = port or os.getenv('PANOPTES_CONFIG_PORT', 6563)
cmd_kwargs = dict(host=host, port=port)
logger.debug(f'Setting up config server process with cmd_kwargs={cmd_kwargs!r}')
server_process = Process(target=start_server,
daemon=True,
kwargs=cmd_kwargs)
if auto_start:
server_process.start()
return server_process
def set_config_entry():
"""Sets an item in the config.
Endpoint that responds to GET and POST requests and sets a
configuration item corresponding to the provided key.
The key entries should be specified in dot-notation, with the names
corresponding to the entries stored in the configuration file. See
the `scalpl <https://pypi.org/project/scalpl/>`_ documentation for details
on the dot-notation.
The endpoint should receive a JSON document with a single key named ``"key"``
and a value that corresponds to the desired key within the configuration.
For example, take the following configuration:
.. code:: javascript
{
'location': {
'elevation': 3400.0,
'latitude': 19.55,
'longitude': 155.12,
}
}
To set the corresponding value for the elevation, pass a JSON document similar to:
.. code:: javascript
'{"location.elevation": "1000 m"}'
Returns:
str: If method is successful, returned json string will be a copy of the set values.
On failure, a json string with ``status`` and ``msg`` keys will be returned.
"""
params = dict()
if request.method == 'GET':
params = request.args
elif request.method == 'POST':
params = request.get_json()
if params is None:
return jsonify({
'success': False,
'msg': "Invalid. Need json request: {'key': <config_entry>, 'value': <new_values>}"
})
try:
app.config['POCS_cut'].update(params)
except KeyError:
for k, v in params.items():
app.config['POCS_cut'].setdefault(k, v)
# Config has been modified so save to file.
save_local = app.config['save_local']
logger.info(f'Setting config save_local={save_local!r}')
if save_local and app.config['config_file'] is not None:
save_config(app.config['config_file'], app.config['POCS_cut'].copy())
return jsonify(params)
def get_rgb_background(fits_fn,
box_size=(84, 84),
filter_size=(3, 3),
camera_bias=0,
estimator='mean',
interpolator='zoom',
sigma=5,
iters=5,
exclude_percentile=100,
return_separate=False,
*args,
**kwargs):
"""Get the background for each color channel.
Most of the options are described in the `photutils.Background2D` page:
https://photutils.readthedocs.io/en/stable/background.html#d-background-and-noise-estimation
>>> from panoptes.utils.images import fits as fits_utils
>>> fits_fn = getfixture('solved_fits_file')
>>> data = fits_utils.getdata(fits_fn)
>>> data.mean()
2236.816...
>>> rgb_back = get_rgb_background(fits_fn)
>>> rgb_back.mean()
2202.392...
>>> rgb_backs = get_rgb_background(fits_fn, return_separate=True)
>>> rgb_backs[0]
<photutils.background.background_2d.Background2D...>
>>> {color:data.background_rms_median for color, data in zip('rgb', rgb_backs)}
{'r': 20.566..., 'g': 32.787..., 'b': 23.820...}
Args:
fits_fn (str): The filename of the FITS image.
box_size (tuple, optional): The box size over which to compute the
2D-Background, default (84, 84).
filter_size (tuple, optional): The filter size for determining the median,
default (3, 3).
camera_bias (int, optional): The built-in camera bias, default 0. A zero camera
bias means the bias will be considered as part of the background.
estimator (str, optional): The estimator object to use, default 'median'.
interpolator (str, optional): The interpolater object to user, default 'zoom'.
sigma (int, optional): The sigma on which to filter values, default 5.
iters (int, optional): The number of iterations to sigma filter, default 5.
exclude_percentile (int, optional): The percentage of the data (per channel)
that can be masked, default 100 (i.e. all).
return_separate (bool, optional): If the function should return a separate array
for color channel, default False.
*args: Description
**kwargs: Description
Returns:
`numpy.array`|list: Either a single numpy array representing the entire
background, or a list of masked numpy arrays in RGB order. The background
for each channel has full interploation across all pixels, but the mask covers
them.
"""
logger.info(f"Getting background for {fits_fn}")
logger.debug(
f"{estimator} {interpolator} {box_size} {filter_size} {camera_bias} σ={sigma} n={iters}"
)
estimators = {
'sexb': SExtractorBackground,
'median': MedianBackground,
'mean': MeanBackground,
'mmm': MMMBackground
}
interpolators = {
'zoom': BkgZoomInterpolator,
}
bkg_estimator = estimators[estimator]()
interp = interpolators[interpolator]()
data = fits_utils.getdata(fits_fn) - camera_bias
# Get the data per color channel.
rgb_data = get_rgb_data(data)
backgrounds = list()
for color, color_data in zip(['R', 'G', 'B'], rgb_data):
logger.debug(f'Performing background {color} for {fits_fn}')
bkg = Background2D(color_data,
box_size,
filter_size=filter_size,
sigma_clip=SigmaClip(sigma=sigma, maxiters=iters),
bkg_estimator=bkg_estimator,
exclude_percentile=exclude_percentile,
mask=color_data.mask,
interpolator=interp)
# Create a masked array for the background
if return_separate:
backgrounds.append(bkg)
else:
backgrounds.append(
np.ma.array(data=bkg.background, mask=color_data.mask))
logger.debug(
f"{color} Value: {bkg.background_median:.02f} RMS: {bkg.background_rms_median:.02f}"
)
if return_separate:
return backgrounds
# Create one array for the backgrounds, where any holes are filled with zeros.
full_background = np.ma.array(backgrounds).sum(0).filled(0)
return full_background
def get_solve_field(fname, replace=True, overwrite=True, timeout=30, **kwargs):
"""Convenience function to wait for `solve_field` to finish.
This function merely passes the `fname` of the image to be solved along to `solve_field`,
which returns a subprocess.Popen object. This function then waits for that command
to complete, populates a dictonary with the EXIF informaiton and returns. This is often
more useful than the raw `solve_field` function.
Example:
>>> from panoptes.utils.images import fits as fits_utils
>>> # Get our fits filename.
>>> fits_fn = getfixture('unsolved_fits_file')
>>> # Perform the solve.
>>> solve_info = fits_utils.get_solve_field(fits_fn)
>>> # Show solved filename.
>>> solve_info['solved_fits_file']
'.../unsolved.fits'
>>> # Pass a suggested location.
>>> ra = 15.23
>>> dec = 90
>>> radius = 5 # deg
>>> solve_info = fits_utils.solve_field(fits_fn, ra=ra, dec=dec, radius=radius)
>>> # Pass kwargs to `solve-field` program.
>>> solve_kwargs = {'--pnm': '/tmp/awesome.bmp', '--overwrite': True}
>>> solve_info = fits_utils.get_solve_field(fits_fn, **solve_kwargs, skip_solved=False)
>>> assert os.path.exists('/tmp/awesome.bmp')
Args:
fname ({str}): Name of FITS file to be solved.
replace (bool, optional): Saves the WCS back to the original file,
otherwise output base filename with `.new` extension. Default True.
overwrite (bool, optional): Clobber file, default True. Required if `replace=True`.
timeout (int, optional): The timeout for solving, default 30 seconds.
**kwargs ({dict}): Options to pass to `solve_field` should start with `--`.
Returns:
dict: Keyword information from the solved field.
"""
skip_solved = kwargs.get('skip_solved', True)
out_dict = {}
output = None
errs = None
header = getheader(fname)
wcs = WCS(header)
# Check for solved file
if skip_solved and wcs.is_celestial:
logger.info(
f"Skipping solved file (use skip_solved=False to solve again): {fname}"
)
out_dict.update(header)
out_dict['solved_fits_file'] = fname
return out_dict
# Set a default radius of 15
if overwrite:
kwargs['--overwrite'] = True
# Use unpacked version of file.
was_compressed = False
if fname.endswith('.fz'):
logger.debug(f'Uncompressing {fname}')
fname = funpack(fname)
logger.debug(f'Using {fname} for solving')
was_compressed = True
logger.debug(f'Solving with: {kwargs!r}')
proc = solve_field(fname, **kwargs)
try:
output, errs = proc.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
proc.kill()
output, errs = proc.communicate()
raise error.Timeout(f'Timeout while solving: {output!r} {errs!r}')
else:
if proc.returncode != 0:
logger.debug(f'Returncode: {proc.returncode}')
for log in [output, errs]:
if log and log > '':
logger.debug(f'Output on {fname}: {log}')
if proc.returncode == 3:
raise error.SolveError(f'solve-field not found: {output}')
new_fname = fname.replace('.fits', '.new')
if replace:
logger.debug(f'Overwriting original {fname}')
os.replace(new_fname, fname)
else:
fname = new_fname
try:
header = getheader(fname)
header.remove('COMMENT', ignore_missing=True, remove_all=True)
header.remove('HISTORY', ignore_missing=True, remove_all=True)
out_dict.update(header)
except OSError:
logger.warning(f"Can't read fits header for: {fname}")
# Check it was solved.
if WCS(header).is_celestial is False:
raise error.SolveError(
'File not properly solved, no WCS header present.')
# Remove WCS file.
os.remove(fname.replace('.fits', '.wcs'))
if was_compressed and replace:
logger.debug(f'Compressing plate-solved {fname}')
fname = fpack(fname)
out_dict['solved_fits_file'] = fname
return out_dict
def stop(context):
"""Stops the config server by setting a flag in the server itself."""
host = context.obj.get('host')
port = context.obj.get('port')
logger.info(f'Shutting down config server on {host}:{port}')
set_config('config_server.running', False, host=host, port=port)
def set_config(key, new_value, host=None, port=None, parse=True):
"""Set config item in config server.
Given a `key` entry, update the config to match. The `key` is a dot accessible
string, as given by `scalpl <https://pypi.org/project/scalpl/>`_. See Examples in
:func:`get_config` for details.
Examples:
.. doctest::
>>> from astropy import units as u
>>> # Can use astropy units.
>>> set_config('location.horizon', 35 * u.degree)
{'location.horizon': <Quantity 35. deg>}
>>> get_config(key='location.horizon')
<Quantity 35. deg>
>>> # String equivalent works for 'deg', 'm', 's'.
>>> set_config('location.horizon', '30 deg')
{'location.horizon': <Quantity 30. deg>}
Args:
key (str): The key to update, see Examples in :func:`get_config` for details.
new_value (scalar|object): The new value for the key, can be any serializable object.
host (str, optional): The config server host. First checks for PANOPTES_CONFIG_HOST
env var, defaults to 'localhost'.
port (str or int, optional): The config server port. First checks for PANOPTES_CONFIG_HOST
env var, defaults to 6563.
parse (bool, optional): If response should be parsed by
:func:`panoptes.utils.serializers.from_json`, default True.
Returns:
dict: The updated config entry.
Raises:
Exception: Raised if the config server is not available.
"""
host = host or os.getenv('PANOPTES_CONFIG_HOST', 'localhost')
port = port or os.getenv('PANOPTES_CONFIG_PORT', 6563)
url = f'http://{host}:{port}/set-config'
json_str = to_json({key: new_value})
config_entry = None
try:
# We use our own serializer so pass as `data` instead of `json`.
logger.info(f'Calling set_config on url={url!r}')
response = requests.post(url,
data=json_str,
headers={'Content-Type': 'application/json'})
if not response.ok: # pragma: no cover
raise Exception(f'Cannot access config server: {response.text}')
except Exception as e:
logger.warning(f'Problem with set_config: {e!r}')
else:
if parse:
config_entry = from_json(response.content.decode('utf8'))
else:
config_entry = response.json()
return config_entry