def save_qc_results(runtime_context, qc_results, image, **kwargs):
"""
Save the Quality Control results to ElasticSearch
Parameters
----------
runtime_context: object
Context instance with runtime values
qc_results : dict
Dictionary of key value pairs to be saved to ElasticSearch
image : banzai.images.Image
Image that should be linked
Notes
-----
File name, site, camera, dayobs and timestamp are always saved in the database.
"""
es_output = {}
if getattr(runtime_context, 'post_to_elasticsearch', False):
filename, results_to_save = format_qc_results(qc_results, image)
es = elasticsearch.Elasticsearch(runtime_context.elasticsearch_url)
try:
es_output = es.update(index=runtime_context.elasticsearch_qc_index,
doc_type=runtime_context.elasticsearch_doc_type,
id=filename, body={'doc': results_to_save, 'doc_as_upsert': True},
retry_on_conflict=5, timestamp=results_to_save['@timestamp'], **kwargs)
except Exception:
error_message = 'Cannot update elasticsearch index to URL \"{url}\": {exception}'
logger.error(error_message.format(url=runtime_context.elasticsearch_url,
exception=logs.format_exception()))
return es_output
def process_master_maker(instrument, frame_type, min_date, max_date,
runtime_context):
extra_tags = {
'type': instrument.type,
'site': instrument.site,
'enclosure': instrument.enclosure,
'telescope': instrument.telescope,
'camera': instrument.camera,
'obstype': frame_type,
'min_date': min_date,
'max_date': max_date
}
logger.info("Making master frames", extra_tags=extra_tags)
image_path_list = dbs.get_individual_calibration_images(
instrument,
frame_type,
min_date,
max_date,
db_address=runtime_context.db_address)
if len(image_path_list) == 0:
logger.info("No calibration frames found to stack",
extra_tags=extra_tags)
try:
run_master_maker(image_path_list, runtime_context, frame_type)
except Exception:
logger.error(logs.format_exception())
logger.info("Finished")
def run_realtime_pipeline():
extra_console_arguments = [{'args': ['--n-processes'],
'kwargs': {'dest': 'n_processes', 'default': 12,
'help': 'Number of listener processes to spawn.', 'type': int}},
{'args': ['--queue-name'],
'kwargs': {'dest': 'queue_name', 'default': 'banzai_pipeline',
'help': 'Name of the queue to listen to from the fits exchange.'}}]
runtime_context = parse_args(extra_console_arguments=extra_console_arguments)
# Need to keep the amqp logger level at least as high as INFO,
# or else it send heartbeat check messages every second
logging.getLogger('amqp').setLevel(max(logger.level, getattr(logging, 'INFO')))
try:
dbs.populate_instrument_tables(db_address=runtime_context.db_address)
except Exception:
logger.error('Could not connect to the configdb: {error}'.format(error=logs.format_exception()))
logger.info('Starting pipeline listener')
fits_exchange = Exchange('fits_files', type='fanout')
listener = RealtimeModeListener(runtime_context)
with Connection(runtime_context.broker_url) as connection:
listener.connection = connection.clone()
listener.queue = Queue(runtime_context.queue_name, fits_exchange)
try:
listener.run()
except listener.connection.connection_errors:
listener.connection = connection.clone()
listener.ensure_connection(max_retries=10)
except KeyboardInterrupt:
logger.info('Shutting down pipeline listener.')
def get_primary_header(filename):
try:
hdulist = open_fits_file(filename)
return hdulist[0].header
except Exception:
logger.error("Unable to open fits file: {}".format(logs.format_exception()), extra_tags={'filename': filename})
return None
def run(self, image):
if image is None:
return image
logger.info('Running {0}'.format(self.stage_name), image=image)
try:
image = self.do_stage(image)
except Exception:
logger.error(logs.format_exception())
return image
def get_primary_header(filename):
try:
hdulist = open_fits_file(filename)
return hdulist[0].header
except Exception:
logger.error("Unable to open fits file: {}".format(
logs.format_exception()),
extra_tags={'filename': filename})
return None
def _post_to_archive(self, filepath, runtime_context):
logger.info('Posting file to the archive', image=self)
try:
file_utils.post_to_archive_queue(filepath,
runtime_context.broker_url)
except Exception:
logger.error("Could not post to ingester: {error}".format(
error=logs.format_exception()),
image=self)
def run(self, image):
if image is None:
return image
logger.info('Running {0}'.format(self.stage_name), image=image)
try:
image = self.do_stage(image)
return image
except Exception:
logger.error(logs.format_exception())
return None
def run(self, images):
images.sort(key=self.get_grouping)
processed_images = []
for _, image_set in itertools.groupby(images, self.get_grouping):
try:
image_set = list(image_set)
logger.info('Running {0}'.format(self.stage_name), image=image_set[0])
processed_images += self.do_stage(image_set)
except Exception:
logger.error(logs.format_exception())
return processed_images
def read_image(filename, runtime_context):
try:
image = FRAME_CLASS(runtime_context, filename=filename)
if image.instrument is None:
logger.error("Image instrument attribute is None, aborting", image=image)
raise IOError
munge(image)
return image
except Exception:
logger.error('Error loading image: {error}'.format(error=logs.format_exception()),
extra_tags={'filename': filename})
def run(self, images):
images.sort(key=self.get_grouping)
processed_images = []
for _, image_set in itertools.groupby(images, self.get_grouping):
try:
image_set = list(image_set)
logger.info('Running {0}'.format(self.stage_name),
image=image_set[0])
processed_images += self.do_stage(image_set)
except Exception:
logger.error(logs.format_exception())
return processed_images
def reduce_single_frame():
extra_console_arguments = [{'args': ['--filepath'],
'kwargs': {'dest': 'path', 'help': 'Full path to the file to process'}}]
runtime_context = parse_directory_args(extra_console_arguments=extra_console_arguments)
# Short circuit
if not image_utils.image_can_be_processed(fits_utils.get_primary_header(runtime_context.path),
runtime_context):
logger.error('Image cannot be processed. Check to make sure the instrument '
'is in the database and that the OBSTYPE is recognized by BANZAI',
extra_tags={'filename': runtime_context.path})
return
try:
run(runtime_context.path, runtime_context)
except Exception:
logger.error(logs.format_exception(), extra_tags={'filepath': runtime_context.path})
def read_image(filename, runtime_context):
try:
frame_class = import_utils.import_attribute(
runtime_context.FRAME_CLASS)
image = frame_class(runtime_context, filename=filename)
if image.instrument is None:
logger.error("Image instrument attribute is None, aborting",
image=image)
raise IOError
munge(image)
return image
except Exception:
logger.error('Error loading image: {error}'.format(
error=logs.format_exception()),
extra_tags={'filename': filename})
def select_images(image_list, image_type, db_address, ignore_schedulability):
images = []
for filename in image_list:
try:
header = get_primary_header(filename)
should_process = image_can_be_processed(header, db_address)
should_process &= (image_type is None or get_obstype(header) == image_type)
if not ignore_schedulability:
instrument = dbs.get_instrument(header, db_address=db_address)
should_process &= instrument.schedulable
if should_process:
images.append(filename)
except Exception:
logger.error(logs.format_exception(), extra_tags={'filename': filename})
return images
def do_stage(self, image):
master_calibration_filename = self.get_calibration_filename(image)
if master_calibration_filename is None:
self.on_missing_master_calibration(image)
return image
master_calibration_image = FRAME_CLASS(self.runtime_context, filename=master_calibration_filename)
try:
image_utils.check_image_homogeneity([image, master_calibration_image], self.master_selection_criteria)
except image_utils.InhomogeneousSetException:
logger.error(logs.format_exception(), image=image)
return None
return self.apply_master_calibration(image, master_calibration_image)
def update_db():
parser = argparse.ArgumentParser(description="Query the configdb to ensure that the instruments table"
"has the most up-to-date information")
parser.add_argument("--log-level", default='debug', choices=['debug', 'info', 'warning',
'critical', 'fatal', 'error'])
parser.add_argument('--db-address', dest='db_address',
default='mysql://*****:*****@localhost/test',
help='Database address: Should be in SQLAlchemy form')
args = parser.parse_args()
logs.set_log_level(args.log_level)
try:
dbs.populate_instrument_tables(db_address=args.db_address)
except Exception:
logger.error('Could not populate instruments table: {error}'.format(error=logs.format_exception()))
def process_image(path: str, runtime_context: dict):
runtime_context = Context(runtime_context)
logger.info('Running process image.')
try:
if realtime_utils.need_to_process_image(path, runtime_context):
logger.info('Reducing frame', extra_tags={'filename': os.path.basename(path)})
# Increment the number of tries for this file
realtime_utils.increment_try_number(path, db_address=runtime_context.db_address)
run(path, runtime_context)
realtime_utils.set_file_as_processed(path, db_address=runtime_context.db_address)
except Exception:
logger.error("Exception processing frame: {error}".format(error=logs.format_exception()),
extra_tags={'filename': os.path.basename(path)})
def process_image(path, runtime_context_dict):
logger.info('Running process image.')
runtime_context = Context(runtime_context_dict)
try:
if realtime_utils.need_to_process_image(path, runtime_context,
db_address=runtime_context.db_address,
max_tries=runtime_context.max_tries):
logger.info('Reducing frame', extra_tags={'filename': os.path.basename(path)})
# Increment the number of tries for this file
realtime_utils.increment_try_number(path, db_address=runtime_context.db_address)
run(path, runtime_context)
realtime_utils.set_file_as_processed(path, db_address=runtime_context.db_address)
except Exception:
logger.error("Exception processing frame: {error}".format(error=logs.format_exception()),
extra_tags={'filename': os.path.basename(path)})
def select_images(image_list, image_type, context):
images = []
for filename in image_list:
try:
header = get_primary_header(filename)
should_process = image_can_be_processed(header, context)
should_process &= (image_type is None
or get_obstype(header) == image_type)
if not context.ignore_schedulability:
instrument = dbs.get_instrument(header,
db_address=context.db_address)
should_process &= instrument.schedulable
if should_process:
images.append(filename)
except Exception:
logger.error(logs.format_exception(),
extra_tags={'filename': filename})
return images
def process_master_maker(runtime_context, instrument, frame_type, min_date, max_date, use_masters=False):
extra_tags = {'type': instrument.type, 'site': instrument.site,
'enclosure': instrument.enclosure, 'telescope': instrument.telescope,
'camera': instrument.camera, 'obstype': frame_type,
'min_date': min_date,
'max_date': max_date}
logger.info("Making master frames", extra_tags=extra_tags)
image_path_list = dbs.get_individual_calibration_images(instrument, frame_type, min_date, max_date,
use_masters=use_masters,
db_address=runtime_context.db_address)
if len(image_path_list) == 0:
logger.info("No calibration frames found to stack", extra_tags=extra_tags)
try:
run_master_maker(image_path_list, runtime_context, frame_type)
except Exception:
logger.error(logs.format_exception())
logger.info("Finished")
def do_stage(self, images):
try:
min_images = settings.CALIBRATION_MIN_FRAMES[self.calibration_type.upper()]
except KeyError:
msg = 'The minimum number of frames required to create a master calibration of type ' \
'{calibration_type} has not been specified in the settings.'
logger.error(msg.format(calibration_type=self.calibration_type.upper()))
return []
if len(images) < min_images:
# Do nothing
msg = 'Number of images less than minimum requirement of {min_images}, not combining'
logger.warning(msg.format(min_images=min_images))
return []
try:
image_utils.check_image_homogeneity(images, self.group_by_attributes())
except image_utils.InhomogeneousSetException:
logger.error(logs.format_exception())
return []
return [self.make_master_calibration_frame(images)]
def save_qc_results(runtime_context, qc_results, image, **kwargs):
"""
Save the Quality Control results to ElasticSearch
Parameters
----------
runtime_context: object
Context instance with runtime values
qc_results : dict
Dictionary of key value pairs to be saved to ElasticSearch
image : banzai.images.Image
Image that should be linked
Notes
-----
File name, site, camera, dayobs and timestamp are always saved in the database.
"""
es_output = {}
if getattr(runtime_context, 'post_to_elasticsearch', False):
filename, results_to_save = format_qc_results(qc_results, image)
es = elasticsearch.Elasticsearch(runtime_context.elasticsearch_url)
try:
es_output = es.update(
index=runtime_context.elasticsearch_qc_index,
doc_type=runtime_context.elasticsearch_doc_type,
id=filename,
body={
'doc': results_to_save,
'doc_as_upsert': True
},
retry_on_conflict=5,
timestamp=results_to_save['@timestamp'],
**kwargs)
except Exception:
error_message = 'Cannot update elasticsearch index to URL \"{url}\": {exception}'
logger.error(
error_message.format(url=runtime_context.elasticsearch_url,
exception=logs.format_exception()))
return es_output
def do_stage(self, images):
try:
min_images = settings.CALIBRATION_MIN_FRAMES[
self.calibration_type.upper()]
except KeyError:
msg = 'The minimum number of frames required to create a master calibration of type ' \
'{calibration_type} has not been specified in the settings.'
logger.error(
msg.format(calibration_type=self.calibration_type.upper()))
return []
if len(images) < min_images:
# Do nothing
msg = 'Number of images less than minimum requirement of {min_images}, not combining'
logger.warning(msg.format(min_images=min_images))
return []
try:
image_utils.check_image_homogeneity(images,
self.group_by_attributes())
except image_utils.InhomogeneousSetException:
logger.error(logs.format_exception())
return []
return [self.make_master_calibration_frame(images)]
def do_stage(self, image):
try:
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))
data = image.data.copy()
error = (np.abs(data) + image.readnoise**2.0)**0.5
mask = image.bpm > 0
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
bkg.subfrom(data)
# Do an initial source detection
# TODO: Add back in masking after we are sure SEP works
sources = sep.extract(data,
self.threshold,
minarea=self.min_area,
err=error,
deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# We remove anything with a detection flag >= 8
# This includes memory overflows and objects that are too close the edge
sources = sources[sources['flag'] < 8]
sources = array_utils.prune_nans_from_table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'],
sources['a'], sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data,
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 * kronrad,
subpix=1,
err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Do circular aperture photometry for diameters of 1" to 6"
for diameter in [1, 2, 3, 4, 5, 6]:
flux, fluxerr, flag = sep.sum_circle(data,
sources['x'],
sources['y'],
diameter / 2.0 /
image.pixel_scale,
gain=1.0,
err=error)
sources['fluxaper{0}'.format(diameter)] = flux
sources['fluxerr{0}'.format(diameter)] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) *
(sources['a']**2.0 + sources['b']**2.0))**0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Measure the flux profile
flux_radii, flag = sep.flux_radius(data,
sources['x'],
sources['y'],
6.0 * sources['a'],
[0.25, 0.5, 0.75],
normflux=sources['flux'],
subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# Calculate the windowed positions
sig = 2.0 / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'],
sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# Calculate the average background at each source
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(),
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 * sources['kronrad'],
subpix=1)
# masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
# sources['a'], sources['b'], np.pi / 2.0,
# 2.5 * kronrad, subpix=1)
background_area = (
2.5 * sources['kronrad']
)**2.0 * sources['a'] * sources['b'] * np.pi # - masksum
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[
background_area > 0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['xwin'] += 1.0
sources['ywin'] += 1.0
sources['theta'] = np.degrees(sources['theta'])
catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak', 'ypeak',
'flux', 'fluxerr', 'peak', 'fluxaper1',
'fluxerr1', 'fluxaper2', 'fluxerr2', 'fluxaper3',
'fluxerr3', 'fluxaper4', 'fluxerr4', 'fluxaper5',
'fluxerr5', 'fluxaper6', 'fluxerr6',
'background', 'fwhm', 'a', 'b', 'theta',
'kronrad', 'ellipticity', 'fluxrad25',
'fluxrad50', 'fluxrad75', 'x2', 'y2', 'xy',
'flag']
# Add the units and description to the catalogs
catalog['x'].unit = 'pixel'
catalog['x'].description = 'X coordinate of the object'
catalog['y'].unit = 'pixel'
catalog['y'].description = 'Y coordinate of the object'
catalog['xwin'].unit = 'pixel'
catalog['xwin'].description = 'Windowed X coordinate of the object'
catalog['ywin'].unit = 'pixel'
catalog['ywin'].description = 'Windowed Y coordinate of the object'
catalog['xpeak'].unit = 'pixel'
catalog['xpeak'].description = 'X coordinate of the peak'
catalog['ypeak'].unit = 'pixel'
catalog['ypeak'].description = 'Windowed Y coordinate of the peak'
catalog['flux'].unit = 'count'
catalog[
'flux'].description = 'Flux within a Kron-like elliptical aperture'
catalog['fluxerr'].unit = 'count'
catalog[
'fluxerr'].description = 'Error on the flux within Kron aperture'
catalog['peak'].unit = 'count'
catalog['peak'].description = 'Peak flux (flux at xpeak, ypeak)'
for diameter in [1, 2, 3, 4, 5, 6]:
catalog['fluxaper{0}'.format(diameter)].unit = 'count'
catalog['fluxaper{0}'.format(
diameter
)].description = 'Flux from fixed circular aperture: {0}" diameter'.format(
diameter)
catalog['fluxerr{0}'.format(diameter)].unit = 'count'
catalog['fluxerr{0}'.format(
diameter
)].description = 'Error on Flux from circular aperture: {0}"'.format(
diameter)
catalog['background'].unit = 'count'
catalog[
'background'].description = 'Average background value in the aperture'
catalog['fwhm'].unit = 'pixel'
catalog['fwhm'].description = 'FWHM of the object'
catalog['a'].unit = 'pixel'
catalog['a'].description = 'Semi-major axis of the object'
catalog['b'].unit = 'pixel'
catalog['b'].description = 'Semi-minor axis of the object'
catalog['theta'].unit = 'degree'
catalog['theta'].description = 'Position angle of the object'
catalog['kronrad'].unit = 'pixel'
catalog['kronrad'].description = 'Kron radius used for extraction'
catalog['ellipticity'].description = 'Ellipticity'
catalog['fluxrad25'].unit = 'pixel'
catalog[
'fluxrad25'].description = 'Radius containing 25% of the flux'
catalog['fluxrad50'].unit = 'pixel'
catalog[
'fluxrad50'].description = 'Radius containing 50% of the flux'
catalog['fluxrad75'].unit = 'pixel'
catalog[
'fluxrad75'].description = 'Radius containing 75% of the flux'
catalog['x2'].unit = 'pixel^2'
catalog[
'x2'].description = 'Variance on X coordinate of the object'
catalog['y2'].unit = 'pixel^2'
catalog[
'y2'].description = 'Variance on Y coordinate of the object'
catalog['xy'].unit = 'pixel^2'
catalog['xy'].description = 'XY covariance of the object'
catalog[
'flag'].description = 'Bit mask of extraction/photometry flags'
catalog.sort('flux')
catalog.reverse()
# Save some background statistics in the header
mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
image.header['L1MEAN'] = (
mean_background,
'[counts] Sigma clipped mean of frame background')
median_background = np.median(bkg.back())
image.header['L1MEDIAN'] = (median_background,
'[counts] Median of frame background')
std_background = stats.robust_standard_deviation(bkg.back())
image.header['L1SIGMA'] = (
std_background, '[counts] Robust std dev of frame background')
# Save some image statistics to the header
good_objects = catalog['flag'] == 0
for quantity in ['fwhm', 'ellipticity', 'theta']:
good_objects = np.logical_and(
good_objects, np.logical_not(np.isnan(catalog[quantity])))
if good_objects.sum() == 0:
image.header['L1FWHM'] = ('NaN',
'[arcsec] Frame FWHM in arcsec')
image.header['L1ELLIP'] = ('NaN',
'Mean image ellipticity (1-B/A)')
image.header['L1ELLIPA'] = (
'NaN', '[deg] PA of mean image ellipticity')
else:
seeing = np.median(
catalog['fwhm'][good_objects]) * image.pixel_scale
image.header['L1FWHM'] = (seeing,
'[arcsec] Frame FWHM in arcsec')
mean_ellipticity = stats.sigma_clipped_mean(
catalog['ellipticity'][good_objects], 3.0)
image.header['L1ELLIP'] = (mean_ellipticity,
'Mean image ellipticity (1-B/A)')
mean_position_angle = stats.sigma_clipped_mean(
catalog['theta'][good_objects], 3.0)
image.header['L1ELLIPA'] = (
mean_position_angle, '[deg] PA of mean image ellipticity')
logging_tags = {
key: float(image.header[key])
for key in [
'L1MEAN', 'L1MEDIAN', 'L1SIGMA', 'L1FWHM', 'L1ELLIP',
'L1ELLIPA'
]
}
logger.info('Extracted sources',
image=image,
extra_tags=logging_tags)
# adding catalog (a data table) to the appropriate images attribute.
image.data_tables['catalog'] = DataTable(data_table=catalog,
name='CAT')
except Exception:
logger.error(logs.format_exception(), image=image)
return image