def test_make_image(test_directory):
"""Use PreviewImage.make_image to create preview images of a sample JWST exposure.
Assert that the number of JPGs created corresponds to the number of integrations.
Parameters
----------
test_directory : str
Path of directory used for testing
"""
filenames = glob.glob(os.path.join(TEST_DATA_DIRECTORY, '*.fits'))
print('\nGenerating preview images for {}.'.format(filenames))
output_directory = test_directory
for filename in filenames:
header = fits.getheader(filename)
# Create and save the preview image or thumbnail
for create_thumbnail in [False, True]:
try:
image = PreviewImage(filename, "SCI")
image.clip_percent = 0.01
image.scaling = 'log'
image.cmap = 'viridis'
image.output_format = 'jpg'
image.thumbnail = create_thumbnail
image.output_directory = output_directory
image.make_image()
except ValueError as error:
print(error)
if create_thumbnail:
extension = 'thumb'
else:
extension = 'jpg'
# list of preview images
preview_image_filenames = glob.glob(
os.path.join(test_directory, '*.{}'.format(extension)))
assert len(preview_image_filenames) == header['NINTS']
# clean up: delete preview images
for file in preview_image_filenames:
os.remove(file)
def get_image_info(file_root, rewrite):
"""Build and return a dictionary containing information for a given
``file_root``.
Parameters
----------
file_root : str
The rootname of the file of interest.
rewrite : bool
``True`` if the corresponding JPEG needs to be rewritten,
``False`` if not.
Returns
-------
image_info : dict
A dictionary containing various information for the given
``file_root``.
"""
# Initialize dictionary to store information
image_info = {}
image_info['all_jpegs'] = []
image_info['suffixes'] = []
image_info['num_ints'] = {}
preview_dir = os.path.join(get_config()['jwql_dir'], 'preview_images')
# Find all of the matching files
dirname = file_root[:7]
search_filepath = os.path.join(FILESYSTEM_DIR, dirname,
file_root + '*.fits')
image_info['all_files'] = glob.glob(search_filepath)
for file in image_info['all_files']:
# Get suffix information
suffix = os.path.basename(file).split('_')[4].split('.')[0]
image_info['suffixes'].append(suffix)
# Determine JPEG file location
jpg_dir = os.path.join(preview_dir, dirname)
jpg_filename = os.path.basename(
os.path.splitext(file)[0] + '_integ0.jpg')
jpg_filepath = os.path.join(jpg_dir, jpg_filename)
# Check that a jpg does not already exist. If it does (and rewrite=False),
# just call the existing jpg file
if os.path.exists(jpg_filepath) and not rewrite:
pass
# If it doesn't, make it using the preview_image module
else:
if not os.path.exists(jpg_dir):
os.makedirs(jpg_dir)
im = PreviewImage(file, 'SCI')
im.output_directory = jpg_dir
im.make_image()
# Record how many integrations there are per filetype
search_jpgs = os.path.join(preview_dir, dirname,
file_root + '_{}_integ*.jpg'.format(suffix))
num_jpgs = len(glob.glob(search_jpgs))
image_info['num_ints'][suffix] = num_jpgs
image_info['all_jpegs'].append(jpg_filepath)
return image_info
def process_program(program):
"""Generate preview images and thumbnails for the given program.
Parameters
----------
program : str
The program identifier (e.g. ``88600``)
"""
# Group together common exposures
filenames = glob.glob(
os.path.join(get_config()['filesystem'], program, '*.fits'))
grouped_filenames = group_filenames(filenames)
logging.info('Found {} filenames'.format(len(filenames)))
for file_list in grouped_filenames:
filename = file_list[0]
# Determine the save location
try:
identifier = 'jw{}'.format(filename_parser(filename)['program_id'])
except ValueError:
identifier = os.path.basename(filename).split('.fits')[0]
preview_output_directory = os.path.join(
get_config()['preview_image_filesystem'], identifier)
thumbnail_output_directory = os.path.join(
get_config()['thumbnail_filesystem'], identifier)
# Check to see if the preview images already exist and skip if they do
file_exists = check_existence(file_list, preview_output_directory)
if file_exists:
logging.info(
"JPG already exists for {}, skipping.".format(filename))
continue
# Create the output directories if necessary
if not os.path.exists(preview_output_directory):
os.makedirs(preview_output_directory)
permissions.set_permissions(preview_output_directory)
logging.info(
'Created directory {}'.format(preview_output_directory))
if not os.path.exists(thumbnail_output_directory):
os.makedirs(thumbnail_output_directory)
permissions.set_permissions(thumbnail_output_directory)
logging.info(
'Created directory {}'.format(thumbnail_output_directory))
# If the exposure contains more than one file (because more
# than one detector was used), then create a mosaic
max_size = 8
numfiles = len(file_list)
if numfiles > 1:
try:
mosaic_image, mosaic_dq = create_mosaic(file_list)
logging.info('Created mosiac for:')
for item in file_list:
logging.info('\t{}'.format(item))
except (ValueError, FileNotFoundError) as error:
logging.error(error)
dummy_file = create_dummy_filename(file_list)
if numfiles in [2, 4]:
max_size = 16
elif numfiles in [8]:
max_size = 32
# Create the nominal preview image and thumbnail
try:
im = PreviewImage(filename, "SCI")
im.clip_percent = 0.01
im.scaling = 'log'
im.cmap = 'viridis'
im.output_format = 'jpg'
im.preview_output_directory = preview_output_directory
im.thumbnail_output_directory = thumbnail_output_directory
# If a mosaic was made from more than one file
# insert it and it's associated DQ array into the
# instance of PreviewImage. Also set the input
# filename to indicate that we have mosaicked data
if numfiles != 1:
im.data = mosaic_image
im.dq = mosaic_dq
im.file = dummy_file
im.make_image(max_img_size=max_size)
logging.info(
'Created preview image and thumbnail for: {}'.format(filename))
except ValueError as error:
logging.warning(error)
def create_mosaic(filenames):
"""If an exposure comprises data from multiple detectors read in all
the appropriate files and create a mosaic so that the preview image
will show all the data together.
Parameters
----------
filenames : list
List of filenames to be combined into a mosaic
Returns
-------
mosaic_filename : str
Name of fits file containing the mosaicked data
"""
# Use preview_image to load data and create difference image
# for each detector. Save in a list
data = []
detector = []
data_lower_left = []
for filename in filenames:
image = PreviewImage(filename, "SCI") # Now have image.data, image.dq
data_dim = len(image.data.shape)
if data_dim == 4:
diff_im = image.difference_image(image.data)
else:
diff_im = image.data
data.append(diff_im)
detector.append(filename_parser(filename)['detector'].upper())
data_lower_left.append((image.xstart, image.ystart))
# Make sure SW and LW data are not being mixed. Create the
# appropriately sized numpy array to hold all the data based
# on the channel, module, and subarray size
mosaic_channel = find_data_channel(detector)
full_xdim, full_ydim, full_lower_left = array_coordinates(
mosaic_channel, detector, data_lower_left)
# Create the array to hold all the data
datashape = data[0].shape
datadim = len(datashape)
if datadim == 2:
full_array = np.zeros((1, full_ydim, full_xdim)) * np.nan
elif datadim == 3:
full_array = np.zeros((datashape[0], full_ydim, full_xdim)) * np.nan
else:
raise ValueError(
('Difference image for {} must be either 2D or 3D.'.format(
filenames[0])))
# Place the data from the individual detectors in the appropriate
# places in the final image
for pixdata, detect in zip(data, detector):
x0, y0 = full_lower_left[detect]
if datadim == 2:
yd, xd = pixdata.shape
full_array[0, y0:y0 + yd, x0:x0 + xd] = pixdata
elif datadim == 3:
ints, yd, xd = pixdata.shape
full_array[:, y0:y0 + yd, x0:x0 + xd] = pixdata
# Create associated DQ array and set unpopulated pixels to be skipped
# in preview image scaling
full_dq = create_dq_array(full_xdim, full_ydim, full_array[0, :, :],
mosaic_channel)
return full_array, full_dq
def generate_preview_images():
"""The main function of the ``generate_preview_image`` module."""
# Begin logging
logging.info("Beginning the script run")
filesystem = get_config()['filesystem']
preview_image_filesystem = get_config()['preview_image_filesystem']
thumbnail_filesystem = get_config()['thumbnail_filesystem']
filenames = glob(os.path.join(filesystem, '*/*.fits'))
grouped_filenames = group_filenames(filenames)
logging.info(f"Found {len(filenames)} filenames")
for file_list in grouped_filenames:
filename = file_list[0]
# Determine the save location
try:
identifier = 'jw{}'.format(filename_parser(filename)['program_id'])
except ValueError as error:
identifier = os.path.basename(filename).split('.fits')[0]
preview_output_directory = os.path.join(preview_image_filesystem,
identifier)
thumbnail_output_directory = os.path.join(thumbnail_filesystem,
identifier)
# Check to see if the preview images already exist and skip
# if they do
file_exists = check_existence(file_list, preview_output_directory)
if file_exists:
logging.info(
"JPG already exists for {}, skipping.".format(filename))
continue
# Create the output directories if necessary
if not os.path.exists(preview_output_directory):
os.makedirs(preview_output_directory)
permissions.set_permissions(preview_output_directory)
logging.info(f'Created directory {preview_output_directory}')
if not os.path.exists(thumbnail_output_directory):
os.makedirs(thumbnail_output_directory)
permissions.set_permissions(thumbnail_output_directory)
logging.info(f'Created directory {thumbnail_output_directory}')
# If the exposure contains more than one file (because more
# than one detector was used), then create a mosaic
max_size = 8
numfiles = len(file_list)
if numfiles != 1:
try:
mosaic_image, mosaic_dq = create_mosaic(file_list)
logging.info('Created mosiac for:')
for item in file_list:
logging.info(f'\t{item}')
except (ValueError, FileNotFoundError) as error:
logging.error(error)
dummy_file = create_dummy_filename(file_list)
if numfiles in [2, 4]:
max_size = 16
elif numfiles in [8]:
max_size = 32
# Create the nominal preview image and thumbnail
try:
im = PreviewImage(filename, "SCI")
im.clip_percent = 0.01
im.scaling = 'log'
im.cmap = 'viridis'
im.output_format = 'jpg'
im.preview_output_directory = preview_output_directory
im.thumbnail_output_directory = thumbnail_output_directory
# If a mosaic was made from more than one file
# insert it and it's associated DQ array into the
# instance of PreviewImage. Also set the input
# filename to indicate that we have mosaicked data
if numfiles != 1:
im.data = mosaic_image
im.dq = mosaic_dq
im.file = dummy_file
im.make_image(max_img_size=max_size)
except ValueError as error:
logging.warning(error)
# Complete logging:
logging.info("Completed.")