def test_make_rgb(self):
"""Test the function that does it all"""
satValue = 1000.0
with tempfile.NamedTemporaryFile(suffix=".png") as temp:
red = saturate(self.image_r, satValue)
green = saturate(self.image_g, satValue)
blue = saturate(self.image_b, satValue)
lupton_rgb.make_lupton_rgb(red, green, blue, self.min_, self.stretch_, self.Q, filename=temp)
assert os.path.exists(temp.name)
def test_make_rgb_saturated_fix(self):
pytest.skip('saturation correction is not implemented')
satValue = 1000.0
# TODO: Cannot test with these options yet, as that part of the code is not implemented.
with tempfile.NamedTemporaryFile(suffix=".png") as temp:
red = saturate(self.image_r, satValue)
green = saturate(self.image_g, satValue)
blue = saturate(self.image_b, satValue)
lupton_rgb.make_lupton_rgb(red, green, blue, self.min_, self.stretch_, self.Q,
saturated_border_width=1, saturated_pixel_value=2000,
filename=temp)
def test_make_rgb(self):
"""Test the function that does it all"""
satValue = 1000.0
with tempfile.NamedTemporaryFile(suffix=".png") as temp:
red = saturate(self.image_r, satValue)
green = saturate(self.image_g, satValue)
blue = saturate(self.image_b, satValue)
lupton_rgb.make_lupton_rgb(red,
green,
blue,
self.min_,
self.stretch_,
self.Q,
filename=temp)
assert os.path.exists(temp.name)
def test_make_rgb_saturated_fix(self):
pytest.skip('saturation correction is not implemented')
satValue = 1000.0
# TODO: Cannot test with these options yet, as that part of the code is not implemented.
with tempfile.NamedTemporaryFile(suffix=".png") as temp:
red = saturate(self.image_r, satValue)
green = saturate(self.image_g, satValue)
blue = saturate(self.image_b, satValue)
lupton_rgb.make_lupton_rgb(red,
green,
blue,
self.min_,
self.stretch_,
self.Q,
saturated_border_width=1,
saturated_pixel_value=2000,
filename=temp)
def test_different_shapes_asserts(self):
with pytest.raises(ValueError) as excinfo:
# just swap the dimensions to get a differently-shaped 'r'
image_r = self.image_r.reshape(self.height, self.width)
lupton_rgb.make_lupton_rgb(image_r, self.image_g, self.image_b)
assert "shapes must match" in str(excinfo.value)
def test_different_shapes_asserts(self):
with pytest.raises(ValueError) as excinfo:
# just swap the dimensions to get a differently-shaped 'r'
image_r = self.image_r.reshape(self.height, self.width)
lupton_rgb.make_lupton_rgb(image_r, self.image_g, self.image_b)
assert "shapes must match" in str(excinfo.value)
def rgb_image(target='galaxy',
file_name='galaxy*.fits',
blue_f='f435w',
green_f='f814w',
red_f='f110w',
plot=False):
"""
Function that creates an RGB image of an input target.
INPUTS:
target: The name of the target, to be used in the output files.
file_name: General form of the file names that contain the RGB filter images.
blue_f: Filter in the blue band.
green_f: Filter in the green band.
red_f: Filter in the red band.
KEYWORDS:
PLOT: Set this keyword to produce a plot of the two-dimensional
RGB image.
OUTPUTS:
Fits file with the RGB image.
"""
import numpy as np
import matplotlib.pyplot as plt
import astropy.io.fits as pyfits
import glob
from grizli import utils
from astropy.visualization import lupton_rgb
print('Target = ', target)
files = glob.glob(file_name)
files.sort()
images = {}
headers = {}
bandpasses = {}
for file in files:
im = pyfits.open(file)
filt = utils.get_hst_filter(im[0].header)
for ext in [0, 1]:
if 'PHOTMODE' in im[ext].header:
photflam = im[1].header['IM2FLAM']
headers[filt.lower()] = im[1].header
break
images[filt.lower()] = im['SCI'].data
blue = images[blue_f] * headers[blue_f]['IM2FLAM'] / 1.e-19
green = images[green_f] * headers[green_f]['IM2FLAM'] / 1.e-19
red = images[red_f] * headers[red_f]['IM2FLAM'] / 1.e-19
rgb = lupton_rgb.make_lupton_rgb(red,
green,
blue,
minimum=-0.1,
stretch=1,
Q=8)
pyfits.writeto('rgb_{0}.fits'.format(target), data=rgb, overwrite=True)
print('RGB image created')
if plot:
fig = plt.figure(figsize=[6, 6])
ax = fig.add_subplot(111)
imsh = ax.imshow(rgb, origin='lower')
def _run_preprocess(self, galaxies, guid, counter, ckpt_objs):
"""Run preprocessing pipeline for given galaxies."""
dl_count = 0
galaxy_count = 0
run_dir = os.getcwd()
redshift_objects = ckpt_objs
objects_on_disk = CheckPoint.get_object_set(self.checkpoint_dir, guid)
last_modified = CheckPoint.last_modified(self.checkpoint_dir, guid)
self.logger.debug(f'Objects on disk: {len(objects_on_disk)}, last Modified: {last_modified}')
for i, galaxy in galaxies.iterrows():
if CheckPoint.last_modified(self.checkpoint_dir, guid) > last_modified:
self.logger.debug('Loading new checkpoint, as the last checkpoint was updated.')
objects_on_disk = CheckPoint.get_object_set(self.checkpoint_dir, guid)
last_modified = CheckPoint.last_modified(self.checkpoint_dir, guid)
self.logger.debug(f'New objects on disk: {len(objects_on_disk)}, last Modified: {last_modified}')
if galaxy['specObjID'] in objects_on_disk:
self.logger.debug(f'Preprocessed image for galaxy with id {galaxy["specObjID"]} already saved, skipping')
continue
download_urls = self._get_formatted_urls(galaxy['rerun'],
galaxy['run'],
galaxy['camcol'],
galaxy['field'])
files = []
# Download the images
for url in download_urls:
filename = url.split('/')[-1]
files.append(self.fits_download_loc + '/' + filename)
if Path(self.fits_download_loc).joinpath(filename.replace('.bz2', '')).exists():
self.logger.debug('Compressed file exists, skipping download.')
else:
self.logger.debug('Downloading compressed file {}'
' from the sdss url to {}'.format(filename,
self.fits_download_loc + '/' + filename))
subprocess.run('wget {0} -O {1}'.format(url,
self.fits_download_loc + '/' + filename),
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=True)
dl_count += 1
os.chdir(self.fits_download_loc)
# Check all the files exist
assert [os.path.exists(file) or os.path.exists(file.replace('.bz2', ''))
for file in files] == [True] * 5, 'Compressed or uncompressed files missing'
# Extract the fits files
fits_files = []
for file in files:
if Path(file).exists():
op = subprocess.run('bzip2 -dkf {}'.format(file),
capture_output=False,
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False)
assert op.returncode == 0, f'Error decompressing the fits file {file}'
os.remove(file)
else:
assert Path(file.replace('.bz2', '')).exists(), "Uncompressed file doesn't exist"
fits_files.append(file.replace('.bz2', ''))
# Check if the files exist
assert [os.path.exists(file) for file in fits_files] == [True] * 5, 'File {} doesnot exist'.format(file)
assert [file.endswith('.fits') for file in fits_files]
self.logger.debug('Successfully uncompressed files for this galaxy')
data_mat = self._apply_swarp(galaxy, fits_files, cleanup=True)
os.chdir(run_dir)
image_filename = '{0}/{1}-{2}-{3}.jpg'.format(os.path.join(self.checkpoint_dir, 'galaxies'),
'galaxy', galaxy['specObjID'], galaxy['z'])
make_lupton_rgb(data_mat[:, :, 3],
data_mat[:, :, 2],
data_mat[:, :, 1],
Q=8, stretch=0.4,
filename=image_filename)
self.logger.debug('Galaxy image saved as {}'.format(image_filename))
assert dl_count == 5, 'Downloaded only {} fits files'.format(dl_count)
self.logger.debug('Completed pre-processing for this galaxy with redshift value {} at index {}'
.format(galaxy['z'], galaxy_count))
redshift_objects.put(dict(key=galaxy['specObjID'],
np_array=data_mat,
redshift=galaxy['z'],
galaxy_meta=galaxy,
image=image_filename,
timestamp=datetime.now()))
galaxy_count += 1
counter.value += 1
self.logger.info(f'Galaxy count: {counter.value}, steps till next checkpoint: '
f'{self.checkpoint_steps - (counter.value % self.checkpoint_steps)}')
dl_count = 0
if counter.value % self.checkpoint_steps == 0:
self.save_checkpoint(objs=redshift_objects, guid=guid)
return redshift_objects
