def make_movie(hdf_file, fps=15, dpi=100, overwrite=False):
"""
Create animation of the contents of a HDF5 files produced by the photometry pipeline.
The function will create a MP4 movie file with the same name as the input file,
placed in the same directory, containing the animation.
Parameters:
hdf_file (string): Path to the HDF5 file to produce movie from.
fps (integer): Frames per second of generated movie. Default=15.
dpi (integer): DPI of the movie. Default=100.
overwrite (boolean): Overwrite existing MP4 files? Default=False.
.. codeauthor:: Rasmus Handberg <*****@*****.**>
"""
logger = logging.getLogger(__name__)
tqdm_settings = {'disable': not logger.isEnabledFor(logging.INFO)}
logger.info("Processing '%s'", hdf_file)
# File to be created:
output_file = os.path.splitext(hdf_file)[0] + '.mp4'
if os.path.exists(output_file):
if overwrite:
logger.debug("Deleting existing output file")
os.remove(output_file)
else:
logger.info("Movie file already exists")
return output_file
# Open HDF5 file:
# We need to have write-privaledges because we are going to updated some attributes
save_image_scales = False
with h5py.File(hdf_file, 'r') as hdf:
# Load the image scales if they have already been calculated:
vmin = hdf['backgrounds'].attrs.get('movie_vmin')
vmax = hdf['backgrounds'].attrs.get('movie_vmax')
vmin2 = hdf['images'].attrs.get('movie_vmin')
vmax2 = hdf['images'].attrs.get('movie_vmax')
# Calculate scales to use for plotting the images:
if not vmin:
logger.info("Calculating image scales...")
numfiles = len(hdf['images'])
vmax = np.empty(numfiles)
vmin = np.empty(numfiles)
vmax2 = np.empty(numfiles)
vmin2 = np.empty(numfiles)
for k in trange(numfiles, **tqdm_settings):
vmin[k], vmax[k] = np.nanpercentile(
hdf['backgrounds/%04d' % k], [1.0, 99.0])
vmin2[k], vmax2[k] = np.nanpercentile(hdf['images/%04d' % k],
[1.0, 99.0])
vmin = np.nanpercentile(vmin, 25.0)
vmax = np.nanpercentile(vmax, 75.0)
vmin2 = np.nanpercentile(vmin2, 25.0)
vmax2 = np.nanpercentile(vmax2, 75.0)
save_image_scales = True
# If needed, reopen the file for saving the attributes:
if save_image_scales:
with h5py.File(hdf_file, 'r+') as hdf:
# Save image scales to HDF5 file:
hdf['backgrounds'].attrs['movie_vmin'] = vmin
hdf['backgrounds'].attrs['movie_vmax'] = vmax
hdf['images'].attrs['movie_vmin'] = vmin2
hdf['images'].attrs['movie_vmax'] = vmax2
hdf.flush()
# We should now be ready for creating the movie, reopen the file as readonly:
logger.info("Creating movie...")
with h5py.File(hdf_file, 'r') as hdf:
numfiles = len(hdf['images'])
dummy_img = np.full_like(hdf['images/0000'], np.NaN)
time = np.asarray(hdf['time'])
cadenceno = np.asarray(hdf['cadenceno'])
sector = hdf['images'].attrs.get('SECTOR')
camera = hdf['images'].attrs.get('CAMERA')
ccd = hdf['images'].attrs.get('CCD')
with plt.style.context('dark_background'):
plt.rc('axes', titlesize=15)
fig, ax = plt.subplots(1, 4, figsize=(20, 6.8), dpi=dpi)
# Colormap to use for FFIs:
cmap = plt.get_cmap('viridis')
cmap.set_bad('k', 1.0)
# Colormap for Flags:
viridis = plt.get_cmap('Dark2')
newcolors = viridis(np.linspace(0, 1, 4))
newcolors[:1, :] = np.array([1, 1, 1, 1])
cmap_flags = ListedColormap(newcolors)
imgs = [None] * 4
imgs[0] = plot_image(dummy_img,
ax=ax[0],
scale='sqrt',
vmin=vmin,
vmax=vmax,
title='Original Image',
cmap=cmap,
cbar='bottom',
cbar_pad=0.05)
imgs[1] = plot_image(dummy_img,
ax=ax[1],
scale='sqrt',
vmin=vmin,
vmax=vmax,
title='Background',
cmap=cmap,
cbar='bottom',
cbar_pad=0.05)
imgs[2] = plot_image(dummy_img,
ax=ax[2],
scale='sqrt',
vmin=vmin2,
vmax=vmax2,
title='Background subtracted',
cmap=cmap,
cbar='bottom',
cbar_pad=0.05)
imgs[3] = plot_image(
dummy_img,
ax=ax[3],
scale='linear',
vmin=-0.5,
vmax=3.5,
title='Pixel Flags',
cmap=cmap_flags,
cbar='bottom',
cbar_pad=0.05,
clabel='Flags',
cbar_ticks=[0, 1, 2, 3],
cbar_ticklabels=['None', 'Not used', 'Man Excl', 'Shenan'])
for a in ax:
a.set_xticks([])
a.set_yticks([])
figtext = fig.suptitle("to come\nt=???????", fontsize=16)
fig.subplots_adjust(left=0.03,
right=0.97,
top=0.95,
bottom=0.03,
wspace=0.05)
set_copyright(fig)
metadata = {
'title':
'TESS Sector {sector:d}, Camera {camera:d}, CCD {ccd:d}'.
format(sector=sector, camera=camera, ccd=ccd),
'artist':
'TASOC'
}
# Set up the writer (FFMpeg)
WriterClass = animation.writers['ffmpeg']
writer = WriterClass(fps=fps,
codec='h264',
bitrate=-1,
metadata=metadata)
with writer.saving(fig, output_file, dpi):
for k in trange(numfiles, **tqdm_settings):
dset_name = '%04d' % k
flux0 = np.asarray(hdf['images/' + dset_name])
bkg = np.asarray(hdf['backgrounds/' + dset_name])
# Plot original image, background and new image:
imgs[0].set_data(flux0 + bkg)
imgs[1].set_data(bkg)
imgs[2].set_data(flux0)
# Background Shenanigans flags, if available:
if 'pixel_flags/' + dset_name in hdf:
img = np.asarray(hdf['pixel_flags/' + dset_name])
flags = np.zeros_like(img, dtype='uint8')
flags[img & PixelQualityFlags.NotUsedForBackground !=
0] = 1
flags[img & PixelQualityFlags.ManualExclude != 0] = 2
flags[img & PixelQualityFlags.BackgroundShenanigans !=
0] = 3
imgs[3].set_data(flags)
# Update figure title with cadence information;
figtext.set_text(
"Sector {sector:d}, Camera {camera:d}, CCD {ccd:d}\ndset={dset:s}, cad={cad:d}, t={time:.6f}"
.format(sector=sector,
camera=camera,
ccd=ccd,
dset=dset_name,
cad=cadenceno[k],
time=time[k]))
writer.grab_frame()
plt.close(fig)
return output_file
def make_combined_movie(input_dir,
mode='images',
fps=15,
dpi=100,
overwrite=False):
"""
Create animation of the combined contents of all HDF5 files in a directory,
produced by the photometry pipeline.
Parameters:
input_dir (string): Path to the directory with HDF5 files to produce movie from.
mode (string): Which images to show.
Choices are `'originals'`, `'images'`, `'backgrounds'` or `'flags'`.
Default=images.
fps (integer): Frames per second of generated movie. Default=15.
dpi (integer): DPI of the movie. Default=100.
overwrite (boolean): Overwrite existing MP4 files? Default=False.
.. codeauthor:: Rasmus Handberg <*****@*****.**>
"""
# Basic input checks:
if mode not in ('originals', 'images', 'backgrounds', 'flags'):
raise ValueError("Invalid MODE specified")
logger = logging.getLogger(__name__)
tqdm_settings = {'disable': not logger.isEnabledFor(logging.INFO)}
logger.info("Processing '%s'", input_dir)
camccdrot = [(1, 3, 1), (1, 2, 3), (2, 3, 1), (2, 2, 3), (3, 1, 1),
(3, 4, 3), (4, 1, 1), (4, 4, 3), (1, 4, 1), (1, 1, 3),
(2, 4, 1), (2, 1, 3), (3, 2, 1), (3, 3, 3), (4, 2, 1),
(4, 3, 3)]
# Find the sectors that are available:
sectors = []
for fname in find_hdf5_files(input_dir):
# Load the sector number from HDF5 file attributes:
with h5py.File(fname, 'r') as hdf:
s = hdf['images'].attrs.get('SECTOR')
if s is not None and int(s) not in sectors:
sectors.append(int(s))
else:
# If the attribute doesn't exist try to find it from
# parsing the file name:
m = re.match(r'^sector(\d+)_camera\d_ccd\d\.hdf5$',
os.path.basename(fname))
if int(m.group(1)) not in sectors:
sectors.append(int(m.group(1)))
# Create one movie per found sector:
for sector in sectors:
# Define the output file, and overwrite it if needed:
output_file = os.path.join(
input_dir,
'sector{sector:03d}_combined_{mode:s}.mp4'.format(sector=sector,
mode=mode))
if os.path.exists(output_file):
if overwrite:
logger.debug("Deleting existing output file")
os.remove(output_file)
else:
logger.info("Movie file already exists")
return output_file
try:
hdf = [None] * 16
vmin = np.full(16, np.NaN)
vmax = np.full(16, np.NaN)
for k, (camera, ccd, rot) in enumerate(camccdrot):
hdf_file = find_hdf5_files(input_dir,
sector=sector,
camera=camera,
ccd=ccd)
if hdf_file:
hdf[k] = h5py.File(hdf_file[0], 'r')
numfiles = len(hdf[k]['images'])
dummy_img = np.full_like(hdf[k]['images/0000'], np.NaN)
time = np.asarray(hdf[k]['time'])
cadenceno = np.asarray(hdf[k]['cadenceno'])
# Load the image scales if they have already been calculated:
if mode == 'backgrounds':
vmin[k] = hdf[k]['backgrounds'].attrs.get(
'movie_vmin', 0)
vmax[k] = hdf[k]['backgrounds'].attrs.get(
'movie_vmax', 500)
elif mode == 'images' or mode == 'originals':
vmin[k] = hdf[k]['images'].attrs.get('movie_vmin', 0)
vmax[k] = hdf[k]['images'].attrs.get('movie_vmax', 500)
# Summarize the different CCDs into common values:
vmin = np.nanpercentile(vmin, 25.0)
vmax = np.nanpercentile(vmax, 75.0)
logger.info("Creating combined %s movie...", mode)
with plt.style.context('dark_background'):
fig, axes = plt.subplots(2, 8, figsize=(25, 6.8), dpi=dpi)
cmap = plt.get_cmap('viridis')
cmap.set_bad('k', 1.0)
# Colormap for Flags:
viridis = plt.get_cmap('Dark2')
newcolors = viridis(np.linspace(0, 1, 4))
newcolors[:1, :] = np.array([1, 1, 1, 1])
cmap_flags = ListedColormap(newcolors)
imgs = [None] * 16
for k, ax in enumerate(axes.flatten()):
if mode == 'flags':
imgs[k] = plot_image(dummy_img,
ax=ax,
scale='linear',
vmin=-0.5,
vmax=4.5,
cmap=cmap_flags)
else:
imgs[k] = plot_image(dummy_img,
ax=ax,
scale='sqrt',
vmin=vmin,
vmax=vmax,
cmap=cmap)
ax.set_xticks([])
ax.set_yticks([])
figtext = fig.suptitle("to come\nt=???????", fontsize=16)
fig.subplots_adjust(left=0.03,
right=0.97,
top=0.90,
bottom=0.05,
wspace=0.05,
hspace=0.05)
set_copyright(fig)
metadata = {
'title':
'TESS Sector {sector:d}, {mode:s}'.format(sector=sector,
mode=mode),
'artist':
'TASOC'
}
# Set up the writer (FFMpeg)
WriterClass = animation.writers['ffmpeg']
writer = WriterClass(fps=fps,
codec='h264',
bitrate=-1,
metadata=metadata)
with writer.saving(fig, output_file, dpi):
for i in trange(numfiles, **tqdm_settings):
dset_name = '%04d' % i
for k in range(16):
if hdf[k] is None:
continue
# Background Shenanigans flags, if available:
if mode == 'flags':
flags = np.asarray(hdf[k]['pixel_flags/' +
dset_name])
img = np.zeros_like(flags, dtype='uint8')
img[flags
& PixelQualityFlags.NotUsedForBackground !=
0] = 1
img[flags
& PixelQualityFlags.ManualExclude != 0] = 2
img[flags
& PixelQualityFlags.BackgroundShenanigans
!= 0] = 3
elif mode == 'originals':
img = np.asarray(hdf[k]['images/' + dset_name])
img += np.asarray(hdf[k]['backgrounds/' +
dset_name])
else:
img = np.asarray(hdf[k][mode + '/' +
dset_name])
# Rotate the image:
cam, ccd, rot = camccdrot[k]
img = np.rot90(img, rot)
# Update the image:
imgs[k].set_data(img)
# Update figure title with cadence information:
figtext.set_text(
"Sector {sector:d} - {mode:s}\ndset={dset:s}, cad={cad:d}, t={time:.6f}"
.format(sector=sector,
mode=mode,
dset=dset_name,
cad=cadenceno[i],
time=time[i]))
writer.grab_frame()
plt.close(fig)
except: # noqa: E722
raise
finally:
for k in range(16):
if hdf[k] is not None:
hdf[k].close()
return output_file
def make_combined_movie(input_dir,
mode='images',
fps=15,
dpi=100,
overwrite=False):
"""
Create animation of the combined contents of all HDF5 files in a directory,
produced by the photometry pipeline.
Parameters:
input_dir (string): Path to the directory with HDF5 files to produce movie from.
mode (string): Which images to show.
Choices are `'images'`, `'backgrounds'` or `'flags'`.
Default=images.
fps (integer): Frames per second of generated movie. Default=15.
dpi (integer): DPI of the movie. Default=100.
overwrite (boolean): Overwrite existing MP4 files? Default=False.
.. codeauthor:: Rasmus Handberg <*****@*****.**>
"""
# Basic input checks:
assert mode in ('images', 'backgrounds', 'flags'), "Invalid MODE specified"
logger = logging.getLogger(__name__)
logger.info("Processing '%s'", input_dir)
camccdrot = [(1, 3, 1), (1, 2, 3), (2, 3, 1), (2, 2, 3), (3, 1, 1),
(3, 4, 3), (4, 1, 1), (4, 4, 3), (1, 4, 1), (1, 1, 3),
(2, 4, 1), (2, 1, 3), (3, 2, 1), (3, 3, 3), (4, 2, 1),
(4, 3, 3)]
# Find the sectors that are available:
# TODO: Could we change this so we don't have to parse the filenames?
sectors = []
for fname in find_hdf5_files(input_dir):
m = re.match(r'^sector(\d+)_camera\d_ccd\d\.hdf5$',
os.path.basename(fname))
if int(m.group(1)) not in sectors:
sectors.append(int(m.group(1)))
# Create one movie per found sector:
for sector in sectors:
# Define the output file, and overwrite it if needed:
output_file = os.path.join(
input_dir,
'sector{sector:03d}_combined_{mode:s}.mp4'.format(sector=sector,
mode=mode))
if os.path.exists(output_file):
if overwrite:
logger.debug("Deleting existing output file")
os.remove(output_file)
else:
logger.info("Movie file already exists")
return output_file
try:
hdf = [None] * 16
vmin = np.full(16, np.NaN)
vmax = np.full(16, np.NaN)
for k, (camera, ccd, rot) in enumerate(camccdrot):
hdf_file = find_hdf5_files(input_dir,
sector=sector,
camera=camera,
ccd=ccd)
if hdf_file:
hdf[k] = h5py.File(hdf_file[0], 'r', libver='latest')
numfiles = len(hdf[k]['images'])
dummy_img = np.zeros_like(hdf[k]['images/0000'])
time = np.asarray(hdf[k]['time'])
cadenceno = np.asarray(hdf[k]['cadenceno'])
# Load the image scales if they have already been calculated:
if mode == 'backgrounds':
vmin[k] = hdf[k]['backgrounds'].attrs.get(
'movie_vmin', 0)
vmax[k] = hdf[k]['backgrounds'].attrs.get(
'movie_vmax', 500)
elif mode == 'images':
vmin[k] = hdf[k]['images'].attrs.get('movie_vmin', 0)
vmax[k] = hdf[k]['images'].attrs.get('movie_vmax', 500)
# Summarize the different CCDs into common values:
vmin = np.nanpercentile(vmin, 25.0)
vmax = np.nanpercentile(vmax, 75.0)
logger.info("Creating movie...")
with plt.style.context('dark_background'):
fig, axes = plt.subplots(2, 8, figsize=(25, 7.3))
cmap = plt.get_cmap('viridis')
cmap.set_bad('k', 1.0)
# Colormap for Flags:
viridis = plt.get_cmap('Dark2')
newcolors = viridis(np.linspace(0, 1, 4))
newcolors[:1, :] = np.array([1, 1, 1, 1])
cmap_flags = ListedColormap(newcolors)
imgs = [None] * 16
for k, ax in enumerate(axes.flatten()):
if mode == 'flags':
imgs[k] = plot_image(dummy_img,
ax=ax,
scale='linear',
vmin=-0.5,
vmax=4.5,
xlabel=None,
ylabel=None,
cmap=cmap_flags,
make_cbar=False)
else:
imgs[k] = plot_image(dummy_img,
ax=ax,
scale='sqrt',
vmin=vmin,
vmax=vmax,
xlabel=None,
ylabel=None,
cmap=cmap,
make_cbar=False)
ax.set_xticks([])
ax.set_yticks([])
figtext = fig.suptitle("to come\nt=???????", fontsize=15)
fig.set_tight_layout('tight')
fig.subplots_adjust(top=0.85)
set_copyright(fig)
writer = animation.FFMpegWriter(fps=fps)
with writer.saving(fig, output_file, dpi):
for k in trange(numfiles):
dset_name = '%04d' % k
for k in range(16):
if hdf[k] is None: continue
# Background Shenanigans flags, if available:
if mode == 'flags':
flags = np.asarray(hdf[k]['pixel_flags/' +
dset_name])
img = np.zeros_like(flags, dtype='uint8')
img[flags
& PixelQualityFlags.NotUsedForBackground !=
0] = 1
img[flags
& PixelQualityFlags.ManualExclude != 0] = 2
img[flags
& PixelQualityFlags.BackgroundShenanigans
!= 0] = 3
else:
img = np.asarray(hdf[k][mode + '/' +
dset_name])
# Rotate the image:
cam, ccd, rot = camccdrot[k]
img = np.rot90(img, rot)
# Update the image:
imgs[k].set_data(img)
# Update figure title with cadence information:
figtext.set_text(
"Sector {sector:d} - {mode:s}\ndset={dset:s}, cad={cad:d}, t={time:.6f}"
.format(sector=sector,
mode=mode,
dset=dset_name,
cad=cadenceno[k],
time=time[k]))
writer.grab_frame()
plt.close(fig)
except:
raise
finally:
for k in range(16):
if hdf[k] is not None:
hdf[k].close()
return output_file
dset_name = '%04d' % k
with h5py.File('sector001_camera1_ccd2.hdf5', 'r') as hdf:
flux0 = np.asarray(hdf['images/' + dset_name])
bkg = np.asarray(hdf['backgrounds/' + dset_name])
#flags = np.asarray(hdf['pixel_flags/' + dset_name])
flags = np.zeros_like(flux0, dtype='int32')
flags[512:1024, 512:1024] = 128
flags = np.where(flags == 0, 0, np.log2(flags) + 1)
#flags = (flags & PixelQualityFlags.BackgroundShenanigans != 0)
white = np.array([1, 1, 1, 1])
viridis = plt.get_cmap('viridis')
newcolors = viridis(np.linspace(0, 1, int(np.max(flags))))
newcolors[:1, :] = white
newcmp = ListedColormap(newcolors)
fig, ax = plt.subplots(1, 4, figsize=(20, 6))
plot_image(flux0 + bkg,
ax=ax[0],
scale='sqrt',
vmin=vmin,
vmax=vmax,
title='Original Image',
xlabel=None,
ylabel=None,
cmap=plt.cm.viridis,
flags[img & PixelQualityFlags.ManualExclude != 0] = 2
flags[img & PixelQualityFlags.BackgroundShenanigans != 0] = 3
vmin = hdf['backgrounds'].attrs.get('movie_vmin')
vmax = hdf['backgrounds'].attrs.get('movie_vmax')
vmin2 = hdf['images'].attrs.get('movie_vmin')
vmax2 = hdf['images'].attrs.get('movie_vmax')
print(vmin, vmax)
print(vmin2, vmax2)
# Colormap for images:
cmap = plt.cm.viridis
# Colormap for Flags:
viridis = plt.get_cmap('Dark2')
newcolors = viridis(np.linspace(0, 1, 4))
newcolors[:1, :] = np.array([1, 1, 1, 1])
cmap_flags = ListedColormap(newcolors)
# Create figures:
fig, ax = plt.subplots(1, 4, figsize=(20, 6.2))
img1 = plot_image(flux0+bkg, ax=ax[0], scale='sqrt', vmin=vmin, vmax=vmax, title='Original Image', xlabel=None, ylabel=None, cmap=cmap, make_cbar=True)
img2 = plot_image(bkg, ax=ax[1], scale='sqrt', vmin=vmin, vmax=vmax, title='Background', xlabel=None, ylabel=None, cmap=cmap, make_cbar=True)
img3 = plot_image(flux0, ax=ax[2], scale='sqrt', vmin=vmin2, vmax=vmax2, title='Background subtracted', xlabel=None, ylabel=None, cmap=cmap, make_cbar=True)
img4 = plot_image(flags, ax=ax[3], scale='linear', vmin=-0.5, vmax=3.5, title='Pixel Flags', xlabel=None, ylabel=None, cmap=cmap_flags, make_cbar=True, clabel='Flags', cbar_ticks=[0,1,2,3], cbar_ticklabels=['None','Not used','Man. Excl.','Shenan'])
# Remove axes ticks:
for a in ax:
a.set_xticks([])
a.set_yticks([])