def generateCalibrationReport(config,
night_dir_path,
match_radius=2.0,
platepar=None,
show_graphs=False):
""" Given the folder of the night, find the Calstars file, check the star fit and generate a report
with the quality of the calibration. The report contains information about both the astrometry and
the photometry calibration. Graphs will be saved in the given directory of the night.
Arguments:
config: [Config instance]
night_dir_path: [str] Full path to the directory of the night.
Keyword arguments:
match_radius: [float] Match radius for star matching between image and catalog stars (px).
platepar: [Platepar instance] Use this platepar instead of finding one in the folder.
show_graphs: [bool] Show the graphs on the screen. False by default.
Return:
None
"""
# Find the CALSTARS file in the given folder
calstars_file = None
for calstars_file in os.listdir(night_dir_path):
if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file):
break
if calstars_file is None:
print('CALSTARS file could not be found in the given directory!')
return None
# Load the calstars file
star_list = readCALSTARS(night_dir_path, calstars_file)
### Load recalibrated platepars, if they exist ###
# Find recalibrated platepars file per FF file
platepars_recalibrated_file = None
for file_name in os.listdir(night_dir_path):
if file_name == config.platepars_recalibrated_name:
platepars_recalibrated_file = file_name
break
# Load all recalibrated platepars if the file is available
recalibrated_platepars = None
if platepars_recalibrated_file:
with open(os.path.join(night_dir_path,
platepars_recalibrated_file)) as f:
recalibrated_platepars = json.load(f)
print(
'Loaded recalibrated platepars JSON file for the calibration report...'
)
### ###
### Load the platepar file ###
# Find the platepar file in the given directory if it was not given
if platepar is None:
# Find the platepar file
platepar_file = None
for file_name in os.listdir(night_dir_path):
if file_name == config.platepar_name:
platepar_file = file_name
break
if platepar_file is None:
print('The platepar cannot be found in the night directory!')
return None
# Load the platepar file
platepar = Platepar()
platepar.read(os.path.join(night_dir_path, platepar_file),
use_flat=config.use_flat)
### ###
night_name = os.path.split(night_dir_path.strip(os.sep))[1]
# Go one mag deeper than in the config
lim_mag = config.catalog_mag_limit + 1
# Load catalog stars (load one magnitude deeper)
catalog_stars, mag_band_str, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(\
config.star_catalog_path, config.star_catalog_file, lim_mag=lim_mag, \
mag_band_ratios=config.star_catalog_band_ratios)
### Take only those CALSTARS entires for which FF files exist in the folder ###
# Get a list of FF files in the folder
ff_list = []
for file_name in os.listdir(night_dir_path):
if validFFName(file_name):
ff_list.append(file_name)
# Filter out calstars entries, generate a star dictionary where the keys are JDs of FFs
star_dict = {}
ff_dict = {}
for entry in star_list:
ff_name, star_data = entry
# Check if the FF from CALSTARS exists in the folder
if ff_name not in ff_list:
continue
dt = getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)
jd = date2JD(*dt)
# Add the time and the stars to the dict
star_dict[jd] = star_data
ff_dict[jd] = ff_name
### ###
# If there are no FF files in the directory, don't generate a report
if len(star_dict) == 0:
print('No FF files from the CALSTARS file in the directory!')
return None
# If the recalibrated platepars file exists, take the one with the most stars
max_jd = 0
using_recalib_platepars = False
if recalibrated_platepars is not None:
max_stars = 0
for ff_name_temp in recalibrated_platepars:
# Compute the Julian date of the FF middle
dt = getMiddleTimeFF(ff_name_temp,
config.fps,
ret_milliseconds=True)
jd = date2JD(*dt)
# Check that this file exists in CALSTARS and the list of FF files
if (jd not in star_dict) or (jd not in ff_dict):
continue
# Check if the number of stars on this FF file is larger than the before
if len(star_dict[jd]) > max_stars:
max_jd = jd
max_stars = len(star_dict[jd])
# Set a flag to indicate if using recalibrated platepars has failed
if max_jd == 0:
using_recalib_platepars = False
else:
print('Using recalibrated platepars, file:', ff_dict[max_jd])
using_recalib_platepars = True
# Select the platepar where the FF file has the most stars
platepar_dict = recalibrated_platepars[ff_dict[max_jd]]
platepar = Platepar()
platepar.loadFromDict(platepar_dict, use_flat=config.use_flat)
filtered_star_dict = {max_jd: star_dict[max_jd]}
# Match stars on the image with the stars in the catalog
n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \
filtered_star_dict, match_radius, ret_nmatch=True, lim_mag=lim_mag)
max_matched_stars = n_matched
# Otherwise take the optimal FF file for evaluation
if (recalibrated_platepars is None) or (not using_recalib_platepars):
# If there are more than a set number of FF files to evaluate, choose only the ones with most stars on
# the image
if len(star_dict) > config.calstars_files_N:
# Find JDs of FF files with most stars on them
top_nstars_indices = np.argsort([len(x) for x in star_dict.values()])[::-1][:config.calstars_files_N \
- 1]
filtered_star_dict = {}
for i in top_nstars_indices:
filtered_star_dict[list(star_dict.keys())[i]] = list(
star_dict.values())[i]
star_dict = filtered_star_dict
# Match stars on the image with the stars in the catalog
n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \
star_dict, match_radius, ret_nmatch=True, lim_mag=lim_mag)
# If no recalibrated platepars where found, find the image with the largest number of matched stars
if (not using_recalib_platepars) or (max_jd == 0):
max_jd = 0
max_matched_stars = 0
for jd in matched_stars:
_, _, distances = matched_stars[jd]
if len(distances) > max_matched_stars:
max_jd = jd
max_matched_stars = len(distances)
# If there are no matched stars, use the image with the largest number of detected stars
if max_matched_stars <= 2:
max_jd = max(star_dict, key=lambda x: len(star_dict[x]))
distances = [np.inf]
# Take the FF file with the largest number of matched stars
ff_name = ff_dict[max_jd]
# Load the FF file
ff = readFF(night_dir_path, ff_name)
img_h, img_w = ff.avepixel.shape
dpi = 200
plt.figure(figsize=(ff.avepixel.shape[1] / dpi,
ff.avepixel.shape[0] / dpi),
dpi=dpi)
# Take the average pixel
img = ff.avepixel
# Slightly adjust the levels
img = Image.adjustLevels(img, np.percentile(img, 1.0), 1.3,
np.percentile(img, 99.99))
plt.imshow(img, cmap='gray', interpolation='nearest')
legend_handles = []
# Plot detected stars
for img_star in star_dict[max_jd]:
y, x, _, _ = img_star
rect_side = 5 * match_radius
square_patch = plt.Rectangle((x - rect_side/2, y - rect_side/2), rect_side, rect_side, color='g', \
fill=False, label='Image stars')
plt.gca().add_artist(square_patch)
legend_handles.append(square_patch)
# If there are matched stars, plot them
if max_matched_stars > 2:
# Take the solution with the largest number of matched stars
image_stars, matched_catalog_stars, distances = matched_stars[max_jd]
# Plot matched stars
for img_star in image_stars:
x, y, _, _ = img_star
circle_patch = plt.Circle((y, x), radius=3*match_radius, color='y', fill=False, \
label='Matched stars')
plt.gca().add_artist(circle_patch)
legend_handles.append(circle_patch)
### Plot match residuals ###
# Compute preducted positions of matched image stars from the catalog
x_predicted, y_predicted = raDecToXYPP(matched_catalog_stars[:, 0], \
matched_catalog_stars[:, 1], max_jd, platepar)
img_y, img_x, _, _ = image_stars.T
delta_x = x_predicted - img_x
delta_y = y_predicted - img_y
# Compute image residual and angle of the error
res_angle = np.arctan2(delta_y, delta_x)
res_distance = np.sqrt(delta_x**2 + delta_y**2)
# Calculate coordinates of the beginning of the residual line
res_x_beg = img_x + 3 * match_radius * np.cos(res_angle)
res_y_beg = img_y + 3 * match_radius * np.sin(res_angle)
# Calculate coordinates of the end of the residual line
res_x_end = img_x + 100 * np.cos(res_angle) * res_distance
res_y_end = img_y + 100 * np.sin(res_angle) * res_distance
# Plot the 100x residuals
for i in range(len(x_predicted)):
res_plot = plt.plot([res_x_beg[i], res_x_end[i]], [res_y_beg[i], res_y_end[i]], color='orange', \
lw=0.5, label='100x residuals')
legend_handles.append(res_plot[0])
### ###
else:
distances = [np.inf]
# If there are no matched stars, plot large text in the middle of the screen
plt.text(img_w / 2,
img_h / 2,
"NO MATCHED STARS!",
color='r',
alpha=0.5,
fontsize=20,
ha='center',
va='center')
### Plot positions of catalog stars to the limiting magnitude of the faintest matched star + 1 mag ###
# Find the faintest magnitude among matched stars
if max_matched_stars > 2:
faintest_mag = np.max(matched_catalog_stars[:, 2]) + 1
else:
# If there are no matched stars, use the limiting magnitude from config
faintest_mag = config.catalog_mag_limit + 1
# Estimate RA,dec of the centre of the FOV
_, RA_c, dec_c, _ = xyToRaDecPP([jd2Date(max_jd)], [platepar.X_res / 2],
[platepar.Y_res / 2], [1], platepar)
RA_c = RA_c[0]
dec_c = dec_c[0]
fov_radius = np.hypot(*computeFOVSize(platepar))
# Get stars from the catalog around the defined center in a given radius
_, extracted_catalog = subsetCatalog(catalog_stars, RA_c, dec_c,
fov_radius, faintest_mag)
ra_catalog, dec_catalog, mag_catalog = extracted_catalog.T
# Compute image positions of all catalog stars that should be on the image
x_catalog, y_catalog = raDecToXYPP(ra_catalog, dec_catalog, max_jd,
platepar)
# Filter all catalog stars outside the image
temp_arr = np.c_[x_catalog, y_catalog, mag_catalog]
temp_arr = temp_arr[temp_arr[:, 0] >= 0]
temp_arr = temp_arr[temp_arr[:, 0] <= ff.avepixel.shape[1]]
temp_arr = temp_arr[temp_arr[:, 1] >= 0]
temp_arr = temp_arr[temp_arr[:, 1] <= ff.avepixel.shape[0]]
x_catalog, y_catalog, mag_catalog = temp_arr.T
# Plot catalog stars on the image
cat_stars_handle = plt.scatter(x_catalog, y_catalog, c='none', marker='D', lw=1.0, alpha=0.4, \
s=((4.0 + (faintest_mag - mag_catalog))/3.0)**(2*2.512), edgecolor='r', label='Catalog stars')
legend_handles.append(cat_stars_handle)
### ###
# Add info text in the corner
info_text = ff_dict[max_jd] + '\n' \
+ "Matched stars within {:.1f} px radius: {:d}/{:d} \n".format(match_radius, max_matched_stars, \
len(star_dict[max_jd])) \
+ "Median distance = {:.2f} px\n".format(np.median(distances)) \
+ "Catalog lim mag = {:.1f}".format(lim_mag)
plt.text(10, 10, info_text, bbox=dict(facecolor='black', alpha=0.5), va='top', ha='left', fontsize=4, \
color='w', family='monospace')
legend = plt.legend(handles=legend_handles,
prop={'size': 4},
loc='upper right')
legend.get_frame().set_facecolor('k')
legend.get_frame().set_edgecolor('k')
for txt in legend.get_texts():
txt.set_color('w')
### Add FOV info (centre, size) ###
# Mark FOV centre
plt.scatter(platepar.X_res / 2,
platepar.Y_res / 2,
marker='+',
s=20,
c='r',
zorder=4)
# Compute FOV centre alt/az
azim_centre, alt_centre = raDec2AltAz(max_jd, platepar.lon, platepar.lat,
RA_c, dec_c)
# Compute FOV size
fov_h, fov_v = computeFOVSize(platepar)
# Compute the rotation wrt. horizon
rot_horizon = rotationWrtHorizon(platepar)
fov_centre_text = "Azim = {:6.2f}$\\degree$\n".format(azim_centre) \
+ "Alt = {:6.2f}$\\degree$\n".format(alt_centre) \
+ "Rot h = {:6.2f}$\\degree$\n".format(rot_horizon) \
+ "FOV h = {:6.2f}$\\degree$\n".format(fov_h) \
+ "FOV v = {:6.2f}$\\degree$".format(fov_v) \
plt.text(10, platepar.Y_res - 10, fov_centre_text, bbox=dict(facecolor='black', alpha=0.5), \
va='bottom', ha='left', fontsize=4, color='w', family='monospace')
### ###
# Plot RA/Dec gridlines #
addEquatorialGrid(plt, platepar, max_jd)
plt.axis('off')
plt.gca().get_xaxis().set_visible(False)
plt.gca().get_yaxis().set_visible(False)
plt.xlim([0, ff.avepixel.shape[1]])
plt.ylim([ff.avepixel.shape[0], 0])
# Remove the margins
plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)
plt.savefig(os.path.join(night_dir_path, night_name + '_calib_report_astrometry.jpg'), \
bbox_inches='tight', pad_inches=0, dpi=dpi)
if show_graphs:
plt.show()
else:
plt.clf()
plt.close()
if max_matched_stars > 2:
### PHOTOMETRY FIT ###
# If a flat is used, set the vignetting coeff to 0
if config.use_flat:
platepar.vignetting_coeff = 0.0
# Extact intensities and mangitudes
star_intensities = image_stars[:, 2]
catalog_mags = matched_catalog_stars[:, 2]
# Compute radius of every star from image centre
radius_arr = np.hypot(image_stars[:, 0] - img_h / 2,
image_stars[:, 1] - img_w / 2)
# Fit the photometry on automated star intensities (use the fixed vignetting coeff, use robust fit)
photom_params, fit_stddev, fit_resid, star_intensities, radius_arr, catalog_mags = \
photometryFitRobust(star_intensities, radius_arr, catalog_mags, \
fixed_vignetting=platepar.vignetting_coeff)
photom_offset, _ = photom_params
### ###
### PLOT PHOTOMETRY ###
# Note: An almost identical code exists in RMS.Astrometry.SkyFit in the PlateTool.photometry function
dpi = 130
fig_p, (ax_p, ax_r) = plt.subplots(nrows=2, facecolor=None, figsize=(6.0, 7.0), dpi=dpi, \
gridspec_kw={'height_ratios':[2, 1]})
# Plot raw star intensities
ax_p.scatter(-2.5 * np.log10(star_intensities),
catalog_mags,
s=5,
c='r',
alpha=0.5,
label="Raw")
# If a flat is used, disregard the vignetting
if not config.use_flat:
# Plot intensities of image stars corrected for vignetting
lsp_corr_arr = np.log10(correctVignetting(star_intensities, radius_arr, \
platepar.vignetting_coeff))
ax_p.scatter(-2.5*lsp_corr_arr, catalog_mags, s=5, c='b', alpha=0.5, \
label="Corrected for vignetting")
# Plot photometric offset from the platepar
x_min, x_max = ax_p.get_xlim()
y_min, y_max = ax_p.get_ylim()
x_min_w = x_min - 3
x_max_w = x_max + 3
y_min_w = y_min - 3
y_max_w = y_max + 3
photometry_info = "Platepar: {:+.1f}*LSP + {:.2f} +/- {:.2f}".format(platepar.mag_0, \
platepar.mag_lev, platepar.mag_lev_stddev) \
+ "\nVignetting coeff = {:.5f}".format(platepar.vignetting_coeff) \
+ "\nGamma = {:.2f}".format(platepar.gamma)
# Plot the photometry calibration from the platepar
logsum_arr = np.linspace(x_min_w, x_max_w, 10)
ax_p.plot(logsum_arr, logsum_arr + platepar.mag_lev, label=photometry_info, linestyle='--', \
color='k', alpha=0.5)
# Plot the fitted photometry calibration
fit_info = "Fit: {:+.1f}*LSP + {:.2f} +/- {:.2f}".format(
-2.5, photom_offset, fit_stddev)
ax_p.plot(logsum_arr,
logsum_arr + photom_offset,
label=fit_info,
linestyle='--',
color='b',
alpha=0.75)
ax_p.legend()
ax_p.set_ylabel("Catalog magnitude ({:s})".format(mag_band_str))
ax_p.set_xlabel("Uncalibrated magnitude")
# Set wider axis limits
ax_p.set_xlim(x_min_w, x_max_w)
ax_p.set_ylim(y_min_w, y_max_w)
ax_p.invert_yaxis()
ax_p.invert_xaxis()
ax_p.grid()
### Plot photometry vs radius ###
img_diagonal = np.hypot(img_h / 2, img_w / 2)
# Plot photometry residuals (including vignetting)
ax_r.scatter(radius_arr, fit_resid, c='b', alpha=0.75, s=5, zorder=3)
# Plot a zero line
ax_r.plot(np.linspace(0, img_diagonal, 10), np.zeros(10), linestyle='dashed', alpha=0.5, \
color='k')
# Plot only when no flat is used
if not config.use_flat:
# Plot radius from centre vs. fit residual
fit_resids_novignetting = catalog_mags - photomLine((np.array(star_intensities), \
np.array(radius_arr)), photom_offset, 0.0)
ax_r.scatter(radius_arr,
fit_resids_novignetting,
s=5,
c='r',
alpha=0.5,
zorder=3)
px_sum_tmp = 1000
radius_arr_tmp = np.linspace(0, img_diagonal, 50)
# Plot vignetting loss curve
vignetting_loss = 2.5*np.log10(px_sum_tmp) \
- 2.5*np.log10(correctVignetting(px_sum_tmp, radius_arr_tmp, \
platepar.vignetting_coeff))
ax_r.plot(radius_arr_tmp,
vignetting_loss,
linestyle='dotted',
alpha=0.5,
color='k')
ax_r.grid()
ax_r.set_ylabel("Fit residuals (mag)")
ax_r.set_xlabel("Radius from centre (px)")
ax_r.set_xlim(0, img_diagonal)
### ###
plt.tight_layout()
plt.savefig(os.path.join(night_dir_path,
night_name + '_calib_report_photometry.png'),
dpi=150)
if show_graphs:
plt.show()
else:
plt.clf()
plt.close()
def recalibrateIndividualFFsAndApplyAstrometry(dir_path, ftpdetectinfo_path, calstars_list, config, platepar,
generate_plot=True):
""" Recalibrate FF files with detections and apply the recalibrated platepar to those detections.
Arguments:
dir_path: [str] Path where the FTPdetectinfo file is.
ftpdetectinfo_path: [str] Name of the FTPdetectinfo file.
calstars_list: [list] A list of entries [[ff_name, star_coordinates], ...].
config: [Config instance]
platepar: [Platepar instance] Initial platepar.
Keyword arguments:
generate_plot: [bool] Generate the calibration variation plot. True by default.
Return:
recalibrated_platepars: [dict] A dictionary where the keys are FF file names and values are
recalibrated platepar instances for every FF file.
"""
# Use a copy of the config file
config = copy.deepcopy(config)
# If the given file does not exits, return nothing
if not os.path.isfile(ftpdetectinfo_path):
print('ERROR! The FTPdetectinfo file does not exist: {:s}'.format(ftpdetectinfo_path))
print(' The recalibration on every file was not done!')
return {}
# Read the FTPdetectinfo data
cam_code, fps, meteor_list = FTPdetectinfo.readFTPdetectinfo(*os.path.split(ftpdetectinfo_path), \
ret_input_format=True)
# Convert the list of stars to a per FF name dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
### Add neighboring FF files for more robust photometry estimation ###
ff_processing_list = []
# Make a list of sorted FF files in CALSTARS
calstars_ffs = sorted([ff_file for ff_file in calstars])
# Go through the list of FF files with detections and add neighboring FFs
for meteor_entry in meteor_list:
ff_name = meteor_entry[0]
if ff_name in calstars_ffs:
# Find the index of the given FF file in the list of calstars
ff_indx = calstars_ffs.index(ff_name)
# Add neighbours to the processing list
for k in range(-(RECALIBRATE_NEIGHBOURHOOD_SIZE//2), RECALIBRATE_NEIGHBOURHOOD_SIZE//2 + 1):
k_indx = ff_indx + k
if (k_indx > 0) and (k_indx < len(calstars_ffs)):
ff_name_tmp = calstars_ffs[k_indx]
if ff_name_tmp not in ff_processing_list:
ff_processing_list.append(ff_name_tmp)
# Sort the processing list of FF files
ff_processing_list = sorted(ff_processing_list)
### ###
# Globally increase catalog limiting magnitude
config.catalog_mag_limit += 1
# Load catalog stars (overwrite the mag band ratios if specific catalog is used)
star_catalog_status = StarCatalog.readStarCatalog(config.star_catalog_path,\
config.star_catalog_file, lim_mag=config.catalog_mag_limit, \
mag_band_ratios=config.star_catalog_band_ratios)
if not star_catalog_status:
print("Could not load the star catalog!")
print(os.path.join(config.star_catalog_path, config.star_catalog_file))
return {}
catalog_stars, _, config.star_catalog_band_ratios = star_catalog_status
# Update the platepar coordinates from the config file
platepar.lat = config.latitude
platepar.lon = config.longitude
platepar.elev = config.elevation
prev_platepar = copy.deepcopy(platepar)
# Go through all FF files with detections, recalibrate and apply astrometry
recalibrated_platepars = {}
for ff_name in ff_processing_list:
working_platepar = copy.deepcopy(prev_platepar)
# Skip this meteor if its FF file was already recalibrated
if ff_name in recalibrated_platepars:
continue
print()
print('Processing: ', ff_name)
print('------------------------------------------------------------------------------')
# Find extracted stars on this image
if not ff_name in calstars:
print('Skipped because it was not in CALSTARS:', ff_name)
continue
# Get stars detected on this FF file (create a dictionaly with only one entry, the residuals function
# needs this format)
calstars_time = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)
jd = date2JD(*calstars_time)
star_dict_ff = {jd: calstars[ff_name]}
# Recalibrate the platepar using star matching
result, min_match_radius = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars)
# If the recalibration failed, try using FFT alignment
if result is None:
print()
print('Running FFT alignment...')
# Run FFT alignment
calstars_coords = np.array(star_dict_ff[jd])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
print(calstars_time)
test_platepar = alignPlatepar(config, prev_platepar, calstars_time, calstars_coords, \
show_plot=False)
# Try to recalibrate after FFT alignment
result, _ = recalibrateFF(config, test_platepar, jd, star_dict_ff, catalog_stars)
# If the FFT alignment failed, align the original platepar using the smallest radius that matched
# and force save the the platepar
if (result is None) and (min_match_radius is not None):
print()
print("Using the old platepar with the minimum match radius of: {:.2f}".format(min_match_radius))
result, _ = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars,
max_match_radius=min_match_radius, force_platepar_save=True)
if result is not None:
working_platepar = result
# If the alignment succeeded, save the result
else:
working_platepar = result
else:
working_platepar = result
# Store the platepar if the fit succeeded
if result is not None:
# Recompute alt/az of the FOV centre
working_platepar.az_centre, working_platepar.alt_centre = raDec2AltAz(working_platepar.RA_d, \
working_platepar.dec_d, working_platepar.JD, working_platepar.lat, working_platepar.lon)
# Recompute the rotation wrt horizon
working_platepar.rotation_from_horiz = rotationWrtHorizon(working_platepar)
# Mark the platepar to indicate that it was automatically recalibrated on an individual FF file
working_platepar.auto_recalibrated = True
recalibrated_platepars[ff_name] = working_platepar
prev_platepar = working_platepar
else:
print('Recalibration of {:s} failed, using the previous platepar...'.format(ff_name))
# Mark the platepar to indicate that autorecalib failed
prev_platepar_tmp = copy.deepcopy(prev_platepar)
prev_platepar_tmp.auto_recalibrated = False
# If the aligning failed, set the previous platepar as the one that should be used for this FF file
recalibrated_platepars[ff_name] = prev_platepar_tmp
### Average out photometric offsets within the given neighbourhood size ###
# Go through the list of FF files with detections
for meteor_entry in meteor_list:
ff_name = meteor_entry[0]
# Make sure the FF was successfuly recalibrated
if ff_name in recalibrated_platepars:
# Find the index of the given FF file in the list of calstars
ff_indx = calstars_ffs.index(ff_name)
# Compute the average photometric offset and the improved standard deviation using all
# neighbors
photom_offset_tmp_list = []
photom_offset_std_tmp_list = []
neighboring_ffs = []
for k in range(-(RECALIBRATE_NEIGHBOURHOOD_SIZE//2), RECALIBRATE_NEIGHBOURHOOD_SIZE//2 + 1):
k_indx = ff_indx + k
if (k_indx > 0) and (k_indx < len(calstars_ffs)):
# Get the name of the FF file
ff_name_tmp = calstars_ffs[k_indx]
# Check that the neighboring FF was successfuly recalibrated
if ff_name_tmp in recalibrated_platepars:
# Get the computed photometric offset and stddev
photom_offset_tmp_list.append(recalibrated_platepars[ff_name_tmp].mag_lev)
photom_offset_std_tmp_list.append(recalibrated_platepars[ff_name_tmp].mag_lev_stddev)
neighboring_ffs.append(ff_name_tmp)
# Compute the new photometric offset and improved standard deviation (assume equal sample size)
# Source: https://stats.stackexchange.com/questions/55999/is-it-possible-to-find-the-combined-standard-deviation
photom_offset_new = np.mean(photom_offset_tmp_list)
photom_offset_std_new = np.sqrt(\
np.sum([st**2 + (mt - photom_offset_new)**2 \
for mt, st in zip(photom_offset_tmp_list, photom_offset_std_tmp_list)]) \
/ len(photom_offset_tmp_list)
)
# Assign the new photometric offset and standard deviation to all FFs used for computation
for ff_name_tmp in neighboring_ffs:
recalibrated_platepars[ff_name_tmp].mag_lev = photom_offset_new
recalibrated_platepars[ff_name_tmp].mag_lev_stddev = photom_offset_std_new
### ###
### Store all recalibrated platepars as a JSON file ###
all_pps = {}
for ff_name in recalibrated_platepars:
json_str = recalibrated_platepars[ff_name].jsonStr()
all_pps[ff_name] = json.loads(json_str)
with open(os.path.join(dir_path, config.platepars_recalibrated_name), 'w') as f:
# Convert all platepars to a JSON file
out_str = json.dumps(all_pps, default=lambda o: o.__dict__, indent=4, sort_keys=True)
f.write(out_str)
### ###
# If no platepars were recalibrated, use the single platepar recalibration procedure
if len(recalibrated_platepars) == 0:
print('No FF images were used for recalibration, using the single platepar calibration function...')
# Use the initial platepar for calibration
applyAstrometryFTPdetectinfo(dir_path, os.path.basename(ftpdetectinfo_path), None, platepar=platepar)
return recalibrated_platepars
### GENERATE PLOTS ###
dt_list = []
ang_dists = []
rot_angles = []
hour_list = []
photom_offset_list = []
photom_offset_std_list = []
first_dt = np.min([FFfile.filenameToDatetime(ff_name) for ff_name in recalibrated_platepars])
for ff_name in recalibrated_platepars:
pp_temp = recalibrated_platepars[ff_name]
# If the fitting failed, skip the platepar
if pp_temp is None:
continue
# Add the datetime of the FF file to the list
ff_dt = FFfile.filenameToDatetime(ff_name)
dt_list.append(ff_dt)
# Compute the angular separation from the reference platepar
ang_dist = np.degrees(angularSeparation(np.radians(platepar.RA_d), np.radians(platepar.dec_d), \
np.radians(pp_temp.RA_d), np.radians(pp_temp.dec_d)))
ang_dists.append(ang_dist*60)
# Compute rotation difference
rot_diff = (platepar.pos_angle_ref - pp_temp.pos_angle_ref + 180)%360 - 180
rot_angles.append(rot_diff*60)
# Compute the hour of the FF used for recalibration
hour_list.append((ff_dt - first_dt).total_seconds()/3600)
# Add the photometric offset to the list
photom_offset_list.append(pp_temp.mag_lev)
photom_offset_std_list.append(pp_temp.mag_lev_stddev)
if generate_plot:
# Generate the name the plots
plot_name = os.path.basename(ftpdetectinfo_path).replace('FTPdetectinfo_', '').replace('.txt', '')
### Plot difference from reference platepar in angular distance from (0, 0) vs rotation ###
plt.figure()
plt.scatter(0, 0, marker='o', edgecolor='k', label='Reference platepar', s=100, c='none', zorder=3)
plt.scatter(ang_dists, rot_angles, c=hour_list, zorder=3)
plt.colorbar(label="Hours from first FF file")
plt.xlabel("Angular distance from reference (arcmin)")
plt.ylabel("Rotation from reference (arcmin)")
plt.title("FOV centre drift starting at {:s}".format(first_dt.strftime("%Y/%m/%d %H:%M:%S")))
plt.grid()
plt.legend()
plt.tight_layout()
plt.savefig(os.path.join(dir_path, plot_name + '_calibration_variation.png'), dpi=150)
# plt.show()
plt.clf()
plt.close()
### ###
### Plot the photometric offset variation ###
plt.figure()
plt.errorbar(dt_list, photom_offset_list, yerr=photom_offset_std_list, fmt="o", \
ecolor='lightgray', elinewidth=2, capsize=0, ms=2)
# Format datetimes
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%H:%M"))
# rotate and align the tick labels so they look better
plt.gcf().autofmt_xdate()
plt.xlabel("UTC time")
plt.ylabel("Photometric offset")
plt.title("Photometric offset variation")
plt.grid()
plt.tight_layout()
plt.savefig(os.path.join(dir_path, plot_name + '_photometry_variation.png'), dpi=150)
plt.clf()
plt.close()
### ###
### Apply platepars to FTPdetectinfo ###
meteor_output_list = []
for meteor_entry in meteor_list:
ff_name, meteor_No, rho, phi, meteor_meas = meteor_entry
# Get the platepar that will be applied to this FF file
if ff_name in recalibrated_platepars:
working_platepar = recalibrated_platepars[ff_name]
else:
print('Using default platepar for:', ff_name)
working_platepar = platepar
# Apply the recalibrated platepar to meteor centroids
meteor_picks = applyPlateparToCentroids(ff_name, fps, meteor_meas, working_platepar, \
add_calstatus=True)
meteor_output_list.append([ff_name, meteor_No, rho, phi, meteor_picks])
# Calibration string to be written to the FTPdetectinfo file
calib_str = 'Recalibrated with RMS on: ' + str(datetime.datetime.utcnow()) + ' UTC'
# If no meteors were detected, set dummpy parameters
if len(meteor_list) == 0:
cam_code = ''
fps = 0
# Back up the old FTPdetectinfo file
try:
shutil.copy(ftpdetectinfo_path, ftpdetectinfo_path.strip('.txt') \
+ '_backup_{:s}.txt'.format(datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S.%f')))
except:
print('ERROR! The FTPdetectinfo file could not be backed up: {:s}'.format(ftpdetectinfo_path))
# Save the updated FTPdetectinfo
FTPdetectinfo.writeFTPdetectinfo(meteor_output_list, dir_path, os.path.basename(ftpdetectinfo_path), \
dir_path, cam_code, fps, calibration=calib_str, celestial_coords_given=True)
### ###
return recalibrated_platepars
def alignPlatepar(config, platepar, calstars_time, calstars_coords, scale_update=False, show_plot=False):
""" Align the platepar using FFT registration between catalog stars and the given list of image stars.
Arguments:
config:
platepar: [Platepar instance] Initial platepar.
calstars_time: [list] A list of (year, month, day, hour, minute, second, millisecond) of the middle of
the FF file used for alignment.
calstars_coords: [ndarray] A 2D numpy array of (x, y) coordinates of image stars.
Keyword arguments:
scale_update: [bool] Update the platepar scale. False by default.
show_plot: [bool] Show the comparison between the reference and image synthetic images.
Return:
platepar_aligned: [Platepar instance] The aligned platepar.
"""
# Try to optimize the catalog limiting magnitude until the number of image and catalog stars are matched
maxiter = 10
search_fainter = True
mag_step = 0.2
for inum in range(maxiter):
# Load the catalog stars
catalog_stars, _, _ = StarCatalog.readStarCatalog(config.star_catalog_path, config.star_catalog_file, \
lim_mag=config.catalog_mag_limit, mag_band_ratios=config.star_catalog_band_ratios)
# Get the RA/Dec of the image centre
_, ra_centre, dec_centre, _ = ApplyAstrometry.xyToRaDecPP([calstars_time], [platepar.X_res/2], \
[platepar.Y_res/2], [1], platepar)
ra_centre = ra_centre[0]
dec_centre = dec_centre[0]
# Calculate the FOV radius in degrees
fov_y, fov_x = ApplyAstrometry.computeFOVSize(platepar)
fov_radius = np.sqrt(fov_x**2 + fov_y**2)
# Take only those stars which are inside the FOV
filtered_indices, _ = subsetCatalog(catalog_stars, ra_centre, dec_centre, \
fov_radius, config.catalog_mag_limit)
# Take those catalog stars which should be inside the FOV
ra_catalog, dec_catalog, _ = catalog_stars[filtered_indices].T
jd = date2JD(*calstars_time)
catalog_xy = ApplyAstrometry.raDecToXYPP(ra_catalog, dec_catalog, jd, platepar)
catalog_x, catalog_y = catalog_xy
catalog_xy = np.c_[catalog_x, catalog_y]
# Cut all stars that are outside image coordinates
catalog_xy = catalog_xy[catalog_xy[:, 0] > 0]
catalog_xy = catalog_xy[catalog_xy[:, 0] < config.width]
catalog_xy = catalog_xy[catalog_xy[:, 1] > 0]
catalog_xy = catalog_xy[catalog_xy[:, 1] < config.height]
# If there are more catalog than image stars, this means that the limiting magnitude is too faint
# and that the search should go in the brighter direction
if len(catalog_xy) > len(calstars_coords):
search_fainter = False
else:
search_fainter = True
# print('Catalog stars:', len(catalog_xy), 'Image stars:', len(calstars_coords), \
# 'Limiting magnitude:', config.catalog_mag_limit)
# Search in mag_step magnitude steps
if search_fainter:
config.catalog_mag_limit += mag_step
else:
config.catalog_mag_limit -= mag_step
print('Final catalog limiting magnitude:', config.catalog_mag_limit)
# Find the transform between the image coordinates and predicted platepar coordinates
res = findStarsTransform(config, calstars_coords, catalog_xy, show_plot=show_plot)
angle, scale, translation_x, translation_y = res
### Update the platepar ###
platepar_aligned = copy.deepcopy(platepar)
# Correct the rotation
platepar_aligned.pos_angle_ref = (platepar_aligned.pos_angle_ref - angle)%360
# Update the scale if needed
if scale_update:
platepar_aligned.F_scale *= scale
# Compute the new reference RA and Dec
# _, ra_centre_new, dec_centre_new, _ = ApplyAstrometry.xyToRaDecPP([jd2Date(platepar.JD)], \
# [platepar.X_res/2 - translation_x], [platepar.Y_res/2 - translation_y], [1], platepar)
_, ra_centre_new, dec_centre_new, _ = ApplyAstrometry.xyToRaDecPP([jd2Date(platepar.JD)], \
[platepar.X_res/2 - platepar.x_poly_fwd[0] - translation_x], \
[platepar.Y_res/2 - platepar.y_poly_fwd[0] - translation_y], [1], platepar)
# Correct RA/Dec
platepar_aligned.RA_d = ra_centre_new[0]
platepar_aligned.dec_d = dec_centre_new[0]
# # Update the reference time and hour angle
# platepar_aligned.JD = jd
# platepar_aligned.Ho = JD2HourAngle(jd)
# Recompute the FOV centre in Alt/Az and update the rotation
platepar_aligned.az_centre, platepar_aligned.alt_centre = ApplyAstrometry.raDec2AltAz(platepar.JD, \
platepar.lon, platepar.lat, platepar.RA_d, platepar.dec_d)
platepar_aligned.rotation_from_horiz = ApplyAstrometry.rotationWrtHorizon(platepar_aligned)
# Indicate that the platepar has been automatically updated
platepar_aligned.auto_check_fit_refined = True
###
return platepar_aligned
def autoCheckFit(config, platepar, calstars_list, _fft_refinement=False):
""" Attempts to refine the astrometry fit with the given stars and and initial astrometry parameters.
Arguments:
config: [Config structure]
platepar: [Platepar structure] Initial astrometry parameters.
calstars_list: [list] A list containing stars extracted from FF files. See RMS.Formats.CALSTARS for
more details.
Keyword arguments:
_fft_refinement: [bool] Internal flag indicating that autoCF is running the second time recursively
after FFT platepar adjustment.
Return:
(platepar, fit_status):
platepar: [Platepar structure] Estimated/refined platepar.
fit_status: [bool] True if fit was successfuly, False if not.
"""
def _handleFailure(config, platepar, calstars_list, catalog_stars,
_fft_refinement):
""" Run FFT alignment before giving up on ACF. """
if not _fft_refinement:
print()
print(
"-------------------------------------------------------------------------------"
)
print(
'The initial platepar is bad, trying to refine it using FFT phase correlation...'
)
print()
# Prepare data for FFT image registration
calstars_dict = {
ff_file: star_data
for ff_file, star_data in calstars_list
}
# Extract star list from CALSTARS file from FF file with most stars
max_len_ff = max(calstars_dict,
key=lambda k: len(calstars_dict[k]))
# Take only X, Y (change order so X is first)
calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
# Get the time of the FF file
calstars_time = FFfile.getMiddleTimeFF(max_len_ff,
config.fps,
ret_milliseconds=True)
# Try aligning the platepar using FFT image registration
platepar_refined = alignPlatepar(config, platepar, calstars_time,
calstars_coords)
print()
### If there are still not enough stars matched, try FFT again ###
min_radius = 10
# Prepare star dictionary to check the match
dt = FFfile.getMiddleTimeFF(max_len_ff,
config.fps,
ret_milliseconds=True)
jd = date2JD(*dt)
star_dict_temp = {}
star_dict_temp[jd] = calstars_dict[max_len_ff]
# Check the number of matched stars
n_matched, _, _, _ = matchStarsResiduals(config, platepar_refined, catalog_stars, \
star_dict_temp, min_radius, ret_nmatch=True, verbose=True)
# Realign again if necessary
if n_matched < config.min_matched_stars:
print()
print(
"-------------------------------------------------------------------------------"
)
print(
'Doing a second FFT pass as the number of matched stars was too small...'
)
print()
platepar_refined = alignPlatepar(config, platepar_refined,
calstars_time,
calstars_coords)
print()
### ###
# Redo autoCF
return autoCheckFit(config,
platepar_refined,
calstars_list,
_fft_refinement=True)
else:
print(
'Auto Check Fit failed completely, please redo the plate manually!'
)
return platepar, False
if _fft_refinement:
print(
'Second ACF run with an updated platepar via FFT phase correlation...'
)
# Load catalog stars (overwrite the mag band ratios if specific catalog is used)
catalog_stars, _, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(config.star_catalog_path, \
config.star_catalog_file, lim_mag=config.catalog_mag_limit, \
mag_band_ratios=config.star_catalog_band_ratios)
# Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars
star_dict = starListToDict(config,
calstars_list,
max_ffs=config.calstars_files_N)
# There has to be a minimum of 200 FF files for star fitting
if len(star_dict) < config.calstars_files_N:
print('Not enough FF files in CALSTARS for ACF!')
return platepar, False
# Calculate the total number of calibration stars used
total_calstars = sum([len(star_dict[key]) for key in star_dict])
print('Total calstars:', total_calstars)
if total_calstars < config.calstars_min_stars:
print('Not enough calibration stars, need at least',
config.calstars_min_stars)
return platepar, False
print()
# A list of matching radiuses to try
min_radius = 0.5
radius_list = [10, 5, 3, 1.5, min_radius]
# Calculate the function tolerance, so the desired precision can be reached (the number is calculated
# in the same regard as the cost function)
fatol, xatol_ang = computeMinimizationTolerances(config, platepar,
len(star_dict))
### If the initial match is good enough, do only quick recalibratoin ###
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
min_radius, ret_nmatch=True)
if n_matched >= config.calstars_files_N:
# Check if the average distance with the tightest radius is close
if avg_dist < config.dist_check_quick_threshold:
print("Using quick fit with smaller radiia...")
# Use a reduced set of initial radius values
radius_list = [1.5, min_radius]
##########
# Match increasingly smaller search radiia around image stars
for i, match_radius in enumerate(radius_list):
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
match_radius, ret_nmatch=True)
print()
print("-------------------------------------------------------------")
print("Refining camera pointing with max pixel deviation = {:.1f} px".
format(match_radius))
print("Initial values:")
print(" Matched stars = {:>6d}".format(n_matched))
print(" Average deviation = {:>6.2f} px".format(avg_dist))
# The initial number of matched stars has to be at least the number of FF imaages, otherwise it means
# that the initial platepar is no good
if n_matched < config.calstars_files_N:
print(
"The total number of initially matched stars is too small! Please manually redo the plate or make sure there are enough calibration stars."
)
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list,
catalog_stars, _fft_refinement)
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config,
platepar,
catalog_stars,
star_dict,
min_radius,
verbose=True):
# Print out notice only if the platepar is good right away
if i == 0:
print("Initial platepar is good enough!")
return platepar, True
# Initial parameters for the astrometric fit
p0 = [
platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref,
platepar.F_scale
]
# Fit the astrometric parameters
res = scipy.optimize.minimize(_calcImageResidualsAstro, p0, args=(config, platepar, catalog_stars, \
star_dict, match_radius), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': xatol_ang})
print(res)
# If the fit was not successful, stop further fitting
if not res.success:
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list,
catalog_stars, _fft_refinement)
else:
# If the fit was successful, use the new parameters from now on
ra_ref, dec_ref, pos_angle_ref, F_scale = res.x
platepar.RA_d = ra_ref
platepar.dec_d = dec_ref
platepar.pos_angle_ref = pos_angle_ref
platepar.F_scale = F_scale
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config,
platepar,
catalog_stars,
star_dict,
min_radius,
verbose=True):
return platepar, True
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \
star_dict, min_radius, ret_nmatch=True)
print("FINAL SOLUTION with radius {:.1} px:".format(min_radius))
print(" Matched stars = {:>6d}".format(n_matched))
print(" Average deviation = {:>6.2f} px".format(avg_dist))
# Mark the platepar to indicate that it was automatically refined with CheckFit
platepar.auto_check_fit_refined = True
# Recompute alt/az of the FOV centre
platepar.az_centre, platepar.alt_centre = raDec2AltAz(platepar.RA_d, platepar.dec_d, platepar.JD, \
platepar.lat, platepar.lon)
# Recompute the rotation wrt horizon
platepar.rotation_from_horiz = rotationWrtHorizon(platepar)
return platepar, True
def alignPlatepar(config,
platepar,
calstars_time,
calstars_coords,
scale_update=False,
show_plot=False):
""" Align the platepar using FFT registration between catalog stars and the given list of image stars.
Arguments:
config:
platepar: [Platepar instance] Initial platepar.
calstars_time: [list] A list of (year, month, day, hour, minute, second, millisecond) of the middle of
the FF file used for alignment.
calstars_coords: [ndarray] A 2D numpy array of (x, y) coordinates of image stars.
Keyword arguments:
scale_update: [bool] Update the platepar scale. False by default.
show_plot: [bool] Show the comparison between the reference and image synthetic images.
Return:
platepar_aligned: [Platepar instance] The aligned platepar.
"""
# Create a copy of the config not to mess with the original config parameters
config = copy.deepcopy(config)
# Try to optimize the catalog limiting magnitude until the number of image and catalog stars are matched
maxiter = 10
search_fainter = True
mag_step = 0.2
for inum in range(maxiter):
# Load the catalog stars
catalog_stars, _, _ = StarCatalog.readStarCatalog(config.star_catalog_path, config.star_catalog_file, \
lim_mag=config.catalog_mag_limit, mag_band_ratios=config.star_catalog_band_ratios)
# Get the RA/Dec of the image centre
_, ra_centre, dec_centre, _ = ApplyAstrometry.xyToRaDecPP([calstars_time], [platepar.X_res/2], \
[platepar.Y_res/2], [1], platepar, extinction_correction=False)
ra_centre = ra_centre[0]
dec_centre = dec_centre[0]
# Compute Julian date
jd = date2JD(*calstars_time)
# Calculate the FOV radius in degrees
fov_y, fov_x = ApplyAstrometry.computeFOVSize(platepar)
fov_radius = np.sqrt(fov_x**2 + fov_y**2)
# Take only those stars which are inside the FOV
filtered_indices, _ = subsetCatalog(catalog_stars, ra_centre, dec_centre, jd, platepar.lat, \
platepar.lon, fov_radius, config.catalog_mag_limit)
# Take those catalog stars which should be inside the FOV
ra_catalog, dec_catalog, _ = catalog_stars[filtered_indices].T
catalog_xy = ApplyAstrometry.raDecToXYPP(ra_catalog, dec_catalog, jd,
platepar)
catalog_x, catalog_y = catalog_xy
catalog_xy = np.c_[catalog_x, catalog_y]
# Cut all stars that are outside image coordinates
catalog_xy = catalog_xy[catalog_xy[:, 0] > 0]
catalog_xy = catalog_xy[catalog_xy[:, 0] < config.width]
catalog_xy = catalog_xy[catalog_xy[:, 1] > 0]
catalog_xy = catalog_xy[catalog_xy[:, 1] < config.height]
# If there are more catalog than image stars, this means that the limiting magnitude is too faint
# and that the search should go in the brighter direction
if len(catalog_xy) > len(calstars_coords):
search_fainter = False
else:
search_fainter = True
# print('Catalog stars:', len(catalog_xy), 'Image stars:', len(calstars_coords), \
# 'Limiting magnitude:', config.catalog_mag_limit)
# Search in mag_step magnitude steps
if search_fainter:
config.catalog_mag_limit += mag_step
else:
config.catalog_mag_limit -= mag_step
print('Final catalog limiting magnitude:', config.catalog_mag_limit)
# Find the transform between the image coordinates and predicted platepar coordinates
res = findStarsTransform(config,
calstars_coords,
catalog_xy,
show_plot=show_plot)
angle, scale, translation_x, translation_y = res
### Update the platepar ###
platepar_aligned = copy.deepcopy(platepar)
# Correct the rotation
platepar_aligned.pos_angle_ref = (platepar_aligned.pos_angle_ref -
angle) % 360
# Update the scale if needed
if scale_update:
platepar_aligned.F_scale *= scale
# Compute the new reference RA and Dec
_, ra_centre_new, dec_centre_new, _ = ApplyAstrometry.xyToRaDecPP([jd2Date(platepar.JD)], \
[platepar.X_res/2 - platepar.x_poly_fwd[0] - translation_x], \
[platepar.Y_res/2 - platepar.y_poly_fwd[0] - translation_y], [1], platepar, \
extinction_correction=False)
# Correct RA/Dec
platepar_aligned.RA_d = ra_centre_new[0]
platepar_aligned.dec_d = dec_centre_new[0]
# # Update the reference time and hour angle
# platepar_aligned.JD = jd
# platepar_aligned.Ho = JD2HourAngle(jd)
# Recompute the FOV centre in Alt/Az and update the rotation
platepar_aligned.az_centre, platepar_aligned.alt_centre = raDec2AltAz(platepar.RA_d, \
platepar.dec_d, platepar.JD, platepar.lat, platepar.lon)
platepar_aligned.rotation_from_horiz = ApplyAstrometry.rotationWrtHorizon(
platepar_aligned)
###
return platepar_aligned
def autoCheckFit(config, platepar, calstars_list, distorsion_refinement=False, _fft_refinement=False):
""" Attempts to refine the astrometry fit with the given stars and and initial astrometry parameters.
Arguments:
config: [Config structure]
platepar: [Platepar structure] Initial astrometry parameters.
calstars_list: [list] A list containing stars extracted from FF files. See RMS.Formats.CALSTARS for
more details.
Keyword arguments:
distorsion_refinement: [bool] Whether the distorsion should be fitted as well. False by default.
_fft_refinement: [bool] Internal flag indicating that autoCF is running the second time recursively
after FFT platepar adjustment.
Return:
(platepar, fit_status):
platepar: [Platepar structure] Estimated/refined platepar.
fit_status: [bool] True if fit was successfuly, False if not.
"""
def _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement):
""" Run FFT alignment before giving up on ACF. """
if not _fft_refinement:
print('The initial platepar is bad, trying to refine it using FFT phase correlation...')
# Prepare data for FFT image registration
calstars_dict = {ff_file: star_data for ff_file, star_data in calstars_list}
# Extract star list from CALSTARS file from FF file with most stars
max_len_ff = max(calstars_dict, key=lambda k: len(calstars_dict[k]))
# Take only X, Y (change order so X is first)
calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
# Get the time of the FF file
calstars_time = FFfile.getMiddleTimeFF(max_len_ff, config.fps, ret_milliseconds=True)
# Try aligning the platepar using FFT image registration
platepar_refined = alignPlatepar(config, platepar, calstars_time, calstars_coords)
# Redo autoCF
return autoCheckFit(config, platepar_refined, calstars_list, \
distorsion_refinement=distorsion_refinement, _fft_refinement=True)
else:
print('Auto Check Fit failed completely, please redo the plate manually!')
return platepar, False
if _fft_refinement:
print('Second ACF run with an updated platepar via FFT phase correlation...')
# Convert the list to a dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
# Load catalog stars (overwrite the mag band ratios if specific catalog is used)
catalog_stars, _, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(config.star_catalog_path, \
config.star_catalog_file, lim_mag=config.catalog_mag_limit, \
mag_band_ratios=config.star_catalog_band_ratios)
# Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars
star_dict = {}
# Take only those files with enough stars on them
for ff_name in calstars:
stars_list = calstars[ff_name]
# Check if there are enough stars on the image
if len(stars_list) >= config.ff_min_stars:
# Calculate the JD time of the FF file
dt = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)
jd = date2JD(*dt)
# Add the time and the stars to the dict
star_dict[jd] = stars_list
# There has to be a minimum of 200 FF files for star fitting, and only 100 will be subset if there are more
if len(star_dict) < config.calstars_files_N:
print('Not enough FF files in CALSTARS for ACF!')
return platepar, False
else:
# Randomly choose calstars_files_N image files from the whole list
rand_keys = random.sample(list(star_dict), config.calstars_files_N)
star_dict = {key: star_dict[key] for key in rand_keys}
# Calculate the total number of calibration stars used
total_calstars = sum([len(star_dict[key]) for key in star_dict])
print('Total calstars:', total_calstars)
if total_calstars < config.calstars_min_stars:
print('Not enough calibration stars, need at least', config.calstars_min_stars)
return platepar, False
# A list of matching radiuses to try, pairs of [radius, fit_distorsion_flag]
# The distorsion will be fitted only if explicity requested
min_radius = 0.5
radius_list = [[10, False],
[5, False],
[3, False],
[1.5, True and distorsion_refinement],
[min_radius, True and distorsion_refinement]]
# Calculate the function tolerance, so the desired precision can be reached (the number is calculated
# in the same regard as the cost function)
fatol, xatol_ang = computeMinimizationTolerances(config, platepar, len(star_dict))
### If the initial match is good enough, do only quick recalibratoin ###
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
min_radius, ret_nmatch=True)
if n_matched >= config.calstars_files_N:
# Check if the average distance with the tightest radius is close
if avg_dist < config.dist_check_quick_threshold:
# Use a reduced set of initial radius values
radius_list = [[1.5, True and distorsion_refinement],
[min_radius, True and distorsion_refinement]]
##########
# Match increasingly smaller search radiia around image stars
for i, (match_radius, fit_distorsion) in enumerate(radius_list):
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
match_radius, ret_nmatch=True)
print('Max radius:', match_radius)
print('Initial values:')
print(' Matched stars:', n_matched)
print(' Average deviation:', avg_dist)
# The initial number of matched stars has to be at least the number of FF imaages, otherwise it means
# that the initial platepar is no good
if n_matched < config.calstars_files_N:
print('The total number of initially matched stars is too small! Please manually redo the plate or make sure there are enough calibration stars.')
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict, min_radius, verbose=True):
# Print out notice only if the platepar is good right away
if i == 0:
print("Initial platepar is good enough!")
return platepar, True
# Initial parameters for the astrometric fit (don't fit the scale if the distorsion is not being fit)
if fit_distorsion:
p0 = [platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref, platepar.F_scale]
else:
p0 = [platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref]
# Fit the astrometric parameters
res = scipy.optimize.minimize(_calcImageResidualsAstro, p0, args=(config, platepar, catalog_stars, \
star_dict, match_radius, fit_distorsion), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': xatol_ang})
print(res)
# If the fit was not successful, stop further fitting
if not res.success:
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
else:
# If the fit was successful, use the new parameters from now on
if fit_distorsion:
ra_ref, dec_ref, pos_angle_ref, F_scale = res.x
else:
ra_ref, dec_ref, pos_angle_ref = res.x
F_scale = platepar.F_scale
platepar.RA_d = ra_ref
platepar.dec_d = dec_ref
platepar.pos_angle_ref = pos_angle_ref
platepar.F_scale = F_scale
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict, min_radius, verbose=True):
return platepar, True
# Fit the lens distorsion parameters
if fit_distorsion:
### REVERSE DISTORSION POLYNOMIALS FIT ###
# Fit the distortion parameters (X axis)
res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.x_poly_rev, args=(config, \
platepar, catalog_stars, star_dict, match_radius, 'x'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
else:
platepar.x_poly_rev = res.x
# Fit the distortion parameters (Y axis)
res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.y_poly_rev, args=(config, \
platepar,catalog_stars, star_dict, match_radius, 'y'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
else:
platepar.y_poly_rev = res.x
### ###
### FORWARD DISTORSION POLYNOMIALS FIT ###
# Fit the distortion parameters (X axis)
res = scipy.optimize.minimize(_calcSkyResidualsDistorsion, platepar.x_poly_fwd, args=(config, \
platepar, catalog_stars, star_dict, match_radius, 'x'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
else:
platepar.x_poly_fwd = res.x
# Fit the distortion parameters (Y axis)
res = scipy.optimize.minimize(_calcSkyResidualsDistorsion, platepar.y_poly_fwd, args=(config, \
platepar,catalog_stars, star_dict, match_radius, 'y'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
return platepar, False
else:
platepar.y_poly_fwd = res.x
### ###
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \
star_dict, min_radius, ret_nmatch=True)
print('FINAL SOLUTION with {:f} px:'.format(min_radius))
print('Matched stars:', n_matched)
print('Average deviation:', avg_dist)
# Mark the platepar to indicate that it was automatically refined with CheckFit
platepar.auto_check_fit_refined = True
return platepar, True
def autoCheckFit(config, platepar, calstars_list):
""" Attempts to refine the astrometry fit with the given stars and and initial astrometry parameters.
Arguments:
config: [Config structure]
platepar: [Platepar structure] Initial astrometry parameters.
calstars_list: [list] A list containing stars extracted from FF files. See RMS.Formats.CALSTARS for
more details.
Return:
(platepar, fit_status):
platepar: [Platepar structure] Estimated/refined platepar.
fit_status: [bool] True if fit was successfuly, False if not.
"""
# Convert the list to a dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
# Load catalog stars
catalog_stars = StarCatalog.readStarCatalog(config.star_catalog_path, config.star_catalog_file, \
lim_mag=config.catalog_mag_limit, mag_band_ratios=config.star_catalog_band_ratios)
# Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars
star_dict = {}
# Take only those files with enough stars on them
for ff_name in calstars:
stars_list = calstars[ff_name]
# Check if there are enough stars on the image
if len(stars_list) >= config.ff_min_stars:
# Calculate the JD time of the FF file
dt = FFfile.getMiddleTimeFF(ff_name,
config.fps,
ret_milliseconds=True)
jd = date2JD(*dt)
# Add the time and the stars to the dict
star_dict[jd] = stars_list
# There has to be a minimum of 200 FF files for star fitting, and only 100 will be subset if there are more
if len(star_dict) < config.calstars_files_N:
print('Not enough FF files in CALSTARS for ACF!')
return platepar, False
else:
# Randomly choose calstars_files_N image files from the whole list
rand_keys = random.sample(list(star_dict), config.calstars_files_N)
star_dict = {key: star_dict[key] for key in rand_keys}
# Calculate the total number of calibration stars used
total_calstars = sum([len(star_dict[key]) for key in star_dict])
print('Total calstars:', total_calstars)
if total_calstars < config.calstars_min_stars:
print('Not enough calibration stars, need at least',
config.calstars_min_stars)
return platepar, False
# A list of matching radiuses to try, pairs of [radius, fit_distorsion_flag]
min_radius = 0.5
radius_list = [[10, False], [5, False], [3, False], [1.5, True],
[min_radius, True]]
# Calculate the function tolerance, so the desired precision can be reached (the number is calculated
# in the same reagrd as the cost function)
fatol = (config.dist_check_threshold**
2) / np.sqrt(len(star_dict) * config.min_matched_stars + 1)
# Parameter estimation tolerance for angular values
fov_w = platepar.X_res / platepar.F_scale
xatol_ang = config.dist_check_threshold * fov_w / platepar.X_res
### If the initial match is good enough, do only quick recalibratoin ###
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
min_radius, ret_nmatch=True)
if n_matched >= config.calstars_files_N:
# Check if the average distance with the tightest radius is close
if avg_dist < config.dist_check_quick_threshold:
# Use a reduced set of initial radius values
radius_list = [[1.5, True], [min_radius, True]]
##########
# Match increasingly smaller search radiia around image stars
for i, (match_radius, fit_distorsion) in enumerate(radius_list):
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
match_radius, ret_nmatch=True)
print('Max radius:', match_radius)
print('Initial values:')
print(' Matched stars:', n_matched)
print(' Average deviation:', avg_dist)
# The initial number of matched stars has to be at least the number of FF imaages, otherwise it means
# that the initial platepar is no good
if n_matched < config.calstars_files_N:
print(
'The total number of initially matched stars is too small! Please manually redo the plate or make sure there are enough calibration stars.'
)
return platepar, False
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict,
min_radius):
# Print out notice only if the platepar is good right away
if i == 0:
print("Initial platepar is good enough!")
return platepar, True
# Initial parameters for the astrometric fit
p0 = [
platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref,
platepar.F_scale
]
# Fit the astrometric parameters
res = scipy.optimize.minimize(_calcImageResidualsAstro, p0, args=(config, platepar, catalog_stars, \
star_dict, match_radius), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': xatol_ang})
print(res)
# If the fit was not successful, stop further fitting
if not res.success:
return platepar, False
else:
# If the fit was successful, use the new parameters from now on
ra_ref, dec_ref, pos_angle_ref, F_scale = res.x
platepar.RA_d = ra_ref
platepar.dec_d = dec_ref
platepar.pos_angle_ref = pos_angle_ref
platepar.F_scale = F_scale
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict,
min_radius):
return platepar, True
# Fit the lens distorsion parameters
if fit_distorsion:
# Fit the distortion parameters (X axis)
res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.x_poly, args=(config, platepar,\
catalog_stars, star_dict, match_radius, 'x'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
return platepar, False
else:
platepar.x_poly = res.x
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict,
min_radius):
return platepar, True
# Fit the distortion parameters (Y axis)
res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.y_poly, args=(config, platepar,\
catalog_stars, star_dict, match_radius, 'y'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
return platepar, False
else:
platepar.y_poly = res.x
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \
star_dict, min_radius, ret_nmatch=True)
print('FINAL SOLUTION with {:f} px:'.format(min_radius))
print('Matched stars:', n_matched)
print('Average deviation:', avg_dist)
return platepar, True
def recalibrateSelectedFF(dir_path, ff_file_names, calstars_list, config, lim_mag, \
pp_recalib_name, ignore_distance_threshold=False):
"""Recalibrate FF files, ignoring whether there are detections.
Arguments:
dir_path: [str] Path where the FF files are.
ff_file_names: [str] List of ff files to recalibrate platepars to
calstars_list: [list] A list of entries [[ff_name, star_coordinates], ...].
config: [Config instance]
lim_mag: [float] Limiting magnitude for the catalog.
pp_recalib_name: [str] Name for the file where the recalibrated platepars will be stored as JSON.
Keyword arguments:
ignore_distance_threshold: [bool] Don't consider the recalib as failed if the median distance
is larger than the threshold.
Return:
recalibrated_platepars: [dict] A dictionary where the keys are FF file names and values are
recalibrated platepar instances for every FF file.
"""
config = copy.deepcopy(config)
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
# load star catalog with increased catalog limiting magnitude
star_catalog_status = StarCatalog.readStarCatalog(
config.star_catalog_path,
config.star_catalog_file,
lim_mag=lim_mag,
mag_band_ratios=config.star_catalog_band_ratios,
)
if not star_catalog_status:
print("Could not load the star catalog!")
print(os.path.join(config.star_catalog_path, config.star_catalog_file))
return {}
catalog_stars, _, config.star_catalog_band_ratios = star_catalog_status
# print(catalog_stars)
prev_platepar = Platepar.Platepar()
prev_platepar.read(os.path.join(dir_path, config.platepar_name),
use_flat=config.use_flat)
# Update the platepar coordinates from the config file
prev_platepar.lat = config.latitude
prev_platepar.lon = config.longitude
prev_platepar.elev = config.elevation
recalibrated_platepars = recalibratePlateparsForFF(
prev_platepar,
ff_file_names,
calstars,
catalog_stars,
config,
lim_mag=lim_mag,
ignore_distance_threshold=ignore_distance_threshold,
)
# Store recalibrated platepars in json
all_pps = {}
for ff_name in recalibrated_platepars:
json_str = recalibrated_platepars[ff_name].jsonStr()
all_pps[ff_name] = json.loads(json_str)
with open(os.path.join(dir_path, pp_recalib_name), 'w') as f:
# Convert all platepars to a JSON file
out_str = json.dumps(all_pps,
default=lambda o: o.__dict__,
indent=4,
sort_keys=True)
f.write(out_str)
return recalibrated_platepars
def generateCalibrationReport(config, night_dir_path, match_radius=2.0, platepar=None, show_graphs=False):
""" Given the folder of the night, find the Calstars file, check the star fit and generate a report
with the quality of the calibration. The report contains information about both the astrometry and
the photometry calibration. Graphs will be saved in the given directory of the night.
Arguments:
config: [Config instance]
night_dir_path: [str] Full path to the directory of the night.
Keyword arguments:
match_radius: [float] Match radius for star matching between image and catalog stars (px).
platepar: [Platepar instance] Use this platepar instead of finding one in the folder.
show_graphs: [bool] Show the graphs on the screen. False by default.
Return:
None
"""
# Find the CALSTARS file in the given folder
calstars_file = None
for calstars_file in os.listdir(night_dir_path):
if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file):
break
if calstars_file is None:
print('CALSTARS file could not be found in the given directory!')
return None
# Load the calstars file
star_list = readCALSTARS(night_dir_path, calstars_file)
### Load recalibrated platepars, if they exist ###
# Find recalibrated platepars file per FF file
platepars_recalibrated_file = None
for file_name in os.listdir(night_dir_path):
if file_name == config.platepars_recalibrated_name:
platepars_recalibrated_file = file_name
break
# Load all recalibrated platepars if the file is available
recalibrated_platepars = None
if platepars_recalibrated_file:
with open(os.path.join(night_dir_path, platepars_recalibrated_file)) as f:
recalibrated_platepars = json.load(f)
print('Loaded recalibrated platepars JSON file for the calibration report...')
### ###
### Load the platepar file ###
# Find the platepar file in the given directory if it was not given
if platepar is None:
# Find the platepar file
platepar_file = None
for file_name in os.listdir(night_dir_path):
if file_name == config.platepar_name:
platepar_file = file_name
break
if platepar_file is None:
print('The platepar cannot be found in the night directory!')
return None
# Load the platepar file
platepar = Platepar()
platepar.read(os.path.join(night_dir_path, platepar_file))
### ###
night_name = os.path.split(night_dir_path.strip(os.sep))[1]
# Go one mag deeper than in the config
lim_mag = config.catalog_mag_limit + 1
# Load catalog stars (load one magnitude deeper)
catalog_stars, mag_band_str, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(\
config.star_catalog_path, config.star_catalog_file, lim_mag=lim_mag, \
mag_band_ratios=config.star_catalog_band_ratios)
### Take only those CALSTARS entires for which FF files exist in the folder ###
# Get a list of FF files in the folder\
ff_list = []
for file_name in os.listdir(night_dir_path):
if validFFName(file_name):
ff_list.append(file_name)
# Filter out calstars entries, generate a star dictionary where the keys are JDs of FFs
star_dict = {}
ff_dict = {}
for entry in star_list:
ff_name, star_data = entry
# Check if the FF from CALSTARS exists in the folder
if ff_name not in ff_list:
continue
dt = getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)
jd = date2JD(*dt)
# Add the time and the stars to the dict
star_dict[jd] = star_data
ff_dict[jd] = ff_name
### ###
# If there are no FF files in the directory, don't generate a report
if len(star_dict) == 0:
print('No FF files from the CALSTARS file in the directory!')
return None
# If the recalibrated platepars file exists, take the one with the most stars
max_jd = 0
if recalibrated_platepars is not None:
max_stars = 0
for ff_name_temp in recalibrated_platepars:
# Compute the Julian date of the FF middle
dt = getMiddleTimeFF(ff_name_temp, config.fps, ret_milliseconds=True)
jd = date2JD(*dt)
# Check that this file exists in CALSTARS and the list of FF files
if (jd not in star_dict) or (jd not in ff_dict):
continue
# Check if the number of stars on this FF file is larger than the before
if len(star_dict[jd]) > max_stars:
max_jd = jd
max_stars = len(star_dict[jd])
# Set a flag to indicate if using recalibrated platepars has failed
if max_jd == 0:
using_recalib_platepars = False
else:
print('Using recalibrated platepars, file:', ff_dict[max_jd])
using_recalib_platepars = True
# Select the platepar where the FF file has the most stars
platepar_dict = recalibrated_platepars[ff_dict[max_jd]]
platepar = Platepar()
platepar.loadFromDict(platepar_dict)
filtered_star_dict = {max_jd: star_dict[max_jd]}
# Match stars on the image with the stars in the catalog
n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \
filtered_star_dict, match_radius, ret_nmatch=True, lim_mag=lim_mag)
max_matched_stars = n_matched
# Otherwise take the optimal FF file for evaluation
if (recalibrated_platepars is None) or (not using_recalib_platepars):
# If there are more than a set number of FF files to evaluate, choose only the ones with most stars on
# the image
if len(star_dict) > config.calstars_files_N:
# Find JDs of FF files with most stars on them
top_nstars_indices = np.argsort([len(x) for x in star_dict.values()])[::-1][:config.calstars_files_N \
- 1]
filtered_star_dict = {}
for i in top_nstars_indices:
filtered_star_dict[list(star_dict.keys())[i]] = list(star_dict.values())[i]
star_dict = filtered_star_dict
# Match stars on the image with the stars in the catalog
n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \
star_dict, match_radius, ret_nmatch=True, lim_mag=lim_mag)
# If no recalibrated platepars where found, find the image with the largest number of matched stars
if (not using_recalib_platepars) or (max_jd == 0):
max_jd = 0
max_matched_stars = 0
for jd in matched_stars:
_, _, distances = matched_stars[jd]
if len(distances) > max_matched_stars:
max_jd = jd
max_matched_stars = len(distances)
# If there are no matched stars, use the image with the largest number of detected stars
if max_matched_stars <= 2:
max_jd = max(star_dict, key=lambda x: len(star_dict[x]))
distances = [np.inf]
# Take the FF file with the largest number of matched stars
ff_name = ff_dict[max_jd]
# Load the FF file
ff = readFF(night_dir_path, ff_name)
img_h, img_w = ff.avepixel.shape
dpi = 200
plt.figure(figsize=(ff.avepixel.shape[1]/dpi, ff.avepixel.shape[0]/dpi), dpi=dpi)
# Take the average pixel
img = ff.avepixel
# Slightly adjust the levels
img = Image.adjustLevels(img, np.percentile(img, 1.0), 1.2, np.percentile(img, 99.99))
plt.imshow(img, cmap='gray', interpolation='nearest')
legend_handles = []
# Plot detected stars
for img_star in star_dict[max_jd]:
y, x, _, _ = img_star
rect_side = 5*match_radius
square_patch = plt.Rectangle((x - rect_side/2, y - rect_side/2), rect_side, rect_side, color='g', \
fill=False, label='Image stars')
plt.gca().add_artist(square_patch)
legend_handles.append(square_patch)
# If there are matched stars, plot them
if max_matched_stars > 2:
# Take the solution with the largest number of matched stars
image_stars, matched_catalog_stars, distances = matched_stars[max_jd]
# Plot matched stars
for img_star in image_stars:
x, y, _, _ = img_star
circle_patch = plt.Circle((y, x), radius=3*match_radius, color='y', fill=False, \
label='Matched stars')
plt.gca().add_artist(circle_patch)
legend_handles.append(circle_patch)
### Plot match residuals ###
# Compute preducted positions of matched image stars from the catalog
x_predicted, y_predicted = raDecToXYPP(matched_catalog_stars[:, 0], \
matched_catalog_stars[:, 1], max_jd, platepar)
img_y, img_x, _, _ = image_stars.T
delta_x = x_predicted - img_x
delta_y = y_predicted - img_y
# Compute image residual and angle of the error
res_angle = np.arctan2(delta_y, delta_x)
res_distance = np.sqrt(delta_x**2 + delta_y**2)
# Calculate coordinates of the beginning of the residual line
res_x_beg = img_x + 3*match_radius*np.cos(res_angle)
res_y_beg = img_y + 3*match_radius*np.sin(res_angle)
# Calculate coordinates of the end of the residual line
res_x_end = img_x + 100*np.cos(res_angle)*res_distance
res_y_end = img_y + 100*np.sin(res_angle)*res_distance
# Plot the 100x residuals
for i in range(len(x_predicted)):
res_plot = plt.plot([res_x_beg[i], res_x_end[i]], [res_y_beg[i], res_y_end[i]], color='orange', \
lw=0.5, label='100x residuals')
legend_handles.append(res_plot[0])
### ###
else:
distances = [np.inf]
# If there are no matched stars, plot large text in the middle of the screen
plt.text(img_w/2, img_h/2, "NO MATCHED STARS!", color='r', alpha=0.5, fontsize=20, ha='center',
va='center')
### Plot positions of catalog stars to the limiting magnitude of the faintest matched star + 1 mag ###
# Find the faintest magnitude among matched stars
if max_matched_stars > 2:
faintest_mag = np.max(matched_catalog_stars[:, 2]) + 1
else:
# If there are no matched stars, use the limiting magnitude from config
faintest_mag = config.catalog_mag_limit + 1
# Estimate RA,dec of the centre of the FOV
_, RA_c, dec_c, _ = xyToRaDecPP([jd2Date(max_jd)], [platepar.X_res/2], [platepar.Y_res/2], [1],
platepar)
RA_c = RA_c[0]
dec_c = dec_c[0]
fov_radius = np.hypot(*computeFOVSize(platepar))
# Get stars from the catalog around the defined center in a given radius
_, extracted_catalog = subsetCatalog(catalog_stars, RA_c, dec_c, fov_radius, faintest_mag)
ra_catalog, dec_catalog, mag_catalog = extracted_catalog.T
# Compute image positions of all catalog stars that should be on the image
x_catalog, y_catalog = raDecToXYPP(ra_catalog, dec_catalog, max_jd, platepar)
# Filter all catalog stars outside the image
temp_arr = np.c_[x_catalog, y_catalog, mag_catalog]
temp_arr = temp_arr[temp_arr[:, 0] >= 0]
temp_arr = temp_arr[temp_arr[:, 0] <= ff.avepixel.shape[1]]
temp_arr = temp_arr[temp_arr[:, 1] >= 0]
temp_arr = temp_arr[temp_arr[:, 1] <= ff.avepixel.shape[0]]
x_catalog, y_catalog, mag_catalog = temp_arr.T
# Plot catalog stars on the image
cat_stars_handle = plt.scatter(x_catalog, y_catalog, c='none', marker='D', lw=1.0, alpha=0.4, \
s=((4.0 + (faintest_mag - mag_catalog))/3.0)**(2*2.512), edgecolor='r', label='Catalog stars')
legend_handles.append(cat_stars_handle)
### ###
# Add info text
info_text = ff_dict[max_jd] + '\n' \
+ "Matched stars: {:d}/{:d}\n".format(max_matched_stars, len(star_dict[max_jd])) \
+ "Median distance: {:.2f} px\n".format(np.median(distances)) \
+ "Catalog limiting magnitude: {:.1f}".format(lim_mag)
plt.text(10, 10, info_text, bbox=dict(facecolor='black', alpha=0.5), va='top', ha='left', fontsize=4, \
color='w')
legend = plt.legend(handles=legend_handles, prop={'size': 4}, loc='upper right')
legend.get_frame().set_facecolor('k')
legend.get_frame().set_edgecolor('k')
for txt in legend.get_texts():
txt.set_color('w')
plt.axis('off')
plt.gca().get_xaxis().set_visible(False)
plt.gca().get_yaxis().set_visible(False)
plt.xlim([0, ff.avepixel.shape[1]])
plt.ylim([ff.avepixel.shape[0], 0])
# Remove the margins
plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)
plt.savefig(os.path.join(night_dir_path, night_name + '_calib_report_astrometry.jpg'), \
bbox_inches='tight', pad_inches=0, dpi=dpi)
if show_graphs:
plt.show()
else:
plt.clf()
plt.close()
if max_matched_stars > 2:
### Plot the photometry ###
plt.figure(dpi=dpi)
# Take only those stars which are inside the 3/4 of the shorter image axis from the center
photom_selection_radius = np.min([img_h, img_w])/3
filter_indices = ((image_stars[:, 0] - img_h/2)**2 + (image_stars[:, 1] \
- img_w/2)**2) <= photom_selection_radius**2
star_intensities = image_stars[filter_indices, 2]
catalog_mags = matched_catalog_stars[filter_indices, 2]
# Plot intensities of image stars
#star_intensities = image_stars[:, 2]
plt.scatter(-2.5*np.log10(star_intensities), catalog_mags, s=5, c='r')
# Fit the photometry on automated star intensities
photom_offset, fit_stddev, _ = photometryFit(np.log10(star_intensities), catalog_mags)
# Plot photometric offset from the platepar
x_min, x_max = plt.gca().get_xlim()
y_min, y_max = plt.gca().get_ylim()
x_min_w = x_min - 3
x_max_w = x_max + 3
y_min_w = y_min - 3
y_max_w = y_max + 3
photometry_info = 'Platepar: {:+.2f}LSP {:+.2f} +/- {:.2f} \nGamma = {:.2f}'.format(platepar.mag_0, \
platepar.mag_lev, platepar.mag_lev_stddev, platepar.gamma)
# Plot the photometry calibration from the platepar
logsum_arr = np.linspace(x_min_w, x_max_w, 10)
plt.plot(logsum_arr, logsum_arr + platepar.mag_lev, label=photometry_info, linestyle='--', \
color='k', alpha=0.5)
# Plot the fitted photometry calibration
fit_info = "Fit: {:+.2f}LSP {:+.2f} +/- {:.2f}".format(-2.5, photom_offset, fit_stddev)
plt.plot(logsum_arr, logsum_arr + photom_offset, label=fit_info, linestyle='--', color='red',
alpha=0.5)
plt.legend()
plt.ylabel("Catalog magnitude ({:s})".format(mag_band_str))
plt.xlabel("Uncalibrated magnitude")
# Set wider axis limits
plt.xlim(x_min_w, x_max_w)
plt.ylim(y_min_w, y_max_w)
plt.gca().invert_yaxis()
plt.gca().invert_xaxis()
plt.grid()
plt.savefig(os.path.join(night_dir_path, night_name + '_calib_report_photometry.png'), dpi=150)
if show_graphs:
plt.show()
else:
plt.clf()
plt.close()
def recalibrateIndividualFFsAndApplyAstrometry(dir_path, ftpdetectinfo_path, calstars_list, config, platepar):
""" Recalibrate FF files with detections and apply the recalibrated platepar to those detections.
Arguments:
dir_path: [str] Path where the FTPdetectinfo file is.
ftpdetectinfo_path: [str] Name of the FTPdetectinfo file.
calstars_list: [list] A list of entries [[ff_name, star_coordinates], ...].
config: [Config instance]
platepar: [Platepar instance] Initial platepar.
Return:
recalibrated_platepars: [dict] A dictionary where the keys are FF file names and values are
recalibrated platepar instances for every FF file.
"""
# Read the FTPdetectinfo data
cam_code, fps, meteor_list = FTPdetectinfo.readFTPdetectinfo(*os.path.split(ftpdetectinfo_path), \
ret_input_format=True)
# Convert the list of stars to a per FF name dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
# Load catalog stars (overwrite the mag band ratios if specific catalog is used)
catalog_stars, _, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(config.star_catalog_path,\
config.star_catalog_file, lim_mag=config.catalog_mag_limit, \
mag_band_ratios=config.star_catalog_band_ratios)
prev_platepar = copy.deepcopy(platepar)
# Go through all FF files with detections, recalibrate and apply astrometry
recalibrated_platepars = {}
for meteor_entry in meteor_list:
working_platepar = copy.deepcopy(prev_platepar)
ff_name, meteor_No, rho, phi, meteor_meas = meteor_entry
# Skip this meteors if its FF file was already recalibrated
if ff_name in recalibrated_platepars:
continue
print()
print('Processing: ', ff_name)
print('------------------------------------------------------------------------------')
# Find extracted stars on this image
if not ff_name in calstars:
print('Skipped because it was not in CALSTARS:', ff_name)
continue
# Get stars detected on this FF file (create a dictionaly with only one entry, the residuals function
# needs this format)
calstars_time = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)
jd = date2JD(*calstars_time)
star_dict_ff = {jd: calstars[ff_name]}
# Recalibrate the platepar using star matching
result = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars)
# If the recalibration failed, try using FFT alignment
if result is None:
print()
print('Running FFT alignment...')
# Run FFT alignment
calstars_coords = np.array(star_dict_ff[jd])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
print(calstars_time)
working_platepar = alignPlatepar(config, prev_platepar, calstars_time, calstars_coords, \
show_plot=False)
# Try to recalibrate after FFT alignment
result = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars)
if result is not None:
working_platepar = result
else:
working_platepar = result
# Store the platepar if the fit succeeded
if result is not None:
recalibrated_platepars[ff_name] = working_platepar
prev_platepar = working_platepar
else:
print('Recalibration of {:s} failed, using the previous platepar...'.format(ff_name))
# If the aligning failed, set the previous platepar as the one that should be used for this FF file
recalibrated_platepars[ff_name] = prev_platepar
### Store all recalibrated platepars as a JSON file ###
all_pps = {}
for ff_name in recalibrated_platepars:
json_str = recalibrated_platepars[ff_name].jsonStr()
all_pps[ff_name] = json.loads(json_str)
with open(os.path.join(dir_path, config.platepars_recalibrated_name), 'w') as f:
# Convert all platepars to a JSON file
out_str = json.dumps(all_pps, default=lambda o: o.__dict__, indent=4, sort_keys=True)
f.write(out_str)
### ###
# If no platepars were recalibrated, use the single platepar recalibration procedure
if len(recalibrated_platepars) == 0:
print('No FF images were used for recalibration, using the single platepar calibration function...')
# Use the initial platepar for calibration
applyAstrometryFTPdetectinfo(dir_path, os.path.basename(ftpdetectinfo_path), None, platepar=platepar)
return recalibrated_platepars
### Plot difference from reference platepar in angular distance from (0, 0) vs rotation ###
ang_dists = []
rot_angles = []
hour_list = []
first_jd = np.min([FFfile.filenameToDatetime(ff_name) for ff_name in recalibrated_platepars])
for ff_name in recalibrated_platepars:
pp_temp = recalibrated_platepars[ff_name]
# If the fitting failed, skip the platepar
if pp_temp is None:
continue
# Compute the angular separation from the reference platepar
ang_dist = np.degrees(angularSeparation(np.radians(platepar.RA_d), np.radians(platepar.dec_d), \
np.radians(pp_temp.RA_d), np.radians(pp_temp.dec_d)))
ang_dists.append(ang_dist*60)
rot_angles.append((platepar.pos_angle_ref - pp_temp.pos_angle_ref)*60)
# Compute the hour of the FF used for recalibration
hour_list.append((FFfile.filenameToDatetime(ff_name) - first_jd).total_seconds()/3600)
plt.figure()
plt.scatter(0, 0, marker='o', edgecolor='k', label='Reference platepar', s=100, c='none', zorder=3)
plt.scatter(ang_dists, rot_angles, c=hour_list, zorder=3)
plt.colorbar(label='Hours from first FF file')
plt.xlabel("Angular distance from reference (arcmin)")
plt.ylabel('Rotation from reference (arcmin)')
plt.grid()
plt.legend()
plt.tight_layout()
# Generate the name for the plot
calib_plot_name = os.path.basename(ftpdetectinfo_path).replace('FTPdetectinfo_', '').replace('.txt', '') \
+ '_calibration_variation.png'
plt.savefig(os.path.join(dir_path, calib_plot_name), dpi=150)
# plt.show()
plt.clf()
plt.close()
### ###
### Apply platepars to FTPdetectinfo ###
meteor_output_list = []
for meteor_entry in meteor_list:
ff_name, meteor_No, rho, phi, meteor_meas = meteor_entry
# Get the platepar that will be applied to this FF file
if ff_name in recalibrated_platepars:
working_platepar = recalibrated_platepars[ff_name]
else:
print('Using default platepar for:', ff_name)
working_platepar = platepar
# Apply the recalibrated platepar to meteor centroids
meteor_picks = applyPlateparToCentroids(ff_name, fps, meteor_meas, working_platepar, \
add_calstatus=True)
meteor_output_list.append([ff_name, meteor_No, rho, phi, meteor_picks])
# Calibration string to be written to the FTPdetectinfo file
calib_str = 'Recalibrated with RMS on: ' + str(datetime.datetime.utcnow()) + ' UTC'
# If no meteors were detected, set dummpy parameters
if len(meteor_list) == 0:
cam_code = ''
fps = 0
# Back up the old FTPdetectinfo file
shutil.copy(ftpdetectinfo_path, ftpdetectinfo_path.strip('.txt') \
+ '_backup_{:s}.txt'.format(datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S.%f')))
# Save the updated FTPdetectinfo
FTPdetectinfo.writeFTPdetectinfo(meteor_output_list, dir_path, os.path.basename(ftpdetectinfo_path), \
dir_path, cam_code, fps, calibration=calib_str, celestial_coords_given=True)
### ###
return recalibrated_platepars