def _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement):
""" Run FFT alignment before giving up on ACF. """
if not _fft_refinement:
print('The initial platepar is bad, trying to refine it using FFT phase correlation...')
# Prepare data for FFT image registration
calstars_dict = {ff_file: star_data for ff_file, star_data in calstars_list}
# Extract star list from CALSTARS file from FF file with most stars
max_len_ff = max(calstars_dict, key=lambda k: len(calstars_dict[k]))
# Take only X, Y (change order so X is first)
calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
# Get the time of the FF file
calstars_time = FFfile.getMiddleTimeFF(max_len_ff, config.fps, ret_milliseconds=True)
# Try aligning the platepar using FFT image registration
platepar_refined = alignPlatepar(config, platepar, calstars_time, calstars_coords)
# Redo autoCF
return autoCheckFit(config, platepar_refined, calstars_list, \
distorsion_refinement=distorsion_refinement, _fft_refinement=True)
else:
print('Auto Check Fit failed completely, please redo the plate manually!')
return platepar, False
def starListToDict(config, calstars_list, max_ffs=None):
""" Converts the list of calstars into dictionary where the keys are FF file JD and the values is
a list of (X, Y, bg_intens, intens) of stars.
"""
# Convert the list to a dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
# Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars
star_dict = {}
# Take only those files with enough stars on them
for ff_name in calstars:
stars_list = calstars[ff_name]
# Check if there are enough stars on the image
if len(stars_list) >= config.ff_min_stars:
# Calculate the JD time of the FF file
dt = FFfile.getMiddleTimeFF(ff_name,
config.fps,
ret_milliseconds=True)
jd = date2JD(*dt)
# Add the time and the stars to the dict
star_dict[jd] = stars_list
if max_ffs is not None:
# Limit the number of FF files used
if len(star_dict) > max_ffs:
# Randomly choose calstars_files_N image files from the whole list
rand_keys = random.sample(list(star_dict), max_ffs)
star_dict = {key: star_dict[key] for key in rand_keys}
return star_dict
def recalibrateIndividualFFsAndApplyAstrometry(dir_path, ftpdetectinfo_path, calstars_list, config, platepar,
generate_plot=True):
""" Recalibrate FF files with detections and apply the recalibrated platepar to those detections.
Arguments:
dir_path: [str] Path where the FTPdetectinfo file is.
ftpdetectinfo_path: [str] Name of the FTPdetectinfo file.
calstars_list: [list] A list of entries [[ff_name, star_coordinates], ...].
config: [Config instance]
platepar: [Platepar instance] Initial platepar.
Keyword arguments:
generate_plot: [bool] Generate the calibration variation plot. True by default.
Return:
recalibrated_platepars: [dict] A dictionary where the keys are FF file names and values are
recalibrated platepar instances for every FF file.
"""
# Use a copy of the config file
config = copy.deepcopy(config)
# If the given file does not exits, return nothing
if not os.path.isfile(ftpdetectinfo_path):
print('ERROR! The FTPdetectinfo file does not exist: {:s}'.format(ftpdetectinfo_path))
print(' The recalibration on every file was not done!')
return {}
# Read the FTPdetectinfo data
cam_code, fps, meteor_list = FTPdetectinfo.readFTPdetectinfo(*os.path.split(ftpdetectinfo_path), \
ret_input_format=True)
# Convert the list of stars to a per FF name dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
### Add neighboring FF files for more robust photometry estimation ###
ff_processing_list = []
# Make a list of sorted FF files in CALSTARS
calstars_ffs = sorted([ff_file for ff_file in calstars])
# Go through the list of FF files with detections and add neighboring FFs
for meteor_entry in meteor_list:
ff_name = meteor_entry[0]
if ff_name in calstars_ffs:
# Find the index of the given FF file in the list of calstars
ff_indx = calstars_ffs.index(ff_name)
# Add neighbours to the processing list
for k in range(-(RECALIBRATE_NEIGHBOURHOOD_SIZE//2), RECALIBRATE_NEIGHBOURHOOD_SIZE//2 + 1):
k_indx = ff_indx + k
if (k_indx > 0) and (k_indx < len(calstars_ffs)):
ff_name_tmp = calstars_ffs[k_indx]
if ff_name_tmp not in ff_processing_list:
ff_processing_list.append(ff_name_tmp)
# Sort the processing list of FF files
ff_processing_list = sorted(ff_processing_list)
### ###
# Globally increase catalog limiting magnitude
config.catalog_mag_limit += 1
# Load catalog stars (overwrite the mag band ratios if specific catalog is used)
star_catalog_status = StarCatalog.readStarCatalog(config.star_catalog_path,\
config.star_catalog_file, lim_mag=config.catalog_mag_limit, \
mag_band_ratios=config.star_catalog_band_ratios)
if not star_catalog_status:
print("Could not load the star catalog!")
print(os.path.join(config.star_catalog_path, config.star_catalog_file))
return {}
catalog_stars, _, config.star_catalog_band_ratios = star_catalog_status
# Update the platepar coordinates from the config file
platepar.lat = config.latitude
platepar.lon = config.longitude
platepar.elev = config.elevation
prev_platepar = copy.deepcopy(platepar)
# Go through all FF files with detections, recalibrate and apply astrometry
recalibrated_platepars = {}
for ff_name in ff_processing_list:
working_platepar = copy.deepcopy(prev_platepar)
# Skip this meteor if its FF file was already recalibrated
if ff_name in recalibrated_platepars:
continue
print()
print('Processing: ', ff_name)
print('------------------------------------------------------------------------------')
# Find extracted stars on this image
if not ff_name in calstars:
print('Skipped because it was not in CALSTARS:', ff_name)
continue
# Get stars detected on this FF file (create a dictionaly with only one entry, the residuals function
# needs this format)
calstars_time = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)
jd = date2JD(*calstars_time)
star_dict_ff = {jd: calstars[ff_name]}
# Recalibrate the platepar using star matching
result, min_match_radius = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars)
# If the recalibration failed, try using FFT alignment
if result is None:
print()
print('Running FFT alignment...')
# Run FFT alignment
calstars_coords = np.array(star_dict_ff[jd])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
print(calstars_time)
test_platepar = alignPlatepar(config, prev_platepar, calstars_time, calstars_coords, \
show_plot=False)
# Try to recalibrate after FFT alignment
result, _ = recalibrateFF(config, test_platepar, jd, star_dict_ff, catalog_stars)
# If the FFT alignment failed, align the original platepar using the smallest radius that matched
# and force save the the platepar
if (result is None) and (min_match_radius is not None):
print()
print("Using the old platepar with the minimum match radius of: {:.2f}".format(min_match_radius))
result, _ = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars,
max_match_radius=min_match_radius, force_platepar_save=True)
if result is not None:
working_platepar = result
# If the alignment succeeded, save the result
else:
working_platepar = result
else:
working_platepar = result
# Store the platepar if the fit succeeded
if result is not None:
# Recompute alt/az of the FOV centre
working_platepar.az_centre, working_platepar.alt_centre = raDec2AltAz(working_platepar.RA_d, \
working_platepar.dec_d, working_platepar.JD, working_platepar.lat, working_platepar.lon)
# Recompute the rotation wrt horizon
working_platepar.rotation_from_horiz = rotationWrtHorizon(working_platepar)
# Mark the platepar to indicate that it was automatically recalibrated on an individual FF file
working_platepar.auto_recalibrated = True
recalibrated_platepars[ff_name] = working_platepar
prev_platepar = working_platepar
else:
print('Recalibration of {:s} failed, using the previous platepar...'.format(ff_name))
# Mark the platepar to indicate that autorecalib failed
prev_platepar_tmp = copy.deepcopy(prev_platepar)
prev_platepar_tmp.auto_recalibrated = False
# If the aligning failed, set the previous platepar as the one that should be used for this FF file
recalibrated_platepars[ff_name] = prev_platepar_tmp
### Average out photometric offsets within the given neighbourhood size ###
# Go through the list of FF files with detections
for meteor_entry in meteor_list:
ff_name = meteor_entry[0]
# Make sure the FF was successfuly recalibrated
if ff_name in recalibrated_platepars:
# Find the index of the given FF file in the list of calstars
ff_indx = calstars_ffs.index(ff_name)
# Compute the average photometric offset and the improved standard deviation using all
# neighbors
photom_offset_tmp_list = []
photom_offset_std_tmp_list = []
neighboring_ffs = []
for k in range(-(RECALIBRATE_NEIGHBOURHOOD_SIZE//2), RECALIBRATE_NEIGHBOURHOOD_SIZE//2 + 1):
k_indx = ff_indx + k
if (k_indx > 0) and (k_indx < len(calstars_ffs)):
# Get the name of the FF file
ff_name_tmp = calstars_ffs[k_indx]
# Check that the neighboring FF was successfuly recalibrated
if ff_name_tmp in recalibrated_platepars:
# Get the computed photometric offset and stddev
photom_offset_tmp_list.append(recalibrated_platepars[ff_name_tmp].mag_lev)
photom_offset_std_tmp_list.append(recalibrated_platepars[ff_name_tmp].mag_lev_stddev)
neighboring_ffs.append(ff_name_tmp)
# Compute the new photometric offset and improved standard deviation (assume equal sample size)
# Source: https://stats.stackexchange.com/questions/55999/is-it-possible-to-find-the-combined-standard-deviation
photom_offset_new = np.mean(photom_offset_tmp_list)
photom_offset_std_new = np.sqrt(\
np.sum([st**2 + (mt - photom_offset_new)**2 \
for mt, st in zip(photom_offset_tmp_list, photom_offset_std_tmp_list)]) \
/ len(photom_offset_tmp_list)
)
# Assign the new photometric offset and standard deviation to all FFs used for computation
for ff_name_tmp in neighboring_ffs:
recalibrated_platepars[ff_name_tmp].mag_lev = photom_offset_new
recalibrated_platepars[ff_name_tmp].mag_lev_stddev = photom_offset_std_new
### ###
### Store all recalibrated platepars as a JSON file ###
all_pps = {}
for ff_name in recalibrated_platepars:
json_str = recalibrated_platepars[ff_name].jsonStr()
all_pps[ff_name] = json.loads(json_str)
with open(os.path.join(dir_path, config.platepars_recalibrated_name), 'w') as f:
# Convert all platepars to a JSON file
out_str = json.dumps(all_pps, default=lambda o: o.__dict__, indent=4, sort_keys=True)
f.write(out_str)
### ###
# If no platepars were recalibrated, use the single platepar recalibration procedure
if len(recalibrated_platepars) == 0:
print('No FF images were used for recalibration, using the single platepar calibration function...')
# Use the initial platepar for calibration
applyAstrometryFTPdetectinfo(dir_path, os.path.basename(ftpdetectinfo_path), None, platepar=platepar)
return recalibrated_platepars
### GENERATE PLOTS ###
dt_list = []
ang_dists = []
rot_angles = []
hour_list = []
photom_offset_list = []
photom_offset_std_list = []
first_dt = np.min([FFfile.filenameToDatetime(ff_name) for ff_name in recalibrated_platepars])
for ff_name in recalibrated_platepars:
pp_temp = recalibrated_platepars[ff_name]
# If the fitting failed, skip the platepar
if pp_temp is None:
continue
# Add the datetime of the FF file to the list
ff_dt = FFfile.filenameToDatetime(ff_name)
dt_list.append(ff_dt)
# Compute the angular separation from the reference platepar
ang_dist = np.degrees(angularSeparation(np.radians(platepar.RA_d), np.radians(platepar.dec_d), \
np.radians(pp_temp.RA_d), np.radians(pp_temp.dec_d)))
ang_dists.append(ang_dist*60)
# Compute rotation difference
rot_diff = (platepar.pos_angle_ref - pp_temp.pos_angle_ref + 180)%360 - 180
rot_angles.append(rot_diff*60)
# Compute the hour of the FF used for recalibration
hour_list.append((ff_dt - first_dt).total_seconds()/3600)
# Add the photometric offset to the list
photom_offset_list.append(pp_temp.mag_lev)
photom_offset_std_list.append(pp_temp.mag_lev_stddev)
if generate_plot:
# Generate the name the plots
plot_name = os.path.basename(ftpdetectinfo_path).replace('FTPdetectinfo_', '').replace('.txt', '')
### Plot difference from reference platepar in angular distance from (0, 0) vs rotation ###
plt.figure()
plt.scatter(0, 0, marker='o', edgecolor='k', label='Reference platepar', s=100, c='none', zorder=3)
plt.scatter(ang_dists, rot_angles, c=hour_list, zorder=3)
plt.colorbar(label="Hours from first FF file")
plt.xlabel("Angular distance from reference (arcmin)")
plt.ylabel("Rotation from reference (arcmin)")
plt.title("FOV centre drift starting at {:s}".format(first_dt.strftime("%Y/%m/%d %H:%M:%S")))
plt.grid()
plt.legend()
plt.tight_layout()
plt.savefig(os.path.join(dir_path, plot_name + '_calibration_variation.png'), dpi=150)
# plt.show()
plt.clf()
plt.close()
### ###
### Plot the photometric offset variation ###
plt.figure()
plt.errorbar(dt_list, photom_offset_list, yerr=photom_offset_std_list, fmt="o", \
ecolor='lightgray', elinewidth=2, capsize=0, ms=2)
# Format datetimes
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%H:%M"))
# rotate and align the tick labels so they look better
plt.gcf().autofmt_xdate()
plt.xlabel("UTC time")
plt.ylabel("Photometric offset")
plt.title("Photometric offset variation")
plt.grid()
plt.tight_layout()
plt.savefig(os.path.join(dir_path, plot_name + '_photometry_variation.png'), dpi=150)
plt.clf()
plt.close()
### ###
### Apply platepars to FTPdetectinfo ###
meteor_output_list = []
for meteor_entry in meteor_list:
ff_name, meteor_No, rho, phi, meteor_meas = meteor_entry
# Get the platepar that will be applied to this FF file
if ff_name in recalibrated_platepars:
working_platepar = recalibrated_platepars[ff_name]
else:
print('Using default platepar for:', ff_name)
working_platepar = platepar
# Apply the recalibrated platepar to meteor centroids
meteor_picks = applyPlateparToCentroids(ff_name, fps, meteor_meas, working_platepar, \
add_calstatus=True)
meteor_output_list.append([ff_name, meteor_No, rho, phi, meteor_picks])
# Calibration string to be written to the FTPdetectinfo file
calib_str = 'Recalibrated with RMS on: ' + str(datetime.datetime.utcnow()) + ' UTC'
# If no meteors were detected, set dummpy parameters
if len(meteor_list) == 0:
cam_code = ''
fps = 0
# Back up the old FTPdetectinfo file
try:
shutil.copy(ftpdetectinfo_path, ftpdetectinfo_path.strip('.txt') \
+ '_backup_{:s}.txt'.format(datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S.%f')))
except:
print('ERROR! The FTPdetectinfo file could not be backed up: {:s}'.format(ftpdetectinfo_path))
# Save the updated FTPdetectinfo
FTPdetectinfo.writeFTPdetectinfo(meteor_output_list, dir_path, os.path.basename(ftpdetectinfo_path), \
dir_path, cam_code, fps, calibration=calib_str, celestial_coords_given=True)
### ###
return recalibrated_platepars
def _handleFailure(config, platepar, calstars_list, catalog_stars,
_fft_refinement):
""" Run FFT alignment before giving up on ACF. """
if not _fft_refinement:
print()
print(
"-------------------------------------------------------------------------------"
)
print(
'The initial platepar is bad, trying to refine it using FFT phase correlation...'
)
print()
# Prepare data for FFT image registration
calstars_dict = {
ff_file: star_data
for ff_file, star_data in calstars_list
}
# Extract star list from CALSTARS file from FF file with most stars
max_len_ff = max(calstars_dict,
key=lambda k: len(calstars_dict[k]))
# Take only X, Y (change order so X is first)
calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
# Get the time of the FF file
calstars_time = FFfile.getMiddleTimeFF(max_len_ff,
config.fps,
ret_milliseconds=True)
# Try aligning the platepar using FFT image registration
platepar_refined = alignPlatepar(config, platepar, calstars_time,
calstars_coords)
print()
### If there are still not enough stars matched, try FFT again ###
min_radius = 10
# Prepare star dictionary to check the match
dt = FFfile.getMiddleTimeFF(max_len_ff,
config.fps,
ret_milliseconds=True)
jd = date2JD(*dt)
star_dict_temp = {}
star_dict_temp[jd] = calstars_dict[max_len_ff]
# Check the number of matched stars
n_matched, _, _, _ = matchStarsResiduals(config, platepar_refined, catalog_stars, \
star_dict_temp, min_radius, ret_nmatch=True, verbose=True)
# Realign again if necessary
if n_matched < config.min_matched_stars:
print()
print(
"-------------------------------------------------------------------------------"
)
print(
'Doing a second FFT pass as the number of matched stars was too small...'
)
print()
platepar_refined = alignPlatepar(config, platepar_refined,
calstars_time,
calstars_coords)
print()
### ###
# Redo autoCF
return autoCheckFit(config,
platepar_refined,
calstars_list,
_fft_refinement=True)
else:
print(
'Auto Check Fit failed completely, please redo the plate manually!'
)
return platepar, False
def autoCheckFit(config, platepar, calstars_list, distorsion_refinement=False, _fft_refinement=False):
""" Attempts to refine the astrometry fit with the given stars and and initial astrometry parameters.
Arguments:
config: [Config structure]
platepar: [Platepar structure] Initial astrometry parameters.
calstars_list: [list] A list containing stars extracted from FF files. See RMS.Formats.CALSTARS for
more details.
Keyword arguments:
distorsion_refinement: [bool] Whether the distorsion should be fitted as well. False by default.
_fft_refinement: [bool] Internal flag indicating that autoCF is running the second time recursively
after FFT platepar adjustment.
Return:
(platepar, fit_status):
platepar: [Platepar structure] Estimated/refined platepar.
fit_status: [bool] True if fit was successfuly, False if not.
"""
def _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement):
""" Run FFT alignment before giving up on ACF. """
if not _fft_refinement:
print('The initial platepar is bad, trying to refine it using FFT phase correlation...')
# Prepare data for FFT image registration
calstars_dict = {ff_file: star_data for ff_file, star_data in calstars_list}
# Extract star list from CALSTARS file from FF file with most stars
max_len_ff = max(calstars_dict, key=lambda k: len(calstars_dict[k]))
# Take only X, Y (change order so X is first)
calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
# Get the time of the FF file
calstars_time = FFfile.getMiddleTimeFF(max_len_ff, config.fps, ret_milliseconds=True)
# Try aligning the platepar using FFT image registration
platepar_refined = alignPlatepar(config, platepar, calstars_time, calstars_coords)
# Redo autoCF
return autoCheckFit(config, platepar_refined, calstars_list, \
distorsion_refinement=distorsion_refinement, _fft_refinement=True)
else:
print('Auto Check Fit failed completely, please redo the plate manually!')
return platepar, False
if _fft_refinement:
print('Second ACF run with an updated platepar via FFT phase correlation...')
# Convert the list to a dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
# Load catalog stars (overwrite the mag band ratios if specific catalog is used)
catalog_stars, _, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(config.star_catalog_path, \
config.star_catalog_file, lim_mag=config.catalog_mag_limit, \
mag_band_ratios=config.star_catalog_band_ratios)
# Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars
star_dict = {}
# Take only those files with enough stars on them
for ff_name in calstars:
stars_list = calstars[ff_name]
# Check if there are enough stars on the image
if len(stars_list) >= config.ff_min_stars:
# Calculate the JD time of the FF file
dt = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)
jd = date2JD(*dt)
# Add the time and the stars to the dict
star_dict[jd] = stars_list
# There has to be a minimum of 200 FF files for star fitting, and only 100 will be subset if there are more
if len(star_dict) < config.calstars_files_N:
print('Not enough FF files in CALSTARS for ACF!')
return platepar, False
else:
# Randomly choose calstars_files_N image files from the whole list
rand_keys = random.sample(list(star_dict), config.calstars_files_N)
star_dict = {key: star_dict[key] for key in rand_keys}
# Calculate the total number of calibration stars used
total_calstars = sum([len(star_dict[key]) for key in star_dict])
print('Total calstars:', total_calstars)
if total_calstars < config.calstars_min_stars:
print('Not enough calibration stars, need at least', config.calstars_min_stars)
return platepar, False
# A list of matching radiuses to try, pairs of [radius, fit_distorsion_flag]
# The distorsion will be fitted only if explicity requested
min_radius = 0.5
radius_list = [[10, False],
[5, False],
[3, False],
[1.5, True and distorsion_refinement],
[min_radius, True and distorsion_refinement]]
# Calculate the function tolerance, so the desired precision can be reached (the number is calculated
# in the same regard as the cost function)
fatol, xatol_ang = computeMinimizationTolerances(config, platepar, len(star_dict))
### If the initial match is good enough, do only quick recalibratoin ###
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
min_radius, ret_nmatch=True)
if n_matched >= config.calstars_files_N:
# Check if the average distance with the tightest radius is close
if avg_dist < config.dist_check_quick_threshold:
# Use a reduced set of initial radius values
radius_list = [[1.5, True and distorsion_refinement],
[min_radius, True and distorsion_refinement]]
##########
# Match increasingly smaller search radiia around image stars
for i, (match_radius, fit_distorsion) in enumerate(radius_list):
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
match_radius, ret_nmatch=True)
print('Max radius:', match_radius)
print('Initial values:')
print(' Matched stars:', n_matched)
print(' Average deviation:', avg_dist)
# The initial number of matched stars has to be at least the number of FF imaages, otherwise it means
# that the initial platepar is no good
if n_matched < config.calstars_files_N:
print('The total number of initially matched stars is too small! Please manually redo the plate or make sure there are enough calibration stars.')
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict, min_radius, verbose=True):
# Print out notice only if the platepar is good right away
if i == 0:
print("Initial platepar is good enough!")
return platepar, True
# Initial parameters for the astrometric fit (don't fit the scale if the distorsion is not being fit)
if fit_distorsion:
p0 = [platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref, platepar.F_scale]
else:
p0 = [platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref]
# Fit the astrometric parameters
res = scipy.optimize.minimize(_calcImageResidualsAstro, p0, args=(config, platepar, catalog_stars, \
star_dict, match_radius, fit_distorsion), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': xatol_ang})
print(res)
# If the fit was not successful, stop further fitting
if not res.success:
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
else:
# If the fit was successful, use the new parameters from now on
if fit_distorsion:
ra_ref, dec_ref, pos_angle_ref, F_scale = res.x
else:
ra_ref, dec_ref, pos_angle_ref = res.x
F_scale = platepar.F_scale
platepar.RA_d = ra_ref
platepar.dec_d = dec_ref
platepar.pos_angle_ref = pos_angle_ref
platepar.F_scale = F_scale
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict, min_radius, verbose=True):
return platepar, True
# Fit the lens distorsion parameters
if fit_distorsion:
### REVERSE DISTORSION POLYNOMIALS FIT ###
# Fit the distortion parameters (X axis)
res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.x_poly_rev, args=(config, \
platepar, catalog_stars, star_dict, match_radius, 'x'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
else:
platepar.x_poly_rev = res.x
# Fit the distortion parameters (Y axis)
res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.y_poly_rev, args=(config, \
platepar,catalog_stars, star_dict, match_radius, 'y'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
else:
platepar.y_poly_rev = res.x
### ###
### FORWARD DISTORSION POLYNOMIALS FIT ###
# Fit the distortion parameters (X axis)
res = scipy.optimize.minimize(_calcSkyResidualsDistorsion, platepar.x_poly_fwd, args=(config, \
platepar, catalog_stars, star_dict, match_radius, 'x'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
else:
platepar.x_poly_fwd = res.x
# Fit the distortion parameters (Y axis)
res = scipy.optimize.minimize(_calcSkyResidualsDistorsion, platepar.y_poly_fwd, args=(config, \
platepar,catalog_stars, star_dict, match_radius, 'y'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
return platepar, False
else:
platepar.y_poly_fwd = res.x
### ###
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \
star_dict, min_radius, ret_nmatch=True)
print('FINAL SOLUTION with {:f} px:'.format(min_radius))
print('Matched stars:', n_matched)
print('Average deviation:', avg_dist)
# Mark the platepar to indicate that it was automatically refined with CheckFit
platepar.auto_check_fit_refined = True
return platepar, True
def autoCheckFit(config, platepar, calstars_list):
""" Attempts to refine the astrometry fit with the given stars and and initial astrometry parameters.
Arguments:
config: [Config structure]
platepar: [Platepar structure] Initial astrometry parameters.
calstars_list: [list] A list containing stars extracted from FF files. See RMS.Formats.CALSTARS for
more details.
Return:
(platepar, fit_status):
platepar: [Platepar structure] Estimated/refined platepar.
fit_status: [bool] True if fit was successfuly, False if not.
"""
# Convert the list to a dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
# Load catalog stars
catalog_stars = StarCatalog.readStarCatalog(config.star_catalog_path, config.star_catalog_file, \
lim_mag=config.catalog_mag_limit, mag_band_ratios=config.star_catalog_band_ratios)
# Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars
star_dict = {}
# Take only those files with enough stars on them
for ff_name in calstars:
stars_list = calstars[ff_name]
# Check if there are enough stars on the image
if len(stars_list) >= config.ff_min_stars:
# Calculate the JD time of the FF file
dt = FFfile.getMiddleTimeFF(ff_name,
config.fps,
ret_milliseconds=True)
jd = date2JD(*dt)
# Add the time and the stars to the dict
star_dict[jd] = stars_list
# There has to be a minimum of 200 FF files for star fitting, and only 100 will be subset if there are more
if len(star_dict) < config.calstars_files_N:
print('Not enough FF files in CALSTARS for ACF!')
return platepar, False
else:
# Randomly choose calstars_files_N image files from the whole list
rand_keys = random.sample(list(star_dict), config.calstars_files_N)
star_dict = {key: star_dict[key] for key in rand_keys}
# Calculate the total number of calibration stars used
total_calstars = sum([len(star_dict[key]) for key in star_dict])
print('Total calstars:', total_calstars)
if total_calstars < config.calstars_min_stars:
print('Not enough calibration stars, need at least',
config.calstars_min_stars)
return platepar, False
# A list of matching radiuses to try, pairs of [radius, fit_distorsion_flag]
min_radius = 0.5
radius_list = [[10, False], [5, False], [3, False], [1.5, True],
[min_radius, True]]
# Calculate the function tolerance, so the desired precision can be reached (the number is calculated
# in the same reagrd as the cost function)
fatol = (config.dist_check_threshold**
2) / np.sqrt(len(star_dict) * config.min_matched_stars + 1)
# Parameter estimation tolerance for angular values
fov_w = platepar.X_res / platepar.F_scale
xatol_ang = config.dist_check_threshold * fov_w / platepar.X_res
### If the initial match is good enough, do only quick recalibratoin ###
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
min_radius, ret_nmatch=True)
if n_matched >= config.calstars_files_N:
# Check if the average distance with the tightest radius is close
if avg_dist < config.dist_check_quick_threshold:
# Use a reduced set of initial radius values
radius_list = [[1.5, True], [min_radius, True]]
##########
# Match increasingly smaller search radiia around image stars
for i, (match_radius, fit_distorsion) in enumerate(radius_list):
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
match_radius, ret_nmatch=True)
print('Max radius:', match_radius)
print('Initial values:')
print(' Matched stars:', n_matched)
print(' Average deviation:', avg_dist)
# The initial number of matched stars has to be at least the number of FF imaages, otherwise it means
# that the initial platepar is no good
if n_matched < config.calstars_files_N:
print(
'The total number of initially matched stars is too small! Please manually redo the plate or make sure there are enough calibration stars.'
)
return platepar, False
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict,
min_radius):
# Print out notice only if the platepar is good right away
if i == 0:
print("Initial platepar is good enough!")
return platepar, True
# Initial parameters for the astrometric fit
p0 = [
platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref,
platepar.F_scale
]
# Fit the astrometric parameters
res = scipy.optimize.minimize(_calcImageResidualsAstro, p0, args=(config, platepar, catalog_stars, \
star_dict, match_radius), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': xatol_ang})
print(res)
# If the fit was not successful, stop further fitting
if not res.success:
return platepar, False
else:
# If the fit was successful, use the new parameters from now on
ra_ref, dec_ref, pos_angle_ref, F_scale = res.x
platepar.RA_d = ra_ref
platepar.dec_d = dec_ref
platepar.pos_angle_ref = pos_angle_ref
platepar.F_scale = F_scale
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict,
min_radius):
return platepar, True
# Fit the lens distorsion parameters
if fit_distorsion:
# Fit the distortion parameters (X axis)
res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.x_poly, args=(config, platepar,\
catalog_stars, star_dict, match_radius, 'x'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
return platepar, False
else:
platepar.x_poly = res.x
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict,
min_radius):
return platepar, True
# Fit the distortion parameters (Y axis)
res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.y_poly, args=(config, platepar,\
catalog_stars, star_dict, match_radius, 'y'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
return platepar, False
else:
platepar.y_poly = res.x
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \
star_dict, min_radius, ret_nmatch=True)
print('FINAL SOLUTION with {:f} px:'.format(min_radius))
print('Matched stars:', n_matched)
print('Average deviation:', avg_dist)
return platepar, True
def recalibratePlateparsForFF(
prev_platepar,
ff_file_names,
calstars,
catalog_stars,
config,
lim_mag=None,
ignore_distance_threshold=False,
):
"""
Recalibrate platepars corresponding to ff files based on the stars.
Arguments:
prev_platepar: [platepar]
ff_file_names: [list] list of ff file names
calstars: [dict] A dictionary with only one entry, where the key is 'jd' and the value is the
list of star coordinates.
catalog_stars: [list] A list of entries [[ff_name, star_coordinates], ...].
config: [config]
Keyword arguments:
lim_mag: [float]
ignore_distance_threshold: [bool] Don't consider the recalib as failed if the median distance
is larger than the threshold.
Returns:
recalibrated_platepars: [dict] A dictionary where one key is ff file name and the value is
a calibrated corresponding platepar.
"""
# Go through all FF files with detections, recalibrate and apply astrometry
recalibrated_platepars = {}
for ff_name in ff_file_names:
working_platepar = copy.deepcopy(prev_platepar)
# Skip this meteor if its FF file was already recalibrated
if ff_name in recalibrated_platepars:
continue
print()
print('Processing: ', ff_name)
print(
'------------------------------------------------------------------------------'
)
# Find extracted stars on this image
if not ff_name in calstars:
print('Skipped because it was not in CALSTARS:', ff_name)
continue
# Get stars detected on this FF file (create a dictionaly with only one entry, the residuals function
# needs this format)
calstars_time = FFfile.getMiddleTimeFF(ff_name,
config.fps,
ret_milliseconds=True)
jd = date2JD(*calstars_time)
star_dict_ff = {jd: calstars[ff_name]}
result = None
# Skip recalibration if less than a minimum number of stars were detected
if (len(calstars[ff_name]) >= config.ff_min_stars) and (len(
calstars[ff_name]) >= config.min_matched_stars):
# Recalibrate the platepar using star matching
result, min_match_radius = recalibrateFF(
config,
working_platepar,
jd,
star_dict_ff,
catalog_stars,
lim_mag=lim_mag,
ignore_distance_threshold=ignore_distance_threshold,
)
# If the recalibration failed, try using FFT alignment
if result is None:
print()
print('Running FFT alignment...')
# Run FFT alignment
calstars_coords = np.array(star_dict_ff[jd])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
print(calstars_time)
test_platepar = alignPlatepar(config,
prev_platepar,
calstars_time,
calstars_coords,
show_plot=False)
# Try to recalibrate after FFT alignment
result, _ = recalibrateFF(config,
test_platepar,
jd,
star_dict_ff,
catalog_stars,
lim_mag=lim_mag)
# If the FFT alignment failed, align the original platepar using the smallest radius that matched
# and force save the the platepar
if (result is None) and (min_match_radius is not None):
print()
print(
"Using the old platepar with the minimum match radius of: {:.2f}"
.format(min_match_radius))
result, _ = recalibrateFF(
config,
working_platepar,
jd,
star_dict_ff,
catalog_stars,
max_match_radius=min_match_radius,
force_platepar_save=True,
lim_mag=lim_mag,
)
if result is not None:
working_platepar = result
# If the alignment succeeded, save the result
else:
working_platepar = result
else:
working_platepar = result
# Store the platepar if the fit succeeded
if result is not None:
# Recompute alt/az of the FOV centre
working_platepar.az_centre, working_platepar.alt_centre = raDec2AltAz(
working_platepar.RA_d,
working_platepar.dec_d,
working_platepar.JD,
working_platepar.lat,
working_platepar.lon,
)
# Recompute the rotation wrt horizon
working_platepar.rotation_from_horiz = rotationWrtHorizon(
working_platepar)
# Mark the platepar to indicate that it was automatically recalibrated on an individual FF file
working_platepar.auto_recalibrated = True
recalibrated_platepars[ff_name] = working_platepar
prev_platepar = working_platepar
else:
print(
'Recalibration of {:s} failed, using the previous platepar...'.
format(ff_name))
# Mark the platepar to indicate that autorecalib failed
prev_platepar_tmp = copy.deepcopy(prev_platepar)
prev_platepar_tmp.auto_recalibrated = False
# If the aligning failed, set the previous platepar as the one that should be used for this FF file
recalibrated_platepars[ff_name] = prev_platepar_tmp
return recalibrated_platepars
def recalibrateIndividualFFsAndApplyAstrometry(dir_path, ftpdetectinfo_path, calstars_list, config, platepar):
""" Recalibrate FF files with detections and apply the recalibrated platepar to those detections.
Arguments:
dir_path: [str] Path where the FTPdetectinfo file is.
ftpdetectinfo_path: [str] Name of the FTPdetectinfo file.
calstars_list: [list] A list of entries [[ff_name, star_coordinates], ...].
config: [Config instance]
platepar: [Platepar instance] Initial platepar.
Return:
recalibrated_platepars: [dict] A dictionary where the keys are FF file names and values are
recalibrated platepar instances for every FF file.
"""
# Read the FTPdetectinfo data
cam_code, fps, meteor_list = FTPdetectinfo.readFTPdetectinfo(*os.path.split(ftpdetectinfo_path), \
ret_input_format=True)
# Convert the list of stars to a per FF name dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
# Load catalog stars (overwrite the mag band ratios if specific catalog is used)
catalog_stars, _, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(config.star_catalog_path,\
config.star_catalog_file, lim_mag=config.catalog_mag_limit, \
mag_band_ratios=config.star_catalog_band_ratios)
prev_platepar = copy.deepcopy(platepar)
# Go through all FF files with detections, recalibrate and apply astrometry
recalibrated_platepars = {}
for meteor_entry in meteor_list:
working_platepar = copy.deepcopy(prev_platepar)
ff_name, meteor_No, rho, phi, meteor_meas = meteor_entry
# Skip this meteors if its FF file was already recalibrated
if ff_name in recalibrated_platepars:
continue
print()
print('Processing: ', ff_name)
print('------------------------------------------------------------------------------')
# Find extracted stars on this image
if not ff_name in calstars:
print('Skipped because it was not in CALSTARS:', ff_name)
continue
# Get stars detected on this FF file (create a dictionaly with only one entry, the residuals function
# needs this format)
calstars_time = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)
jd = date2JD(*calstars_time)
star_dict_ff = {jd: calstars[ff_name]}
# Recalibrate the platepar using star matching
result = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars)
# If the recalibration failed, try using FFT alignment
if result is None:
print()
print('Running FFT alignment...')
# Run FFT alignment
calstars_coords = np.array(star_dict_ff[jd])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
print(calstars_time)
working_platepar = alignPlatepar(config, prev_platepar, calstars_time, calstars_coords, \
show_plot=False)
# Try to recalibrate after FFT alignment
result = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars)
if result is not None:
working_platepar = result
else:
working_platepar = result
# Store the platepar if the fit succeeded
if result is not None:
recalibrated_platepars[ff_name] = working_platepar
prev_platepar = working_platepar
else:
print('Recalibration of {:s} failed, using the previous platepar...'.format(ff_name))
# If the aligning failed, set the previous platepar as the one that should be used for this FF file
recalibrated_platepars[ff_name] = prev_platepar
### Store all recalibrated platepars as a JSON file ###
all_pps = {}
for ff_name in recalibrated_platepars:
json_str = recalibrated_platepars[ff_name].jsonStr()
all_pps[ff_name] = json.loads(json_str)
with open(os.path.join(dir_path, config.platepars_recalibrated_name), 'w') as f:
# Convert all platepars to a JSON file
out_str = json.dumps(all_pps, default=lambda o: o.__dict__, indent=4, sort_keys=True)
f.write(out_str)
### ###
# If no platepars were recalibrated, use the single platepar recalibration procedure
if len(recalibrated_platepars) == 0:
print('No FF images were used for recalibration, using the single platepar calibration function...')
# Use the initial platepar for calibration
applyAstrometryFTPdetectinfo(dir_path, os.path.basename(ftpdetectinfo_path), None, platepar=platepar)
return recalibrated_platepars
### Plot difference from reference platepar in angular distance from (0, 0) vs rotation ###
ang_dists = []
rot_angles = []
hour_list = []
first_jd = np.min([FFfile.filenameToDatetime(ff_name) for ff_name in recalibrated_platepars])
for ff_name in recalibrated_platepars:
pp_temp = recalibrated_platepars[ff_name]
# If the fitting failed, skip the platepar
if pp_temp is None:
continue
# Compute the angular separation from the reference platepar
ang_dist = np.degrees(angularSeparation(np.radians(platepar.RA_d), np.radians(platepar.dec_d), \
np.radians(pp_temp.RA_d), np.radians(pp_temp.dec_d)))
ang_dists.append(ang_dist*60)
rot_angles.append((platepar.pos_angle_ref - pp_temp.pos_angle_ref)*60)
# Compute the hour of the FF used for recalibration
hour_list.append((FFfile.filenameToDatetime(ff_name) - first_jd).total_seconds()/3600)
plt.figure()
plt.scatter(0, 0, marker='o', edgecolor='k', label='Reference platepar', s=100, c='none', zorder=3)
plt.scatter(ang_dists, rot_angles, c=hour_list, zorder=3)
plt.colorbar(label='Hours from first FF file')
plt.xlabel("Angular distance from reference (arcmin)")
plt.ylabel('Rotation from reference (arcmin)')
plt.grid()
plt.legend()
plt.tight_layout()
# Generate the name for the plot
calib_plot_name = os.path.basename(ftpdetectinfo_path).replace('FTPdetectinfo_', '').replace('.txt', '') \
+ '_calibration_variation.png'
plt.savefig(os.path.join(dir_path, calib_plot_name), dpi=150)
# plt.show()
plt.clf()
plt.close()
### ###
### Apply platepars to FTPdetectinfo ###
meteor_output_list = []
for meteor_entry in meteor_list:
ff_name, meteor_No, rho, phi, meteor_meas = meteor_entry
# Get the platepar that will be applied to this FF file
if ff_name in recalibrated_platepars:
working_platepar = recalibrated_platepars[ff_name]
else:
print('Using default platepar for:', ff_name)
working_platepar = platepar
# Apply the recalibrated platepar to meteor centroids
meteor_picks = applyPlateparToCentroids(ff_name, fps, meteor_meas, working_platepar, \
add_calstatus=True)
meteor_output_list.append([ff_name, meteor_No, rho, phi, meteor_picks])
# Calibration string to be written to the FTPdetectinfo file
calib_str = 'Recalibrated with RMS on: ' + str(datetime.datetime.utcnow()) + ' UTC'
# If no meteors were detected, set dummpy parameters
if len(meteor_list) == 0:
cam_code = ''
fps = 0
# Back up the old FTPdetectinfo file
shutil.copy(ftpdetectinfo_path, ftpdetectinfo_path.strip('.txt') \
+ '_backup_{:s}.txt'.format(datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S.%f')))
# Save the updated FTPdetectinfo
FTPdetectinfo.writeFTPdetectinfo(meteor_output_list, dir_path, os.path.basename(ftpdetectinfo_path), \
dir_path, cam_code, fps, calibration=calib_str, celestial_coords_given=True)
### ###
return recalibrated_platepars