def saveFF(self, arr, startTime, N):
""" Write metadata and data array to FF file.
Arguments:
arr: [3D ndarray] 3D numpy array in format: (N, y, x) where N is [0, 4)
startTime: [float] seconds and fractions of a second from epoch to first frame
N: [int] frame counter (ie. 0000512)
"""
# Generate the name for the file
date_string = time.strftime("%Y%m%d_%H%M%S", time.gmtime(startTime))
# Calculate miliseconds
millis = int((startTime - floor(startTime))*1000)
filename = str(self.config.stationID).zfill(3) + "_" + date_string + "_" + str(millis).zfill(3) \
+ "_" + str(N).zfill(7)
ff = FFStruct.FFStruct()
ff.array = arr
ff.nrows = arr.shape[1]
ff.ncols = arr.shape[2]
ff.nbits = self.config.bit_depth
ff.nframes = 256
ff.first = N + 256
ff.camno = self.config.stationID
ff.fps = self.config.fps
# Write the FF file
FFfile.write(ff, self.data_dir, filename, fmt=self.config.ff_format)
return filename
def saveFF(self, arr, startTime, N):
""" Write metadata and data array to FF file.
Arguments:
arr: [3D ndarray] 3D numpy array in format: (N, y, x) where N is [0, 4)
startTime: [float] seconds and fractions of a second from epoch to first frame
N: [int] frame counter (ie. 0000512)
"""
# Generate the name for the file
date_string = time.strftime("%Y%m%d_%H%M%S", time.gmtime(startTime))
# Calculate miliseconds
millis = int((startTime - floor(startTime)) * 1000)
filename = str(self.config.stationID).zfill(3) + "_" + date_string + "_" + str(millis).zfill(3) \
+ "_" + str(N).zfill(7)
ff = FFStruct.FFStruct()
ff.array = arr
ff.nrows = arr.shape[1]
ff.ncols = arr.shape[2]
ff.nbits = self.config.bit_depth
ff.nframes = 256
ff.first = N + 256
ff.camno = self.config.stationID
ff.fps = self.config.fps
# Write the FF file
FFfile.write(ff, self.data_dir, filename, fmt=self.config.ff_format)
return filename
def _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement):
""" Run FFT alignment before giving up on ACF. """
if not _fft_refinement:
print('The initial platepar is bad, trying to refine it using FFT phase correlation...')
# Prepare data for FFT image registration
calstars_dict = {ff_file: star_data for ff_file, star_data in calstars_list}
# Extract star list from CALSTARS file from FF file with most stars
max_len_ff = max(calstars_dict, key=lambda k: len(calstars_dict[k]))
# Take only X, Y (change order so X is first)
calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
# Get the time of the FF file
calstars_time = FFfile.getMiddleTimeFF(max_len_ff, config.fps, ret_milliseconds=True)
# Try aligning the platepar using FFT image registration
platepar_refined = alignPlatepar(config, platepar, calstars_time, calstars_coords)
# Redo autoCF
return autoCheckFit(config, platepar_refined, calstars_list, \
distorsion_refinement=distorsion_refinement, _fft_refinement=True)
else:
print('Auto Check Fit failed completely, please redo the plate manually!')
return platepar, False
def starListToDict(config, calstars_list, max_ffs=None):
""" Converts the list of calstars into dictionary where the keys are FF file JD and the values is
a list of (X, Y, bg_intens, intens) of stars.
"""
# Convert the list to a dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
# Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars
star_dict = {}
# Take only those files with enough stars on them
for ff_name in calstars:
stars_list = calstars[ff_name]
# Check if there are enough stars on the image
if len(stars_list) >= config.ff_min_stars:
# Calculate the JD time of the FF file
dt = FFfile.getMiddleTimeFF(ff_name,
config.fps,
ret_milliseconds=True)
jd = date2JD(*dt)
# Add the time and the stars to the dict
star_dict[jd] = stars_list
if max_ffs is not None:
# Limit the number of FF files used
if len(star_dict) > max_ffs:
# Randomly choose calstars_files_N image files from the whole list
rand_keys = random.sample(list(star_dict), max_ffs)
star_dict = {key: star_dict[key] for key in rand_keys}
return star_dict
def recalibrateIndividualFFsAndApplyAstrometry(dir_path, ftpdetectinfo_path, calstars_list, config, platepar,
generate_plot=True):
""" Recalibrate FF files with detections and apply the recalibrated platepar to those detections.
Arguments:
dir_path: [str] Path where the FTPdetectinfo file is.
ftpdetectinfo_path: [str] Name of the FTPdetectinfo file.
calstars_list: [list] A list of entries [[ff_name, star_coordinates], ...].
config: [Config instance]
platepar: [Platepar instance] Initial platepar.
Keyword arguments:
generate_plot: [bool] Generate the calibration variation plot. True by default.
Return:
recalibrated_platepars: [dict] A dictionary where the keys are FF file names and values are
recalibrated platepar instances for every FF file.
"""
# Use a copy of the config file
config = copy.deepcopy(config)
# If the given file does not exits, return nothing
if not os.path.isfile(ftpdetectinfo_path):
print('ERROR! The FTPdetectinfo file does not exist: {:s}'.format(ftpdetectinfo_path))
print(' The recalibration on every file was not done!')
return {}
# Read the FTPdetectinfo data
cam_code, fps, meteor_list = FTPdetectinfo.readFTPdetectinfo(*os.path.split(ftpdetectinfo_path), \
ret_input_format=True)
# Convert the list of stars to a per FF name dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
### Add neighboring FF files for more robust photometry estimation ###
ff_processing_list = []
# Make a list of sorted FF files in CALSTARS
calstars_ffs = sorted([ff_file for ff_file in calstars])
# Go through the list of FF files with detections and add neighboring FFs
for meteor_entry in meteor_list:
ff_name = meteor_entry[0]
if ff_name in calstars_ffs:
# Find the index of the given FF file in the list of calstars
ff_indx = calstars_ffs.index(ff_name)
# Add neighbours to the processing list
for k in range(-(RECALIBRATE_NEIGHBOURHOOD_SIZE//2), RECALIBRATE_NEIGHBOURHOOD_SIZE//2 + 1):
k_indx = ff_indx + k
if (k_indx > 0) and (k_indx < len(calstars_ffs)):
ff_name_tmp = calstars_ffs[k_indx]
if ff_name_tmp not in ff_processing_list:
ff_processing_list.append(ff_name_tmp)
# Sort the processing list of FF files
ff_processing_list = sorted(ff_processing_list)
### ###
# Globally increase catalog limiting magnitude
config.catalog_mag_limit += 1
# Load catalog stars (overwrite the mag band ratios if specific catalog is used)
star_catalog_status = StarCatalog.readStarCatalog(config.star_catalog_path,\
config.star_catalog_file, lim_mag=config.catalog_mag_limit, \
mag_band_ratios=config.star_catalog_band_ratios)
if not star_catalog_status:
print("Could not load the star catalog!")
print(os.path.join(config.star_catalog_path, config.star_catalog_file))
return {}
catalog_stars, _, config.star_catalog_band_ratios = star_catalog_status
# Update the platepar coordinates from the config file
platepar.lat = config.latitude
platepar.lon = config.longitude
platepar.elev = config.elevation
prev_platepar = copy.deepcopy(platepar)
# Go through all FF files with detections, recalibrate and apply astrometry
recalibrated_platepars = {}
for ff_name in ff_processing_list:
working_platepar = copy.deepcopy(prev_platepar)
# Skip this meteor if its FF file was already recalibrated
if ff_name in recalibrated_platepars:
continue
print()
print('Processing: ', ff_name)
print('------------------------------------------------------------------------------')
# Find extracted stars on this image
if not ff_name in calstars:
print('Skipped because it was not in CALSTARS:', ff_name)
continue
# Get stars detected on this FF file (create a dictionaly with only one entry, the residuals function
# needs this format)
calstars_time = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)
jd = date2JD(*calstars_time)
star_dict_ff = {jd: calstars[ff_name]}
# Recalibrate the platepar using star matching
result, min_match_radius = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars)
# If the recalibration failed, try using FFT alignment
if result is None:
print()
print('Running FFT alignment...')
# Run FFT alignment
calstars_coords = np.array(star_dict_ff[jd])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
print(calstars_time)
test_platepar = alignPlatepar(config, prev_platepar, calstars_time, calstars_coords, \
show_plot=False)
# Try to recalibrate after FFT alignment
result, _ = recalibrateFF(config, test_platepar, jd, star_dict_ff, catalog_stars)
# If the FFT alignment failed, align the original platepar using the smallest radius that matched
# and force save the the platepar
if (result is None) and (min_match_radius is not None):
print()
print("Using the old platepar with the minimum match radius of: {:.2f}".format(min_match_radius))
result, _ = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars,
max_match_radius=min_match_radius, force_platepar_save=True)
if result is not None:
working_platepar = result
# If the alignment succeeded, save the result
else:
working_platepar = result
else:
working_platepar = result
# Store the platepar if the fit succeeded
if result is not None:
# Recompute alt/az of the FOV centre
working_platepar.az_centre, working_platepar.alt_centre = raDec2AltAz(working_platepar.RA_d, \
working_platepar.dec_d, working_platepar.JD, working_platepar.lat, working_platepar.lon)
# Recompute the rotation wrt horizon
working_platepar.rotation_from_horiz = rotationWrtHorizon(working_platepar)
# Mark the platepar to indicate that it was automatically recalibrated on an individual FF file
working_platepar.auto_recalibrated = True
recalibrated_platepars[ff_name] = working_platepar
prev_platepar = working_platepar
else:
print('Recalibration of {:s} failed, using the previous platepar...'.format(ff_name))
# Mark the platepar to indicate that autorecalib failed
prev_platepar_tmp = copy.deepcopy(prev_platepar)
prev_platepar_tmp.auto_recalibrated = False
# If the aligning failed, set the previous platepar as the one that should be used for this FF file
recalibrated_platepars[ff_name] = prev_platepar_tmp
### Average out photometric offsets within the given neighbourhood size ###
# Go through the list of FF files with detections
for meteor_entry in meteor_list:
ff_name = meteor_entry[0]
# Make sure the FF was successfuly recalibrated
if ff_name in recalibrated_platepars:
# Find the index of the given FF file in the list of calstars
ff_indx = calstars_ffs.index(ff_name)
# Compute the average photometric offset and the improved standard deviation using all
# neighbors
photom_offset_tmp_list = []
photom_offset_std_tmp_list = []
neighboring_ffs = []
for k in range(-(RECALIBRATE_NEIGHBOURHOOD_SIZE//2), RECALIBRATE_NEIGHBOURHOOD_SIZE//2 + 1):
k_indx = ff_indx + k
if (k_indx > 0) and (k_indx < len(calstars_ffs)):
# Get the name of the FF file
ff_name_tmp = calstars_ffs[k_indx]
# Check that the neighboring FF was successfuly recalibrated
if ff_name_tmp in recalibrated_platepars:
# Get the computed photometric offset and stddev
photom_offset_tmp_list.append(recalibrated_platepars[ff_name_tmp].mag_lev)
photom_offset_std_tmp_list.append(recalibrated_platepars[ff_name_tmp].mag_lev_stddev)
neighboring_ffs.append(ff_name_tmp)
# Compute the new photometric offset and improved standard deviation (assume equal sample size)
# Source: https://stats.stackexchange.com/questions/55999/is-it-possible-to-find-the-combined-standard-deviation
photom_offset_new = np.mean(photom_offset_tmp_list)
photom_offset_std_new = np.sqrt(\
np.sum([st**2 + (mt - photom_offset_new)**2 \
for mt, st in zip(photom_offset_tmp_list, photom_offset_std_tmp_list)]) \
/ len(photom_offset_tmp_list)
)
# Assign the new photometric offset and standard deviation to all FFs used for computation
for ff_name_tmp in neighboring_ffs:
recalibrated_platepars[ff_name_tmp].mag_lev = photom_offset_new
recalibrated_platepars[ff_name_tmp].mag_lev_stddev = photom_offset_std_new
### ###
### Store all recalibrated platepars as a JSON file ###
all_pps = {}
for ff_name in recalibrated_platepars:
json_str = recalibrated_platepars[ff_name].jsonStr()
all_pps[ff_name] = json.loads(json_str)
with open(os.path.join(dir_path, config.platepars_recalibrated_name), 'w') as f:
# Convert all platepars to a JSON file
out_str = json.dumps(all_pps, default=lambda o: o.__dict__, indent=4, sort_keys=True)
f.write(out_str)
### ###
# If no platepars were recalibrated, use the single platepar recalibration procedure
if len(recalibrated_platepars) == 0:
print('No FF images were used for recalibration, using the single platepar calibration function...')
# Use the initial platepar for calibration
applyAstrometryFTPdetectinfo(dir_path, os.path.basename(ftpdetectinfo_path), None, platepar=platepar)
return recalibrated_platepars
### GENERATE PLOTS ###
dt_list = []
ang_dists = []
rot_angles = []
hour_list = []
photom_offset_list = []
photom_offset_std_list = []
first_dt = np.min([FFfile.filenameToDatetime(ff_name) for ff_name in recalibrated_platepars])
for ff_name in recalibrated_platepars:
pp_temp = recalibrated_platepars[ff_name]
# If the fitting failed, skip the platepar
if pp_temp is None:
continue
# Add the datetime of the FF file to the list
ff_dt = FFfile.filenameToDatetime(ff_name)
dt_list.append(ff_dt)
# Compute the angular separation from the reference platepar
ang_dist = np.degrees(angularSeparation(np.radians(platepar.RA_d), np.radians(platepar.dec_d), \
np.radians(pp_temp.RA_d), np.radians(pp_temp.dec_d)))
ang_dists.append(ang_dist*60)
# Compute rotation difference
rot_diff = (platepar.pos_angle_ref - pp_temp.pos_angle_ref + 180)%360 - 180
rot_angles.append(rot_diff*60)
# Compute the hour of the FF used for recalibration
hour_list.append((ff_dt - first_dt).total_seconds()/3600)
# Add the photometric offset to the list
photom_offset_list.append(pp_temp.mag_lev)
photom_offset_std_list.append(pp_temp.mag_lev_stddev)
if generate_plot:
# Generate the name the plots
plot_name = os.path.basename(ftpdetectinfo_path).replace('FTPdetectinfo_', '').replace('.txt', '')
### Plot difference from reference platepar in angular distance from (0, 0) vs rotation ###
plt.figure()
plt.scatter(0, 0, marker='o', edgecolor='k', label='Reference platepar', s=100, c='none', zorder=3)
plt.scatter(ang_dists, rot_angles, c=hour_list, zorder=3)
plt.colorbar(label="Hours from first FF file")
plt.xlabel("Angular distance from reference (arcmin)")
plt.ylabel("Rotation from reference (arcmin)")
plt.title("FOV centre drift starting at {:s}".format(first_dt.strftime("%Y/%m/%d %H:%M:%S")))
plt.grid()
plt.legend()
plt.tight_layout()
plt.savefig(os.path.join(dir_path, plot_name + '_calibration_variation.png'), dpi=150)
# plt.show()
plt.clf()
plt.close()
### ###
### Plot the photometric offset variation ###
plt.figure()
plt.errorbar(dt_list, photom_offset_list, yerr=photom_offset_std_list, fmt="o", \
ecolor='lightgray', elinewidth=2, capsize=0, ms=2)
# Format datetimes
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%H:%M"))
# rotate and align the tick labels so they look better
plt.gcf().autofmt_xdate()
plt.xlabel("UTC time")
plt.ylabel("Photometric offset")
plt.title("Photometric offset variation")
plt.grid()
plt.tight_layout()
plt.savefig(os.path.join(dir_path, plot_name + '_photometry_variation.png'), dpi=150)
plt.clf()
plt.close()
### ###
### Apply platepars to FTPdetectinfo ###
meteor_output_list = []
for meteor_entry in meteor_list:
ff_name, meteor_No, rho, phi, meteor_meas = meteor_entry
# Get the platepar that will be applied to this FF file
if ff_name in recalibrated_platepars:
working_platepar = recalibrated_platepars[ff_name]
else:
print('Using default platepar for:', ff_name)
working_platepar = platepar
# Apply the recalibrated platepar to meteor centroids
meteor_picks = applyPlateparToCentroids(ff_name, fps, meteor_meas, working_platepar, \
add_calstatus=True)
meteor_output_list.append([ff_name, meteor_No, rho, phi, meteor_picks])
# Calibration string to be written to the FTPdetectinfo file
calib_str = 'Recalibrated with RMS on: ' + str(datetime.datetime.utcnow()) + ' UTC'
# If no meteors were detected, set dummpy parameters
if len(meteor_list) == 0:
cam_code = ''
fps = 0
# Back up the old FTPdetectinfo file
try:
shutil.copy(ftpdetectinfo_path, ftpdetectinfo_path.strip('.txt') \
+ '_backup_{:s}.txt'.format(datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S.%f')))
except:
print('ERROR! The FTPdetectinfo file could not be backed up: {:s}'.format(ftpdetectinfo_path))
# Save the updated FTPdetectinfo
FTPdetectinfo.writeFTPdetectinfo(meteor_output_list, dir_path, os.path.basename(ftpdetectinfo_path), \
dir_path, cam_code, fps, calibration=calib_str, celestial_coords_given=True)
### ###
return recalibrated_platepars
config = cr.parse(".config")
# Get the list of FR bin files (fireball detections) in the given directory
fr_list = [
fr for fr in os.listdir(dir_path)
if fr[0:2] == "FR" and fr.endswith('bin')
]
fr_list = sorted(fr_list)
if not fr_list:
print("No files found!")
sys.exit()
# Get the list of FF bin files (compressed video frames)
ff_list = [ff for ff in os.listdir(dir_path) if FFfile.validFFName(ff)]
ff_list = sorted(ff_list)
i = 0
while True:
# Break the loop if at the end
if i >= len(fr_list):
break
fr = fr_list[i]
ff_match = None
# Strip extensions
# Import old morph
from RMS.OLD import MorphologicalOperations as morph
# Cython init
import pyximport
pyximport.install(setup_args={'include_dirs': [np.get_include()]})
from RMS.Routines.MorphCy import morphApply
# Run tests
# Extract file and directory
head, ff_name = os.path.split(sys.argv[1])
ff_path = os.path.abspath(head) + os.sep
# Load the FF bin file
ff = FFfile.read(ff_path, ff_name)
img_thresh = thresholdImg(ff, 1.8, 9)
show('thresh', img_thresh)
# Convert img to integer
img = img_thresh.astype(np.uint8)
# Old morph
img_old = np.copy(img)
t1 = time.clock()
img_old = morph.clean(img_old)
img_old = morph.bridge(img_old)
img_old = morph.close(img_old)
def _handleFailure(config, platepar, calstars_list, catalog_stars,
_fft_refinement):
""" Run FFT alignment before giving up on ACF. """
if not _fft_refinement:
print()
print(
"-------------------------------------------------------------------------------"
)
print(
'The initial platepar is bad, trying to refine it using FFT phase correlation...'
)
print()
# Prepare data for FFT image registration
calstars_dict = {
ff_file: star_data
for ff_file, star_data in calstars_list
}
# Extract star list from CALSTARS file from FF file with most stars
max_len_ff = max(calstars_dict,
key=lambda k: len(calstars_dict[k]))
# Take only X, Y (change order so X is first)
calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
# Get the time of the FF file
calstars_time = FFfile.getMiddleTimeFF(max_len_ff,
config.fps,
ret_milliseconds=True)
# Try aligning the platepar using FFT image registration
platepar_refined = alignPlatepar(config, platepar, calstars_time,
calstars_coords)
print()
### If there are still not enough stars matched, try FFT again ###
min_radius = 10
# Prepare star dictionary to check the match
dt = FFfile.getMiddleTimeFF(max_len_ff,
config.fps,
ret_milliseconds=True)
jd = date2JD(*dt)
star_dict_temp = {}
star_dict_temp[jd] = calstars_dict[max_len_ff]
# Check the number of matched stars
n_matched, _, _, _ = matchStarsResiduals(config, platepar_refined, catalog_stars, \
star_dict_temp, min_radius, ret_nmatch=True, verbose=True)
# Realign again if necessary
if n_matched < config.min_matched_stars:
print()
print(
"-------------------------------------------------------------------------------"
)
print(
'Doing a second FFT pass as the number of matched stars was too small...'
)
print()
platepar_refined = alignPlatepar(config, platepar_refined,
calstars_time,
calstars_coords)
print()
### ###
# Redo autoCF
return autoCheckFit(config,
platepar_refined,
calstars_list,
_fft_refinement=True)
else:
print(
'Auto Check Fit failed completely, please redo the plate manually!'
)
return platepar, False
def autoCheckFit(config, platepar, calstars_list, distorsion_refinement=False, _fft_refinement=False):
""" Attempts to refine the astrometry fit with the given stars and and initial astrometry parameters.
Arguments:
config: [Config structure]
platepar: [Platepar structure] Initial astrometry parameters.
calstars_list: [list] A list containing stars extracted from FF files. See RMS.Formats.CALSTARS for
more details.
Keyword arguments:
distorsion_refinement: [bool] Whether the distorsion should be fitted as well. False by default.
_fft_refinement: [bool] Internal flag indicating that autoCF is running the second time recursively
after FFT platepar adjustment.
Return:
(platepar, fit_status):
platepar: [Platepar structure] Estimated/refined platepar.
fit_status: [bool] True if fit was successfuly, False if not.
"""
def _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement):
""" Run FFT alignment before giving up on ACF. """
if not _fft_refinement:
print('The initial platepar is bad, trying to refine it using FFT phase correlation...')
# Prepare data for FFT image registration
calstars_dict = {ff_file: star_data for ff_file, star_data in calstars_list}
# Extract star list from CALSTARS file from FF file with most stars
max_len_ff = max(calstars_dict, key=lambda k: len(calstars_dict[k]))
# Take only X, Y (change order so X is first)
calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
# Get the time of the FF file
calstars_time = FFfile.getMiddleTimeFF(max_len_ff, config.fps, ret_milliseconds=True)
# Try aligning the platepar using FFT image registration
platepar_refined = alignPlatepar(config, platepar, calstars_time, calstars_coords)
# Redo autoCF
return autoCheckFit(config, platepar_refined, calstars_list, \
distorsion_refinement=distorsion_refinement, _fft_refinement=True)
else:
print('Auto Check Fit failed completely, please redo the plate manually!')
return platepar, False
if _fft_refinement:
print('Second ACF run with an updated platepar via FFT phase correlation...')
# Convert the list to a dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
# Load catalog stars (overwrite the mag band ratios if specific catalog is used)
catalog_stars, _, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(config.star_catalog_path, \
config.star_catalog_file, lim_mag=config.catalog_mag_limit, \
mag_band_ratios=config.star_catalog_band_ratios)
# Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars
star_dict = {}
# Take only those files with enough stars on them
for ff_name in calstars:
stars_list = calstars[ff_name]
# Check if there are enough stars on the image
if len(stars_list) >= config.ff_min_stars:
# Calculate the JD time of the FF file
dt = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)
jd = date2JD(*dt)
# Add the time and the stars to the dict
star_dict[jd] = stars_list
# There has to be a minimum of 200 FF files for star fitting, and only 100 will be subset if there are more
if len(star_dict) < config.calstars_files_N:
print('Not enough FF files in CALSTARS for ACF!')
return platepar, False
else:
# Randomly choose calstars_files_N image files from the whole list
rand_keys = random.sample(list(star_dict), config.calstars_files_N)
star_dict = {key: star_dict[key] for key in rand_keys}
# Calculate the total number of calibration stars used
total_calstars = sum([len(star_dict[key]) for key in star_dict])
print('Total calstars:', total_calstars)
if total_calstars < config.calstars_min_stars:
print('Not enough calibration stars, need at least', config.calstars_min_stars)
return platepar, False
# A list of matching radiuses to try, pairs of [radius, fit_distorsion_flag]
# The distorsion will be fitted only if explicity requested
min_radius = 0.5
radius_list = [[10, False],
[5, False],
[3, False],
[1.5, True and distorsion_refinement],
[min_radius, True and distorsion_refinement]]
# Calculate the function tolerance, so the desired precision can be reached (the number is calculated
# in the same regard as the cost function)
fatol, xatol_ang = computeMinimizationTolerances(config, platepar, len(star_dict))
### If the initial match is good enough, do only quick recalibratoin ###
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
min_radius, ret_nmatch=True)
if n_matched >= config.calstars_files_N:
# Check if the average distance with the tightest radius is close
if avg_dist < config.dist_check_quick_threshold:
# Use a reduced set of initial radius values
radius_list = [[1.5, True and distorsion_refinement],
[min_radius, True and distorsion_refinement]]
##########
# Match increasingly smaller search radiia around image stars
for i, (match_radius, fit_distorsion) in enumerate(radius_list):
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
match_radius, ret_nmatch=True)
print('Max radius:', match_radius)
print('Initial values:')
print(' Matched stars:', n_matched)
print(' Average deviation:', avg_dist)
# The initial number of matched stars has to be at least the number of FF imaages, otherwise it means
# that the initial platepar is no good
if n_matched < config.calstars_files_N:
print('The total number of initially matched stars is too small! Please manually redo the plate or make sure there are enough calibration stars.')
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict, min_radius, verbose=True):
# Print out notice only if the platepar is good right away
if i == 0:
print("Initial platepar is good enough!")
return platepar, True
# Initial parameters for the astrometric fit (don't fit the scale if the distorsion is not being fit)
if fit_distorsion:
p0 = [platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref, platepar.F_scale]
else:
p0 = [platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref]
# Fit the astrometric parameters
res = scipy.optimize.minimize(_calcImageResidualsAstro, p0, args=(config, platepar, catalog_stars, \
star_dict, match_radius, fit_distorsion), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': xatol_ang})
print(res)
# If the fit was not successful, stop further fitting
if not res.success:
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
else:
# If the fit was successful, use the new parameters from now on
if fit_distorsion:
ra_ref, dec_ref, pos_angle_ref, F_scale = res.x
else:
ra_ref, dec_ref, pos_angle_ref = res.x
F_scale = platepar.F_scale
platepar.RA_d = ra_ref
platepar.dec_d = dec_ref
platepar.pos_angle_ref = pos_angle_ref
platepar.F_scale = F_scale
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict, min_radius, verbose=True):
return platepar, True
# Fit the lens distorsion parameters
if fit_distorsion:
### REVERSE DISTORSION POLYNOMIALS FIT ###
# Fit the distortion parameters (X axis)
res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.x_poly_rev, args=(config, \
platepar, catalog_stars, star_dict, match_radius, 'x'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
else:
platepar.x_poly_rev = res.x
# Fit the distortion parameters (Y axis)
res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.y_poly_rev, args=(config, \
platepar,catalog_stars, star_dict, match_radius, 'y'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
else:
platepar.y_poly_rev = res.x
### ###
### FORWARD DISTORSION POLYNOMIALS FIT ###
# Fit the distortion parameters (X axis)
res = scipy.optimize.minimize(_calcSkyResidualsDistorsion, platepar.x_poly_fwd, args=(config, \
platepar, catalog_stars, star_dict, match_radius, 'x'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
# Try to refine the platepar with FFT phase correlation and redo the ACF
return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement)
else:
platepar.x_poly_fwd = res.x
# Fit the distortion parameters (Y axis)
res = scipy.optimize.minimize(_calcSkyResidualsDistorsion, platepar.y_poly_fwd, args=(config, \
platepar,catalog_stars, star_dict, match_radius, 'y'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
return platepar, False
else:
platepar.y_poly_fwd = res.x
### ###
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \
star_dict, min_radius, ret_nmatch=True)
print('FINAL SOLUTION with {:f} px:'.format(min_radius))
print('Matched stars:', n_matched)
print('Average deviation:', avg_dist)
# Mark the platepar to indicate that it was automatically refined with CheckFit
platepar.auto_check_fit_refined = True
return platepar, True
def extractStarsAndSave(config, ff_dir):
""" Extract stars in the given folder and save the CALSTARS file.
Arguments:
config: [config object] configuration object (loaded from the .config file)
ff_dir: [str] Path to directory where FF files are.
Return:
star_list: [list] A list of [ff_name, star_data] entries, where star_data contains a list of
(column, row, amplitude, intensity, fwhm) values for every star.
"""
time_start = time.time()
# Load mask, dark, flat
mask, dark, flat_struct = loadImageCalibration(ff_dir, config)
extraction_list = []
# Go through all files in the directory and add them to the detection list
for ff_name in sorted(os.listdir(ff_dir)):
# Check if the given file is a valid FF file
if not FFfile.validFFName(ff_name):
continue
extraction_list.append(ff_name)
# Run the QueuedPool for detection
workpool = QueuedPool(extractStars, cores=-1, backup_dir=ff_dir)
# Add jobs for the pool
for ff_name in extraction_list:
print('Adding for extraction:', ff_name)
workpool.addJob([
ff_dir, ff_name, config, None, None, None, None, flat_struct, dark,
mask
])
print('Starting pool...')
# Start the detection
workpool.startPool()
print('Waiting for the detection to finish...')
# Wait for the detector to finish and close it
workpool.closePool()
# Get extraction results
star_list = []
for result in workpool.getResults():
ff_name, x2, y2, amplitude, intensity, fwhm_data = result
# Skip if no stars were found
if not x2:
continue
# Construct the table of the star parameters
star_data = list(zip(x2, y2, amplitude, intensity, fwhm_data))
# Add star info to the star list
star_list.append([ff_name, star_data])
dir_name = os.path.basename(os.path.abspath(ff_dir))
if dir_name.startswith(config.stationID):
prefix = dir_name
else:
prefix = "{:s}_{:s}".format(config.stationID, dir_name)
# Generate the name for the CALSTARS file
calstars_name = 'CALSTARS_' + prefix + '.txt'
# Write detected stars to the CALSTARS file
CALSTARS.writeCALSTARS(star_list, ff_dir, calstars_name, config.stationID,
config.height, config.width)
# Delete QueudPool backed up files
workpool.deleteBackupFiles()
print('Total time taken: {:.2f} s'.format(time.time() - time_start))
return star_list
def generateThumbnails(dir_path,
config,
mosaic_type,
file_list=None,
no_stack=False):
""" Generates a mosaic of thumbnails from all FF files in the given folder and saves it as a JPG image.
Arguments:
dir_path: [str] Path of the night directory.
config: [Conf object] Configuration.
mosaic_type: [str] Type of the mosaic (e.g. "Captured" or "Detected")
Keyword arguments:
file_list: [list] A list of file names (without full path) which will be searched for FF files. This
is used when generating separate thumbnails for captured and detected files.
Return:
file_name: [str] Name of the thumbnail file.
no_stack: [bool] Don't stack the images using the config.thumb_stack option. A max of 1000 images
are supported with this option. If there are more, stacks will be done according to the
config.thumb_stack option.
"""
if file_list is None:
file_list = sorted(os.listdir(dir_path))
# Make a list of all FF files in the night directory
ff_list = []
for file_name in file_list:
if FFfile.validFFName(file_name):
ff_list.append(file_name)
# Calculate the dimensions of the binned image
bin_w = int(config.width / config.thumb_bin)
bin_h = int(config.height / config.thumb_bin)
### RESIZE AND STACK THUMBNAILS ###
##########################################################################################################
timestamps = []
stacked_imgs = []
thumb_stack = config.thumb_stack
# Check if no stacks should be done (max 1000 images for no stack)
if no_stack and (len(ff_list) < 1000):
thumb_stack = 1
for i in range(0, len(ff_list), thumb_stack):
img_stack = np.zeros((bin_h, bin_w))
# Stack thumb_stack images using the 'if lighter' method
for j in range(thumb_stack):
if (i + j) < len(ff_list):
tmp_file_name = ff_list[i + j]
# Read the FF file
ff = FFfile.read(dir_path, tmp_file_name)
# Skip the FF if it is corruped
if ff is None:
continue
img = ff.maxpixel
# Resize the image
img = cv2.resize(img, (bin_w, bin_h))
# Stack the image
img_stack = stackIfLighter(img_stack, img)
else:
break
# Save the timestamp of the first image in the stack
timestamps.append(FFfile.filenameToDatetime(ff_list[i]))
# Save the stacked image
stacked_imgs.append(img_stack)
# cv2.imshow('test', img_stack)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
##########################################################################################################
### ADD THUMBS TO ONE MOSAIC IMAGE ###
##########################################################################################################
header_height = 20
timestamp_height = 10
# Calculate the number of rows for the thumbnail image
n_rows = int(
np.ceil(float(len(ff_list)) / thumb_stack / config.thumb_n_width))
# Calculate the size of the mosaic
mosaic_w = int(config.thumb_n_width * bin_w)
mosaic_h = int((bin_h + timestamp_height) * n_rows + header_height)
mosaic_img = np.zeros((mosaic_h, mosaic_w), dtype=np.uint8)
# Write header text
header_text = 'Station: ' + str(config.stationID) + ' Night: ' + os.path.basename(dir_path) \
+ ' Type: ' + mosaic_type
cv2.putText(mosaic_img, header_text, (0, header_height//2), \
cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255,255,255), 1)
for row in range(n_rows):
for col in range(config.thumb_n_width):
# Calculate image index
indx = row * config.thumb_n_width + col
if indx < len(stacked_imgs):
# Calculate position of the text
text_x = col * bin_w
text_y = row * bin_h + (
row + 1) * timestamp_height - 1 + header_height
# Add timestamp text
cv2.putText(mosaic_img, timestamps[indx].strftime('%H:%M:%S'), (text_x, text_y), \
cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1)
# Add the image to the mosaic
img_pos_x = col * bin_w
img_pos_y = row * bin_h + (
row + 1) * timestamp_height + header_height
mosaic_img[img_pos_y:img_pos_y + bin_h,
img_pos_x:img_pos_x + bin_w] = stacked_imgs[indx]
else:
break
##########################################################################################################
# Only add the station ID if the dir name already doesn't start with it
dir_name = os.path.basename(os.path.abspath(dir_path))
if dir_name.startswith(config.stationID):
prefix = dir_name
else:
prefix = "{:s}_{:s}".format(config.stationID, dir_name)
thumb_name = "{:s}_{:s}_thumbs.jpg".format(prefix, mosaic_type)
# Save the mosaic
if USING_IMAGEIO:
# Use imageio to write the image
imwrite(os.path.join(dir_path, thumb_name), mosaic_img, quality=80)
else:
# Use OpenCV to save the image
imwrite(os.path.join(dir_path, thumb_name), mosaic_img,
[int(cv2.IMWRITE_JPEG_QUALITY), 80])
return thumb_name
# RPi Meteor Station
# Copyright (C) 2015 Dario Zubovic
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import cv2
import sys, os
from RMS.Formats import FFfile
if __name__ == "__main__":
ff = FFfile.read(sys.argv[1], sys.argv[2])
cv2.imwrite(sys.argv[2] + "_max.png", ff.maxpixel)
cv2.imwrite(sys.argv[2] + "_frame.png", ff.maxframe)
cv2.imwrite(sys.argv[2] + "_avg.png", ff.avepixel)
cv2.imwrite(sys.argv[2] + "_stddev.png", ff.stdpixel)
def view(dir_path, ff_path, fr_path, config, save_frames=False):
""" Shows the detected fireball stored in the FR file.
Arguments:
dir_path: [str] Current directory.
ff: [str] path to the FF bin file
fr: [str] path to the FR bin file
config: [conf object] configuration structure
"""
name = fr_path
fr = FRbin.read(dir_path, fr_path)
print('------------------------')
print('Showing file:', fr_path)
if ff_path is None:
#background = np.zeros((config.height, config.width), np.uint8)
# Get the maximum extent of meteor frames
y_size = max([max(np.array(fr.yc[i]) + np.array(fr.size[i])//2) for i in range(fr.lines)])
x_size = max([max(np.array(fr.xc[i]) + np.array(fr.size[i])//2) for i in range(fr.lines)])
# Make the image square
img_size = max(y_size, x_size)
background = np.zeros((img_size, img_size), np.uint8)
else:
background = FFfile.read(dir_path, ff_path).maxpixel
print("Number of lines:", fr.lines)
first_image = True
wait_time = 2*int(1000.0/config.fps)
pause_flag = False
for current_line in range(fr.lines):
print('Frame, Y , X , size')
for z in range(fr.frameNum[current_line]):
# Get the center position of the detection on the current frame
yc = fr.yc[current_line][z]
xc = fr.xc[current_line][z]
# Get the frame number
t = fr.t[current_line][z]
# Get the size of the window
size = fr.size[current_line][z]
print(" {:3d}, {:3d}, {:3d}, {:d}".format(t, yc, xc, size))
img = np.copy(background)
# Paste the frames onto the big image
y_img = np.arange(yc - size//2, yc + size//2)
x_img = np.arange(xc - size//2, xc + size//2)
Y_img, X_img = np.meshgrid(y_img, x_img)
y_frame = np.arange(len(y_img))
x_frame = np.arange(len(x_img))
Y_frame, X_frame = np.meshgrid(y_frame, x_frame)
img[Y_img, X_img] = fr.frames[current_line][z][Y_frame, X_frame]
# Save frame to disk
if save_frames:
frame_file_name = fr_path.replace('.bin', '') + "_line_{:02d}_frame_{:03d}.png".format(current_line, t)
cv2.imwrite(os.path.join(dir_path, frame_file_name), img)
# Show the frame
cv2.imshow(name, img)
# If this is the first image, move it to the upper left corner
if first_image:
cv2.moveWindow(name, 0, 0)
first_image = False
if pause_flag:
wait_time = 0
else:
wait_time = 2*int(1000.0/config.fps)
# Space key: pause display.
# 1: previous file.
# 2: next line.
# q: Quit.
key = cv2.waitKey(wait_time) & 0xFF
if key == ord("1"):
cv2.destroyWindow(name)
return -1
elif key == ord("2"):
break
elif key == ord(" "):
# Pause/unpause video
pause_flag = not pause_flag
elif key == ord("q") :
os._exit(0)
cv2.destroyWindow(name)
# Load the configuration file
config = cr.parse(".config")
# Get the list of FR bin files (fireball detections) in the given directory
fr_list = [fr for fr in os.listdir(dir_path) if fr[0:2]=="FR" and fr.endswith('bin')]
fr_list = sorted(fr_list)
if not fr_list:
print("No files found!")
sys.exit()
# Get the list of FF bin files (compressed video frames)
ff_list = [ff for ff in os.listdir(dir_path) if FFfile.validFFName(ff)]
ff_list = sorted(ff_list)
i = 0
while True:
# Break the loop if at the end
if i >= len(fr_list):
break
fr = fr_list[i]
ff_match = None
if __name__ == "__main__":
time_start = time.clock()
# Load config file
config = cr.parse(".config")
if not len(sys.argv) == 2:
print("Usage: python -m RMS.ExtractStars /path/to/FF/files/")
sys.exit()
# Get paths to every FF bin file in a directory
ff_dir = os.path.abspath(sys.argv[1])
ff_list = [ff_name for ff_name in os.listdir(ff_dir) if FFfile.validFFName(ff_name)]
# Check if there are any file in the directory
if(len(ff_list) == None):
print("No files found!")
sys.exit()
# Try loading a flat field image
flat_struct = None
if config.use_flat:
# Check if there is flat in the data directory
if os.path.exists(os.path.join(ff_dir, config.flat_file)):
def recalibrateIndividualFFsAndApplyAstrometry(dir_path, ftpdetectinfo_path, calstars_list, config, platepar):
""" Recalibrate FF files with detections and apply the recalibrated platepar to those detections.
Arguments:
dir_path: [str] Path where the FTPdetectinfo file is.
ftpdetectinfo_path: [str] Name of the FTPdetectinfo file.
calstars_list: [list] A list of entries [[ff_name, star_coordinates], ...].
config: [Config instance]
platepar: [Platepar instance] Initial platepar.
Return:
recalibrated_platepars: [dict] A dictionary where the keys are FF file names and values are
recalibrated platepar instances for every FF file.
"""
# Read the FTPdetectinfo data
cam_code, fps, meteor_list = FTPdetectinfo.readFTPdetectinfo(*os.path.split(ftpdetectinfo_path), \
ret_input_format=True)
# Convert the list of stars to a per FF name dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
# Load catalog stars (overwrite the mag band ratios if specific catalog is used)
catalog_stars, _, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(config.star_catalog_path,\
config.star_catalog_file, lim_mag=config.catalog_mag_limit, \
mag_band_ratios=config.star_catalog_band_ratios)
prev_platepar = copy.deepcopy(platepar)
# Go through all FF files with detections, recalibrate and apply astrometry
recalibrated_platepars = {}
for meteor_entry in meteor_list:
working_platepar = copy.deepcopy(prev_platepar)
ff_name, meteor_No, rho, phi, meteor_meas = meteor_entry
# Skip this meteors if its FF file was already recalibrated
if ff_name in recalibrated_platepars:
continue
print()
print('Processing: ', ff_name)
print('------------------------------------------------------------------------------')
# Find extracted stars on this image
if not ff_name in calstars:
print('Skipped because it was not in CALSTARS:', ff_name)
continue
# Get stars detected on this FF file (create a dictionaly with only one entry, the residuals function
# needs this format)
calstars_time = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)
jd = date2JD(*calstars_time)
star_dict_ff = {jd: calstars[ff_name]}
# Recalibrate the platepar using star matching
result = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars)
# If the recalibration failed, try using FFT alignment
if result is None:
print()
print('Running FFT alignment...')
# Run FFT alignment
calstars_coords = np.array(star_dict_ff[jd])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
print(calstars_time)
working_platepar = alignPlatepar(config, prev_platepar, calstars_time, calstars_coords, \
show_plot=False)
# Try to recalibrate after FFT alignment
result = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars)
if result is not None:
working_platepar = result
else:
working_platepar = result
# Store the platepar if the fit succeeded
if result is not None:
recalibrated_platepars[ff_name] = working_platepar
prev_platepar = working_platepar
else:
print('Recalibration of {:s} failed, using the previous platepar...'.format(ff_name))
# If the aligning failed, set the previous platepar as the one that should be used for this FF file
recalibrated_platepars[ff_name] = prev_platepar
### Store all recalibrated platepars as a JSON file ###
all_pps = {}
for ff_name in recalibrated_platepars:
json_str = recalibrated_platepars[ff_name].jsonStr()
all_pps[ff_name] = json.loads(json_str)
with open(os.path.join(dir_path, config.platepars_recalibrated_name), 'w') as f:
# Convert all platepars to a JSON file
out_str = json.dumps(all_pps, default=lambda o: o.__dict__, indent=4, sort_keys=True)
f.write(out_str)
### ###
# If no platepars were recalibrated, use the single platepar recalibration procedure
if len(recalibrated_platepars) == 0:
print('No FF images were used for recalibration, using the single platepar calibration function...')
# Use the initial platepar for calibration
applyAstrometryFTPdetectinfo(dir_path, os.path.basename(ftpdetectinfo_path), None, platepar=platepar)
return recalibrated_platepars
### Plot difference from reference platepar in angular distance from (0, 0) vs rotation ###
ang_dists = []
rot_angles = []
hour_list = []
first_jd = np.min([FFfile.filenameToDatetime(ff_name) for ff_name in recalibrated_platepars])
for ff_name in recalibrated_platepars:
pp_temp = recalibrated_platepars[ff_name]
# If the fitting failed, skip the platepar
if pp_temp is None:
continue
# Compute the angular separation from the reference platepar
ang_dist = np.degrees(angularSeparation(np.radians(platepar.RA_d), np.radians(platepar.dec_d), \
np.radians(pp_temp.RA_d), np.radians(pp_temp.dec_d)))
ang_dists.append(ang_dist*60)
rot_angles.append((platepar.pos_angle_ref - pp_temp.pos_angle_ref)*60)
# Compute the hour of the FF used for recalibration
hour_list.append((FFfile.filenameToDatetime(ff_name) - first_jd).total_seconds()/3600)
plt.figure()
plt.scatter(0, 0, marker='o', edgecolor='k', label='Reference platepar', s=100, c='none', zorder=3)
plt.scatter(ang_dists, rot_angles, c=hour_list, zorder=3)
plt.colorbar(label='Hours from first FF file')
plt.xlabel("Angular distance from reference (arcmin)")
plt.ylabel('Rotation from reference (arcmin)')
plt.grid()
plt.legend()
plt.tight_layout()
# Generate the name for the plot
calib_plot_name = os.path.basename(ftpdetectinfo_path).replace('FTPdetectinfo_', '').replace('.txt', '') \
+ '_calibration_variation.png'
plt.savefig(os.path.join(dir_path, calib_plot_name), dpi=150)
# plt.show()
plt.clf()
plt.close()
### ###
### Apply platepars to FTPdetectinfo ###
meteor_output_list = []
for meteor_entry in meteor_list:
ff_name, meteor_No, rho, phi, meteor_meas = meteor_entry
# Get the platepar that will be applied to this FF file
if ff_name in recalibrated_platepars:
working_platepar = recalibrated_platepars[ff_name]
else:
print('Using default platepar for:', ff_name)
working_platepar = platepar
# Apply the recalibrated platepar to meteor centroids
meteor_picks = applyPlateparToCentroids(ff_name, fps, meteor_meas, working_platepar, \
add_calstatus=True)
meteor_output_list.append([ff_name, meteor_No, rho, phi, meteor_picks])
# Calibration string to be written to the FTPdetectinfo file
calib_str = 'Recalibrated with RMS on: ' + str(datetime.datetime.utcnow()) + ' UTC'
# If no meteors were detected, set dummpy parameters
if len(meteor_list) == 0:
cam_code = ''
fps = 0
# Back up the old FTPdetectinfo file
shutil.copy(ftpdetectinfo_path, ftpdetectinfo_path.strip('.txt') \
+ '_backup_{:s}.txt'.format(datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S.%f')))
# Save the updated FTPdetectinfo
FTPdetectinfo.writeFTPdetectinfo(meteor_output_list, dir_path, os.path.basename(ftpdetectinfo_path), \
dir_path, cam_code, fps, calibration=calib_str, celestial_coords_given=True)
### ###
return recalibrated_platepars
def plot(points, y_dim, x_dim):
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
plt.title(name)
y = points[:, 0]
x = points[:, 1]
z = points[:, 2]
# Plot points in 3D
ax.scatter(x, y, z)
# Set axes limits
ax.set_zlim(0, 255)
plt.xlim([0, x_dim])
plt.ylim([0, y_dim])
ax.set_ylabel("Y")
ax.set_xlabel("X")
ax.set_zlabel("Time")
plt.show()
if __name__ == "__main__":
ff = FFfile.read(sys.argv[1], sys.argv[2], array=True)
view(ff)
# Adjust levels
img_data = adjustLevels(img_data, 100, 1.2, 240)
plt.imshow(img_data, cmap='gray')
plt.show()
#### Apply the flat
# Load an FF file
dir_path = "/home/dvida/Dropbox/Apps/Elginfield RPi RMS data/ArchivedFiles/CA0001_20171018_230520_894458_detected"
file_name = "FF_CA0001_20171019_092744_161_1118976.fits"
ff = FFfile.read(dir_path, file_name)
# Load the flat
flat_struct = loadFlat(os.getcwd(), config.flat_file)
t1 = time.clock()
# Apply the flat
img = applyFlat(ff.maxpixel, flat_struct)
print('Flat time:', time.clock() - t1)
plt.imshow(img, cmap='gray', vmin=0, vmax=255)
plt.show()
def extractStars(ff_dir, ff_name, config=None, max_global_intensity=150, border=10, neighborhood_size=10,
intensity_threshold=5, flat_struct=None, dark=None, mask=None):
""" Extracts stars on a given FF bin by searching for local maxima and applying PSF fit for star
confirmation.
Source of one part of the code:
http://stackoverflow.com/questions/9111711/get-coordinates-of-local-maxima-in-2d-array-above-certain-value
Arguments:
ff: [ff bin struct] FF bin file loaded in the FF bin structure
config: [config object] configuration object (loaded from the .config file)
max_global_intensity: [int] maximum mean intensity of an image before it is discared as too bright
border: [int] apply a mask on the detections by removing all that are too close to the given image
border (in pixels)
neighborhood_size: [int] size of the neighbourhood for the maximum search (in pixels)
intensity_threshold: [float] a threshold for cutting the detections which are too faint (0-255)
flat_struct: [Flat struct] Structure containing the flat field. None by default.
dark: [ndarray] Dark frame. None by default.
mask: [ndarray] Mask image. None by default.
Return:
x2, y2, background, intensity, sigma_fitted: [list of ndarrays]
- x2: X axis coordinates of the star
- y2: Y axis coordinates of the star
- background: background intensity
- intensity: intensity of the star
- Gaussian stddev of fitted stars
"""
# This will be returned if there was an error
error_return = [[], [], [], []]
# Load parameters from config if given
if config:
max_global_intensity = config.max_global_intensity
border = config.border
neighborhood_size = config.neighborhood_size
intensity_threshold = config.intensity_threshold
# Load the FF bin file
ff = FFfile.read(ff_dir, ff_name)
# If the FF file could not be read, skip star extraction
if ff is None:
return error_return
# Apply the dark frame
if dark is not None:
ff.avepixel = Image.applyDark(ff.avepixel, dark)
# Apply the flat
if flat_struct is not None:
ff.avepixel = Image.applyFlat(ff.avepixel, flat_struct)
# Mask the FF file
if mask is not None:
ff = MaskImage.applyMask(ff, mask, ff_flag=True)
# Calculate image mean and stddev
global_mean = np.mean(ff.avepixel)
# Check if the image is too bright and skip the image
if global_mean > max_global_intensity:
return error_return
data = ff.avepixel.astype(np.float32)
# Apply a mean filter to the image to reduce noise
data = ndimage.filters.convolve(data, weights=np.full((2, 2), 1.0/4))
# Locate local maxima on the image
data_max = filters.maximum_filter(data, neighborhood_size)
maxima = (data == data_max)
data_min = filters.minimum_filter(data, neighborhood_size)
diff = ((data_max - data_min) > intensity_threshold)
maxima[diff == 0] = 0
# Apply a border mask
border_mask = np.ones_like(maxima)*255
border_mask[:border,:] = 0
border_mask[-border:,:] = 0
border_mask[:,:border] = 0
border_mask[:,-border:] = 0
maxima = MaskImage.applyMask(maxima, border_mask, image=True)
# Find and label the maxima
labeled, num_objects = ndimage.label(maxima)
# Skip the image if there are too many maxima to process
if num_objects > config.max_stars:
print('Too many candidate stars to process! {:d}/{:d}'.format(num_objects, config.max_stars))
return error_return
# Find centres of mass of each labeled objects
xy = np.array(ndimage.center_of_mass(data, labeled, range(1, num_objects+1)))
# Remove all detection on the border
#xy = xy[np.where((xy[:, 1] > border) & (xy[:,1] < ff.ncols - border) & (xy[:,0] > border) & (xy[:,0] < ff.nrows - border))]
# Unpack star coordinates
y, x = np.hsplit(xy, 2)
# # Plot stars before the PSF fit
# plotStars(ff, x, y)
# Fit a PSF to each star
x2, y2, amplitude, intensity, sigma_y_fitted, sigma_x_fitted = fitPSF(ff, global_mean, x, y, config)
# x2, y2, amplitude, intensity = list(x), list(y), [], [] # Skip PSF fit
# # Plot stars after PSF fit filtering
# plotStars(ff, x2, y2)
# Compute one dimensional sigma
sigma_x_fitted = np.array(sigma_x_fitted)
sigma_y_fitted = np.array(sigma_y_fitted)
sigma_fitted = np.sqrt(sigma_x_fitted**2 + sigma_y_fitted**2)
return ff_name, x2, y2, amplitude, intensity, sigma_fitted
def FTPdetectinfo2UFOOrbitInput(dir_path,
file_name,
platepar_path,
platepar_dict=None):
""" Convert the FTPdetectinfo file into UFOOrbit input CSV file.
Arguments:
dir_path: [str] Path of the directory which contains the FTPdetectinfo file.
file_name: [str] Name of the FTPdetectinfo file.
platepar_path: [str] Full path to the platepar file.
Keyword arguments:
platepar_dict: [dict] Dictionary of Platepar instances where keys are FF file names. This will be
used instead of the platepar at platepar_path. None by default.
"""
# Load the FTPdetecinfo file
meteor_list = FTPdetectinfo.readFTPdetectinfo(dir_path, file_name)
# Load the platepar file
if platepar_dict is None:
pp = RMS.Formats.Platepar.Platepar()
pp.read(platepar_path, use_flat=None)
# Init the UFO format list
ufo_meteor_list = []
# Go through every meteor in the list
for meteor in meteor_list:
ff_name, cam_code, meteor_No, n_segments, fps, hnr, mle, binn, px_fm, rho, phi, \
meteor_meas = meteor
# Load the platepar from the platepar dictionary, if given
if platepar_dict is not None:
if ff_name in platepar_dict:
pp = platepar_dict[ff_name]
else:
print(
'Skipping {:s} becuase no platepar was found for this FF file!'
.format(ff_name))
continue
# Convert the FF file name into time
dt = FFfile.filenameToDatetime(ff_name)
# Extract measurements
calib_status, frame_n, x, y, ra, dec, azim, elev, inten, mag = np.array(
meteor_meas).T
# If the meteor wasn't calibrated, skip it
if not np.all(calib_status):
print('Meteor {:d} was not calibrated, skipping it...'.format(
meteor_No))
continue
# Compute the peak magnitude
peak_mag = np.min(mag)
# Compute the total duration
first_frame = np.min(frame_n)
last_frame = np.max(frame_n)
duration = (last_frame - first_frame) / fps
# Compute times of first and last points
dt1 = dt + datetime.timedelta(seconds=first_frame / fps)
dt2 = dt + datetime.timedelta(seconds=last_frame / fps)
### Fit a great circle to Az/Alt measurements and compute model beg/end RA and Dec ###
# Convert the measurement Az/Alt to cartesian coordinates
# NOTE: All values that are used for Great Circle computation are:
# theta - the zenith angle (90 deg - altitude)
# phi - azimuth +N of due E, which is (90 deg - azim)
x, y, z = Math.polarToCartesian(np.radians((90 - azim) % 360),
np.radians(90 - elev))
# Fit a great circle
C, theta0, phi0 = GreatCircle.fitGreatCircle(x, y, z)
# Get the first point on the great circle
phase1 = GreatCircle.greatCirclePhase(np.radians(90 - elev[0]), np.radians((90 - azim[0])%360), \
theta0, phi0)
alt1, azim1 = Math.cartesianToPolar(
*GreatCircle.greatCircle(phase1, theta0, phi0))
alt1 = 90 - np.degrees(alt1)
azim1 = (90 - np.degrees(azim1)) % 360
# Get the last point on the great circle
phase2 = GreatCircle.greatCirclePhase(np.radians(90 - elev[-1]), np.radians((90 - azim[-1])%360),\
theta0, phi0)
alt2, azim2 = Math.cartesianToPolar(
*GreatCircle.greatCircle(phase2, theta0, phi0))
alt2 = 90 - np.degrees(alt2)
azim2 = (90 - np.degrees(azim2)) % 360
# Compute RA/Dec from Alt/Az
_, ra1, dec1 = RMS.Astrometry.ApplyAstrometry.altAzToRADec(pp.lat, pp.lon, pp.UT_corr, [dt1], \
[azim1], [alt1], dt_time=True)
_, ra2, dec2 = RMS.Astrometry.ApplyAstrometry.altAzToRADec(pp.lat, pp.lon, pp.UT_corr, [dt2], \
[azim2], [alt2], dt_time=True)
### ###
ufo_meteor_list.append([dt1, peak_mag, duration, azim1[0], alt1[0], azim2[0], alt2[0], \
ra1[0][0], dec1[0][0], ra2[0][0], dec2[0][0], cam_code, pp.lon, pp.lat, pp.elev, pp.UT_corr])
# Construct a file name for the UFO file, which is the FTPdetectinfo file without the FTPdetectinfo
# part
ufo_file_name = file_name.replace('FTPdetectinfo_', '').replace(
'.txt', '') + '.csv'
# Write the UFOorbit file
UFOOrbit.writeUFOOrbit(dir_path, ufo_file_name, ufo_meteor_list)
action="store_true",
help="""Show a histogram of stddevs of PSFs of all detected stars. """)
# Parse the command line arguments
cml_args = arg_parser.parse_args()
#########################
# Load the config file
config = cr.loadConfigFromDirectory(cml_args.config, cml_args.dir_path)
# Get paths to every FF bin file in a directory
ff_dir = os.path.abspath(cml_args.dir_path[0])
ff_list = [
ff_name for ff_name in os.listdir(ff_dir)
if FFfile.validFFName(ff_name)
]
# Check if there are any file in the directory
if (len(ff_list) == None):
print("No files found!")
sys.exit()
# Run extraction and save the resulting CALSTARS file
star_list = extractStarsAndSave(config, ff_dir)
fwhm_list = []
intensity_list = []
x_list = []
y_list = []
def FTPdetectinfo2UFOOrbitInput(dir_path, file_name, platepar_path, platepar_dict=None):
""" Convert the FTPdetectinfo file into UFOOrbit input CSV file.
Arguments:
dir_path: [str] Path of the directory which contains the FTPdetectinfo file.
file_name: [str] Name of the FTPdetectinfo file.
platepar_path: [str] Full path to the platepar file.
Keyword arguments:
platepar_dict: [dict] Dictionary of Platepar instances where keys are FF file names. This will be
used instead of the platepar at platepar_path. None by default.
"""
# Load the FTPdetecinfo file
meteor_list = FTPdetectinfo.readFTPdetectinfo(dir_path, file_name)
# Load the platepar file
if platepar_dict is None:
pp = RMS.Formats.Platepar.Platepar()
pp.read(platepar_path)
# Init the UFO format list
ufo_meteor_list = []
# Go through every meteor in the list
for meteor in meteor_list:
ff_name, cam_code, meteor_No, n_segments, fps, hnr, mle, binn, px_fm, rho, phi, \
meteor_meas = meteor
# Load the platepar from the platepar dictionary, if given
if platepar_dict is not None:
if ff_name in platepar_dict:
pp = platepar_dict[ff_name]
else:
print('Skipping {:s} becuase no platepar was found for this FF file!'.format(ff_name))
# Convert the FF file name into time
dt = FFfile.filenameToDatetime(ff_name)
# Extract measurements
calib_status, frame_n, x, y, ra, dec, azim, elev, inten, mag = np.array(meteor_meas).T
# If the meteor wasn't calibrated, skip it
if not np.all(calib_status):
print('Meteor {:d} was not calibrated, skipping it...'.format(meteor_No))
continue
# Compute the peak magnitude
peak_mag = np.min(mag)
# Compute the total duration
first_frame = np.min(frame_n)
last_frame = np.max(frame_n)
duration = (last_frame - first_frame)/fps
# Compute times of first and last points
dt1 = dt + datetime.timedelta(seconds=first_frame/fps)
dt2 = dt + datetime.timedelta(seconds=last_frame/fps)
### Fit a great circle to Az/Alt measurements and compute model beg/end RA and Dec ###
# Convert the measurement Az/Alt to cartesian coordinates
# NOTE: All values that are used for Great Circle computation are:
# theta - the zenith angle (90 deg - altitude)
# phi - azimuth +N of due E, which is (90 deg - azim)
x, y, z = Math.polarToCartesian(np.radians((90 - azim)%360), np.radians(90 - elev))
# Fit a great circle
C, theta0, phi0 = GreatCircle.fitGreatCircle(x, y, z)
# Get the first point on the great circle
phase1 = GreatCircle.greatCirclePhase(np.radians(90 - elev[0]), np.radians((90 - azim[0])%360), \
theta0, phi0)
alt1, azim1 = Math.cartesianToPolar(*GreatCircle.greatCircle(phase1, theta0, phi0))
alt1 = 90 - np.degrees(alt1)
azim1 = (90 - np.degrees(azim1))%360
# Get the last point on the great circle
phase2 = GreatCircle.greatCirclePhase(np.radians(90 - elev[-1]), np.radians((90 - azim[-1])%360),\
theta0, phi0)
alt2, azim2 = Math.cartesianToPolar(*GreatCircle.greatCircle(phase2, theta0, phi0))
alt2 = 90 - np.degrees(alt2)
azim2 = (90 - np.degrees(azim2))%360
# Compute RA/Dec from Alt/Az
_, ra1, dec1 = RMS.Astrometry.ApplyAstrometry.altAzToRADec(pp.lat, pp.lon, pp.UT_corr, [dt1], \
[azim1], [alt1], dt_time=True)
_, ra2, dec2 = RMS.Astrometry.ApplyAstrometry.altAzToRADec(pp.lat, pp.lon, pp.UT_corr, [dt2], \
[azim2], [alt2], dt_time=True)
### ###
ufo_meteor_list.append([dt1, peak_mag, duration, azim1[0], alt1[0], azim2[0], alt2[0], \
ra1[0][0], dec1[0][0], ra2[0][0], dec2[0][0], cam_code, pp.lon, pp.lat, pp.elev, pp.UT_corr])
# Construct a file name for the UFO file, which is the FTPdetectinfo file without the FTPdetectinfo
# part
ufo_file_name = file_name.replace('FTPdetectinfo_', '').replace('.txt', '') + '.csv'
# Write the UFOorbit file
UFOOrbit.writeUFOOrbit(dir_path, ufo_file_name, ufo_meteor_list)
def extractStars(ff_dir,
ff_name,
config=None,
max_global_intensity=150,
border=10,
neighborhood_size=10,
intensity_threshold=5,
flat_struct=None,
dark=None,
mask=None):
""" Extracts stars on a given FF bin by searching for local maxima and applying PSF fit for star
confirmation.
Source of one part of the code:
http://stackoverflow.com/questions/9111711/get-coordinates-of-local-maxima-in-2d-array-above-certain-value
Arguments:
ff_dir: [str] Path to directory where FF files are.
ff_name: [str] Name of the FF file.
config: [config object] configuration object (loaded from the .config file)
max_global_intensity: [int] maximum mean intensity of an image before it is discared as too bright
border: [int] apply a mask on the detections by removing all that are too close to the given image
border (in pixels)
neighborhood_size: [int] size of the neighbourhood for the maximum search (in pixels)
intensity_threshold: [float] a threshold for cutting the detections which are too faint (0-255)
flat_struct: [Flat struct] Structure containing the flat field. None by default.
dark: [ndarray] Dark frame. None by default.
mask: [ndarray] Mask image. None by default.
Return:
x2, y2, background, intensity, fwhm: [list of ndarrays]
- x2: X axis coordinates of the star
- y2: Y axis coordinates of the star
- background: background intensity
- intensity: intensity of the star
- Gaussian Full width at half maximum (FWHM) of fitted stars
"""
# This will be returned if there was an error
error_return = [[], [], [], [], [], []]
# Load parameters from config if given
if config:
max_global_intensity = config.max_global_intensity
border = config.border
neighborhood_size = config.neighborhood_size
intensity_threshold = config.intensity_threshold
# Load the FF bin file
ff = FFfile.read(ff_dir, ff_name)
# If the FF file could not be read, skip star extraction
if ff is None:
return error_return
# Apply the dark frame
if dark is not None:
ff.avepixel = Image.applyDark(ff.avepixel, dark)
# Apply the flat
if flat_struct is not None:
ff.avepixel = Image.applyFlat(ff.avepixel, flat_struct)
# Mask the FF file
if mask is not None:
ff = MaskImage.applyMask(ff, mask, ff_flag=True)
# Calculate image mean and stddev
global_mean = np.mean(ff.avepixel)
# Check if the image is too bright and skip the image
if global_mean > max_global_intensity:
return error_return
data = ff.avepixel.astype(np.float32)
# Apply a mean filter to the image to reduce noise
data = ndimage.filters.convolve(data, weights=np.full((2, 2), 1.0 / 4))
# Locate local maxima on the image
data_max = filters.maximum_filter(data, neighborhood_size)
maxima = (data == data_max)
data_min = filters.minimum_filter(data, neighborhood_size)
diff = ((data_max - data_min) > intensity_threshold)
maxima[diff == 0] = 0
# Apply a border mask
border_mask = np.ones_like(maxima) * 255
border_mask[:border, :] = 0
border_mask[-border:, :] = 0
border_mask[:, :border] = 0
border_mask[:, -border:] = 0
maxima = MaskImage.applyMask(maxima, border_mask, image=True)
# Remove all detections close to the mask image
if mask is not None:
erosion_kernel = np.ones((5, 5), mask.img.dtype)
mask_eroded = cv2.erode(mask.img, erosion_kernel, iterations=1)
maxima = MaskImage.applyMask(maxima, mask_eroded, image=True)
# Find and label the maxima
labeled, num_objects = ndimage.label(maxima)
# Skip the image if there are too many maxima to process
if num_objects > config.max_stars:
print('Too many candidate stars to process! {:d}/{:d}'.format(
num_objects, config.max_stars))
return error_return
# Find centres of mass of each labeled objects
xy = np.array(
ndimage.center_of_mass(data, labeled, range(1, num_objects + 1)))
# Remove all detection on the border
#xy = xy[np.where((xy[:, 1] > border) & (xy[:,1] < ff.ncols - border) & (xy[:,0] > border) & (xy[:,0] < ff.nrows - border))]
# Unpack star coordinates
y, x = np.hsplit(xy, 2)
# # Plot stars before the PSF fit
# plotStars(ff, x, y)
# Fit a PSF to each star
x2, y2, amplitude, intensity, sigma_y_fitted, sigma_x_fitted = fitPSF(
ff, global_mean, x, y, config)
# x2, y2, amplitude, intensity = list(x), list(y), [], [] # Skip PSF fit
# # Plot stars after PSF fit filtering
# plotStars(ff, x2, y2)
# Compute FWHM from one dimensional sigma
sigma_x_fitted = np.array(sigma_x_fitted)
sigma_y_fitted = np.array(sigma_y_fitted)
sigma_fitted = np.sqrt(sigma_x_fitted**2 + sigma_y_fitted**2)
fwhm = 2.355 * sigma_fitted
return ff_name, x2, y2, amplitude, intensity, fwhm
def recalibratePlateparsForFF(
prev_platepar,
ff_file_names,
calstars,
catalog_stars,
config,
lim_mag=None,
ignore_distance_threshold=False,
):
"""
Recalibrate platepars corresponding to ff files based on the stars.
Arguments:
prev_platepar: [platepar]
ff_file_names: [list] list of ff file names
calstars: [dict] A dictionary with only one entry, where the key is 'jd' and the value is the
list of star coordinates.
catalog_stars: [list] A list of entries [[ff_name, star_coordinates], ...].
config: [config]
Keyword arguments:
lim_mag: [float]
ignore_distance_threshold: [bool] Don't consider the recalib as failed if the median distance
is larger than the threshold.
Returns:
recalibrated_platepars: [dict] A dictionary where one key is ff file name and the value is
a calibrated corresponding platepar.
"""
# Go through all FF files with detections, recalibrate and apply astrometry
recalibrated_platepars = {}
for ff_name in ff_file_names:
working_platepar = copy.deepcopy(prev_platepar)
# Skip this meteor if its FF file was already recalibrated
if ff_name in recalibrated_platepars:
continue
print()
print('Processing: ', ff_name)
print(
'------------------------------------------------------------------------------'
)
# Find extracted stars on this image
if not ff_name in calstars:
print('Skipped because it was not in CALSTARS:', ff_name)
continue
# Get stars detected on this FF file (create a dictionaly with only one entry, the residuals function
# needs this format)
calstars_time = FFfile.getMiddleTimeFF(ff_name,
config.fps,
ret_milliseconds=True)
jd = date2JD(*calstars_time)
star_dict_ff = {jd: calstars[ff_name]}
result = None
# Skip recalibration if less than a minimum number of stars were detected
if (len(calstars[ff_name]) >= config.ff_min_stars) and (len(
calstars[ff_name]) >= config.min_matched_stars):
# Recalibrate the platepar using star matching
result, min_match_radius = recalibrateFF(
config,
working_platepar,
jd,
star_dict_ff,
catalog_stars,
lim_mag=lim_mag,
ignore_distance_threshold=ignore_distance_threshold,
)
# If the recalibration failed, try using FFT alignment
if result is None:
print()
print('Running FFT alignment...')
# Run FFT alignment
calstars_coords = np.array(star_dict_ff[jd])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
print(calstars_time)
test_platepar = alignPlatepar(config,
prev_platepar,
calstars_time,
calstars_coords,
show_plot=False)
# Try to recalibrate after FFT alignment
result, _ = recalibrateFF(config,
test_platepar,
jd,
star_dict_ff,
catalog_stars,
lim_mag=lim_mag)
# If the FFT alignment failed, align the original platepar using the smallest radius that matched
# and force save the the platepar
if (result is None) and (min_match_radius is not None):
print()
print(
"Using the old platepar with the minimum match radius of: {:.2f}"
.format(min_match_radius))
result, _ = recalibrateFF(
config,
working_platepar,
jd,
star_dict_ff,
catalog_stars,
max_match_radius=min_match_radius,
force_platepar_save=True,
lim_mag=lim_mag,
)
if result is not None:
working_platepar = result
# If the alignment succeeded, save the result
else:
working_platepar = result
else:
working_platepar = result
# Store the platepar if the fit succeeded
if result is not None:
# Recompute alt/az of the FOV centre
working_platepar.az_centre, working_platepar.alt_centre = raDec2AltAz(
working_platepar.RA_d,
working_platepar.dec_d,
working_platepar.JD,
working_platepar.lat,
working_platepar.lon,
)
# Recompute the rotation wrt horizon
working_platepar.rotation_from_horiz = rotationWrtHorizon(
working_platepar)
# Mark the platepar to indicate that it was automatically recalibrated on an individual FF file
working_platepar.auto_recalibrated = True
recalibrated_platepars[ff_name] = working_platepar
prev_platepar = working_platepar
else:
print(
'Recalibration of {:s} failed, using the previous platepar...'.
format(ff_name))
# Mark the platepar to indicate that autorecalib failed
prev_platepar_tmp = copy.deepcopy(prev_platepar)
prev_platepar_tmp.auto_recalibrated = False
# If the aligning failed, set the previous platepar as the one that should be used for this FF file
recalibrated_platepars[ff_name] = prev_platepar_tmp
return recalibrated_platepars
def extractStars(ff_dir,
ff_name,
config=None,
max_global_intensity=150,
border=10,
neighborhood_size=10,
intensity_threshold=5,
flat_struct=None):
""" Extracts stars on a given FF bin by searching for local maxima and applying PSF fit for star
confirmation.
Source of one part of the code:
http://stackoverflow.com/questions/9111711/get-coordinates-of-local-maxima-in-2d-array-above-certain-value
Arguments:
ff: [ff bin struct] FF bin file loaded in the FF bin structure
config: [config object] configuration object (loaded from the .config file)
max_global_intensity: [int] maximum mean intensity of an image before it is discared as too bright
border: [int] apply a mask on the detections by removing all that are too close to the given image
border (in pixels)
neighborhood_size: [int] size of the neighbourhood for the maximum search (in pixels)
intensity_threshold: [float] a threshold for cutting the detections which are too faint (0-255)
flat_struct: [Flat struct] Structure containing the flat field. None by default.
Return:
x2, y2, background, intensity: [list of ndarrays]
- x2: X axis coordinates of the star
- y2: Y axis coordinates of the star
- background: background intensity
- intensity: intensity of the star
"""
# Load parameters from config if given
if config:
max_global_intensity = config.max_global_intensity
border = config.border
neighborhood_size = config.neighborhood_size
intensity_threshold = config.intensity_threshold
# Load the FF bin file
ff = FFfile.read(ff_dir, ff_name)
# Load the mask file
mask = MaskImage.loadMask(config.mask_file)
# Mask the FF file
ff = MaskImage.applyMask(ff, mask, ff_flag=True)
# Apply the flat to maxpixel and avepixel
if flat_struct is not None:
ff.maxpixel = Image.applyFlat(ff.maxpixel, flat_struct)
ff.avepixel = Image.applyFlat(ff.avepixel, flat_struct)
# Calculate image mean and stddev
global_mean = np.mean(ff.avepixel)
# Check if the image is too bright and skip the image
if global_mean > max_global_intensity:
return [[], [], [], []]
data = ff.avepixel.astype(np.float32)
# Apply a mean filter to the image to reduce noise
data = ndimage.filters.convolve(data, weights=np.full((2, 2), 1.0 / 4))
# Locate local maxima on the image
data_max = filters.maximum_filter(data, neighborhood_size)
maxima = (data == data_max)
data_min = filters.minimum_filter(data, neighborhood_size)
diff = ((data_max - data_min) > intensity_threshold)
maxima[diff == 0] = 0
# Apply a border mask
border_mask = np.ones_like(maxima) * 255
border_mask[:border, :] = 0
border_mask[-border:, :] = 0
border_mask[:, :border] = 0
border_mask[:, -border:] = 0
maxima = MaskImage.applyMask(maxima, (True, border_mask))
# Find and label the maxima
labeled, num_objects = ndimage.label(maxima)
# Skip the image if there are too many maxima to process
if num_objects > config.max_stars:
print('Too many candidate stars to process! {:d}/{:d}'.format(
num_objects, config.max_stars))
return [[], [], [], []]
# Find centres of mass of each labeled objects
xy = np.array(
ndimage.center_of_mass(data, labeled, range(1, num_objects + 1)))
# Remove all detection on the border
#xy = xy[np.where((xy[:, 1] > border) & (xy[:,1] < ff.ncols - border) & (xy[:,0] > border) & (xy[:,0] < ff.nrows - border))]
# Unpack star coordinates
y, x = np.hsplit(xy, 2)
# # Plot stars before the PSF fit
# plotStars(ff, x, y)
# Fit a PSF to each star
x2, y2, amplitude, intensity = fitPSF(ff, global_mean, x, y, config=config)
# x2, y2, amplitude, intensity = list(x), list(y), [], [] # Skip PSF fit
# # Plot stars after PSF fit filtering
# plotStars(ff, x2, y2)
return x2, y2, amplitude, intensity
def view(dir_path, ff_path, fr_path, config, save_frames=False):
""" Shows the detected fireball stored in the FR file.
Arguments:
dir_path: [str] Current directory.
ff: [str] path to the FF bin file
fr: [str] path to the FR bin file
config: [conf object] configuration structure
"""
name = fr_path
fr = FRbin.read(dir_path, fr_path)
if ff_path is None:
#background = np.zeros((config.height, config.width), np.uint8)
# Get the maximum extent of the meteor frames
y_size = max(
max(np.array(fr.yc[0]) + np.array(fr.size[0]) // 2)
for i in range(fr.lines))
x_size = max(
max(np.array(fr.xc[0]) + np.array(fr.size[0]) // 2)
for i in range(fr.lines))
# Make the image square
img_size = max(y_size, x_size)
background = np.zeros((img_size, img_size), np.uint8)
else:
background = FFfile.read(dir_path, ff_path).maxpixel
print("Number of lines:", fr.lines)
first_image = True
for current_line in range(fr.lines):
print('Frame, Y , X , size')
for z in range(fr.frameNum[current_line]):
# Get the center position of the detection on the current frame
yc = fr.yc[current_line][z]
xc = fr.xc[current_line][z]
# Get the frame number
t = fr.t[current_line][z]
# Get the size of the window
size = fr.size[current_line][z]
print(" {:3d}, {:3d}, {:3d}, {:d}".format(t, yc, xc, size))
img = np.copy(background)
# Paste the frames onto the big image
y_img = np.arange(yc - size // 2, yc + size // 2)
x_img = np.arange(xc - size // 2, xc + size // 2)
Y_img, X_img = np.meshgrid(y_img, x_img)
y_frame = np.arange(len(y_img))
x_frame = np.arange(len(x_img))
Y_frame, X_frame = np.meshgrid(y_frame, x_frame)
img[Y_img, X_img] = fr.frames[current_line][z][Y_frame, X_frame]
# Save frame to disk
if save_frames:
frame_file_name = fr_path.replace(
'.bin', '') + "_frame_{:03d}.png".format(t)
cv2.imwrite(os.path.join(dir_path, frame_file_name), img)
# Show the frame
cv2.imshow(name, img)
# If this is the first image, move it to the upper left corner
if first_image:
cv2.moveWindow(name, 0, 0)
first_image = False
cv2.waitKey(2 * int(1000.0 / config.fps))
cv2.destroyWindow(name)
if __name__ == "__main__":
time_start = time.clock()
# Load config file
config = cr.parse(".config")
if not len(sys.argv) == 2:
print("Usage: python -m RMS.ExtractStars /path/to/FF/files/")
sys.exit()
# Get paths to every FF bin file in a directory
ff_dir = os.path.abspath(sys.argv[1])
ff_list = [
ff_name for ff_name in os.listdir(ff_dir)
if FFfile.validFFName(ff_name)
]
# Check if there are any file in the directory
if (len(ff_list) == None):
print("No files found!")
sys.exit()
# Try loading a flat field image
flat_struct = None
if config.use_flat:
# Check if there is flat in the data directory
if os.path.exists(os.path.join(ff_dir, config.flat_file)):
flat_struct = Image.loadFlat(ff_dir, config.flat_file)
def view(dir_path, ff_path, fr_path, config):
""" Shows the detected fireball stored in the FR file.
Arguments:
dir_path: [str] Current directory.
ff: [str] path to the FF bin file
fr: [str] path to the FR bin file
config: [conf object] configuration structure
"""
name = fr_path
fr = FRbin.read(dir_path, fr_path)
if ff_path is None:
#background = np.zeros((config.height, config.width), np.uint8)
# Get the maximum extent of the meteor frames
y_size = max(max(np.array(fr.yc[i]) + np.array(fr.size[i])//2) for i in range(fr.lines))
x_size = max(max(np.array(fr.xc[i]) + np.array(fr.size[i])//2) for i in range(fr.lines))
# Make the image square
img_size = max(y_size, x_size)
background = np.zeros((img_size, img_size), np.uint8)
else:
background = FFfile.read(dir_path, ff_path).maxpixel
print("Number of lines:", fr.lines)
first_image = True
for i in range(fr.lines):
print('Frame, Y , X , size')
for z in range(fr.frameNum[i]):
# Get the center position of the detection on the current frame
yc = fr.yc[i][z]
xc = fr.xc[i][z]
# Get the frame number
t = fr.t[i][z]
# Get the size of the window
size = fr.size[i][z]
print(" {:3d}, {:3d}, {:3d}, {:d}".format(t, yc, xc, size))
y2 = 0
# Assign the detection pixels to the background image
for y in range(yc - size//2, yc + size//2):
x2 = 0
for x in range(xc - size//2, xc + size//2):
background[y, x] = fr.frames[i][z][y2, x2]
x2 += 1
y2 += 1
cv2.imshow(name, background)
# If this is the first image, move it to the upper left corner
if first_image:
cv2.moveWindow(name, 0, 0)
first_image = False
cv2.waitKey(2*int(1000.0/config.fps))
cv2.destroyWindow(name)
def generateThumbnails(dir_path, config, mosaic_type, file_list=None):
""" Generates a mosaic of thumbnails from all FF files in the given folder and saves it as a JPG image.
Arguments:
dir_path: [str] Path of the night directory.
config: [Conf object] Configuration.
mosaic_type: [str] Type of the mosaic (e.g. "Captured" or "Detected")
Keyword arguments:
file_list: [list] A list of file names (without full path) which will be searched for FF files. This
is used when generating separate thumbnails for captured and detected files.
Return:
file_name: [str] Name of the thumbnail file.
"""
if file_list is None:
file_list = sorted(os.listdir(dir_path))
# Make a list of all FF files in the night directory
ff_list = []
for file_name in file_list:
if FFfile.validFFName(file_name):
ff_list.append(file_name)
# Calculate the dimensions of the binned image
bin_w = int(config.width/config.thumb_bin)
bin_h = int(config.height/config.thumb_bin)
### RESIZE AND STACK THUMBNAILS ###
##########################################################################################################
timestamps = []
stacked_imgs = []
for i in range(0, len(ff_list), config.thumb_stack):
img_stack = np.zeros((bin_h, bin_w))
# Stack thumb_stack images using the 'if lighter' method
for j in range(config.thumb_stack):
if (i + j) < len(ff_list):
# Read maxpixel image
img = FFfile.read(dir_path, ff_list[i + j]).maxpixel
# Resize the image
img = cv2.resize(img, (bin_w, bin_h))
# Stack the image
img_stack = stackIfLighter(img_stack, img)
else:
break
# Save the timestamp of the first image in the stack
timestamps.append(FFfile.filenameToDatetime(ff_list[i]))
# Save the stacked image
stacked_imgs.append(img_stack)
# cv2.imshow('test', img_stack)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
##########################################################################################################
### ADD THUMBS TO ONE MOSAIC IMAGE ###
##########################################################################################################
header_height = 20
timestamp_height = 10
# Calculate the number of rows for the thumbnail image
n_rows = int(np.ceil(float(len(ff_list))/config.thumb_stack/config.thumb_n_width))
# Calculate the size of the mosaic
mosaic_w = int(config.thumb_n_width*bin_w)
mosaic_h = int((bin_h + timestamp_height)*n_rows + header_height)
mosaic_img = np.zeros((mosaic_h, mosaic_w), dtype=np.uint8)
# Write header text
header_text = 'Station: ' + str(config.stationID) + ' Night: ' + os.path.basename(dir_path) \
+ ' Type: ' + mosaic_type
cv2.putText(mosaic_img, header_text, (0, header_height//2), \
cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255,255,255), 1)
for row in range(n_rows):
for col in range(config.thumb_n_width):
# Calculate image index
indx = row*config.thumb_n_width + col
if indx < len(stacked_imgs):
# Calculate position of the text
text_x = col*bin_w
text_y = row*bin_h + (row + 1)*timestamp_height - 1 + header_height
# Add timestamp text
cv2.putText(mosaic_img, timestamps[indx].strftime('%H:%M:%S'), (text_x, text_y), \
cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1)
# Add the image to the mosaic
img_pos_x = col*bin_w
img_pos_y = row*bin_h + (row + 1)*timestamp_height + header_height
mosaic_img[img_pos_y : img_pos_y + bin_h, img_pos_x : img_pos_x + bin_w] = stacked_imgs[indx]
else:
break
##########################################################################################################
thumb_name = "{:s}_{:s}_{:s}_thumbs.jpg".format(str(config.stationID), os.path.basename(dir_path), \
mosaic_type)
# Save the mosaic
cv2.imwrite(os.path.join(dir_path, thumb_name), mosaic_img, [int(cv2.IMWRITE_JPEG_QUALITY), 80])
return thumb_name
plot(points, ff.nrows//config.f, ff.ncols//config.f)
def plot(points, y_dim, x_dim):
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
plt.title(name)
y = points[:,0]
x = points[:,1]
z = points[:,2]
# Plot points in 3D
ax.scatter(x, y, z)
# Set axes limits
ax.set_zlim(0, 255)
plt.xlim([0, x_dim])
plt.ylim([0, y_dim])
ax.set_ylabel("Y")
ax.set_xlabel("X")
ax.set_zlabel("Time")
plt.show()
if __name__ == "__main__":
ff = FFfile.read(sys.argv[1], sys.argv[2], array=True)
view(ff)
def detectMeteors(ff_directory, ff_name, config, flat_struct=None):
""" Detect meteors on the given FF bin image. Here are the steps in the detection:
- input image (FF bin format file) is thresholded (converted to black and white)
- several morphological operations are applied to clean the image
- image is then broken into several image "windows" (these "windows" are reconstructed from the input FF file, given
an input frame range (e.g. 64-128) which helps reduce the noise further)
- on each "window" the Kernel-based Hough transform is performed to find any lines on the image
- similar lines are joined
- stripe around the lines is extracted
- 3D line finding (third dimension is time) is applied to check if the line propagates in time
- centroiding is performed, which calculates the position and intensity of meteor on each frame
Arguments:
ff_directory: [string] an absolute path to the input FF bin file
ff_name: [string] file name of the FF bin file on which to run the detection on
config: [config object] configuration object (loaded from the .config file)
Keyword arguments:
flat_struct: [Flat struct] Structure containing the flat field. None by default.
Return:
meteor_detections: [list] a list of detected meteors, with these elements:
- rho: [float] meteor line distance from image center (polar coordinates, in pixels)
- theta: [float] meteor line angle from image center (polar coordinates, in degrees)
- centroids: [list] [frame, X, Y, level] list of meteor points
"""
t1 = time()
t_all = time()
# Load the FF bin file
ff = FFfile.read(ff_directory, ff_name)
# Load the mask file
mask = MaskImage.loadMask(config.mask_file)
# Mask the FF file
ff = MaskImage.applyMask(ff, mask, ff_flag=True)
# Apply the flat to maxpixel and avepixel
if flat_struct is not None:
ff.maxpixel = Image.applyFlat(ff.maxpixel, flat_struct)
ff.avepixel = Image.applyFlat(ff.avepixel, flat_struct)
# At the end, a check that the detection has a surface brightness above the background will be performed.
# The assumption here is that the peak of the meteor should have the intensity which is at least
# that of a patch of 4x4 pixels that are of the mean background brightness
min_patch_intensity = 4 * 4 * (np.mean(ff.maxpixel - ff.avepixel) +
config.k1_det * np.mean(ff.stdpixel) +
config.j1)
# # Show the maxpixel image
# show2(ff_name+' maxpixel', ff.maxpixel)
# Get lines on the image
line_list = getLines(ff, config.k1_det, config.j1_det, config.time_slide,
config.time_window_size, config.max_lines_det,
config.max_white_ratio, config.kht_lib_path)
logDebug('List of lines:', line_list)
# Init meteor list
meteor_detections = []
# Only if there are some lines in the image
if len(line_list):
# Join similar lines
line_list = mergeLines(line_list, config.line_min_dist, ff.ncols,
ff.nrows)
logDebug('Time for finding lines:', time() - t1)
logDebug('Number of KHT lines: ', len(line_list))
logDebug(line_list)
# Plot lines
# plotLines(ff, line_list)
# Threshold the image
img_thres = thresholdImg(ff, config.k1_det, config.j1_det)
filtered_lines = []
# Analyze stripes of each line
for line in line_list:
rho, theta, frame_min, frame_max = line
logDebug('rho, theta, frame_min, frame_max')
logDebug(rho, theta, frame_min, frame_max)
# Bounded the thresholded image by min and max frames
img = selectFrames(np.copy(img_thres), ff, frame_min, frame_max)
# Remove lonely pixels
img = morph.clean(img)
# Get indices of stripe pixels around the line
stripe_indices = getStripeIndices(rho, theta, config.stripe_width,
img.shape[0], img.shape[1])
# Extract the stripe from the thresholded image
stripe = np.zeros((ff.nrows, ff.ncols), np.uint8)
stripe[stripe_indices] = img[stripe_indices]
# Show stripe
#COMMENTED
# show2("stripe", stripe*255)
# Show 3D could
# show3DCloud(ff, stripe)
# Get stripe positions
stripe_positions = stripe.nonzero()
xs = stripe_positions[1]
ys = stripe_positions[0]
zs = ff.maxframe[stripe_positions]
# Limit the number of points to search if too large
if len(zs) > config.max_points_det:
# Extract weights of each point
maxpix_elements = ff.maxpixel[ys, xs].astype(np.float64)
weights = maxpix_elements / np.sum(maxpix_elements)
# Random sample the point, sampling is weighted by pixel intensity
indices = np.random.choice(len(zs),
config.max_points_det,
replace=False,
p=weights)
ys = ys[indices]
xs = xs[indices]
zs = zs[indices]
# Make an array to feed into the gropuing algorithm
stripe_points = np.vstack((xs, ys, zs))
stripe_points = np.swapaxes(stripe_points, 0, 1)
# Sort stripe points by frame
stripe_points = stripe_points[stripe_points[:, 2].argsort()]
t1 = time()
logDebug('finding lines...')
# Find a single line in the point cloud
detected_line = find3DLines(stripe_points,
time(),
config,
fireball_detection=False)
logDebug('time for GROUPING: ', time() - t1)
# Extract the first and only line if any
if detected_line:
detected_line = detected_line[0]
# logDebug(detected_line)
# Show 3D cloud
# show3DCloud(ff, stripe, detected_line, stripe_points, config)
# Add the line to the results list
filtered_lines.append(detected_line)
# Merge similar lines in 3D
filtered_lines = merge3DLines(filtered_lines, config.vect_angle_thresh)
logDebug('after filtering:')
logDebug(filtered_lines)
for detected_line in filtered_lines:
# Get frame range
frame_min = detected_line[4]
frame_max = detected_line[5]
# Check if the line covers a minimum frame range
if (abs(frame_max - frame_min) + 1 <
config.line_minimum_frame_range_det):
continue
# Extand the frame range for several frames, just to be sure to catch all parts of a meteor
frame_min -= config.frame_extension
frame_max += config.frame_extension
# Cap values to 0-255
frame_min = max(frame_min, 0)
frame_max = min(frame_max, 255)
logDebug(detected_line)
# Get coordinates of 2 points that describe the line
x1, y1, z1 = detected_line[0]
x2, y2, z2 = detected_line[1]
# Convert Cartesian line coordinates to polar
rho, theta = getPolarLine(x1, y1, x2, y2, ff.nrows, ff.ncols)
# Convert Cartesian line coordinate to CAMS compatible polar coordinates (flipped Y axis)
rho_cams, theta_cams = getPolarLine(x1, ff.nrows - y1, x2,
ff.nrows - y2, ff.nrows,
ff.ncols)
logDebug('converted rho, theta')
logDebug(rho, theta)
# Bounded the thresholded image by min and max frames
img = selectFrames(np.copy(img_thres), ff, frame_min, frame_max)
# Remove lonely pixels
img = morph.clean(img)
# Get indices of stripe pixels around the line
stripe_indices = getStripeIndices(rho, theta,
int(config.stripe_width * 1.5),
img.shape[0], img.shape[1])
# Extract the stripe from the thresholded image
stripe = np.zeros((ff.nrows, ff.ncols), np.uint8)
stripe[stripe_indices] = img[stripe_indices]
# Show detected line
# show('detected line: '+str(frame_min)+'-'+str(frame_max), stripe)
# Get stripe positions
stripe_positions = stripe.nonzero()
xs = stripe_positions[1]
ys = stripe_positions[0]
zs = ff.maxframe[stripe_positions]
# Make an array to feed into the centroiding algorithm
stripe_points = np.vstack((xs, ys, zs))
stripe_points = np.swapaxes(stripe_points, 0, 1)
# Sort stripe points by frame
stripe_points = stripe_points[stripe_points[:, 2].argsort()]
# Show 3D cloud
# show3DCloud(ff, stripe, detected_line, stripe_points, config)
# Get points of the given line
line_points = getAllPoints(stripe_points,
x1,
y1,
z1,
x2,
y2,
z2,
config,
fireball_detection=False)
# Skip if no points were returned
if not line_points.any():
continue
# Skip if the points cover too small a frame range
if abs(np.max(line_points[:, 2]) - np.min(line_points[:, 2])
) + 1 < config.line_minimum_frame_range_det:
continue
# Calculate centroids
centroids = []
for i in range(frame_min, frame_max + 1):
# Select pixel indicies belonging to a given frame
frame_pixels_inds = np.where(line_points[:, 2] == i)
# Get pixel positions in a given frame (pixels belonging to a found line)
frame_pixels = line_points[frame_pixels_inds].astype(np.int64)
# Get pixel positions in a given frame (pixels belonging to the whole stripe)
frame_pixels_stripe = stripe_points[np.where(
stripe_points[:, 2] == i)].astype(np.int64)
# Skip if there are no pixels in the frame
if not len(frame_pixels):
continue
# Calculate weights for centroiding
max_avg_corrected = ff.maxpixel - ff.avepixel
flattened_weights = (max_avg_corrected).astype(
np.float32) / ff.stdpixel
# Calculate centroids by half-frame
for half_frame in range(2):
# Apply deinterlacing if it is present in the video
if config.deinterlace_order >= 0:
# Deinterlace by fields (line lixels)
half_frame_pixels = frame_pixels[
frame_pixels[:, 1] %
2 == (config.deinterlace_order + half_frame) % 2]
# Deinterlace by fields (stripe pixels)
half_frame_pixels_stripe = frame_pixels_stripe[
frame_pixels_stripe[:, 1] %
2 == (config.deinterlace_order + half_frame) % 2]
# Skip if there are no pixels in the half-frame
if not len(half_frame_pixels):
continue
# Calculate half-frame value
frame_no = i + half_frame * 0.5
# No deinterlacing
else:
# Skip the second half frame
if half_frame == 1:
continue
half_frame_pixels = frame_pixels
half_frame_pixels_stripe = frame_pixels_stripe
frame_no = i
# Get maxpixel-avepixel values of given pixel indices (this will be used as weights)
max_weights = flattened_weights[half_frame_pixels[:, 1],
half_frame_pixels[:, 0]]
# Calculate weighted centroids
x_weighted = half_frame_pixels[:, 0] * np.transpose(
max_weights)
x_centroid = np.sum(x_weighted) / float(
np.sum(max_weights))
y_weighted = half_frame_pixels[:, 1] * np.transpose(
max_weights)
y_centroid = np.sum(y_weighted) / float(
np.sum(max_weights))
# Calculate intensity as the sum of threshold passer pixels on the stripe
#intensity_values = max_avg_corrected[half_frame_pixels[:,1], half_frame_pixels[:,0]]
intensity_values = max_avg_corrected[
half_frame_pixels_stripe[:, 1],
half_frame_pixels_stripe[:, 0]]
intensity = np.sum(intensity_values)
logDebug("centroid: ", frame_no, x_centroid, y_centroid,
intensity)
centroids.append(
[frame_no, x_centroid, y_centroid, intensity])
# Filter centroids
centroids = filterCentroids(centroids,
config.centroids_max_deviation,
config.centroids_max_distance)
# Convert to numpy array for easy slicing
centroids = np.array(centroids)
# Reject the solution if there are too few centroids
if len(centroids) < config.line_minimum_frame_range_det:
continue
# Check that the detection has a surface brightness above the background
# The assumption here is that the peak of the meteor should have the intensity which is at least
# that of a patch of 4x4 pixels that are of the mean background brightness
if np.max(centroids[:, 3]) < min_patch_intensity:
continue
# Check the detection if it has the proper angular velocity
if not checkAngularVelocity(centroids, config):
continue
# Append the result to the meteor detections
meteor_detections.append([rho_cams, theta_cams, centroids])
logDebug('time for processing:', time() - t_all)
# # Plot centroids to image
# fig, (ax1, ax2) = plt.subplots(nrows=2)
# ax1.imshow(ff.maxpixel - ff.avepixel, cmap='gray')
# ax1.scatter(centroids[:,1], centroids[:,2], s=5, c='r', edgecolors='none')
# # Plot lightcurve
# ax2.plot(centroids[:,0], centroids[:,3])
# # # Plot relative angular velocity
# # ang_vels = []
# # fr_prev, x_prev, y_prev, _ = centroids[0]
# # for fr, x, y, _ in centroids[1:]:
# # dx = x - x_prev
# # dy = y - y_prev
# # dfr = fr - fr_prev
# # ddist = np.sqrt(dx**2 + dy**2)
# # dt = dfr/config.fps
# # ang_vels.append(ddist/dt)
# # x_prev = x
# # y_prev = y
# # fr_prev = fr
# # ax2.plot(ang_vels)
# plt.show()
return meteor_detections
# Extract the directory name from the given argument
bin_dir = sys.argv[1]
# Load config file
config = cr.parse(".config")
print('Directory:', bin_dir)
for ff_name in os.listdir(bin_dir):
if 'FF' in ff_name:
print(ff_name)
# Load compressed file
compressed = FFfile.read(bin_dir,
ff_name,
array=True,
full_filename=True).array
# Show maxpixel
ff = FFfile.read(bin_dir, ff_name, full_filename=True)
plt.imshow(ff.maxpixel, cmap='gray')
plt.show()
plt.clf()
plt.close()
# Dummy frames (empty)
frames = np.zeros(
shape=(256, compressed.shape[1], compressed.shape[2]),
dtype=np.uint8) + 255
def view(dir_path,
ff_path,
fr_path,
config,
save_frames=False,
extract_format='png',
hide=False):
""" Shows the detected fireball stored in the FR file.
Arguments:
dir_path: [str] Current directory.
ff: [str] path to the FF bin file
fr: [str] path to the FR bin file
config: [conf object] configuration structure
Keyword arguments:
save_frames: [bool] Save FR frames to disk. False by defualt.
extract_format: [str] Format of saved images. png by default.
hide: [bool] Don't show frames on the screen.
"""
if extract_format is None:
extract_format = 'png'
name = fr_path
fr = FRbin.read(dir_path, fr_path)
print('------------------------')
print('Showing file:', fr_path)
if ff_path is None:
#background = np.zeros((config.height, config.width), np.uint8)
# Get the maximum extent of meteor frames
y_size = max([
max(np.array(fr.yc[i]) + np.array(fr.size[i]) // 2)
for i in range(fr.lines)
])
x_size = max([
max(np.array(fr.xc[i]) + np.array(fr.size[i]) // 2)
for i in range(fr.lines)
])
# Make the image square
img_size = max(y_size, x_size)
background = np.zeros((img_size, img_size), np.uint8)
else:
background = FFfile.read(dir_path, ff_path).maxpixel
print("Number of lines:", fr.lines)
first_image = True
wait_time = 2 * int(1000.0 / config.fps)
pause_flag = False
for current_line in range(fr.lines):
print('Frame, Y , X , size')
for z in range(fr.frameNum[current_line]):
# Get the center position of the detection on the current frame
yc = fr.yc[current_line][z]
xc = fr.xc[current_line][z]
# Get the frame number
t = fr.t[current_line][z]
# Get the size of the window
size = fr.size[current_line][z]
print(" {:3d}, {:3d}, {:3d}, {:d}".format(t, yc, xc, size))
img = np.copy(background)
# Paste the frames onto the big image
y_img = np.arange(yc - size // 2, yc + size // 2)
x_img = np.arange(xc - size // 2, xc + size // 2)
Y_img, X_img = np.meshgrid(y_img, x_img)
y_frame = np.arange(len(y_img))
x_frame = np.arange(len(x_img))
Y_frame, X_frame = np.meshgrid(y_frame, x_frame)
img[Y_img, X_img] = fr.frames[current_line][z][Y_frame, X_frame]
# Save frame to disk
if save_frames:
frame_file_name = fr_path.replace('.bin', '') \
+ "_line_{:02d}_frame_{:03d}.{:s}".format(current_line, t, extract_format)
cv2.imwrite(os.path.join(dir_path, frame_file_name), img)
if not hide:
# Show the frame
cv2.imshow(name, img)
# If this is the first image, move it to the upper left corner
if first_image:
cv2.moveWindow(name, 0, 0)
first_image = False
if pause_flag:
wait_time = 0
else:
wait_time = 2 * int(1000.0 / config.fps)
# Space key: pause display.
# 1: previous file.
# 2: next line.
# q: Quit.
key = cv2.waitKey(wait_time) & 0xFF
if key == ord("1"):
cv2.destroyWindow(name)
return -1
elif key == ord("2"):
break
elif key == ord(" "):
# Pause/unpause video
pause_flag = not pause_flag
elif key == ord("q"):
os._exit(0)
if not hide:
cv2.destroyWindow(name)
def autoCheckFit(config, platepar, calstars_list):
""" Attempts to refine the astrometry fit with the given stars and and initial astrometry parameters.
Arguments:
config: [Config structure]
platepar: [Platepar structure] Initial astrometry parameters.
calstars_list: [list] A list containing stars extracted from FF files. See RMS.Formats.CALSTARS for
more details.
Return:
(platepar, fit_status):
platepar: [Platepar structure] Estimated/refined platepar.
fit_status: [bool] True if fit was successfuly, False if not.
"""
# Convert the list to a dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
# Load catalog stars
catalog_stars = StarCatalog.readStarCatalog(config.star_catalog_path, config.star_catalog_file, \
lim_mag=config.catalog_mag_limit, mag_band_ratios=config.star_catalog_band_ratios)
# Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars
star_dict = {}
# Take only those files with enough stars on them
for ff_name in calstars:
stars_list = calstars[ff_name]
# Check if there are enough stars on the image
if len(stars_list) >= config.ff_min_stars:
# Calculate the JD time of the FF file
dt = FFfile.getMiddleTimeFF(ff_name,
config.fps,
ret_milliseconds=True)
jd = date2JD(*dt)
# Add the time and the stars to the dict
star_dict[jd] = stars_list
# There has to be a minimum of 200 FF files for star fitting, and only 100 will be subset if there are more
if len(star_dict) < config.calstars_files_N:
print('Not enough FF files in CALSTARS for ACF!')
return platepar, False
else:
# Randomly choose calstars_files_N image files from the whole list
rand_keys = random.sample(list(star_dict), config.calstars_files_N)
star_dict = {key: star_dict[key] for key in rand_keys}
# Calculate the total number of calibration stars used
total_calstars = sum([len(star_dict[key]) for key in star_dict])
print('Total calstars:', total_calstars)
if total_calstars < config.calstars_min_stars:
print('Not enough calibration stars, need at least',
config.calstars_min_stars)
return platepar, False
# A list of matching radiuses to try, pairs of [radius, fit_distorsion_flag]
min_radius = 0.5
radius_list = [[10, False], [5, False], [3, False], [1.5, True],
[min_radius, True]]
# Calculate the function tolerance, so the desired precision can be reached (the number is calculated
# in the same reagrd as the cost function)
fatol = (config.dist_check_threshold**
2) / np.sqrt(len(star_dict) * config.min_matched_stars + 1)
# Parameter estimation tolerance for angular values
fov_w = platepar.X_res / platepar.F_scale
xatol_ang = config.dist_check_threshold * fov_w / platepar.X_res
### If the initial match is good enough, do only quick recalibratoin ###
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
min_radius, ret_nmatch=True)
if n_matched >= config.calstars_files_N:
# Check if the average distance with the tightest radius is close
if avg_dist < config.dist_check_quick_threshold:
# Use a reduced set of initial radius values
radius_list = [[1.5, True], [min_radius, True]]
##########
# Match increasingly smaller search radiia around image stars
for i, (match_radius, fit_distorsion) in enumerate(radius_list):
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
match_radius, ret_nmatch=True)
print('Max radius:', match_radius)
print('Initial values:')
print(' Matched stars:', n_matched)
print(' Average deviation:', avg_dist)
# The initial number of matched stars has to be at least the number of FF imaages, otherwise it means
# that the initial platepar is no good
if n_matched < config.calstars_files_N:
print(
'The total number of initially matched stars is too small! Please manually redo the plate or make sure there are enough calibration stars.'
)
return platepar, False
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict,
min_radius):
# Print out notice only if the platepar is good right away
if i == 0:
print("Initial platepar is good enough!")
return platepar, True
# Initial parameters for the astrometric fit
p0 = [
platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref,
platepar.F_scale
]
# Fit the astrometric parameters
res = scipy.optimize.minimize(_calcImageResidualsAstro, p0, args=(config, platepar, catalog_stars, \
star_dict, match_radius), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': xatol_ang})
print(res)
# If the fit was not successful, stop further fitting
if not res.success:
return platepar, False
else:
# If the fit was successful, use the new parameters from now on
ra_ref, dec_ref, pos_angle_ref, F_scale = res.x
platepar.RA_d = ra_ref
platepar.dec_d = dec_ref
platepar.pos_angle_ref = pos_angle_ref
platepar.F_scale = F_scale
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict,
min_radius):
return platepar, True
# Fit the lens distorsion parameters
if fit_distorsion:
# Fit the distortion parameters (X axis)
res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.x_poly, args=(config, platepar,\
catalog_stars, star_dict, match_radius, 'x'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
return platepar, False
else:
platepar.x_poly = res.x
# Check if the platepar is good enough and do not estimate further parameters
if checkFitGoodness(config, platepar, catalog_stars, star_dict,
min_radius):
return platepar, True
# Fit the distortion parameters (Y axis)
res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.y_poly, args=(config, platepar,\
catalog_stars, star_dict, match_radius, 'y'), method='Nelder-Mead', \
options={'fatol': fatol, 'xatol': 0.1})
print(res)
# If the fit was not successfull, stop further fitting
if not res.success:
return platepar, False
else:
platepar.y_poly = res.x
# Match the stars and calculate the residuals
n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \
star_dict, min_radius, ret_nmatch=True)
print('FINAL SOLUTION with {:f} px:'.format(min_radius))
print('Matched stars:', n_matched)
print('Average deviation:', avg_dist)
return platepar, True
# Extract the directory name from the given argument
bin_dir = sys.argv[1]
# Load config file
config = cr.parse(".config")
print('Directory:', bin_dir)
for ff_name in os.listdir(bin_dir):
if 'FF' in ff_name:
print(ff_name)
# Load compressed file
compressed = FFfile.read(bin_dir,
ff_name,
array=True,
full_filename=True).array
# Show maxpixel
ff = FFfile.read(bin_dir, ff_name, full_filename=True)
plt.imshow(ff.maxpixel, cmap='gray')
plt.show()
plt.clf()
plt.close()
# Dummy frames from FF file
frames = FFfile.reconstruct(ff)
# Add avepixel to all reconstructed frames
frames += ff.avepixel
def generateThumbnails(dir_path, config, mosaic_type, file_list=None):
""" Generates a mosaic of thumbnails from all FF files in the given folder and saves it as a JPG image.
Arguments:
dir_path: [str] Path of the night directory.
config: [Conf object] Configuration.
mosaic_type: [str] Type of the mosaic (e.g. "Captured" or "Detected")
Keyword arguments:
file_list: [list] A list of file names (without full path) which will be searched for FF files. This
is used when generating separate thumbnails for captured and detected files.
Return:
file_name: [str] Name of the thumbnail file.
"""
if file_list is None:
file_list = sorted(os.listdir(dir_path))
# Make a list of all FF files in the night directory
ff_list = []
for file_name in file_list:
if FFfile.validFFName(file_name):
ff_list.append(file_name)
# Calculate the dimensions of the binned image
bin_w = int(config.width/config.thumb_bin)
bin_h = int(config.height/config.thumb_bin)
### RESIZE AND STACK THUMBNAILS ###
##########################################################################################################
timestamps = []
stacked_imgs = []
for i in range(0, len(ff_list), config.thumb_stack):
img_stack = np.zeros((bin_h, bin_w))
# Stack thumb_stack images using the 'if lighter' method
for j in range(config.thumb_stack):
if (i + j) < len(ff_list):
tmp_file_name = ff_list[i + j]
# Read the FF file
ff = FFfile.read(dir_path, tmp_file_name)
# Skip the FF if it is corruped
if ff is None:
continue
img = ff.maxpixel
# Resize the image
img = cv2.resize(img, (bin_w, bin_h))
# Stack the image
img_stack = stackIfLighter(img_stack, img)
else:
break
# Save the timestamp of the first image in the stack
timestamps.append(FFfile.filenameToDatetime(ff_list[i]))
# Save the stacked image
stacked_imgs.append(img_stack)
# cv2.imshow('test', img_stack)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
##########################################################################################################
### ADD THUMBS TO ONE MOSAIC IMAGE ###
##########################################################################################################
header_height = 20
timestamp_height = 10
# Calculate the number of rows for the thumbnail image
n_rows = int(np.ceil(float(len(ff_list))/config.thumb_stack/config.thumb_n_width))
# Calculate the size of the mosaic
mosaic_w = int(config.thumb_n_width*bin_w)
mosaic_h = int((bin_h + timestamp_height)*n_rows + header_height)
mosaic_img = np.zeros((mosaic_h, mosaic_w), dtype=np.uint8)
# Write header text
header_text = 'Station: ' + str(config.stationID) + ' Night: ' + os.path.basename(dir_path) \
+ ' Type: ' + mosaic_type
cv2.putText(mosaic_img, header_text, (0, header_height//2), \
cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255,255,255), 1)
for row in range(n_rows):
for col in range(config.thumb_n_width):
# Calculate image index
indx = row*config.thumb_n_width + col
if indx < len(stacked_imgs):
# Calculate position of the text
text_x = col*bin_w
text_y = row*bin_h + (row + 1)*timestamp_height - 1 + header_height
# Add timestamp text
cv2.putText(mosaic_img, timestamps[indx].strftime('%H:%M:%S'), (text_x, text_y), \
cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1)
# Add the image to the mosaic
img_pos_x = col*bin_w
img_pos_y = row*bin_h + (row + 1)*timestamp_height + header_height
mosaic_img[img_pos_y : img_pos_y + bin_h, img_pos_x : img_pos_x + bin_w] = stacked_imgs[indx]
else:
break
##########################################################################################################
thumb_name = "{:s}_{:s}_{:s}_thumbs.jpg".format(str(config.stationID), os.path.basename(dir_path), \
mosaic_type)
# Save the mosaic
cv2.imwrite(os.path.join(dir_path, thumb_name), mosaic_img, [int(cv2.IMWRITE_JPEG_QUALITY), 80])
return thumb_name
def sensorCharacterization(config, dir_path):
""" Characterize the standard deviation of the background and the FWHM of stars on every image. """
# Find the CALSTARS file in the given folder that has FWHM information
found_good_calstars = False
for cal_file in os.listdir(dir_path):
if ('CALSTARS' in cal_file) and ('.txt' in cal_file) and (not found_good_calstars):
# Load the calstars file
calstars_list = CALSTARS.readCALSTARS(dir_path, cal_file)
if len(calstars_list) > 0:
# Check that at least one image has good FWHM measurements
for ff_name, star_data in calstars_list:
if len(star_data) > 1:
star_data = np.array(star_data)
# Check if the calstars file have FWHM information
fwhm = star_data[:, 4]
# Check that FWHM values have been computed well
if np.all(fwhm > 1):
found_good_calstars = True
print('CALSTARS file: ' + cal_file + ' loaded!')
break
# If the FWHM information is not present, run the star extraction
if not found_good_calstars:
print()
print("No FWHM information found in existing CALSTARS files!")
print()
print("Rerunning star detection...")
print()
found_good_calstars = False
# Run star extraction again, and now FWHM will be computed
calstars_list = extractStarsAndSave(config, dir_path)
if len(calstars_list) == 0:
found_good_calstars = False
# Check for a minimum of detected stars
for ff_name, star_data in calstars_list:
if len(star_data) >= config.ff_min_stars:
found_good_calstars = True
break
# If no good calstars exist, stop computing the flux
if not found_good_calstars:
print("No stars were detected in the data!")
return False
# Dictionary which holds information about FWHM and standard deviation of the image background
sensor_data = {}
# Compute median FWHM per FF file
for ff_name, star_data in calstars_list:
# Check that the FF file exists in the data directory
if ff_name not in os.listdir(dir_path):
continue
star_data = np.array(star_data)
# Compute the median star FWHM
fwhm_median = np.median(star_data[:, 4])
# Load the FF file and compute the standard deviation of the background
ff = FFfile.read(dir_path, ff_name)
# Compute the median stddev of the background
stddev_median = np.median(ff.stdpixel)
# Store the values to the dictionary
sensor_data[ff_name] = [fwhm_median, stddev_median]
print("{:s}, {:5.2f}, {:5.2f}".format(ff_name, fwhm_median, stddev_median))
return sensor_data
help="Path to a config file which will be used instead of the default one.")
arg_parser.add_argument('-s', '--showstd', action="store_true", help="""Show a histogram of stddevs of PSFs of all detected stars. """)
# Parse the command line arguments
cml_args = arg_parser.parse_args()
#########################
# Load the config file
config = cr.loadConfigFromDirectory(cml_args.config, cml_args.dir_path)
# Get paths to every FF bin file in a directory
ff_dir = os.path.abspath(cml_args.dir_path[0])
ff_list = [ff_name for ff_name in os.listdir(ff_dir) if FFfile.validFFName(ff_name)]
# Check if there are any file in the directory
if(len(ff_list) == None):
print("No files found!")
sys.exit()
# Try loading a flat field image
flat_struct = None
if config.use_flat:
# Check if there is flat in the data directory
if os.path.exists(os.path.join(ff_dir, config.flat_file)):
