def saveFF(self, arr, startTime, N): """ Write metadata and data array to FF file. Arguments: arr: [3D ndarray] 3D numpy array in format: (N, y, x) where N is [0, 4) startTime: [float] seconds and fractions of a second from epoch to first frame N: [int] frame counter (ie. 0000512) """ # Generate the name for the file date_string = time.strftime("%Y%m%d_%H%M%S", time.gmtime(startTime)) # Calculate miliseconds millis = int((startTime - floor(startTime))*1000) filename = str(self.config.stationID).zfill(3) + "_" + date_string + "_" + str(millis).zfill(3) \ + "_" + str(N).zfill(7) ff = FFStruct.FFStruct() ff.array = arr ff.nrows = arr.shape[1] ff.ncols = arr.shape[2] ff.nbits = self.config.bit_depth ff.nframes = 256 ff.first = N + 256 ff.camno = self.config.stationID ff.fps = self.config.fps # Write the FF file FFfile.write(ff, self.data_dir, filename, fmt=self.config.ff_format) return filename
def saveFF(self, arr, startTime, N): """ Write metadata and data array to FF file. Arguments: arr: [3D ndarray] 3D numpy array in format: (N, y, x) where N is [0, 4) startTime: [float] seconds and fractions of a second from epoch to first frame N: [int] frame counter (ie. 0000512) """ # Generate the name for the file date_string = time.strftime("%Y%m%d_%H%M%S", time.gmtime(startTime)) # Calculate miliseconds millis = int((startTime - floor(startTime)) * 1000) filename = str(self.config.stationID).zfill(3) + "_" + date_string + "_" + str(millis).zfill(3) \ + "_" + str(N).zfill(7) ff = FFStruct.FFStruct() ff.array = arr ff.nrows = arr.shape[1] ff.ncols = arr.shape[2] ff.nbits = self.config.bit_depth ff.nframes = 256 ff.first = N + 256 ff.camno = self.config.stationID ff.fps = self.config.fps # Write the FF file FFfile.write(ff, self.data_dir, filename, fmt=self.config.ff_format) return filename
def _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement): """ Run FFT alignment before giving up on ACF. """ if not _fft_refinement: print('The initial platepar is bad, trying to refine it using FFT phase correlation...') # Prepare data for FFT image registration calstars_dict = {ff_file: star_data for ff_file, star_data in calstars_list} # Extract star list from CALSTARS file from FF file with most stars max_len_ff = max(calstars_dict, key=lambda k: len(calstars_dict[k])) # Take only X, Y (change order so X is first) calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2] calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]] # Get the time of the FF file calstars_time = FFfile.getMiddleTimeFF(max_len_ff, config.fps, ret_milliseconds=True) # Try aligning the platepar using FFT image registration platepar_refined = alignPlatepar(config, platepar, calstars_time, calstars_coords) # Redo autoCF return autoCheckFit(config, platepar_refined, calstars_list, \ distorsion_refinement=distorsion_refinement, _fft_refinement=True) else: print('Auto Check Fit failed completely, please redo the plate manually!') return platepar, False
def starListToDict(config, calstars_list, max_ffs=None): """ Converts the list of calstars into dictionary where the keys are FF file JD and the values is a list of (X, Y, bg_intens, intens) of stars. """ # Convert the list to a dictionary calstars = {ff_file: star_data for ff_file, star_data in calstars_list} # Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars star_dict = {} # Take only those files with enough stars on them for ff_name in calstars: stars_list = calstars[ff_name] # Check if there are enough stars on the image if len(stars_list) >= config.ff_min_stars: # Calculate the JD time of the FF file dt = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True) jd = date2JD(*dt) # Add the time and the stars to the dict star_dict[jd] = stars_list if max_ffs is not None: # Limit the number of FF files used if len(star_dict) > max_ffs: # Randomly choose calstars_files_N image files from the whole list rand_keys = random.sample(list(star_dict), max_ffs) star_dict = {key: star_dict[key] for key in rand_keys} return star_dict
def recalibrateIndividualFFsAndApplyAstrometry(dir_path, ftpdetectinfo_path, calstars_list, config, platepar, generate_plot=True): """ Recalibrate FF files with detections and apply the recalibrated platepar to those detections. Arguments: dir_path: [str] Path where the FTPdetectinfo file is. ftpdetectinfo_path: [str] Name of the FTPdetectinfo file. calstars_list: [list] A list of entries [[ff_name, star_coordinates], ...]. config: [Config instance] platepar: [Platepar instance] Initial platepar. Keyword arguments: generate_plot: [bool] Generate the calibration variation plot. True by default. Return: recalibrated_platepars: [dict] A dictionary where the keys are FF file names and values are recalibrated platepar instances for every FF file. """ # Use a copy of the config file config = copy.deepcopy(config) # If the given file does not exits, return nothing if not os.path.isfile(ftpdetectinfo_path): print('ERROR! The FTPdetectinfo file does not exist: {:s}'.format(ftpdetectinfo_path)) print(' The recalibration on every file was not done!') return {} # Read the FTPdetectinfo data cam_code, fps, meteor_list = FTPdetectinfo.readFTPdetectinfo(*os.path.split(ftpdetectinfo_path), \ ret_input_format=True) # Convert the list of stars to a per FF name dictionary calstars = {ff_file: star_data for ff_file, star_data in calstars_list} ### Add neighboring FF files for more robust photometry estimation ### ff_processing_list = [] # Make a list of sorted FF files in CALSTARS calstars_ffs = sorted([ff_file for ff_file in calstars]) # Go through the list of FF files with detections and add neighboring FFs for meteor_entry in meteor_list: ff_name = meteor_entry[0] if ff_name in calstars_ffs: # Find the index of the given FF file in the list of calstars ff_indx = calstars_ffs.index(ff_name) # Add neighbours to the processing list for k in range(-(RECALIBRATE_NEIGHBOURHOOD_SIZE//2), RECALIBRATE_NEIGHBOURHOOD_SIZE//2 + 1): k_indx = ff_indx + k if (k_indx > 0) and (k_indx < len(calstars_ffs)): ff_name_tmp = calstars_ffs[k_indx] if ff_name_tmp not in ff_processing_list: ff_processing_list.append(ff_name_tmp) # Sort the processing list of FF files ff_processing_list = sorted(ff_processing_list) ### ### # Globally increase catalog limiting magnitude config.catalog_mag_limit += 1 # Load catalog stars (overwrite the mag band ratios if specific catalog is used) star_catalog_status = StarCatalog.readStarCatalog(config.star_catalog_path,\ config.star_catalog_file, lim_mag=config.catalog_mag_limit, \ mag_band_ratios=config.star_catalog_band_ratios) if not star_catalog_status: print("Could not load the star catalog!") print(os.path.join(config.star_catalog_path, config.star_catalog_file)) return {} catalog_stars, _, config.star_catalog_band_ratios = star_catalog_status # Update the platepar coordinates from the config file platepar.lat = config.latitude platepar.lon = config.longitude platepar.elev = config.elevation prev_platepar = copy.deepcopy(platepar) # Go through all FF files with detections, recalibrate and apply astrometry recalibrated_platepars = {} for ff_name in ff_processing_list: working_platepar = copy.deepcopy(prev_platepar) # Skip this meteor if its FF file was already recalibrated if ff_name in recalibrated_platepars: continue print() print('Processing: ', ff_name) print('------------------------------------------------------------------------------') # Find extracted stars on this image if not ff_name in calstars: print('Skipped because it was not in CALSTARS:', ff_name) continue # Get stars detected on this FF file (create a dictionaly with only one entry, the residuals function # needs this format) calstars_time = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True) jd = date2JD(*calstars_time) star_dict_ff = {jd: calstars[ff_name]} # Recalibrate the platepar using star matching result, min_match_radius = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars) # If the recalibration failed, try using FFT alignment if result is None: print() print('Running FFT alignment...') # Run FFT alignment calstars_coords = np.array(star_dict_ff[jd])[:, :2] calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]] print(calstars_time) test_platepar = alignPlatepar(config, prev_platepar, calstars_time, calstars_coords, \ show_plot=False) # Try to recalibrate after FFT alignment result, _ = recalibrateFF(config, test_platepar, jd, star_dict_ff, catalog_stars) # If the FFT alignment failed, align the original platepar using the smallest radius that matched # and force save the the platepar if (result is None) and (min_match_radius is not None): print() print("Using the old platepar with the minimum match radius of: {:.2f}".format(min_match_radius)) result, _ = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars, max_match_radius=min_match_radius, force_platepar_save=True) if result is not None: working_platepar = result # If the alignment succeeded, save the result else: working_platepar = result else: working_platepar = result # Store the platepar if the fit succeeded if result is not None: # Recompute alt/az of the FOV centre working_platepar.az_centre, working_platepar.alt_centre = raDec2AltAz(working_platepar.RA_d, \ working_platepar.dec_d, working_platepar.JD, working_platepar.lat, working_platepar.lon) # Recompute the rotation wrt horizon working_platepar.rotation_from_horiz = rotationWrtHorizon(working_platepar) # Mark the platepar to indicate that it was automatically recalibrated on an individual FF file working_platepar.auto_recalibrated = True recalibrated_platepars[ff_name] = working_platepar prev_platepar = working_platepar else: print('Recalibration of {:s} failed, using the previous platepar...'.format(ff_name)) # Mark the platepar to indicate that autorecalib failed prev_platepar_tmp = copy.deepcopy(prev_platepar) prev_platepar_tmp.auto_recalibrated = False # If the aligning failed, set the previous platepar as the one that should be used for this FF file recalibrated_platepars[ff_name] = prev_platepar_tmp ### Average out photometric offsets within the given neighbourhood size ### # Go through the list of FF files with detections for meteor_entry in meteor_list: ff_name = meteor_entry[0] # Make sure the FF was successfuly recalibrated if ff_name in recalibrated_platepars: # Find the index of the given FF file in the list of calstars ff_indx = calstars_ffs.index(ff_name) # Compute the average photometric offset and the improved standard deviation using all # neighbors photom_offset_tmp_list = [] photom_offset_std_tmp_list = [] neighboring_ffs = [] for k in range(-(RECALIBRATE_NEIGHBOURHOOD_SIZE//2), RECALIBRATE_NEIGHBOURHOOD_SIZE//2 + 1): k_indx = ff_indx + k if (k_indx > 0) and (k_indx < len(calstars_ffs)): # Get the name of the FF file ff_name_tmp = calstars_ffs[k_indx] # Check that the neighboring FF was successfuly recalibrated if ff_name_tmp in recalibrated_platepars: # Get the computed photometric offset and stddev photom_offset_tmp_list.append(recalibrated_platepars[ff_name_tmp].mag_lev) photom_offset_std_tmp_list.append(recalibrated_platepars[ff_name_tmp].mag_lev_stddev) neighboring_ffs.append(ff_name_tmp) # Compute the new photometric offset and improved standard deviation (assume equal sample size) # Source: https://stats.stackexchange.com/questions/55999/is-it-possible-to-find-the-combined-standard-deviation photom_offset_new = np.mean(photom_offset_tmp_list) photom_offset_std_new = np.sqrt(\ np.sum([st**2 + (mt - photom_offset_new)**2 \ for mt, st in zip(photom_offset_tmp_list, photom_offset_std_tmp_list)]) \ / len(photom_offset_tmp_list) ) # Assign the new photometric offset and standard deviation to all FFs used for computation for ff_name_tmp in neighboring_ffs: recalibrated_platepars[ff_name_tmp].mag_lev = photom_offset_new recalibrated_platepars[ff_name_tmp].mag_lev_stddev = photom_offset_std_new ### ### ### Store all recalibrated platepars as a JSON file ### all_pps = {} for ff_name in recalibrated_platepars: json_str = recalibrated_platepars[ff_name].jsonStr() all_pps[ff_name] = json.loads(json_str) with open(os.path.join(dir_path, config.platepars_recalibrated_name), 'w') as f: # Convert all platepars to a JSON file out_str = json.dumps(all_pps, default=lambda o: o.__dict__, indent=4, sort_keys=True) f.write(out_str) ### ### # If no platepars were recalibrated, use the single platepar recalibration procedure if len(recalibrated_platepars) == 0: print('No FF images were used for recalibration, using the single platepar calibration function...') # Use the initial platepar for calibration applyAstrometryFTPdetectinfo(dir_path, os.path.basename(ftpdetectinfo_path), None, platepar=platepar) return recalibrated_platepars ### GENERATE PLOTS ### dt_list = [] ang_dists = [] rot_angles = [] hour_list = [] photom_offset_list = [] photom_offset_std_list = [] first_dt = np.min([FFfile.filenameToDatetime(ff_name) for ff_name in recalibrated_platepars]) for ff_name in recalibrated_platepars: pp_temp = recalibrated_platepars[ff_name] # If the fitting failed, skip the platepar if pp_temp is None: continue # Add the datetime of the FF file to the list ff_dt = FFfile.filenameToDatetime(ff_name) dt_list.append(ff_dt) # Compute the angular separation from the reference platepar ang_dist = np.degrees(angularSeparation(np.radians(platepar.RA_d), np.radians(platepar.dec_d), \ np.radians(pp_temp.RA_d), np.radians(pp_temp.dec_d))) ang_dists.append(ang_dist*60) # Compute rotation difference rot_diff = (platepar.pos_angle_ref - pp_temp.pos_angle_ref + 180)%360 - 180 rot_angles.append(rot_diff*60) # Compute the hour of the FF used for recalibration hour_list.append((ff_dt - first_dt).total_seconds()/3600) # Add the photometric offset to the list photom_offset_list.append(pp_temp.mag_lev) photom_offset_std_list.append(pp_temp.mag_lev_stddev) if generate_plot: # Generate the name the plots plot_name = os.path.basename(ftpdetectinfo_path).replace('FTPdetectinfo_', '').replace('.txt', '') ### Plot difference from reference platepar in angular distance from (0, 0) vs rotation ### plt.figure() plt.scatter(0, 0, marker='o', edgecolor='k', label='Reference platepar', s=100, c='none', zorder=3) plt.scatter(ang_dists, rot_angles, c=hour_list, zorder=3) plt.colorbar(label="Hours from first FF file") plt.xlabel("Angular distance from reference (arcmin)") plt.ylabel("Rotation from reference (arcmin)") plt.title("FOV centre drift starting at {:s}".format(first_dt.strftime("%Y/%m/%d %H:%M:%S"))) plt.grid() plt.legend() plt.tight_layout() plt.savefig(os.path.join(dir_path, plot_name + '_calibration_variation.png'), dpi=150) # plt.show() plt.clf() plt.close() ### ### ### Plot the photometric offset variation ### plt.figure() plt.errorbar(dt_list, photom_offset_list, yerr=photom_offset_std_list, fmt="o", \ ecolor='lightgray', elinewidth=2, capsize=0, ms=2) # Format datetimes plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%H:%M")) # rotate and align the tick labels so they look better plt.gcf().autofmt_xdate() plt.xlabel("UTC time") plt.ylabel("Photometric offset") plt.title("Photometric offset variation") plt.grid() plt.tight_layout() plt.savefig(os.path.join(dir_path, plot_name + '_photometry_variation.png'), dpi=150) plt.clf() plt.close() ### ### ### Apply platepars to FTPdetectinfo ### meteor_output_list = [] for meteor_entry in meteor_list: ff_name, meteor_No, rho, phi, meteor_meas = meteor_entry # Get the platepar that will be applied to this FF file if ff_name in recalibrated_platepars: working_platepar = recalibrated_platepars[ff_name] else: print('Using default platepar for:', ff_name) working_platepar = platepar # Apply the recalibrated platepar to meteor centroids meteor_picks = applyPlateparToCentroids(ff_name, fps, meteor_meas, working_platepar, \ add_calstatus=True) meteor_output_list.append([ff_name, meteor_No, rho, phi, meteor_picks]) # Calibration string to be written to the FTPdetectinfo file calib_str = 'Recalibrated with RMS on: ' + str(datetime.datetime.utcnow()) + ' UTC' # If no meteors were detected, set dummpy parameters if len(meteor_list) == 0: cam_code = '' fps = 0 # Back up the old FTPdetectinfo file try: shutil.copy(ftpdetectinfo_path, ftpdetectinfo_path.strip('.txt') \ + '_backup_{:s}.txt'.format(datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S.%f'))) except: print('ERROR! The FTPdetectinfo file could not be backed up: {:s}'.format(ftpdetectinfo_path)) # Save the updated FTPdetectinfo FTPdetectinfo.writeFTPdetectinfo(meteor_output_list, dir_path, os.path.basename(ftpdetectinfo_path), \ dir_path, cam_code, fps, calibration=calib_str, celestial_coords_given=True) ### ### return recalibrated_platepars
config = cr.parse(".config") # Get the list of FR bin files (fireball detections) in the given directory fr_list = [ fr for fr in os.listdir(dir_path) if fr[0:2] == "FR" and fr.endswith('bin') ] fr_list = sorted(fr_list) if not fr_list: print("No files found!") sys.exit() # Get the list of FF bin files (compressed video frames) ff_list = [ff for ff in os.listdir(dir_path) if FFfile.validFFName(ff)] ff_list = sorted(ff_list) i = 0 while True: # Break the loop if at the end if i >= len(fr_list): break fr = fr_list[i] ff_match = None # Strip extensions
# Import old morph from RMS.OLD import MorphologicalOperations as morph # Cython init import pyximport pyximport.install(setup_args={'include_dirs': [np.get_include()]}) from RMS.Routines.MorphCy import morphApply # Run tests # Extract file and directory head, ff_name = os.path.split(sys.argv[1]) ff_path = os.path.abspath(head) + os.sep # Load the FF bin file ff = FFfile.read(ff_path, ff_name) img_thresh = thresholdImg(ff, 1.8, 9) show('thresh', img_thresh) # Convert img to integer img = img_thresh.astype(np.uint8) # Old morph img_old = np.copy(img) t1 = time.clock() img_old = morph.clean(img_old) img_old = morph.bridge(img_old) img_old = morph.close(img_old)
def _handleFailure(config, platepar, calstars_list, catalog_stars, _fft_refinement): """ Run FFT alignment before giving up on ACF. """ if not _fft_refinement: print() print( "-------------------------------------------------------------------------------" ) print( 'The initial platepar is bad, trying to refine it using FFT phase correlation...' ) print() # Prepare data for FFT image registration calstars_dict = { ff_file: star_data for ff_file, star_data in calstars_list } # Extract star list from CALSTARS file from FF file with most stars max_len_ff = max(calstars_dict, key=lambda k: len(calstars_dict[k])) # Take only X, Y (change order so X is first) calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2] calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]] # Get the time of the FF file calstars_time = FFfile.getMiddleTimeFF(max_len_ff, config.fps, ret_milliseconds=True) # Try aligning the platepar using FFT image registration platepar_refined = alignPlatepar(config, platepar, calstars_time, calstars_coords) print() ### If there are still not enough stars matched, try FFT again ### min_radius = 10 # Prepare star dictionary to check the match dt = FFfile.getMiddleTimeFF(max_len_ff, config.fps, ret_milliseconds=True) jd = date2JD(*dt) star_dict_temp = {} star_dict_temp[jd] = calstars_dict[max_len_ff] # Check the number of matched stars n_matched, _, _, _ = matchStarsResiduals(config, platepar_refined, catalog_stars, \ star_dict_temp, min_radius, ret_nmatch=True, verbose=True) # Realign again if necessary if n_matched < config.min_matched_stars: print() print( "-------------------------------------------------------------------------------" ) print( 'Doing a second FFT pass as the number of matched stars was too small...' ) print() platepar_refined = alignPlatepar(config, platepar_refined, calstars_time, calstars_coords) print() ### ### # Redo autoCF return autoCheckFit(config, platepar_refined, calstars_list, _fft_refinement=True) else: print( 'Auto Check Fit failed completely, please redo the plate manually!' ) return platepar, False
def autoCheckFit(config, platepar, calstars_list, distorsion_refinement=False, _fft_refinement=False): """ Attempts to refine the astrometry fit with the given stars and and initial astrometry parameters. Arguments: config: [Config structure] platepar: [Platepar structure] Initial astrometry parameters. calstars_list: [list] A list containing stars extracted from FF files. See RMS.Formats.CALSTARS for more details. Keyword arguments: distorsion_refinement: [bool] Whether the distorsion should be fitted as well. False by default. _fft_refinement: [bool] Internal flag indicating that autoCF is running the second time recursively after FFT platepar adjustment. Return: (platepar, fit_status): platepar: [Platepar structure] Estimated/refined platepar. fit_status: [bool] True if fit was successfuly, False if not. """ def _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement): """ Run FFT alignment before giving up on ACF. """ if not _fft_refinement: print('The initial platepar is bad, trying to refine it using FFT phase correlation...') # Prepare data for FFT image registration calstars_dict = {ff_file: star_data for ff_file, star_data in calstars_list} # Extract star list from CALSTARS file from FF file with most stars max_len_ff = max(calstars_dict, key=lambda k: len(calstars_dict[k])) # Take only X, Y (change order so X is first) calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2] calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]] # Get the time of the FF file calstars_time = FFfile.getMiddleTimeFF(max_len_ff, config.fps, ret_milliseconds=True) # Try aligning the platepar using FFT image registration platepar_refined = alignPlatepar(config, platepar, calstars_time, calstars_coords) # Redo autoCF return autoCheckFit(config, platepar_refined, calstars_list, \ distorsion_refinement=distorsion_refinement, _fft_refinement=True) else: print('Auto Check Fit failed completely, please redo the plate manually!') return platepar, False if _fft_refinement: print('Second ACF run with an updated platepar via FFT phase correlation...') # Convert the list to a dictionary calstars = {ff_file: star_data for ff_file, star_data in calstars_list} # Load catalog stars (overwrite the mag band ratios if specific catalog is used) catalog_stars, _, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(config.star_catalog_path, \ config.star_catalog_file, lim_mag=config.catalog_mag_limit, \ mag_band_ratios=config.star_catalog_band_ratios) # Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars star_dict = {} # Take only those files with enough stars on them for ff_name in calstars: stars_list = calstars[ff_name] # Check if there are enough stars on the image if len(stars_list) >= config.ff_min_stars: # Calculate the JD time of the FF file dt = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True) jd = date2JD(*dt) # Add the time and the stars to the dict star_dict[jd] = stars_list # There has to be a minimum of 200 FF files for star fitting, and only 100 will be subset if there are more if len(star_dict) < config.calstars_files_N: print('Not enough FF files in CALSTARS for ACF!') return platepar, False else: # Randomly choose calstars_files_N image files from the whole list rand_keys = random.sample(list(star_dict), config.calstars_files_N) star_dict = {key: star_dict[key] for key in rand_keys} # Calculate the total number of calibration stars used total_calstars = sum([len(star_dict[key]) for key in star_dict]) print('Total calstars:', total_calstars) if total_calstars < config.calstars_min_stars: print('Not enough calibration stars, need at least', config.calstars_min_stars) return platepar, False # A list of matching radiuses to try, pairs of [radius, fit_distorsion_flag] # The distorsion will be fitted only if explicity requested min_radius = 0.5 radius_list = [[10, False], [5, False], [3, False], [1.5, True and distorsion_refinement], [min_radius, True and distorsion_refinement]] # Calculate the function tolerance, so the desired precision can be reached (the number is calculated # in the same regard as the cost function) fatol, xatol_ang = computeMinimizationTolerances(config, platepar, len(star_dict)) ### If the initial match is good enough, do only quick recalibratoin ### # Match the stars and calculate the residuals n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \ min_radius, ret_nmatch=True) if n_matched >= config.calstars_files_N: # Check if the average distance with the tightest radius is close if avg_dist < config.dist_check_quick_threshold: # Use a reduced set of initial radius values radius_list = [[1.5, True and distorsion_refinement], [min_radius, True and distorsion_refinement]] ########## # Match increasingly smaller search radiia around image stars for i, (match_radius, fit_distorsion) in enumerate(radius_list): # Match the stars and calculate the residuals n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \ match_radius, ret_nmatch=True) print('Max radius:', match_radius) print('Initial values:') print(' Matched stars:', n_matched) print(' Average deviation:', avg_dist) # The initial number of matched stars has to be at least the number of FF imaages, otherwise it means # that the initial platepar is no good if n_matched < config.calstars_files_N: print('The total number of initially matched stars is too small! Please manually redo the plate or make sure there are enough calibration stars.') # Try to refine the platepar with FFT phase correlation and redo the ACF return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement) # Check if the platepar is good enough and do not estimate further parameters if checkFitGoodness(config, platepar, catalog_stars, star_dict, min_radius, verbose=True): # Print out notice only if the platepar is good right away if i == 0: print("Initial platepar is good enough!") return platepar, True # Initial parameters for the astrometric fit (don't fit the scale if the distorsion is not being fit) if fit_distorsion: p0 = [platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref, platepar.F_scale] else: p0 = [platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref] # Fit the astrometric parameters res = scipy.optimize.minimize(_calcImageResidualsAstro, p0, args=(config, platepar, catalog_stars, \ star_dict, match_radius, fit_distorsion), method='Nelder-Mead', \ options={'fatol': fatol, 'xatol': xatol_ang}) print(res) # If the fit was not successful, stop further fitting if not res.success: # Try to refine the platepar with FFT phase correlation and redo the ACF return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement) else: # If the fit was successful, use the new parameters from now on if fit_distorsion: ra_ref, dec_ref, pos_angle_ref, F_scale = res.x else: ra_ref, dec_ref, pos_angle_ref = res.x F_scale = platepar.F_scale platepar.RA_d = ra_ref platepar.dec_d = dec_ref platepar.pos_angle_ref = pos_angle_ref platepar.F_scale = F_scale # Check if the platepar is good enough and do not estimate further parameters if checkFitGoodness(config, platepar, catalog_stars, star_dict, min_radius, verbose=True): return platepar, True # Fit the lens distorsion parameters if fit_distorsion: ### REVERSE DISTORSION POLYNOMIALS FIT ### # Fit the distortion parameters (X axis) res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.x_poly_rev, args=(config, \ platepar, catalog_stars, star_dict, match_radius, 'x'), method='Nelder-Mead', \ options={'fatol': fatol, 'xatol': 0.1}) print(res) # If the fit was not successfull, stop further fitting if not res.success: # Try to refine the platepar with FFT phase correlation and redo the ACF return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement) else: platepar.x_poly_rev = res.x # Fit the distortion parameters (Y axis) res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.y_poly_rev, args=(config, \ platepar,catalog_stars, star_dict, match_radius, 'y'), method='Nelder-Mead', \ options={'fatol': fatol, 'xatol': 0.1}) print(res) # If the fit was not successfull, stop further fitting if not res.success: # Try to refine the platepar with FFT phase correlation and redo the ACF return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement) else: platepar.y_poly_rev = res.x ### ### ### FORWARD DISTORSION POLYNOMIALS FIT ### # Fit the distortion parameters (X axis) res = scipy.optimize.minimize(_calcSkyResidualsDistorsion, platepar.x_poly_fwd, args=(config, \ platepar, catalog_stars, star_dict, match_radius, 'x'), method='Nelder-Mead', \ options={'fatol': fatol, 'xatol': 0.1}) print(res) # If the fit was not successfull, stop further fitting if not res.success: # Try to refine the platepar with FFT phase correlation and redo the ACF return _handleFailure(config, platepar, calstars_list, distorsion_refinement, _fft_refinement) else: platepar.x_poly_fwd = res.x # Fit the distortion parameters (Y axis) res = scipy.optimize.minimize(_calcSkyResidualsDistorsion, platepar.y_poly_fwd, args=(config, \ platepar,catalog_stars, star_dict, match_radius, 'y'), method='Nelder-Mead', \ options={'fatol': fatol, 'xatol': 0.1}) print(res) # If the fit was not successfull, stop further fitting if not res.success: return platepar, False else: platepar.y_poly_fwd = res.x ### ### # Match the stars and calculate the residuals n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \ star_dict, min_radius, ret_nmatch=True) print('FINAL SOLUTION with {:f} px:'.format(min_radius)) print('Matched stars:', n_matched) print('Average deviation:', avg_dist) # Mark the platepar to indicate that it was automatically refined with CheckFit platepar.auto_check_fit_refined = True return platepar, True
def extractStarsAndSave(config, ff_dir): """ Extract stars in the given folder and save the CALSTARS file. Arguments: config: [config object] configuration object (loaded from the .config file) ff_dir: [str] Path to directory where FF files are. Return: star_list: [list] A list of [ff_name, star_data] entries, where star_data contains a list of (column, row, amplitude, intensity, fwhm) values for every star. """ time_start = time.time() # Load mask, dark, flat mask, dark, flat_struct = loadImageCalibration(ff_dir, config) extraction_list = [] # Go through all files in the directory and add them to the detection list for ff_name in sorted(os.listdir(ff_dir)): # Check if the given file is a valid FF file if not FFfile.validFFName(ff_name): continue extraction_list.append(ff_name) # Run the QueuedPool for detection workpool = QueuedPool(extractStars, cores=-1, backup_dir=ff_dir) # Add jobs for the pool for ff_name in extraction_list: print('Adding for extraction:', ff_name) workpool.addJob([ ff_dir, ff_name, config, None, None, None, None, flat_struct, dark, mask ]) print('Starting pool...') # Start the detection workpool.startPool() print('Waiting for the detection to finish...') # Wait for the detector to finish and close it workpool.closePool() # Get extraction results star_list = [] for result in workpool.getResults(): ff_name, x2, y2, amplitude, intensity, fwhm_data = result # Skip if no stars were found if not x2: continue # Construct the table of the star parameters star_data = list(zip(x2, y2, amplitude, intensity, fwhm_data)) # Add star info to the star list star_list.append([ff_name, star_data]) dir_name = os.path.basename(os.path.abspath(ff_dir)) if dir_name.startswith(config.stationID): prefix = dir_name else: prefix = "{:s}_{:s}".format(config.stationID, dir_name) # Generate the name for the CALSTARS file calstars_name = 'CALSTARS_' + prefix + '.txt' # Write detected stars to the CALSTARS file CALSTARS.writeCALSTARS(star_list, ff_dir, calstars_name, config.stationID, config.height, config.width) # Delete QueudPool backed up files workpool.deleteBackupFiles() print('Total time taken: {:.2f} s'.format(time.time() - time_start)) return star_list
def generateThumbnails(dir_path, config, mosaic_type, file_list=None, no_stack=False): """ Generates a mosaic of thumbnails from all FF files in the given folder and saves it as a JPG image. Arguments: dir_path: [str] Path of the night directory. config: [Conf object] Configuration. mosaic_type: [str] Type of the mosaic (e.g. "Captured" or "Detected") Keyword arguments: file_list: [list] A list of file names (without full path) which will be searched for FF files. This is used when generating separate thumbnails for captured and detected files. Return: file_name: [str] Name of the thumbnail file. no_stack: [bool] Don't stack the images using the config.thumb_stack option. A max of 1000 images are supported with this option. If there are more, stacks will be done according to the config.thumb_stack option. """ if file_list is None: file_list = sorted(os.listdir(dir_path)) # Make a list of all FF files in the night directory ff_list = [] for file_name in file_list: if FFfile.validFFName(file_name): ff_list.append(file_name) # Calculate the dimensions of the binned image bin_w = int(config.width / config.thumb_bin) bin_h = int(config.height / config.thumb_bin) ### RESIZE AND STACK THUMBNAILS ### ########################################################################################################## timestamps = [] stacked_imgs = [] thumb_stack = config.thumb_stack # Check if no stacks should be done (max 1000 images for no stack) if no_stack and (len(ff_list) < 1000): thumb_stack = 1 for i in range(0, len(ff_list), thumb_stack): img_stack = np.zeros((bin_h, bin_w)) # Stack thumb_stack images using the 'if lighter' method for j in range(thumb_stack): if (i + j) < len(ff_list): tmp_file_name = ff_list[i + j] # Read the FF file ff = FFfile.read(dir_path, tmp_file_name) # Skip the FF if it is corruped if ff is None: continue img = ff.maxpixel # Resize the image img = cv2.resize(img, (bin_w, bin_h)) # Stack the image img_stack = stackIfLighter(img_stack, img) else: break # Save the timestamp of the first image in the stack timestamps.append(FFfile.filenameToDatetime(ff_list[i])) # Save the stacked image stacked_imgs.append(img_stack) # cv2.imshow('test', img_stack) # cv2.waitKey(0) # cv2.destroyAllWindows() ########################################################################################################## ### ADD THUMBS TO ONE MOSAIC IMAGE ### ########################################################################################################## header_height = 20 timestamp_height = 10 # Calculate the number of rows for the thumbnail image n_rows = int( np.ceil(float(len(ff_list)) / thumb_stack / config.thumb_n_width)) # Calculate the size of the mosaic mosaic_w = int(config.thumb_n_width * bin_w) mosaic_h = int((bin_h + timestamp_height) * n_rows + header_height) mosaic_img = np.zeros((mosaic_h, mosaic_w), dtype=np.uint8) # Write header text header_text = 'Station: ' + str(config.stationID) + ' Night: ' + os.path.basename(dir_path) \ + ' Type: ' + mosaic_type cv2.putText(mosaic_img, header_text, (0, header_height//2), \ cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255,255,255), 1) for row in range(n_rows): for col in range(config.thumb_n_width): # Calculate image index indx = row * config.thumb_n_width + col if indx < len(stacked_imgs): # Calculate position of the text text_x = col * bin_w text_y = row * bin_h + ( row + 1) * timestamp_height - 1 + header_height # Add timestamp text cv2.putText(mosaic_img, timestamps[indx].strftime('%H:%M:%S'), (text_x, text_y), \ cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1) # Add the image to the mosaic img_pos_x = col * bin_w img_pos_y = row * bin_h + ( row + 1) * timestamp_height + header_height mosaic_img[img_pos_y:img_pos_y + bin_h, img_pos_x:img_pos_x + bin_w] = stacked_imgs[indx] else: break ########################################################################################################## # Only add the station ID if the dir name already doesn't start with it dir_name = os.path.basename(os.path.abspath(dir_path)) if dir_name.startswith(config.stationID): prefix = dir_name else: prefix = "{:s}_{:s}".format(config.stationID, dir_name) thumb_name = "{:s}_{:s}_thumbs.jpg".format(prefix, mosaic_type) # Save the mosaic if USING_IMAGEIO: # Use imageio to write the image imwrite(os.path.join(dir_path, thumb_name), mosaic_img, quality=80) else: # Use OpenCV to save the image imwrite(os.path.join(dir_path, thumb_name), mosaic_img, [int(cv2.IMWRITE_JPEG_QUALITY), 80]) return thumb_name
# RPi Meteor Station # Copyright (C) 2015 Dario Zubovic # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. import cv2 import sys, os from RMS.Formats import FFfile if __name__ == "__main__": ff = FFfile.read(sys.argv[1], sys.argv[2]) cv2.imwrite(sys.argv[2] + "_max.png", ff.maxpixel) cv2.imwrite(sys.argv[2] + "_frame.png", ff.maxframe) cv2.imwrite(sys.argv[2] + "_avg.png", ff.avepixel) cv2.imwrite(sys.argv[2] + "_stddev.png", ff.stdpixel)
def view(dir_path, ff_path, fr_path, config, save_frames=False): """ Shows the detected fireball stored in the FR file. Arguments: dir_path: [str] Current directory. ff: [str] path to the FF bin file fr: [str] path to the FR bin file config: [conf object] configuration structure """ name = fr_path fr = FRbin.read(dir_path, fr_path) print('------------------------') print('Showing file:', fr_path) if ff_path is None: #background = np.zeros((config.height, config.width), np.uint8) # Get the maximum extent of meteor frames y_size = max([max(np.array(fr.yc[i]) + np.array(fr.size[i])//2) for i in range(fr.lines)]) x_size = max([max(np.array(fr.xc[i]) + np.array(fr.size[i])//2) for i in range(fr.lines)]) # Make the image square img_size = max(y_size, x_size) background = np.zeros((img_size, img_size), np.uint8) else: background = FFfile.read(dir_path, ff_path).maxpixel print("Number of lines:", fr.lines) first_image = True wait_time = 2*int(1000.0/config.fps) pause_flag = False for current_line in range(fr.lines): print('Frame, Y , X , size') for z in range(fr.frameNum[current_line]): # Get the center position of the detection on the current frame yc = fr.yc[current_line][z] xc = fr.xc[current_line][z] # Get the frame number t = fr.t[current_line][z] # Get the size of the window size = fr.size[current_line][z] print(" {:3d}, {:3d}, {:3d}, {:d}".format(t, yc, xc, size)) img = np.copy(background) # Paste the frames onto the big image y_img = np.arange(yc - size//2, yc + size//2) x_img = np.arange(xc - size//2, xc + size//2) Y_img, X_img = np.meshgrid(y_img, x_img) y_frame = np.arange(len(y_img)) x_frame = np.arange(len(x_img)) Y_frame, X_frame = np.meshgrid(y_frame, x_frame) img[Y_img, X_img] = fr.frames[current_line][z][Y_frame, X_frame] # Save frame to disk if save_frames: frame_file_name = fr_path.replace('.bin', '') + "_line_{:02d}_frame_{:03d}.png".format(current_line, t) cv2.imwrite(os.path.join(dir_path, frame_file_name), img) # Show the frame cv2.imshow(name, img) # If this is the first image, move it to the upper left corner if first_image: cv2.moveWindow(name, 0, 0) first_image = False if pause_flag: wait_time = 0 else: wait_time = 2*int(1000.0/config.fps) # Space key: pause display. # 1: previous file. # 2: next line. # q: Quit. key = cv2.waitKey(wait_time) & 0xFF if key == ord("1"): cv2.destroyWindow(name) return -1 elif key == ord("2"): break elif key == ord(" "): # Pause/unpause video pause_flag = not pause_flag elif key == ord("q") : os._exit(0) cv2.destroyWindow(name)
# Load the configuration file config = cr.parse(".config") # Get the list of FR bin files (fireball detections) in the given directory fr_list = [fr for fr in os.listdir(dir_path) if fr[0:2]=="FR" and fr.endswith('bin')] fr_list = sorted(fr_list) if not fr_list: print("No files found!") sys.exit() # Get the list of FF bin files (compressed video frames) ff_list = [ff for ff in os.listdir(dir_path) if FFfile.validFFName(ff)] ff_list = sorted(ff_list) i = 0 while True: # Break the loop if at the end if i >= len(fr_list): break fr = fr_list[i] ff_match = None
if __name__ == "__main__": time_start = time.clock() # Load config file config = cr.parse(".config") if not len(sys.argv) == 2: print("Usage: python -m RMS.ExtractStars /path/to/FF/files/") sys.exit() # Get paths to every FF bin file in a directory ff_dir = os.path.abspath(sys.argv[1]) ff_list = [ff_name for ff_name in os.listdir(ff_dir) if FFfile.validFFName(ff_name)] # Check if there are any file in the directory if(len(ff_list) == None): print("No files found!") sys.exit() # Try loading a flat field image flat_struct = None if config.use_flat: # Check if there is flat in the data directory if os.path.exists(os.path.join(ff_dir, config.flat_file)):
def recalibrateIndividualFFsAndApplyAstrometry(dir_path, ftpdetectinfo_path, calstars_list, config, platepar): """ Recalibrate FF files with detections and apply the recalibrated platepar to those detections. Arguments: dir_path: [str] Path where the FTPdetectinfo file is. ftpdetectinfo_path: [str] Name of the FTPdetectinfo file. calstars_list: [list] A list of entries [[ff_name, star_coordinates], ...]. config: [Config instance] platepar: [Platepar instance] Initial platepar. Return: recalibrated_platepars: [dict] A dictionary where the keys are FF file names and values are recalibrated platepar instances for every FF file. """ # Read the FTPdetectinfo data cam_code, fps, meteor_list = FTPdetectinfo.readFTPdetectinfo(*os.path.split(ftpdetectinfo_path), \ ret_input_format=True) # Convert the list of stars to a per FF name dictionary calstars = {ff_file: star_data for ff_file, star_data in calstars_list} # Load catalog stars (overwrite the mag band ratios if specific catalog is used) catalog_stars, _, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(config.star_catalog_path,\ config.star_catalog_file, lim_mag=config.catalog_mag_limit, \ mag_band_ratios=config.star_catalog_band_ratios) prev_platepar = copy.deepcopy(platepar) # Go through all FF files with detections, recalibrate and apply astrometry recalibrated_platepars = {} for meteor_entry in meteor_list: working_platepar = copy.deepcopy(prev_platepar) ff_name, meteor_No, rho, phi, meteor_meas = meteor_entry # Skip this meteors if its FF file was already recalibrated if ff_name in recalibrated_platepars: continue print() print('Processing: ', ff_name) print('------------------------------------------------------------------------------') # Find extracted stars on this image if not ff_name in calstars: print('Skipped because it was not in CALSTARS:', ff_name) continue # Get stars detected on this FF file (create a dictionaly with only one entry, the residuals function # needs this format) calstars_time = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True) jd = date2JD(*calstars_time) star_dict_ff = {jd: calstars[ff_name]} # Recalibrate the platepar using star matching result = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars) # If the recalibration failed, try using FFT alignment if result is None: print() print('Running FFT alignment...') # Run FFT alignment calstars_coords = np.array(star_dict_ff[jd])[:, :2] calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]] print(calstars_time) working_platepar = alignPlatepar(config, prev_platepar, calstars_time, calstars_coords, \ show_plot=False) # Try to recalibrate after FFT alignment result = recalibrateFF(config, working_platepar, jd, star_dict_ff, catalog_stars) if result is not None: working_platepar = result else: working_platepar = result # Store the platepar if the fit succeeded if result is not None: recalibrated_platepars[ff_name] = working_platepar prev_platepar = working_platepar else: print('Recalibration of {:s} failed, using the previous platepar...'.format(ff_name)) # If the aligning failed, set the previous platepar as the one that should be used for this FF file recalibrated_platepars[ff_name] = prev_platepar ### Store all recalibrated platepars as a JSON file ### all_pps = {} for ff_name in recalibrated_platepars: json_str = recalibrated_platepars[ff_name].jsonStr() all_pps[ff_name] = json.loads(json_str) with open(os.path.join(dir_path, config.platepars_recalibrated_name), 'w') as f: # Convert all platepars to a JSON file out_str = json.dumps(all_pps, default=lambda o: o.__dict__, indent=4, sort_keys=True) f.write(out_str) ### ### # If no platepars were recalibrated, use the single platepar recalibration procedure if len(recalibrated_platepars) == 0: print('No FF images were used for recalibration, using the single platepar calibration function...') # Use the initial platepar for calibration applyAstrometryFTPdetectinfo(dir_path, os.path.basename(ftpdetectinfo_path), None, platepar=platepar) return recalibrated_platepars ### Plot difference from reference platepar in angular distance from (0, 0) vs rotation ### ang_dists = [] rot_angles = [] hour_list = [] first_jd = np.min([FFfile.filenameToDatetime(ff_name) for ff_name in recalibrated_platepars]) for ff_name in recalibrated_platepars: pp_temp = recalibrated_platepars[ff_name] # If the fitting failed, skip the platepar if pp_temp is None: continue # Compute the angular separation from the reference platepar ang_dist = np.degrees(angularSeparation(np.radians(platepar.RA_d), np.radians(platepar.dec_d), \ np.radians(pp_temp.RA_d), np.radians(pp_temp.dec_d))) ang_dists.append(ang_dist*60) rot_angles.append((platepar.pos_angle_ref - pp_temp.pos_angle_ref)*60) # Compute the hour of the FF used for recalibration hour_list.append((FFfile.filenameToDatetime(ff_name) - first_jd).total_seconds()/3600) plt.figure() plt.scatter(0, 0, marker='o', edgecolor='k', label='Reference platepar', s=100, c='none', zorder=3) plt.scatter(ang_dists, rot_angles, c=hour_list, zorder=3) plt.colorbar(label='Hours from first FF file') plt.xlabel("Angular distance from reference (arcmin)") plt.ylabel('Rotation from reference (arcmin)') plt.grid() plt.legend() plt.tight_layout() # Generate the name for the plot calib_plot_name = os.path.basename(ftpdetectinfo_path).replace('FTPdetectinfo_', '').replace('.txt', '') \ + '_calibration_variation.png' plt.savefig(os.path.join(dir_path, calib_plot_name), dpi=150) # plt.show() plt.clf() plt.close() ### ### ### Apply platepars to FTPdetectinfo ### meteor_output_list = [] for meteor_entry in meteor_list: ff_name, meteor_No, rho, phi, meteor_meas = meteor_entry # Get the platepar that will be applied to this FF file if ff_name in recalibrated_platepars: working_platepar = recalibrated_platepars[ff_name] else: print('Using default platepar for:', ff_name) working_platepar = platepar # Apply the recalibrated platepar to meteor centroids meteor_picks = applyPlateparToCentroids(ff_name, fps, meteor_meas, working_platepar, \ add_calstatus=True) meteor_output_list.append([ff_name, meteor_No, rho, phi, meteor_picks]) # Calibration string to be written to the FTPdetectinfo file calib_str = 'Recalibrated with RMS on: ' + str(datetime.datetime.utcnow()) + ' UTC' # If no meteors were detected, set dummpy parameters if len(meteor_list) == 0: cam_code = '' fps = 0 # Back up the old FTPdetectinfo file shutil.copy(ftpdetectinfo_path, ftpdetectinfo_path.strip('.txt') \ + '_backup_{:s}.txt'.format(datetime.datetime.utcnow().strftime('%Y%m%d_%H%M%S.%f'))) # Save the updated FTPdetectinfo FTPdetectinfo.writeFTPdetectinfo(meteor_output_list, dir_path, os.path.basename(ftpdetectinfo_path), \ dir_path, cam_code, fps, calibration=calib_str, celestial_coords_given=True) ### ### return recalibrated_platepars
def plot(points, y_dim, x_dim): fig = plt.figure() ax = fig.add_subplot(111, projection='3d') plt.title(name) y = points[:, 0] x = points[:, 1] z = points[:, 2] # Plot points in 3D ax.scatter(x, y, z) # Set axes limits ax.set_zlim(0, 255) plt.xlim([0, x_dim]) plt.ylim([0, y_dim]) ax.set_ylabel("Y") ax.set_xlabel("X") ax.set_zlabel("Time") plt.show() if __name__ == "__main__": ff = FFfile.read(sys.argv[1], sys.argv[2], array=True) view(ff)
# Adjust levels img_data = adjustLevels(img_data, 100, 1.2, 240) plt.imshow(img_data, cmap='gray') plt.show() #### Apply the flat # Load an FF file dir_path = "/home/dvida/Dropbox/Apps/Elginfield RPi RMS data/ArchivedFiles/CA0001_20171018_230520_894458_detected" file_name = "FF_CA0001_20171019_092744_161_1118976.fits" ff = FFfile.read(dir_path, file_name) # Load the flat flat_struct = loadFlat(os.getcwd(), config.flat_file) t1 = time.clock() # Apply the flat img = applyFlat(ff.maxpixel, flat_struct) print('Flat time:', time.clock() - t1) plt.imshow(img, cmap='gray', vmin=0, vmax=255) plt.show()
def extractStars(ff_dir, ff_name, config=None, max_global_intensity=150, border=10, neighborhood_size=10, intensity_threshold=5, flat_struct=None, dark=None, mask=None): """ Extracts stars on a given FF bin by searching for local maxima and applying PSF fit for star confirmation. Source of one part of the code: http://stackoverflow.com/questions/9111711/get-coordinates-of-local-maxima-in-2d-array-above-certain-value Arguments: ff: [ff bin struct] FF bin file loaded in the FF bin structure config: [config object] configuration object (loaded from the .config file) max_global_intensity: [int] maximum mean intensity of an image before it is discared as too bright border: [int] apply a mask on the detections by removing all that are too close to the given image border (in pixels) neighborhood_size: [int] size of the neighbourhood for the maximum search (in pixels) intensity_threshold: [float] a threshold for cutting the detections which are too faint (0-255) flat_struct: [Flat struct] Structure containing the flat field. None by default. dark: [ndarray] Dark frame. None by default. mask: [ndarray] Mask image. None by default. Return: x2, y2, background, intensity, sigma_fitted: [list of ndarrays] - x2: X axis coordinates of the star - y2: Y axis coordinates of the star - background: background intensity - intensity: intensity of the star - Gaussian stddev of fitted stars """ # This will be returned if there was an error error_return = [[], [], [], []] # Load parameters from config if given if config: max_global_intensity = config.max_global_intensity border = config.border neighborhood_size = config.neighborhood_size intensity_threshold = config.intensity_threshold # Load the FF bin file ff = FFfile.read(ff_dir, ff_name) # If the FF file could not be read, skip star extraction if ff is None: return error_return # Apply the dark frame if dark is not None: ff.avepixel = Image.applyDark(ff.avepixel, dark) # Apply the flat if flat_struct is not None: ff.avepixel = Image.applyFlat(ff.avepixel, flat_struct) # Mask the FF file if mask is not None: ff = MaskImage.applyMask(ff, mask, ff_flag=True) # Calculate image mean and stddev global_mean = np.mean(ff.avepixel) # Check if the image is too bright and skip the image if global_mean > max_global_intensity: return error_return data = ff.avepixel.astype(np.float32) # Apply a mean filter to the image to reduce noise data = ndimage.filters.convolve(data, weights=np.full((2, 2), 1.0/4)) # Locate local maxima on the image data_max = filters.maximum_filter(data, neighborhood_size) maxima = (data == data_max) data_min = filters.minimum_filter(data, neighborhood_size) diff = ((data_max - data_min) > intensity_threshold) maxima[diff == 0] = 0 # Apply a border mask border_mask = np.ones_like(maxima)*255 border_mask[:border,:] = 0 border_mask[-border:,:] = 0 border_mask[:,:border] = 0 border_mask[:,-border:] = 0 maxima = MaskImage.applyMask(maxima, border_mask, image=True) # Find and label the maxima labeled, num_objects = ndimage.label(maxima) # Skip the image if there are too many maxima to process if num_objects > config.max_stars: print('Too many candidate stars to process! {:d}/{:d}'.format(num_objects, config.max_stars)) return error_return # Find centres of mass of each labeled objects xy = np.array(ndimage.center_of_mass(data, labeled, range(1, num_objects+1))) # Remove all detection on the border #xy = xy[np.where((xy[:, 1] > border) & (xy[:,1] < ff.ncols - border) & (xy[:,0] > border) & (xy[:,0] < ff.nrows - border))] # Unpack star coordinates y, x = np.hsplit(xy, 2) # # Plot stars before the PSF fit # plotStars(ff, x, y) # Fit a PSF to each star x2, y2, amplitude, intensity, sigma_y_fitted, sigma_x_fitted = fitPSF(ff, global_mean, x, y, config) # x2, y2, amplitude, intensity = list(x), list(y), [], [] # Skip PSF fit # # Plot stars after PSF fit filtering # plotStars(ff, x2, y2) # Compute one dimensional sigma sigma_x_fitted = np.array(sigma_x_fitted) sigma_y_fitted = np.array(sigma_y_fitted) sigma_fitted = np.sqrt(sigma_x_fitted**2 + sigma_y_fitted**2) return ff_name, x2, y2, amplitude, intensity, sigma_fitted
def FTPdetectinfo2UFOOrbitInput(dir_path, file_name, platepar_path, platepar_dict=None): """ Convert the FTPdetectinfo file into UFOOrbit input CSV file. Arguments: dir_path: [str] Path of the directory which contains the FTPdetectinfo file. file_name: [str] Name of the FTPdetectinfo file. platepar_path: [str] Full path to the platepar file. Keyword arguments: platepar_dict: [dict] Dictionary of Platepar instances where keys are FF file names. This will be used instead of the platepar at platepar_path. None by default. """ # Load the FTPdetecinfo file meteor_list = FTPdetectinfo.readFTPdetectinfo(dir_path, file_name) # Load the platepar file if platepar_dict is None: pp = RMS.Formats.Platepar.Platepar() pp.read(platepar_path, use_flat=None) # Init the UFO format list ufo_meteor_list = [] # Go through every meteor in the list for meteor in meteor_list: ff_name, cam_code, meteor_No, n_segments, fps, hnr, mle, binn, px_fm, rho, phi, \ meteor_meas = meteor # Load the platepar from the platepar dictionary, if given if platepar_dict is not None: if ff_name in platepar_dict: pp = platepar_dict[ff_name] else: print( 'Skipping {:s} becuase no platepar was found for this FF file!' .format(ff_name)) continue # Convert the FF file name into time dt = FFfile.filenameToDatetime(ff_name) # Extract measurements calib_status, frame_n, x, y, ra, dec, azim, elev, inten, mag = np.array( meteor_meas).T # If the meteor wasn't calibrated, skip it if not np.all(calib_status): print('Meteor {:d} was not calibrated, skipping it...'.format( meteor_No)) continue # Compute the peak magnitude peak_mag = np.min(mag) # Compute the total duration first_frame = np.min(frame_n) last_frame = np.max(frame_n) duration = (last_frame - first_frame) / fps # Compute times of first and last points dt1 = dt + datetime.timedelta(seconds=first_frame / fps) dt2 = dt + datetime.timedelta(seconds=last_frame / fps) ### Fit a great circle to Az/Alt measurements and compute model beg/end RA and Dec ### # Convert the measurement Az/Alt to cartesian coordinates # NOTE: All values that are used for Great Circle computation are: # theta - the zenith angle (90 deg - altitude) # phi - azimuth +N of due E, which is (90 deg - azim) x, y, z = Math.polarToCartesian(np.radians((90 - azim) % 360), np.radians(90 - elev)) # Fit a great circle C, theta0, phi0 = GreatCircle.fitGreatCircle(x, y, z) # Get the first point on the great circle phase1 = GreatCircle.greatCirclePhase(np.radians(90 - elev[0]), np.radians((90 - azim[0])%360), \ theta0, phi0) alt1, azim1 = Math.cartesianToPolar( *GreatCircle.greatCircle(phase1, theta0, phi0)) alt1 = 90 - np.degrees(alt1) azim1 = (90 - np.degrees(azim1)) % 360 # Get the last point on the great circle phase2 = GreatCircle.greatCirclePhase(np.radians(90 - elev[-1]), np.radians((90 - azim[-1])%360),\ theta0, phi0) alt2, azim2 = Math.cartesianToPolar( *GreatCircle.greatCircle(phase2, theta0, phi0)) alt2 = 90 - np.degrees(alt2) azim2 = (90 - np.degrees(azim2)) % 360 # Compute RA/Dec from Alt/Az _, ra1, dec1 = RMS.Astrometry.ApplyAstrometry.altAzToRADec(pp.lat, pp.lon, pp.UT_corr, [dt1], \ [azim1], [alt1], dt_time=True) _, ra2, dec2 = RMS.Astrometry.ApplyAstrometry.altAzToRADec(pp.lat, pp.lon, pp.UT_corr, [dt2], \ [azim2], [alt2], dt_time=True) ### ### ufo_meteor_list.append([dt1, peak_mag, duration, azim1[0], alt1[0], azim2[0], alt2[0], \ ra1[0][0], dec1[0][0], ra2[0][0], dec2[0][0], cam_code, pp.lon, pp.lat, pp.elev, pp.UT_corr]) # Construct a file name for the UFO file, which is the FTPdetectinfo file without the FTPdetectinfo # part ufo_file_name = file_name.replace('FTPdetectinfo_', '').replace( '.txt', '') + '.csv' # Write the UFOorbit file UFOOrbit.writeUFOOrbit(dir_path, ufo_file_name, ufo_meteor_list)
action="store_true", help="""Show a histogram of stddevs of PSFs of all detected stars. """) # Parse the command line arguments cml_args = arg_parser.parse_args() ######################### # Load the config file config = cr.loadConfigFromDirectory(cml_args.config, cml_args.dir_path) # Get paths to every FF bin file in a directory ff_dir = os.path.abspath(cml_args.dir_path[0]) ff_list = [ ff_name for ff_name in os.listdir(ff_dir) if FFfile.validFFName(ff_name) ] # Check if there are any file in the directory if (len(ff_list) == None): print("No files found!") sys.exit() # Run extraction and save the resulting CALSTARS file star_list = extractStarsAndSave(config, ff_dir) fwhm_list = [] intensity_list = [] x_list = [] y_list = []
def FTPdetectinfo2UFOOrbitInput(dir_path, file_name, platepar_path, platepar_dict=None): """ Convert the FTPdetectinfo file into UFOOrbit input CSV file. Arguments: dir_path: [str] Path of the directory which contains the FTPdetectinfo file. file_name: [str] Name of the FTPdetectinfo file. platepar_path: [str] Full path to the platepar file. Keyword arguments: platepar_dict: [dict] Dictionary of Platepar instances where keys are FF file names. This will be used instead of the platepar at platepar_path. None by default. """ # Load the FTPdetecinfo file meteor_list = FTPdetectinfo.readFTPdetectinfo(dir_path, file_name) # Load the platepar file if platepar_dict is None: pp = RMS.Formats.Platepar.Platepar() pp.read(platepar_path) # Init the UFO format list ufo_meteor_list = [] # Go through every meteor in the list for meteor in meteor_list: ff_name, cam_code, meteor_No, n_segments, fps, hnr, mle, binn, px_fm, rho, phi, \ meteor_meas = meteor # Load the platepar from the platepar dictionary, if given if platepar_dict is not None: if ff_name in platepar_dict: pp = platepar_dict[ff_name] else: print('Skipping {:s} becuase no platepar was found for this FF file!'.format(ff_name)) # Convert the FF file name into time dt = FFfile.filenameToDatetime(ff_name) # Extract measurements calib_status, frame_n, x, y, ra, dec, azim, elev, inten, mag = np.array(meteor_meas).T # If the meteor wasn't calibrated, skip it if not np.all(calib_status): print('Meteor {:d} was not calibrated, skipping it...'.format(meteor_No)) continue # Compute the peak magnitude peak_mag = np.min(mag) # Compute the total duration first_frame = np.min(frame_n) last_frame = np.max(frame_n) duration = (last_frame - first_frame)/fps # Compute times of first and last points dt1 = dt + datetime.timedelta(seconds=first_frame/fps) dt2 = dt + datetime.timedelta(seconds=last_frame/fps) ### Fit a great circle to Az/Alt measurements and compute model beg/end RA and Dec ### # Convert the measurement Az/Alt to cartesian coordinates # NOTE: All values that are used for Great Circle computation are: # theta - the zenith angle (90 deg - altitude) # phi - azimuth +N of due E, which is (90 deg - azim) x, y, z = Math.polarToCartesian(np.radians((90 - azim)%360), np.radians(90 - elev)) # Fit a great circle C, theta0, phi0 = GreatCircle.fitGreatCircle(x, y, z) # Get the first point on the great circle phase1 = GreatCircle.greatCirclePhase(np.radians(90 - elev[0]), np.radians((90 - azim[0])%360), \ theta0, phi0) alt1, azim1 = Math.cartesianToPolar(*GreatCircle.greatCircle(phase1, theta0, phi0)) alt1 = 90 - np.degrees(alt1) azim1 = (90 - np.degrees(azim1))%360 # Get the last point on the great circle phase2 = GreatCircle.greatCirclePhase(np.radians(90 - elev[-1]), np.radians((90 - azim[-1])%360),\ theta0, phi0) alt2, azim2 = Math.cartesianToPolar(*GreatCircle.greatCircle(phase2, theta0, phi0)) alt2 = 90 - np.degrees(alt2) azim2 = (90 - np.degrees(azim2))%360 # Compute RA/Dec from Alt/Az _, ra1, dec1 = RMS.Astrometry.ApplyAstrometry.altAzToRADec(pp.lat, pp.lon, pp.UT_corr, [dt1], \ [azim1], [alt1], dt_time=True) _, ra2, dec2 = RMS.Astrometry.ApplyAstrometry.altAzToRADec(pp.lat, pp.lon, pp.UT_corr, [dt2], \ [azim2], [alt2], dt_time=True) ### ### ufo_meteor_list.append([dt1, peak_mag, duration, azim1[0], alt1[0], azim2[0], alt2[0], \ ra1[0][0], dec1[0][0], ra2[0][0], dec2[0][0], cam_code, pp.lon, pp.lat, pp.elev, pp.UT_corr]) # Construct a file name for the UFO file, which is the FTPdetectinfo file without the FTPdetectinfo # part ufo_file_name = file_name.replace('FTPdetectinfo_', '').replace('.txt', '') + '.csv' # Write the UFOorbit file UFOOrbit.writeUFOOrbit(dir_path, ufo_file_name, ufo_meteor_list)
def extractStars(ff_dir, ff_name, config=None, max_global_intensity=150, border=10, neighborhood_size=10, intensity_threshold=5, flat_struct=None, dark=None, mask=None): """ Extracts stars on a given FF bin by searching for local maxima and applying PSF fit for star confirmation. Source of one part of the code: http://stackoverflow.com/questions/9111711/get-coordinates-of-local-maxima-in-2d-array-above-certain-value Arguments: ff_dir: [str] Path to directory where FF files are. ff_name: [str] Name of the FF file. config: [config object] configuration object (loaded from the .config file) max_global_intensity: [int] maximum mean intensity of an image before it is discared as too bright border: [int] apply a mask on the detections by removing all that are too close to the given image border (in pixels) neighborhood_size: [int] size of the neighbourhood for the maximum search (in pixels) intensity_threshold: [float] a threshold for cutting the detections which are too faint (0-255) flat_struct: [Flat struct] Structure containing the flat field. None by default. dark: [ndarray] Dark frame. None by default. mask: [ndarray] Mask image. None by default. Return: x2, y2, background, intensity, fwhm: [list of ndarrays] - x2: X axis coordinates of the star - y2: Y axis coordinates of the star - background: background intensity - intensity: intensity of the star - Gaussian Full width at half maximum (FWHM) of fitted stars """ # This will be returned if there was an error error_return = [[], [], [], [], [], []] # Load parameters from config if given if config: max_global_intensity = config.max_global_intensity border = config.border neighborhood_size = config.neighborhood_size intensity_threshold = config.intensity_threshold # Load the FF bin file ff = FFfile.read(ff_dir, ff_name) # If the FF file could not be read, skip star extraction if ff is None: return error_return # Apply the dark frame if dark is not None: ff.avepixel = Image.applyDark(ff.avepixel, dark) # Apply the flat if flat_struct is not None: ff.avepixel = Image.applyFlat(ff.avepixel, flat_struct) # Mask the FF file if mask is not None: ff = MaskImage.applyMask(ff, mask, ff_flag=True) # Calculate image mean and stddev global_mean = np.mean(ff.avepixel) # Check if the image is too bright and skip the image if global_mean > max_global_intensity: return error_return data = ff.avepixel.astype(np.float32) # Apply a mean filter to the image to reduce noise data = ndimage.filters.convolve(data, weights=np.full((2, 2), 1.0 / 4)) # Locate local maxima on the image data_max = filters.maximum_filter(data, neighborhood_size) maxima = (data == data_max) data_min = filters.minimum_filter(data, neighborhood_size) diff = ((data_max - data_min) > intensity_threshold) maxima[diff == 0] = 0 # Apply a border mask border_mask = np.ones_like(maxima) * 255 border_mask[:border, :] = 0 border_mask[-border:, :] = 0 border_mask[:, :border] = 0 border_mask[:, -border:] = 0 maxima = MaskImage.applyMask(maxima, border_mask, image=True) # Remove all detections close to the mask image if mask is not None: erosion_kernel = np.ones((5, 5), mask.img.dtype) mask_eroded = cv2.erode(mask.img, erosion_kernel, iterations=1) maxima = MaskImage.applyMask(maxima, mask_eroded, image=True) # Find and label the maxima labeled, num_objects = ndimage.label(maxima) # Skip the image if there are too many maxima to process if num_objects > config.max_stars: print('Too many candidate stars to process! {:d}/{:d}'.format( num_objects, config.max_stars)) return error_return # Find centres of mass of each labeled objects xy = np.array( ndimage.center_of_mass(data, labeled, range(1, num_objects + 1))) # Remove all detection on the border #xy = xy[np.where((xy[:, 1] > border) & (xy[:,1] < ff.ncols - border) & (xy[:,0] > border) & (xy[:,0] < ff.nrows - border))] # Unpack star coordinates y, x = np.hsplit(xy, 2) # # Plot stars before the PSF fit # plotStars(ff, x, y) # Fit a PSF to each star x2, y2, amplitude, intensity, sigma_y_fitted, sigma_x_fitted = fitPSF( ff, global_mean, x, y, config) # x2, y2, amplitude, intensity = list(x), list(y), [], [] # Skip PSF fit # # Plot stars after PSF fit filtering # plotStars(ff, x2, y2) # Compute FWHM from one dimensional sigma sigma_x_fitted = np.array(sigma_x_fitted) sigma_y_fitted = np.array(sigma_y_fitted) sigma_fitted = np.sqrt(sigma_x_fitted**2 + sigma_y_fitted**2) fwhm = 2.355 * sigma_fitted return ff_name, x2, y2, amplitude, intensity, fwhm
def recalibratePlateparsForFF( prev_platepar, ff_file_names, calstars, catalog_stars, config, lim_mag=None, ignore_distance_threshold=False, ): """ Recalibrate platepars corresponding to ff files based on the stars. Arguments: prev_platepar: [platepar] ff_file_names: [list] list of ff file names calstars: [dict] A dictionary with only one entry, where the key is 'jd' and the value is the list of star coordinates. catalog_stars: [list] A list of entries [[ff_name, star_coordinates], ...]. config: [config] Keyword arguments: lim_mag: [float] ignore_distance_threshold: [bool] Don't consider the recalib as failed if the median distance is larger than the threshold. Returns: recalibrated_platepars: [dict] A dictionary where one key is ff file name and the value is a calibrated corresponding platepar. """ # Go through all FF files with detections, recalibrate and apply astrometry recalibrated_platepars = {} for ff_name in ff_file_names: working_platepar = copy.deepcopy(prev_platepar) # Skip this meteor if its FF file was already recalibrated if ff_name in recalibrated_platepars: continue print() print('Processing: ', ff_name) print( '------------------------------------------------------------------------------' ) # Find extracted stars on this image if not ff_name in calstars: print('Skipped because it was not in CALSTARS:', ff_name) continue # Get stars detected on this FF file (create a dictionaly with only one entry, the residuals function # needs this format) calstars_time = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True) jd = date2JD(*calstars_time) star_dict_ff = {jd: calstars[ff_name]} result = None # Skip recalibration if less than a minimum number of stars were detected if (len(calstars[ff_name]) >= config.ff_min_stars) and (len( calstars[ff_name]) >= config.min_matched_stars): # Recalibrate the platepar using star matching result, min_match_radius = recalibrateFF( config, working_platepar, jd, star_dict_ff, catalog_stars, lim_mag=lim_mag, ignore_distance_threshold=ignore_distance_threshold, ) # If the recalibration failed, try using FFT alignment if result is None: print() print('Running FFT alignment...') # Run FFT alignment calstars_coords = np.array(star_dict_ff[jd])[:, :2] calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]] print(calstars_time) test_platepar = alignPlatepar(config, prev_platepar, calstars_time, calstars_coords, show_plot=False) # Try to recalibrate after FFT alignment result, _ = recalibrateFF(config, test_platepar, jd, star_dict_ff, catalog_stars, lim_mag=lim_mag) # If the FFT alignment failed, align the original platepar using the smallest radius that matched # and force save the the platepar if (result is None) and (min_match_radius is not None): print() print( "Using the old platepar with the minimum match radius of: {:.2f}" .format(min_match_radius)) result, _ = recalibrateFF( config, working_platepar, jd, star_dict_ff, catalog_stars, max_match_radius=min_match_radius, force_platepar_save=True, lim_mag=lim_mag, ) if result is not None: working_platepar = result # If the alignment succeeded, save the result else: working_platepar = result else: working_platepar = result # Store the platepar if the fit succeeded if result is not None: # Recompute alt/az of the FOV centre working_platepar.az_centre, working_platepar.alt_centre = raDec2AltAz( working_platepar.RA_d, working_platepar.dec_d, working_platepar.JD, working_platepar.lat, working_platepar.lon, ) # Recompute the rotation wrt horizon working_platepar.rotation_from_horiz = rotationWrtHorizon( working_platepar) # Mark the platepar to indicate that it was automatically recalibrated on an individual FF file working_platepar.auto_recalibrated = True recalibrated_platepars[ff_name] = working_platepar prev_platepar = working_platepar else: print( 'Recalibration of {:s} failed, using the previous platepar...'. format(ff_name)) # Mark the platepar to indicate that autorecalib failed prev_platepar_tmp = copy.deepcopy(prev_platepar) prev_platepar_tmp.auto_recalibrated = False # If the aligning failed, set the previous platepar as the one that should be used for this FF file recalibrated_platepars[ff_name] = prev_platepar_tmp return recalibrated_platepars
def extractStars(ff_dir, ff_name, config=None, max_global_intensity=150, border=10, neighborhood_size=10, intensity_threshold=5, flat_struct=None): """ Extracts stars on a given FF bin by searching for local maxima and applying PSF fit for star confirmation. Source of one part of the code: http://stackoverflow.com/questions/9111711/get-coordinates-of-local-maxima-in-2d-array-above-certain-value Arguments: ff: [ff bin struct] FF bin file loaded in the FF bin structure config: [config object] configuration object (loaded from the .config file) max_global_intensity: [int] maximum mean intensity of an image before it is discared as too bright border: [int] apply a mask on the detections by removing all that are too close to the given image border (in pixels) neighborhood_size: [int] size of the neighbourhood for the maximum search (in pixels) intensity_threshold: [float] a threshold for cutting the detections which are too faint (0-255) flat_struct: [Flat struct] Structure containing the flat field. None by default. Return: x2, y2, background, intensity: [list of ndarrays] - x2: X axis coordinates of the star - y2: Y axis coordinates of the star - background: background intensity - intensity: intensity of the star """ # Load parameters from config if given if config: max_global_intensity = config.max_global_intensity border = config.border neighborhood_size = config.neighborhood_size intensity_threshold = config.intensity_threshold # Load the FF bin file ff = FFfile.read(ff_dir, ff_name) # Load the mask file mask = MaskImage.loadMask(config.mask_file) # Mask the FF file ff = MaskImage.applyMask(ff, mask, ff_flag=True) # Apply the flat to maxpixel and avepixel if flat_struct is not None: ff.maxpixel = Image.applyFlat(ff.maxpixel, flat_struct) ff.avepixel = Image.applyFlat(ff.avepixel, flat_struct) # Calculate image mean and stddev global_mean = np.mean(ff.avepixel) # Check if the image is too bright and skip the image if global_mean > max_global_intensity: return [[], [], [], []] data = ff.avepixel.astype(np.float32) # Apply a mean filter to the image to reduce noise data = ndimage.filters.convolve(data, weights=np.full((2, 2), 1.0 / 4)) # Locate local maxima on the image data_max = filters.maximum_filter(data, neighborhood_size) maxima = (data == data_max) data_min = filters.minimum_filter(data, neighborhood_size) diff = ((data_max - data_min) > intensity_threshold) maxima[diff == 0] = 0 # Apply a border mask border_mask = np.ones_like(maxima) * 255 border_mask[:border, :] = 0 border_mask[-border:, :] = 0 border_mask[:, :border] = 0 border_mask[:, -border:] = 0 maxima = MaskImage.applyMask(maxima, (True, border_mask)) # Find and label the maxima labeled, num_objects = ndimage.label(maxima) # Skip the image if there are too many maxima to process if num_objects > config.max_stars: print('Too many candidate stars to process! {:d}/{:d}'.format( num_objects, config.max_stars)) return [[], [], [], []] # Find centres of mass of each labeled objects xy = np.array( ndimage.center_of_mass(data, labeled, range(1, num_objects + 1))) # Remove all detection on the border #xy = xy[np.where((xy[:, 1] > border) & (xy[:,1] < ff.ncols - border) & (xy[:,0] > border) & (xy[:,0] < ff.nrows - border))] # Unpack star coordinates y, x = np.hsplit(xy, 2) # # Plot stars before the PSF fit # plotStars(ff, x, y) # Fit a PSF to each star x2, y2, amplitude, intensity = fitPSF(ff, global_mean, x, y, config=config) # x2, y2, amplitude, intensity = list(x), list(y), [], [] # Skip PSF fit # # Plot stars after PSF fit filtering # plotStars(ff, x2, y2) return x2, y2, amplitude, intensity
def view(dir_path, ff_path, fr_path, config, save_frames=False): """ Shows the detected fireball stored in the FR file. Arguments: dir_path: [str] Current directory. ff: [str] path to the FF bin file fr: [str] path to the FR bin file config: [conf object] configuration structure """ name = fr_path fr = FRbin.read(dir_path, fr_path) if ff_path is None: #background = np.zeros((config.height, config.width), np.uint8) # Get the maximum extent of the meteor frames y_size = max( max(np.array(fr.yc[0]) + np.array(fr.size[0]) // 2) for i in range(fr.lines)) x_size = max( max(np.array(fr.xc[0]) + np.array(fr.size[0]) // 2) for i in range(fr.lines)) # Make the image square img_size = max(y_size, x_size) background = np.zeros((img_size, img_size), np.uint8) else: background = FFfile.read(dir_path, ff_path).maxpixel print("Number of lines:", fr.lines) first_image = True for current_line in range(fr.lines): print('Frame, Y , X , size') for z in range(fr.frameNum[current_line]): # Get the center position of the detection on the current frame yc = fr.yc[current_line][z] xc = fr.xc[current_line][z] # Get the frame number t = fr.t[current_line][z] # Get the size of the window size = fr.size[current_line][z] print(" {:3d}, {:3d}, {:3d}, {:d}".format(t, yc, xc, size)) img = np.copy(background) # Paste the frames onto the big image y_img = np.arange(yc - size // 2, yc + size // 2) x_img = np.arange(xc - size // 2, xc + size // 2) Y_img, X_img = np.meshgrid(y_img, x_img) y_frame = np.arange(len(y_img)) x_frame = np.arange(len(x_img)) Y_frame, X_frame = np.meshgrid(y_frame, x_frame) img[Y_img, X_img] = fr.frames[current_line][z][Y_frame, X_frame] # Save frame to disk if save_frames: frame_file_name = fr_path.replace( '.bin', '') + "_frame_{:03d}.png".format(t) cv2.imwrite(os.path.join(dir_path, frame_file_name), img) # Show the frame cv2.imshow(name, img) # If this is the first image, move it to the upper left corner if first_image: cv2.moveWindow(name, 0, 0) first_image = False cv2.waitKey(2 * int(1000.0 / config.fps)) cv2.destroyWindow(name)
if __name__ == "__main__": time_start = time.clock() # Load config file config = cr.parse(".config") if not len(sys.argv) == 2: print("Usage: python -m RMS.ExtractStars /path/to/FF/files/") sys.exit() # Get paths to every FF bin file in a directory ff_dir = os.path.abspath(sys.argv[1]) ff_list = [ ff_name for ff_name in os.listdir(ff_dir) if FFfile.validFFName(ff_name) ] # Check if there are any file in the directory if (len(ff_list) == None): print("No files found!") sys.exit() # Try loading a flat field image flat_struct = None if config.use_flat: # Check if there is flat in the data directory if os.path.exists(os.path.join(ff_dir, config.flat_file)): flat_struct = Image.loadFlat(ff_dir, config.flat_file)
def view(dir_path, ff_path, fr_path, config): """ Shows the detected fireball stored in the FR file. Arguments: dir_path: [str] Current directory. ff: [str] path to the FF bin file fr: [str] path to the FR bin file config: [conf object] configuration structure """ name = fr_path fr = FRbin.read(dir_path, fr_path) if ff_path is None: #background = np.zeros((config.height, config.width), np.uint8) # Get the maximum extent of the meteor frames y_size = max(max(np.array(fr.yc[i]) + np.array(fr.size[i])//2) for i in range(fr.lines)) x_size = max(max(np.array(fr.xc[i]) + np.array(fr.size[i])//2) for i in range(fr.lines)) # Make the image square img_size = max(y_size, x_size) background = np.zeros((img_size, img_size), np.uint8) else: background = FFfile.read(dir_path, ff_path).maxpixel print("Number of lines:", fr.lines) first_image = True for i in range(fr.lines): print('Frame, Y , X , size') for z in range(fr.frameNum[i]): # Get the center position of the detection on the current frame yc = fr.yc[i][z] xc = fr.xc[i][z] # Get the frame number t = fr.t[i][z] # Get the size of the window size = fr.size[i][z] print(" {:3d}, {:3d}, {:3d}, {:d}".format(t, yc, xc, size)) y2 = 0 # Assign the detection pixels to the background image for y in range(yc - size//2, yc + size//2): x2 = 0 for x in range(xc - size//2, xc + size//2): background[y, x] = fr.frames[i][z][y2, x2] x2 += 1 y2 += 1 cv2.imshow(name, background) # If this is the first image, move it to the upper left corner if first_image: cv2.moveWindow(name, 0, 0) first_image = False cv2.waitKey(2*int(1000.0/config.fps)) cv2.destroyWindow(name)
def generateThumbnails(dir_path, config, mosaic_type, file_list=None): """ Generates a mosaic of thumbnails from all FF files in the given folder and saves it as a JPG image. Arguments: dir_path: [str] Path of the night directory. config: [Conf object] Configuration. mosaic_type: [str] Type of the mosaic (e.g. "Captured" or "Detected") Keyword arguments: file_list: [list] A list of file names (without full path) which will be searched for FF files. This is used when generating separate thumbnails for captured and detected files. Return: file_name: [str] Name of the thumbnail file. """ if file_list is None: file_list = sorted(os.listdir(dir_path)) # Make a list of all FF files in the night directory ff_list = [] for file_name in file_list: if FFfile.validFFName(file_name): ff_list.append(file_name) # Calculate the dimensions of the binned image bin_w = int(config.width/config.thumb_bin) bin_h = int(config.height/config.thumb_bin) ### RESIZE AND STACK THUMBNAILS ### ########################################################################################################## timestamps = [] stacked_imgs = [] for i in range(0, len(ff_list), config.thumb_stack): img_stack = np.zeros((bin_h, bin_w)) # Stack thumb_stack images using the 'if lighter' method for j in range(config.thumb_stack): if (i + j) < len(ff_list): # Read maxpixel image img = FFfile.read(dir_path, ff_list[i + j]).maxpixel # Resize the image img = cv2.resize(img, (bin_w, bin_h)) # Stack the image img_stack = stackIfLighter(img_stack, img) else: break # Save the timestamp of the first image in the stack timestamps.append(FFfile.filenameToDatetime(ff_list[i])) # Save the stacked image stacked_imgs.append(img_stack) # cv2.imshow('test', img_stack) # cv2.waitKey(0) # cv2.destroyAllWindows() ########################################################################################################## ### ADD THUMBS TO ONE MOSAIC IMAGE ### ########################################################################################################## header_height = 20 timestamp_height = 10 # Calculate the number of rows for the thumbnail image n_rows = int(np.ceil(float(len(ff_list))/config.thumb_stack/config.thumb_n_width)) # Calculate the size of the mosaic mosaic_w = int(config.thumb_n_width*bin_w) mosaic_h = int((bin_h + timestamp_height)*n_rows + header_height) mosaic_img = np.zeros((mosaic_h, mosaic_w), dtype=np.uint8) # Write header text header_text = 'Station: ' + str(config.stationID) + ' Night: ' + os.path.basename(dir_path) \ + ' Type: ' + mosaic_type cv2.putText(mosaic_img, header_text, (0, header_height//2), \ cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255,255,255), 1) for row in range(n_rows): for col in range(config.thumb_n_width): # Calculate image index indx = row*config.thumb_n_width + col if indx < len(stacked_imgs): # Calculate position of the text text_x = col*bin_w text_y = row*bin_h + (row + 1)*timestamp_height - 1 + header_height # Add timestamp text cv2.putText(mosaic_img, timestamps[indx].strftime('%H:%M:%S'), (text_x, text_y), \ cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1) # Add the image to the mosaic img_pos_x = col*bin_w img_pos_y = row*bin_h + (row + 1)*timestamp_height + header_height mosaic_img[img_pos_y : img_pos_y + bin_h, img_pos_x : img_pos_x + bin_w] = stacked_imgs[indx] else: break ########################################################################################################## thumb_name = "{:s}_{:s}_{:s}_thumbs.jpg".format(str(config.stationID), os.path.basename(dir_path), \ mosaic_type) # Save the mosaic cv2.imwrite(os.path.join(dir_path, thumb_name), mosaic_img, [int(cv2.IMWRITE_JPEG_QUALITY), 80]) return thumb_name
plot(points, ff.nrows//config.f, ff.ncols//config.f) def plot(points, y_dim, x_dim): fig = plt.figure() ax = fig.add_subplot(111, projection='3d') plt.title(name) y = points[:,0] x = points[:,1] z = points[:,2] # Plot points in 3D ax.scatter(x, y, z) # Set axes limits ax.set_zlim(0, 255) plt.xlim([0, x_dim]) plt.ylim([0, y_dim]) ax.set_ylabel("Y") ax.set_xlabel("X") ax.set_zlabel("Time") plt.show() if __name__ == "__main__": ff = FFfile.read(sys.argv[1], sys.argv[2], array=True) view(ff)
def detectMeteors(ff_directory, ff_name, config, flat_struct=None): """ Detect meteors on the given FF bin image. Here are the steps in the detection: - input image (FF bin format file) is thresholded (converted to black and white) - several morphological operations are applied to clean the image - image is then broken into several image "windows" (these "windows" are reconstructed from the input FF file, given an input frame range (e.g. 64-128) which helps reduce the noise further) - on each "window" the Kernel-based Hough transform is performed to find any lines on the image - similar lines are joined - stripe around the lines is extracted - 3D line finding (third dimension is time) is applied to check if the line propagates in time - centroiding is performed, which calculates the position and intensity of meteor on each frame Arguments: ff_directory: [string] an absolute path to the input FF bin file ff_name: [string] file name of the FF bin file on which to run the detection on config: [config object] configuration object (loaded from the .config file) Keyword arguments: flat_struct: [Flat struct] Structure containing the flat field. None by default. Return: meteor_detections: [list] a list of detected meteors, with these elements: - rho: [float] meteor line distance from image center (polar coordinates, in pixels) - theta: [float] meteor line angle from image center (polar coordinates, in degrees) - centroids: [list] [frame, X, Y, level] list of meteor points """ t1 = time() t_all = time() # Load the FF bin file ff = FFfile.read(ff_directory, ff_name) # Load the mask file mask = MaskImage.loadMask(config.mask_file) # Mask the FF file ff = MaskImage.applyMask(ff, mask, ff_flag=True) # Apply the flat to maxpixel and avepixel if flat_struct is not None: ff.maxpixel = Image.applyFlat(ff.maxpixel, flat_struct) ff.avepixel = Image.applyFlat(ff.avepixel, flat_struct) # At the end, a check that the detection has a surface brightness above the background will be performed. # The assumption here is that the peak of the meteor should have the intensity which is at least # that of a patch of 4x4 pixels that are of the mean background brightness min_patch_intensity = 4 * 4 * (np.mean(ff.maxpixel - ff.avepixel) + config.k1_det * np.mean(ff.stdpixel) + config.j1) # # Show the maxpixel image # show2(ff_name+' maxpixel', ff.maxpixel) # Get lines on the image line_list = getLines(ff, config.k1_det, config.j1_det, config.time_slide, config.time_window_size, config.max_lines_det, config.max_white_ratio, config.kht_lib_path) logDebug('List of lines:', line_list) # Init meteor list meteor_detections = [] # Only if there are some lines in the image if len(line_list): # Join similar lines line_list = mergeLines(line_list, config.line_min_dist, ff.ncols, ff.nrows) logDebug('Time for finding lines:', time() - t1) logDebug('Number of KHT lines: ', len(line_list)) logDebug(line_list) # Plot lines # plotLines(ff, line_list) # Threshold the image img_thres = thresholdImg(ff, config.k1_det, config.j1_det) filtered_lines = [] # Analyze stripes of each line for line in line_list: rho, theta, frame_min, frame_max = line logDebug('rho, theta, frame_min, frame_max') logDebug(rho, theta, frame_min, frame_max) # Bounded the thresholded image by min and max frames img = selectFrames(np.copy(img_thres), ff, frame_min, frame_max) # Remove lonely pixels img = morph.clean(img) # Get indices of stripe pixels around the line stripe_indices = getStripeIndices(rho, theta, config.stripe_width, img.shape[0], img.shape[1]) # Extract the stripe from the thresholded image stripe = np.zeros((ff.nrows, ff.ncols), np.uint8) stripe[stripe_indices] = img[stripe_indices] # Show stripe #COMMENTED # show2("stripe", stripe*255) # Show 3D could # show3DCloud(ff, stripe) # Get stripe positions stripe_positions = stripe.nonzero() xs = stripe_positions[1] ys = stripe_positions[0] zs = ff.maxframe[stripe_positions] # Limit the number of points to search if too large if len(zs) > config.max_points_det: # Extract weights of each point maxpix_elements = ff.maxpixel[ys, xs].astype(np.float64) weights = maxpix_elements / np.sum(maxpix_elements) # Random sample the point, sampling is weighted by pixel intensity indices = np.random.choice(len(zs), config.max_points_det, replace=False, p=weights) ys = ys[indices] xs = xs[indices] zs = zs[indices] # Make an array to feed into the gropuing algorithm stripe_points = np.vstack((xs, ys, zs)) stripe_points = np.swapaxes(stripe_points, 0, 1) # Sort stripe points by frame stripe_points = stripe_points[stripe_points[:, 2].argsort()] t1 = time() logDebug('finding lines...') # Find a single line in the point cloud detected_line = find3DLines(stripe_points, time(), config, fireball_detection=False) logDebug('time for GROUPING: ', time() - t1) # Extract the first and only line if any if detected_line: detected_line = detected_line[0] # logDebug(detected_line) # Show 3D cloud # show3DCloud(ff, stripe, detected_line, stripe_points, config) # Add the line to the results list filtered_lines.append(detected_line) # Merge similar lines in 3D filtered_lines = merge3DLines(filtered_lines, config.vect_angle_thresh) logDebug('after filtering:') logDebug(filtered_lines) for detected_line in filtered_lines: # Get frame range frame_min = detected_line[4] frame_max = detected_line[5] # Check if the line covers a minimum frame range if (abs(frame_max - frame_min) + 1 < config.line_minimum_frame_range_det): continue # Extand the frame range for several frames, just to be sure to catch all parts of a meteor frame_min -= config.frame_extension frame_max += config.frame_extension # Cap values to 0-255 frame_min = max(frame_min, 0) frame_max = min(frame_max, 255) logDebug(detected_line) # Get coordinates of 2 points that describe the line x1, y1, z1 = detected_line[0] x2, y2, z2 = detected_line[1] # Convert Cartesian line coordinates to polar rho, theta = getPolarLine(x1, y1, x2, y2, ff.nrows, ff.ncols) # Convert Cartesian line coordinate to CAMS compatible polar coordinates (flipped Y axis) rho_cams, theta_cams = getPolarLine(x1, ff.nrows - y1, x2, ff.nrows - y2, ff.nrows, ff.ncols) logDebug('converted rho, theta') logDebug(rho, theta) # Bounded the thresholded image by min and max frames img = selectFrames(np.copy(img_thres), ff, frame_min, frame_max) # Remove lonely pixels img = morph.clean(img) # Get indices of stripe pixels around the line stripe_indices = getStripeIndices(rho, theta, int(config.stripe_width * 1.5), img.shape[0], img.shape[1]) # Extract the stripe from the thresholded image stripe = np.zeros((ff.nrows, ff.ncols), np.uint8) stripe[stripe_indices] = img[stripe_indices] # Show detected line # show('detected line: '+str(frame_min)+'-'+str(frame_max), stripe) # Get stripe positions stripe_positions = stripe.nonzero() xs = stripe_positions[1] ys = stripe_positions[0] zs = ff.maxframe[stripe_positions] # Make an array to feed into the centroiding algorithm stripe_points = np.vstack((xs, ys, zs)) stripe_points = np.swapaxes(stripe_points, 0, 1) # Sort stripe points by frame stripe_points = stripe_points[stripe_points[:, 2].argsort()] # Show 3D cloud # show3DCloud(ff, stripe, detected_line, stripe_points, config) # Get points of the given line line_points = getAllPoints(stripe_points, x1, y1, z1, x2, y2, z2, config, fireball_detection=False) # Skip if no points were returned if not line_points.any(): continue # Skip if the points cover too small a frame range if abs(np.max(line_points[:, 2]) - np.min(line_points[:, 2]) ) + 1 < config.line_minimum_frame_range_det: continue # Calculate centroids centroids = [] for i in range(frame_min, frame_max + 1): # Select pixel indicies belonging to a given frame frame_pixels_inds = np.where(line_points[:, 2] == i) # Get pixel positions in a given frame (pixels belonging to a found line) frame_pixels = line_points[frame_pixels_inds].astype(np.int64) # Get pixel positions in a given frame (pixels belonging to the whole stripe) frame_pixels_stripe = stripe_points[np.where( stripe_points[:, 2] == i)].astype(np.int64) # Skip if there are no pixels in the frame if not len(frame_pixels): continue # Calculate weights for centroiding max_avg_corrected = ff.maxpixel - ff.avepixel flattened_weights = (max_avg_corrected).astype( np.float32) / ff.stdpixel # Calculate centroids by half-frame for half_frame in range(2): # Apply deinterlacing if it is present in the video if config.deinterlace_order >= 0: # Deinterlace by fields (line lixels) half_frame_pixels = frame_pixels[ frame_pixels[:, 1] % 2 == (config.deinterlace_order + half_frame) % 2] # Deinterlace by fields (stripe pixels) half_frame_pixels_stripe = frame_pixels_stripe[ frame_pixels_stripe[:, 1] % 2 == (config.deinterlace_order + half_frame) % 2] # Skip if there are no pixels in the half-frame if not len(half_frame_pixels): continue # Calculate half-frame value frame_no = i + half_frame * 0.5 # No deinterlacing else: # Skip the second half frame if half_frame == 1: continue half_frame_pixels = frame_pixels half_frame_pixels_stripe = frame_pixels_stripe frame_no = i # Get maxpixel-avepixel values of given pixel indices (this will be used as weights) max_weights = flattened_weights[half_frame_pixels[:, 1], half_frame_pixels[:, 0]] # Calculate weighted centroids x_weighted = half_frame_pixels[:, 0] * np.transpose( max_weights) x_centroid = np.sum(x_weighted) / float( np.sum(max_weights)) y_weighted = half_frame_pixels[:, 1] * np.transpose( max_weights) y_centroid = np.sum(y_weighted) / float( np.sum(max_weights)) # Calculate intensity as the sum of threshold passer pixels on the stripe #intensity_values = max_avg_corrected[half_frame_pixels[:,1], half_frame_pixels[:,0]] intensity_values = max_avg_corrected[ half_frame_pixels_stripe[:, 1], half_frame_pixels_stripe[:, 0]] intensity = np.sum(intensity_values) logDebug("centroid: ", frame_no, x_centroid, y_centroid, intensity) centroids.append( [frame_no, x_centroid, y_centroid, intensity]) # Filter centroids centroids = filterCentroids(centroids, config.centroids_max_deviation, config.centroids_max_distance) # Convert to numpy array for easy slicing centroids = np.array(centroids) # Reject the solution if there are too few centroids if len(centroids) < config.line_minimum_frame_range_det: continue # Check that the detection has a surface brightness above the background # The assumption here is that the peak of the meteor should have the intensity which is at least # that of a patch of 4x4 pixels that are of the mean background brightness if np.max(centroids[:, 3]) < min_patch_intensity: continue # Check the detection if it has the proper angular velocity if not checkAngularVelocity(centroids, config): continue # Append the result to the meteor detections meteor_detections.append([rho_cams, theta_cams, centroids]) logDebug('time for processing:', time() - t_all) # # Plot centroids to image # fig, (ax1, ax2) = plt.subplots(nrows=2) # ax1.imshow(ff.maxpixel - ff.avepixel, cmap='gray') # ax1.scatter(centroids[:,1], centroids[:,2], s=5, c='r', edgecolors='none') # # Plot lightcurve # ax2.plot(centroids[:,0], centroids[:,3]) # # # Plot relative angular velocity # # ang_vels = [] # # fr_prev, x_prev, y_prev, _ = centroids[0] # # for fr, x, y, _ in centroids[1:]: # # dx = x - x_prev # # dy = y - y_prev # # dfr = fr - fr_prev # # ddist = np.sqrt(dx**2 + dy**2) # # dt = dfr/config.fps # # ang_vels.append(ddist/dt) # # x_prev = x # # y_prev = y # # fr_prev = fr # # ax2.plot(ang_vels) # plt.show() return meteor_detections
# Extract the directory name from the given argument bin_dir = sys.argv[1] # Load config file config = cr.parse(".config") print('Directory:', bin_dir) for ff_name in os.listdir(bin_dir): if 'FF' in ff_name: print(ff_name) # Load compressed file compressed = FFfile.read(bin_dir, ff_name, array=True, full_filename=True).array # Show maxpixel ff = FFfile.read(bin_dir, ff_name, full_filename=True) plt.imshow(ff.maxpixel, cmap='gray') plt.show() plt.clf() plt.close() # Dummy frames (empty) frames = np.zeros( shape=(256, compressed.shape[1], compressed.shape[2]), dtype=np.uint8) + 255
def view(dir_path, ff_path, fr_path, config, save_frames=False, extract_format='png', hide=False): """ Shows the detected fireball stored in the FR file. Arguments: dir_path: [str] Current directory. ff: [str] path to the FF bin file fr: [str] path to the FR bin file config: [conf object] configuration structure Keyword arguments: save_frames: [bool] Save FR frames to disk. False by defualt. extract_format: [str] Format of saved images. png by default. hide: [bool] Don't show frames on the screen. """ if extract_format is None: extract_format = 'png' name = fr_path fr = FRbin.read(dir_path, fr_path) print('------------------------') print('Showing file:', fr_path) if ff_path is None: #background = np.zeros((config.height, config.width), np.uint8) # Get the maximum extent of meteor frames y_size = max([ max(np.array(fr.yc[i]) + np.array(fr.size[i]) // 2) for i in range(fr.lines) ]) x_size = max([ max(np.array(fr.xc[i]) + np.array(fr.size[i]) // 2) for i in range(fr.lines) ]) # Make the image square img_size = max(y_size, x_size) background = np.zeros((img_size, img_size), np.uint8) else: background = FFfile.read(dir_path, ff_path).maxpixel print("Number of lines:", fr.lines) first_image = True wait_time = 2 * int(1000.0 / config.fps) pause_flag = False for current_line in range(fr.lines): print('Frame, Y , X , size') for z in range(fr.frameNum[current_line]): # Get the center position of the detection on the current frame yc = fr.yc[current_line][z] xc = fr.xc[current_line][z] # Get the frame number t = fr.t[current_line][z] # Get the size of the window size = fr.size[current_line][z] print(" {:3d}, {:3d}, {:3d}, {:d}".format(t, yc, xc, size)) img = np.copy(background) # Paste the frames onto the big image y_img = np.arange(yc - size // 2, yc + size // 2) x_img = np.arange(xc - size // 2, xc + size // 2) Y_img, X_img = np.meshgrid(y_img, x_img) y_frame = np.arange(len(y_img)) x_frame = np.arange(len(x_img)) Y_frame, X_frame = np.meshgrid(y_frame, x_frame) img[Y_img, X_img] = fr.frames[current_line][z][Y_frame, X_frame] # Save frame to disk if save_frames: frame_file_name = fr_path.replace('.bin', '') \ + "_line_{:02d}_frame_{:03d}.{:s}".format(current_line, t, extract_format) cv2.imwrite(os.path.join(dir_path, frame_file_name), img) if not hide: # Show the frame cv2.imshow(name, img) # If this is the first image, move it to the upper left corner if first_image: cv2.moveWindow(name, 0, 0) first_image = False if pause_flag: wait_time = 0 else: wait_time = 2 * int(1000.0 / config.fps) # Space key: pause display. # 1: previous file. # 2: next line. # q: Quit. key = cv2.waitKey(wait_time) & 0xFF if key == ord("1"): cv2.destroyWindow(name) return -1 elif key == ord("2"): break elif key == ord(" "): # Pause/unpause video pause_flag = not pause_flag elif key == ord("q"): os._exit(0) if not hide: cv2.destroyWindow(name)
def autoCheckFit(config, platepar, calstars_list): """ Attempts to refine the astrometry fit with the given stars and and initial astrometry parameters. Arguments: config: [Config structure] platepar: [Platepar structure] Initial astrometry parameters. calstars_list: [list] A list containing stars extracted from FF files. See RMS.Formats.CALSTARS for more details. Return: (platepar, fit_status): platepar: [Platepar structure] Estimated/refined platepar. fit_status: [bool] True if fit was successfuly, False if not. """ # Convert the list to a dictionary calstars = {ff_file: star_data for ff_file, star_data in calstars_list} # Load catalog stars catalog_stars = StarCatalog.readStarCatalog(config.star_catalog_path, config.star_catalog_file, \ lim_mag=config.catalog_mag_limit, mag_band_ratios=config.star_catalog_band_ratios) # Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars star_dict = {} # Take only those files with enough stars on them for ff_name in calstars: stars_list = calstars[ff_name] # Check if there are enough stars on the image if len(stars_list) >= config.ff_min_stars: # Calculate the JD time of the FF file dt = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True) jd = date2JD(*dt) # Add the time and the stars to the dict star_dict[jd] = stars_list # There has to be a minimum of 200 FF files for star fitting, and only 100 will be subset if there are more if len(star_dict) < config.calstars_files_N: print('Not enough FF files in CALSTARS for ACF!') return platepar, False else: # Randomly choose calstars_files_N image files from the whole list rand_keys = random.sample(list(star_dict), config.calstars_files_N) star_dict = {key: star_dict[key] for key in rand_keys} # Calculate the total number of calibration stars used total_calstars = sum([len(star_dict[key]) for key in star_dict]) print('Total calstars:', total_calstars) if total_calstars < config.calstars_min_stars: print('Not enough calibration stars, need at least', config.calstars_min_stars) return platepar, False # A list of matching radiuses to try, pairs of [radius, fit_distorsion_flag] min_radius = 0.5 radius_list = [[10, False], [5, False], [3, False], [1.5, True], [min_radius, True]] # Calculate the function tolerance, so the desired precision can be reached (the number is calculated # in the same reagrd as the cost function) fatol = (config.dist_check_threshold** 2) / np.sqrt(len(star_dict) * config.min_matched_stars + 1) # Parameter estimation tolerance for angular values fov_w = platepar.X_res / platepar.F_scale xatol_ang = config.dist_check_threshold * fov_w / platepar.X_res ### If the initial match is good enough, do only quick recalibratoin ### # Match the stars and calculate the residuals n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \ min_radius, ret_nmatch=True) if n_matched >= config.calstars_files_N: # Check if the average distance with the tightest radius is close if avg_dist < config.dist_check_quick_threshold: # Use a reduced set of initial radius values radius_list = [[1.5, True], [min_radius, True]] ########## # Match increasingly smaller search radiia around image stars for i, (match_radius, fit_distorsion) in enumerate(radius_list): # Match the stars and calculate the residuals n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \ match_radius, ret_nmatch=True) print('Max radius:', match_radius) print('Initial values:') print(' Matched stars:', n_matched) print(' Average deviation:', avg_dist) # The initial number of matched stars has to be at least the number of FF imaages, otherwise it means # that the initial platepar is no good if n_matched < config.calstars_files_N: print( 'The total number of initially matched stars is too small! Please manually redo the plate or make sure there are enough calibration stars.' ) return platepar, False # Check if the platepar is good enough and do not estimate further parameters if checkFitGoodness(config, platepar, catalog_stars, star_dict, min_radius): # Print out notice only if the platepar is good right away if i == 0: print("Initial platepar is good enough!") return platepar, True # Initial parameters for the astrometric fit p0 = [ platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref, platepar.F_scale ] # Fit the astrometric parameters res = scipy.optimize.minimize(_calcImageResidualsAstro, p0, args=(config, platepar, catalog_stars, \ star_dict, match_radius), method='Nelder-Mead', \ options={'fatol': fatol, 'xatol': xatol_ang}) print(res) # If the fit was not successful, stop further fitting if not res.success: return platepar, False else: # If the fit was successful, use the new parameters from now on ra_ref, dec_ref, pos_angle_ref, F_scale = res.x platepar.RA_d = ra_ref platepar.dec_d = dec_ref platepar.pos_angle_ref = pos_angle_ref platepar.F_scale = F_scale # Check if the platepar is good enough and do not estimate further parameters if checkFitGoodness(config, platepar, catalog_stars, star_dict, min_radius): return platepar, True # Fit the lens distorsion parameters if fit_distorsion: # Fit the distortion parameters (X axis) res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.x_poly, args=(config, platepar,\ catalog_stars, star_dict, match_radius, 'x'), method='Nelder-Mead', \ options={'fatol': fatol, 'xatol': 0.1}) print(res) # If the fit was not successfull, stop further fitting if not res.success: return platepar, False else: platepar.x_poly = res.x # Check if the platepar is good enough and do not estimate further parameters if checkFitGoodness(config, platepar, catalog_stars, star_dict, min_radius): return platepar, True # Fit the distortion parameters (Y axis) res = scipy.optimize.minimize(_calcImageResidualsDistorsion, platepar.y_poly, args=(config, platepar,\ catalog_stars, star_dict, match_radius, 'y'), method='Nelder-Mead', \ options={'fatol': fatol, 'xatol': 0.1}) print(res) # If the fit was not successfull, stop further fitting if not res.success: return platepar, False else: platepar.y_poly = res.x # Match the stars and calculate the residuals n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \ star_dict, min_radius, ret_nmatch=True) print('FINAL SOLUTION with {:f} px:'.format(min_radius)) print('Matched stars:', n_matched) print('Average deviation:', avg_dist) return platepar, True
# Extract the directory name from the given argument bin_dir = sys.argv[1] # Load config file config = cr.parse(".config") print('Directory:', bin_dir) for ff_name in os.listdir(bin_dir): if 'FF' in ff_name: print(ff_name) # Load compressed file compressed = FFfile.read(bin_dir, ff_name, array=True, full_filename=True).array # Show maxpixel ff = FFfile.read(bin_dir, ff_name, full_filename=True) plt.imshow(ff.maxpixel, cmap='gray') plt.show() plt.clf() plt.close() # Dummy frames from FF file frames = FFfile.reconstruct(ff) # Add avepixel to all reconstructed frames frames += ff.avepixel
def generateThumbnails(dir_path, config, mosaic_type, file_list=None): """ Generates a mosaic of thumbnails from all FF files in the given folder and saves it as a JPG image. Arguments: dir_path: [str] Path of the night directory. config: [Conf object] Configuration. mosaic_type: [str] Type of the mosaic (e.g. "Captured" or "Detected") Keyword arguments: file_list: [list] A list of file names (without full path) which will be searched for FF files. This is used when generating separate thumbnails for captured and detected files. Return: file_name: [str] Name of the thumbnail file. """ if file_list is None: file_list = sorted(os.listdir(dir_path)) # Make a list of all FF files in the night directory ff_list = [] for file_name in file_list: if FFfile.validFFName(file_name): ff_list.append(file_name) # Calculate the dimensions of the binned image bin_w = int(config.width/config.thumb_bin) bin_h = int(config.height/config.thumb_bin) ### RESIZE AND STACK THUMBNAILS ### ########################################################################################################## timestamps = [] stacked_imgs = [] for i in range(0, len(ff_list), config.thumb_stack): img_stack = np.zeros((bin_h, bin_w)) # Stack thumb_stack images using the 'if lighter' method for j in range(config.thumb_stack): if (i + j) < len(ff_list): tmp_file_name = ff_list[i + j] # Read the FF file ff = FFfile.read(dir_path, tmp_file_name) # Skip the FF if it is corruped if ff is None: continue img = ff.maxpixel # Resize the image img = cv2.resize(img, (bin_w, bin_h)) # Stack the image img_stack = stackIfLighter(img_stack, img) else: break # Save the timestamp of the first image in the stack timestamps.append(FFfile.filenameToDatetime(ff_list[i])) # Save the stacked image stacked_imgs.append(img_stack) # cv2.imshow('test', img_stack) # cv2.waitKey(0) # cv2.destroyAllWindows() ########################################################################################################## ### ADD THUMBS TO ONE MOSAIC IMAGE ### ########################################################################################################## header_height = 20 timestamp_height = 10 # Calculate the number of rows for the thumbnail image n_rows = int(np.ceil(float(len(ff_list))/config.thumb_stack/config.thumb_n_width)) # Calculate the size of the mosaic mosaic_w = int(config.thumb_n_width*bin_w) mosaic_h = int((bin_h + timestamp_height)*n_rows + header_height) mosaic_img = np.zeros((mosaic_h, mosaic_w), dtype=np.uint8) # Write header text header_text = 'Station: ' + str(config.stationID) + ' Night: ' + os.path.basename(dir_path) \ + ' Type: ' + mosaic_type cv2.putText(mosaic_img, header_text, (0, header_height//2), \ cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255,255,255), 1) for row in range(n_rows): for col in range(config.thumb_n_width): # Calculate image index indx = row*config.thumb_n_width + col if indx < len(stacked_imgs): # Calculate position of the text text_x = col*bin_w text_y = row*bin_h + (row + 1)*timestamp_height - 1 + header_height # Add timestamp text cv2.putText(mosaic_img, timestamps[indx].strftime('%H:%M:%S'), (text_x, text_y), \ cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1) # Add the image to the mosaic img_pos_x = col*bin_w img_pos_y = row*bin_h + (row + 1)*timestamp_height + header_height mosaic_img[img_pos_y : img_pos_y + bin_h, img_pos_x : img_pos_x + bin_w] = stacked_imgs[indx] else: break ########################################################################################################## thumb_name = "{:s}_{:s}_{:s}_thumbs.jpg".format(str(config.stationID), os.path.basename(dir_path), \ mosaic_type) # Save the mosaic cv2.imwrite(os.path.join(dir_path, thumb_name), mosaic_img, [int(cv2.IMWRITE_JPEG_QUALITY), 80]) return thumb_name
def sensorCharacterization(config, dir_path): """ Characterize the standard deviation of the background and the FWHM of stars on every image. """ # Find the CALSTARS file in the given folder that has FWHM information found_good_calstars = False for cal_file in os.listdir(dir_path): if ('CALSTARS' in cal_file) and ('.txt' in cal_file) and (not found_good_calstars): # Load the calstars file calstars_list = CALSTARS.readCALSTARS(dir_path, cal_file) if len(calstars_list) > 0: # Check that at least one image has good FWHM measurements for ff_name, star_data in calstars_list: if len(star_data) > 1: star_data = np.array(star_data) # Check if the calstars file have FWHM information fwhm = star_data[:, 4] # Check that FWHM values have been computed well if np.all(fwhm > 1): found_good_calstars = True print('CALSTARS file: ' + cal_file + ' loaded!') break # If the FWHM information is not present, run the star extraction if not found_good_calstars: print() print("No FWHM information found in existing CALSTARS files!") print() print("Rerunning star detection...") print() found_good_calstars = False # Run star extraction again, and now FWHM will be computed calstars_list = extractStarsAndSave(config, dir_path) if len(calstars_list) == 0: found_good_calstars = False # Check for a minimum of detected stars for ff_name, star_data in calstars_list: if len(star_data) >= config.ff_min_stars: found_good_calstars = True break # If no good calstars exist, stop computing the flux if not found_good_calstars: print("No stars were detected in the data!") return False # Dictionary which holds information about FWHM and standard deviation of the image background sensor_data = {} # Compute median FWHM per FF file for ff_name, star_data in calstars_list: # Check that the FF file exists in the data directory if ff_name not in os.listdir(dir_path): continue star_data = np.array(star_data) # Compute the median star FWHM fwhm_median = np.median(star_data[:, 4]) # Load the FF file and compute the standard deviation of the background ff = FFfile.read(dir_path, ff_name) # Compute the median stddev of the background stddev_median = np.median(ff.stdpixel) # Store the values to the dictionary sensor_data[ff_name] = [fwhm_median, stddev_median] print("{:s}, {:5.2f}, {:5.2f}".format(ff_name, fwhm_median, stddev_median)) return sensor_data
help="Path to a config file which will be used instead of the default one.") arg_parser.add_argument('-s', '--showstd', action="store_true", help="""Show a histogram of stddevs of PSFs of all detected stars. """) # Parse the command line arguments cml_args = arg_parser.parse_args() ######################### # Load the config file config = cr.loadConfigFromDirectory(cml_args.config, cml_args.dir_path) # Get paths to every FF bin file in a directory ff_dir = os.path.abspath(cml_args.dir_path[0]) ff_list = [ff_name for ff_name in os.listdir(ff_dir) if FFfile.validFFName(ff_name)] # Check if there are any file in the directory if(len(ff_list) == None): print("No files found!") sys.exit() # Try loading a flat field image flat_struct = None if config.use_flat: # Check if there is flat in the data directory if os.path.exists(os.path.join(ff_dir, config.flat_file)):