def run(self): """ Loop through image files in the directory """ self.index = 0 self.last = -1 if ignore_last_frame: self.last = -2 while True: self.get_filenames() while self.index < len(self.files)+self.last: try: if VERBOSE: print(self.files[self.index]) if self.files[self.index].endswith('fits'): image = readFF(*os.path.split(self.files[self.index])).maxpixel self.producerQueue.put(NamedImage(image, self.files[self.index]), block=True) else: self.producerQueue.put(NamedImage(cv2.imread(self.files[self.index],cv2.IMREAD_COLOR), self.files[self.index]), block=True) self.index += 1 except: print("Error getting image") self.index = 0 time.sleep(1)
def monitorDir(self): """ Monitor the given directory and show new FF files on the screen. """ # Create a list of FF files in the given directory ff_list = [] showing_empty = False # Repeat until the process is killed from the outside while not self.exit.is_set(): # Monitor the given folder for new FF files new_ffs = [file_name for file_name in sorted(os.listdir(self.dir_path)) \ if validFFName(file_name) and (file_name not in ff_list)] # If there are no FF files in the directory, show an empty image if (not len(ff_list)) and (not len(new_ffs)) and (not showing_empty): text = "No FF files found in the given directory as of yet: {:s}".format(self.dir_path) img = np.zeros((720, 1280)) showing_empty = None # If there are new FF files, update the image if len(new_ffs): new_ff = new_ffs[-1] text = new_ff # Load the new FF ff = readFF(self.dir_path, new_ff, verbose=False) if ff is not None: img = ff.maxpixel else: time.sleep(self.update_interval) continue showing_empty = False # Add new FF files to the list ff_list += new_ffs # If there are no FF files, wait else: if showing_empty is not None: time.sleep(self.update_interval) continue if showing_empty is not True: self.updateImage(img, text, self.update_interval, banner_text=self.banner_text) # Set the proper flag if not showing any FF files if showing_empty is None: showing_empty = True
def startSlideshow(self): """ Start a slideshow. """ # Make a list of FF files in the given directory ff_list = [ file_name for file_name in sorted(os.listdir(self.dir_path)) if validFFName(file_name) ] # Exit if no FF files were found if not ff_list: print("No FF files in the given directory to use for a slideshow!") self.exit.set() return None # Go through the list of FF files and show them on the screen first_run = True while not self.exit.is_set(): for ff_name in ff_list: # Stop the loop if the slideshow should stop if self.exit.is_set(): break # Load the FF file ff = readFF(self.dir_path, ff_name, verbose=False) text = ff_name # If the FF files was loaded, show the maxpixel if ff is not None: img = ff.maxpixel else: # If an FF files could not be loaded on the first run, show an empty image if first_run: img = np.zeros((720, 1280)) text = "The FF file {:s} could not be loaded.".format( ff_name) # Otherwise, just wait one more pause interval else: time.sleep(self.slideshow_pause) continue # Update the image on the screen self.updateImage(img, text, self.slideshow_pause, banner_text=self.banner_text) first_run = False
def batchFFtoImage(dir_path, fmt): # Go through all files in the given folder for file_name in os.listdir(dir_path): # Check if the file is an FF file if validFFName(file_name): # Read the FF file ff = readFF(dir_path, file_name) # Skip the file if it could not be read if ff is None: continue # Make a filename for the image img_file_name = file_name.replace('fits', '') + fmt print('Saving: ', img_file_name) # Save the maxpixel to disk saveImage(os.path.join(dir_path, img_file_name), ff.maxpixel)
def __init__(self, dir_path, img_type): # Take FF files if the the image type was not given if img_type is None: # Get all FF files in the given folder self.filenames = sorted([os.path.abspath(os.path.join(dir_path, filename)) for filename \ in os.listdir(dir_path) if validFFName(filename)]) else: # Get all images of the given extension self.filenames = sorted([os.path.abspath(os.path.join(dir_path, filename)) for filename \ in os.listdir(dir_path) if filename.lower().endswith(img_type.lower())]) # If no files were given, take if (self.filenames is None) or (len(self.filenames) == 0): print('No files in the directory that match the pattern!') sys.exit() self.files = self.filenames # Load an image for filename in self.filenames: if validFFName(os.path.basename(filename)): self.im8u = readFF(*os.path.split(filename)).maxpixel else: self.im8u = cv2.imread(filename, cv2.IMREAD_COLOR) break if VERBOSE: print(self.im8u.shape) self.HEIGHT = self.im8u.shape[0] self.WIDTH = self.im8u.shape[1] if VERBOSE: print("Width =", self.WIDTH, "Height = ", self.HEIGHT) if self.WIDTH > 2600: self.scale = 0.25 elif self.WIDTH > 1280: self.scale = 0.5 else: self.scale = 1.0 if len(self.im8u.shape) == 3: self.im8u_grey = cv2.cvtColor(self.im8u, cv2.COLOR_BGR2GRAY) self.last_im8u = cv2.cvtColor(self.im8u, cv2.COLOR_BGR2GRAY) else: self.im8u_grey = np.copy(self.im8u) self.last_im8u = np.copy(self.im8u) self.diff = np.copy(self.im8u) self.prev_image = np.copy(self.im8u) self.short_max_im8u = np.copy(self.im8u) self.long_max_im8u = np.copy(self.im8u) self.short_coadd = np.copy(self.im8u) self.short_coadd_scaled = np.copy(self.im8u) self.trigger_list = [] self.flip = False self.contrast = False self.clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8)) # Set the font for overlay self.font = cv2.FONT_HERSHEY_SIMPLEX self.index = 0 self.pause = False self.short_max_im8u.fill(0) cv2.imshow('CheckNight', cv2.resize(self.short_max_im8u, (0,0), fx=self.scale, fy=self.scale)) cv2.moveWindow("CheckNight", 0, 0)
def __init__(self, config, input1, input2, first_frame=None, fps=None, deinterlace_mode=-1, png_mode=False): """ Tool for manually picking meteor centroids and photometry. Arguments: config: [config] Configuration structure. input1: [str] Path to the FF or FR file. Or, path to the directory with PNGs. inptu2: [str] Path to the FR file, if given (can be None). In PNG mode this must be the datetime of the first frame. Keyword Arguments: first_frame: [int] First frame to start with. None by default, which will start with the first one. fps: [float] Frames per second. None by default, which will read the fps from the config file. deinterlace_mode: [int] -1 - no deinterlace 0 - odd first 1 - even first png_mode: [bool] If True, the frames are taken from a directory which contains a list of PNG images. False by default, in which case FF and/or FR files are used. """ self.config = config self.fps = fps self.deinterlace_mode = deinterlace_mode self.png_mode = png_mode # Compute the frame step if self.deinterlace_mode > -1: self.frame_step = 0.5 else: self.frame_step = 1 # Load the PNG files if in PNG mode if self.png_mode: self.png_dir = input1 self.frame0_time = input2 print('PNG dir:', self.png_dir) self.png_list = sorted([fname for fname in os.listdir(self.png_dir) \ if fname.lower().endswith('.png')]) self.nframes = len(self.png_list) self.png_img_path = '' # Variables for FF mode that are not used self.ff = None self.fr = None # FF mode else: # Variables for PNG mode that are not used self.png_list = None self.frame0_time = None self.ff_file = input1 self.fr_file = input2 # Load the FF file if given if self.ff_file is not None: self.ff = readFF(*os.path.split(self.ff_file)) else: self.ff = None # Load the FR file is given if self.fr_file is not None: self.fr = readFR(*os.path.split(self.fr_file)) print('Total lines:', self.fr.lines) else: self.fr = None self.nframes = 256 ########### self.img_gamma = 1.0 self.show_key_help = True self.current_image = None self.fr_xmin = None self.fr_xmax = None self.fr_ymin = None self.fr_ymax = None # Previous zoom self.prev_xlim = None self.prev_ylim = None self.circle_aperture = None self.circle_aperture_outer = None self.aperture_radius = 5 self.scroll_counter = 0 self.centroid_handle = None self.photometry_coloring_mode = False self.photometry_coloring_color = False self.photometry_aperture_radius = 3 self.photometry_add = True self.photometry_coloring_handle = None self.pick_list = [] ########### # Each FR bin can have multiple detections, the first one is by default self.current_line = 0 if first_frame is not None: self.current_frame = first_frame % self.nframes else: # Set the first frame to the first frame in FR, if given if self.fr is not None: self.current_frame = self.fr.t[self.current_line][0] else: self.current_frame = 0 ### INIT IMAGE ### plt.figure(facecolor='black') # Init the first image self.updateImage() self.printStatus() self.ax = plt.gca() # Set the bacground color to black #matplotlib.rcParams['axes.facecolor'] = 'k' # Disable standard matplotlib keyboard shortcuts plt.rcParams['keymap.save'] = '' plt.rcParams['keymap.fullscreen'] = '' plt.rcParams['keymap.all_axes'] = '' plt.rcParams['keymap.quit'] = '' # Register event handlers self.ax.figure.canvas.mpl_connect('key_press_event', self.onKeyPress) self.ax.figure.canvas.mpl_connect('key_release_event', self.onKeyRelease) self.ax.figure.canvas.mpl_connect('motion_notify_event', self.onMouseMotion) self.ax.figure.canvas.mpl_connect('button_press_event', self.onMousePress) self.ax.figure.canvas.mpl_connect('button_release_event', self.onMouseRelease) self.ax.figure.canvas.mpl_connect('scroll_event', self.onScroll)
# Parse the command line arguments cml_args = arg_parser.parse_args() ######################### dir_path = cml_args.dir_path[0] # Go through all files in the given folder for file_name in os.listdir(dir_path): # Check if the file is an FF file if validFFName(file_name): # Read the FF file ff = readFF(dir_path, file_name) # Skip the file if it could not be read if ff is None: continue # Make a filename for the image img_file_name = file_name.replace('fits', '') + cml_args.file_format[0] print('Saving: ', img_file_name) # Save the maxpixel to disk scipy.misc.imsave(os.path.join(dir_path, img_file_name), ff.maxpixel)
def generateMP4s(dir_path, ftpfile_name): t1 = datetime.datetime.utcnow() # Load the font for labeling try: font = ImageFont.truetype("/usr/share/fonts/dejavu/DejaVuSans.ttf", 18) except: font = ImageFont.load_default() print("Preparing files for the timelapse...") # load the ftpfile so we know which frames we want meteor_list = FTPdetectinfo.readFTPdetectinfo(dir_path, ftpfile_name) for meteor in meteor_list: ff_name, _, _, n_segments, _, _, _, _, _, _, _, \ meteor_meas = meteor # determine which frames we want first_frame = int(meteor_meas[0][1]) - 30 last_frame = first_frame + 60 if first_frame < 0: first_frame = 0 if (n_segments > 1): lastseg = int(n_segments) - 1 last_frame = int(meteor_meas[lastseg][1]) + 30 #if last_frame > 255 : # last_frame = 255 if last_frame < first_frame + 60: last_frame = first_frame + 60 print(ff_name, ' frames ', first_frame, last_frame) # Read the FF file ff = readFF(dir_path, ff_name) # Skip the file if it could not be read if ff is None: continue # Create temporary directory dir_tmp_path = os.path.join(dir_path, "temp_img_dir") if os.path.exists(dir_tmp_path): shutil.rmtree(dir_tmp_path) print("Deleted directory : " + dir_tmp_path) mkdirP(dir_tmp_path) print("Created directory : " + dir_tmp_path) # extract the individual frames name_time_list = f2f.FFtoFrames(dir_path + '/' + ff_name, dir_tmp_path, 'jpg', -1, first_frame, last_frame) # Get id cam from the file name # e.g. FF499_20170626_020520_353_0005120.bin # or FF_CA0001_20170626_020520_353_0005120.fits file_split = ff_name.split('_') # Check the number of list elements, and the new fits format has one more underscore i = 0 if len(file_split[0]) == 2: i = 1 camid = file_split[i] font = cv2.FONT_HERSHEY_SIMPLEX # add datestamp to each frame for img_file_name, timestamp in name_time_list: img = cv2.imread(os.path.join(dir_tmp_path, img_file_name)) # Draw text to image text = camid + " " + timestamp.strftime( "%Y-%m-%d %H:%M:%S") + " UTC" cv2.putText(img, text, (10, ff.nrows - 6), font, 0.4, (255, 255, 255), 1, cv2.LINE_AA) # Save the labelled image to disk cv2.imwrite(os.path.join(dir_tmp_path, img_file_name), img, [cv2.IMWRITE_JPEG_QUALITY, 100]) ffbasename = os.path.splitext(ff_name)[0] mp4_path = ffbasename + ".mp4" temp_img_path = os.path.join(dir_tmp_path, ffbasename + "_%03d.jpg") # If running on Windows, use ffmpeg.exe if platform.system() == 'Windows': # ffmpeg.exe path root = os.path.dirname(__file__) ffmpeg_path = os.path.join(root, "ffmpeg.exe") # Construct the ecommand for ffmpeg com = ffmpeg_path + " -y -f image2 -pattern_type sequence -start_number " + str( first_frame) + " -i " + temp_img_path + " " + mp4_path print("Creating timelapse using ffmpeg...") else: # If avconv is not found, try using ffmpeg software_name = "avconv" print("Checking if avconv is available...") if os.system(software_name + " --help > /dev/null"): software_name = "ffmpeg" # Construct the ecommand for ffmpeg com = software_name + " -y -f image2 -pattern_type sequence -start_number " + str( first_frame) + " -i " + temp_img_path + " " + mp4_path print("Creating timelapse using ffmpeg...") else: print("Creating timelapse using avconv...") com = "cd " + dir_path + ";" \ + software_name + " -v quiet -r 30 -y -start_number " + str(first_frame) + " -i " + temp_img_path \ + " -vcodec libx264 -pix_fmt yuv420p -crf 25 -movflags faststart -g 15 -vf \"hqdn3d=4:3:6:4.5,lutyuv=y=gammaval(0.97)\" " \ + mp4_path #print(com) subprocess.call(com, shell=True, cwd=dir_path) #Delete temporary directory and files inside if os.path.exists(dir_tmp_path): try: shutil.rmtree(dir_tmp_path) except: # may occasionally fail due to ffmpeg thread still terminating # so catch this and wait a bit time.sleep(2) shutil.rmtree(dir_tmp_path) print("Deleted temporary directory : " + dir_tmp_path) print("Total time:", datetime.datetime.utcnow() - t1)
def novaAstrometryNetSolve(ff_file_path=None, img=None, x_data=None, y_data=None, fov_w_range=None, api_key=None): """ Find an astrometric solution of X, Y image coordinates of stars detected on an image using the nova.astrometry.net service. Keyword arguments: ff_file_path: [str] Path to the FF file to load. img: [ndarray] Numpy array containing image data. x_data: [list] A list of star x image coordiantes. y_data: [list] A list of star y image coordiantes fov_w_range: [2 element tuple] A tuple of scale_lower and scale_upper, i.e. the estimate of the width of the FOV in degrees. api_key: [str] nova.astrometry.net user API key. None by default, in which case the default API key will be used. Return: (ra, dec, orientation, scale, fov_w, fov_h): [tuple of floats] All in degrees, scale in px/deg. """ def _printWebLink(stat, first_status=None): if first_status is not None: if not len(stat.get("user_images", "")): stat = first_status if len(stat.get("user_images", "")): print( "Link to web page: http://nova.astrometry.net/user_images/{:d}" .format(stat.get("user_images", "")[0])) # Read the FF file, if given if ff_file_path is not None: # Read the FF file ff = readFF(*os.path.split(cml_args.file_path[0])) img = ff.avepixel else: file_handle = None # Convert an image to a file handle if img is not None: # Save the avepixel as a memory file file_handle = BytesIO() pil_img = Image.fromarray(img.T) # Save image to memory as JPG pil_img.save(file_handle, format='JPEG') img_data = file_handle.getvalue() # Upload the image to imgur image_url = imgurUpload('skyfit_image.jpg', image_data=img_data) c = Client() # Log in to nova.astrometry.net if api_key is None: api_key = API_KEY c.login(api_key) # Add keyword arguments kwargs = {} kwargs['publicly_visible'] = 'y' kwargs['crpix_center'] = True kwargs['tweak_order'] = 3 # Add the scale to keyword arguments, if given if fov_w_range is not None: scale_lower, scale_upper = fov_w_range kwargs['scale_lower'] = scale_lower kwargs['scale_upper'] = scale_upper # Upload image or the list of stars if file_handle is not None: upres = c.url_upload(image_url, **kwargs) elif x_data is not None: upres = c.upload(x=x_data, y=y_data, **kwargs) else: print('No input given to the funtion!') if upres is None: print('Upload failed!') return None stat = upres['status'] if stat != 'success': print('Upload failed: status', stat) print(upres) return False # Submission ID sub_id = upres['subid'] # Wait until the plate is solved solution_tries = 20 tries = 0 while True: # Limit the number of checking if the field is solved, so the script does not get stuck if tries > solution_tries: _printWebLink(stat) return None stat = c.sub_status(sub_id, justdict=True) print('Got status:', stat) jobs = stat.get('jobs', []) if len(jobs): for j in jobs: if j is not None: break if j is not None: print('Selecting job id', j) solved_id = j break time.sleep(5) tries += 1 first_status = copy.deepcopy(stat) # Get results get_results_tries = 10 get_solution_tries = 30 results_tries = 0 solution_tries = 0 while True: # Limit the number of tries of getting the results, so the script does not get stuck if results_tries > get_results_tries: print('Too many tries in getting the results!') _printWebLink(stat, first_status=first_status) return None if solution_tries > get_solution_tries: print('Waiting too long for the solution!') _printWebLink(stat, first_status=first_status) return None # Get the job status stat = c.job_status(solved_id, justdict=True) # Check if the solution is done if stat.get('status', '') in ['success']: # Get the calibration result = c.send_request('jobs/%s/calibration' % solved_id) print(result) break elif stat.get('status', '') in ['failure']: print('Failed to find a solution!') _printWebLink(stat, first_status=first_status) return None # Wait until the job is solved elif stat.get('status', '') in ['solving']: print('Solving... Try {:d}/{:d}'.format(solution_tries, get_solution_tries)) time.sleep(5) solution_tries += 1 continue # Print other error messages else: time.sleep(5) print('Got job status:', stat) results_tries += 1 # RA/Dec of centre ra = result['ra'] dec = result['dec'] # Orientation +E of N orientation = result['orientation'] # Image scale in px/deg scale = 3600 / result['pixscale'] # FOV in deg fov_w = result['width_arcsec'] / 3600 fov_h = result['height_arcsec'] / 3600 return ra, dec, orientation, scale, fov_w, fov_h
def stackFFs(dir_path, file_format, deinterlace=False, subavg=False, filter_bright=False, flat_path=None, file_list=None, mask=None): """ Stack FF files in the given folder. Arguments: dir_path: [str] Path to the directory with FF files. file_format: [str] Image format for the stack. E.g. jpg, png, bmp Keyword arguments: deinterlace: [bool] True if the image shoud be deinterlaced prior to stacking. False by default. subavg: [bool] Whether the average pixel image should be subtracted form the max pixel image. False by default. filter_bright: [bool] Whether images with bright backgrounds (after average subtraction) should be skipped. False by defualt. flat_path: [str] Path to the flat calibration file. None by default. Will only be used if subavg is False. file_list: [list] A list of file for stacking. False by default, in which case all FF files in the given directory will be used. mask: [MaskStructure] Mask to apply to the stack. None by default. Return: stack_path, merge_img: - stack_path: [str] Path of the save stack. - merge_img: [ndarray] Numpy array of the stacked image. """ # Load the flat if it was given flat = None if flat_path != '': # Try finding the default flat if flat_path is None: flat_path = dir_path flat_file = 'flat.bmp' else: flat_path, flat_file = os.path.split(flat_path) flat_full_path = os.path.join(flat_path, flat_file) if os.path.isfile(flat_full_path): # Load the flat flat = loadFlat(flat_path, flat_file) print('Loaded flat:', flat_full_path) first_img = True n_stacked = 0 total_ff_files = 0 merge_img = None # If the list of files was not given, take all files in the given folder if file_list is None: file_list = sorted(os.listdir(dir_path)) # List all FF files in the current dir for ff_name in file_list: if validFFName(ff_name): # Load FF file ff = readFF(dir_path, ff_name) # Skip the file if it is corruped if ff is None: continue total_ff_files += 1 maxpixel = ff.maxpixel avepixel = ff.avepixel # Dinterlace the images if deinterlace: maxpixel = deinterlaceBlend(maxpixel) avepixel = deinterlaceBlend(avepixel) # If the flat was given, apply it to the image, only if no subtraction is done if (flat is not None) and not subavg: maxpixel = applyFlat(maxpixel, flat) avepixel = applyFlat(avepixel, flat) # Reject the image if the median subtracted image is too bright. This usually means that there # are clouds on the image which can ruin the stack if filter_bright: img = maxpixel - avepixel # Compute surface brightness median = np.median(img) # Compute top detection pixels top_brightness = np.percentile(img, 99.9) # Reject all images where the median brightness is high # Preserve images with very bright detections if (median > 10) and (top_brightness < (2**(8*img.itemsize) - 10)): print('Skipping: ', ff_name, 'median:', median, 'top brightness:', top_brightness) continue # Subtract the average from maxpixel if subavg: img = maxpixel - avepixel else: img = maxpixel if first_img: merge_img = np.copy(img) first_img = False continue print('Stacking: ', ff_name) # Blend images 'if lighter' merge_img = blendLighten(merge_img, img) n_stacked += 1 # If the number of stacked image is less than 20% of the given images, stack without filtering if filter_bright and (n_stacked < 0.2*total_ff_files): return stackFFs(dir_path, file_format, deinterlace=deinterlace, subavg=subavg, filter_bright=False, flat_path=flat_path, file_list=file_list) # If no images were stacked, do nothing if n_stacked == 0: return None, None # Extract the name of the night directory which contains the FF files night_dir = os.path.basename(dir_path) stack_path = os.path.join(dir_path, night_dir + '_stack_{:d}_meteors.'.format(n_stacked) + file_format) print("Saving stack to:", stack_path) # Stretch the levels merge_img = adjustLevels(merge_img, np.percentile(merge_img, 0.5), 1.3, np.percentile(merge_img, 99.9)) # Apply the mask, if given if mask is not None: merge_img = MaskImage.applyMask(merge_img, mask) # Save the blended image scipy.misc.imsave(stack_path, merge_img) return stack_path, merge_img
def makeFlat(dir_path, config, nostars=False, use_images=False): """ Makes a flat field from the files in the given folder. CALSTARS file is needed to estimate the quality of every image by counting the number of detected stars. Arguments: dir_path: [str] Path to the directory which contains the FF files and a CALSTARS file. config: [config object] Keyword arguments: nostars: [bool] If True, all files will be taken regardless of if they have stars on them or not. use_images: [bool] Use image files instead of FF files. False by default. Return: [2d ndarray] Flat field image as a numpy array. If the flat generation failed, None will be returned. """ # If only images are used, then don't look for a CALSTARS file if use_images: nostars = True # Load the calstars file if it should be used if not nostars: # Find the CALSTARS file in the given folder calstars_file = None for calstars_file in os.listdir(dir_path): if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file): break if calstars_file is None: print('CALSTARS file could not be found in the given directory!') return None # Load the calstars file calstars_list = CALSTARS.readCALSTARS(dir_path, calstars_file) # Convert the list to a dictionary calstars = {ff_file: star_data for ff_file, star_data in calstars_list} print('CALSTARS file: ' + calstars_file + ' loaded!') # A list of FF files which have any stars on them calstars_ff_files = [line[0] for line in calstars_list] else: calstars = {} calstars_ff_files = [] # Use image files if use_images: # Find the file type with the highest file frequency in the given folder file_extensions = [] for file_name in os.listdir(dir_path): file_ext = file_name.split('.')[-1] if file_ext.lower() in ['jpg', 'png', 'bmp']: file_extensions.append(file_ext) # Get only the most frequent file type file_freqs = np.unique(file_extensions, return_counts=True) most_freq_type = file_freqs[0][0] print('Using image type:', most_freq_type) # Take only files of that file type ff_list = [file_name for file_name in sorted(os.listdir(dir_path)) \ if file_name.lower().endswith(most_freq_type)] # Use FF files else: ff_list = [] # Get a list of FF files in the folder for file_name in os.listdir(dir_path): if validFFName(file_name) and ((file_name in calstars_ff_files) or nostars): ff_list.append(file_name) # Check that there are any FF files in the folder if not ff_list: print('No valid FF files in the selected folder!') return None ff_list_good = [] ff_times = [] # Take only those FF files with enough stars on them for ff_name in ff_list: if (ff_name in calstars) or nostars: # Disable requiring minimum number of stars if specified if not nostars: # Get the number of stars detected on the FF image ff_nstars = len(calstars[ff_name]) else: ff_nstars = 0 # Check if the number of stars on the image is over the detection threshold if (ff_nstars > config.ff_min_stars) or nostars: # Add the FF file to the list of FF files to be used to make a flat ff_list_good.append(ff_name) # If images are used, don't compute the time if use_images: ff_time = 0 else: # Calculate the time of the FF files ff_time = date2JD(*getMiddleTimeFF( ff_name, config.fps, ret_milliseconds=True)) ff_times.append(ff_time) # Check that there are enough good FF files in the folder if (len(ff_times) < config.flat_min_imgs) and (not nostars): print('Not enough FF files have enough stars on them!') return None # Make sure the files cover at least 2 hours if (not (max(ff_times) - min(ff_times)) * 24 > 2) and (not nostars): print('Good FF files cover less than 2 hours!') return None # Sample FF files if there are more than 200 max_ff_flat = 200 if len(ff_list_good) > max_ff_flat: ff_list_good = sorted(random.sample(ff_list_good, max_ff_flat)) print('Using {:d} files for flat...'.format(len(ff_list_good))) c = 0 img_list = [] median_list = [] # Median combine all good FF files for i in range(len(ff_list_good)): # Load 10 files at the time and median combine them, which conserves memory if c < 10: # Use images if use_images: img = loadImage(os.path.join(dir_path, ff_list_good[i]), -1) # Use FF files else: ff = readFF(dir_path, ff_list_good[i]) # Skip the file if it is corruped if ff is None: continue img = ff.avepixel img_list.append(img) c += 1 else: img_list = np.array(img_list) # Median combine the loaded 10 (or less) images ff_median = np.median(img_list, axis=0) median_list.append(ff_median) img_list = [] c = 0 # If there are more than 1 calculated median image, combine them if len(median_list) > 1: # Median combine all median images median_list = np.array(median_list) ff_median = np.median(median_list, axis=0) else: if len(median_list) > 0: ff_median = median_list[0] else: ff_median = np.median(np.array(img_list), axis=0) # Stretch flat to 0-255 ff_median = ff_median / np.max(ff_median) * 255 # Convert the flat to 8 bits ff_median = ff_median.astype(np.uint8) return ff_median
def generateCalibrationReport(config, night_dir_path, match_radius=2.0, platepar=None, show_graphs=False): """ Given the folder of the night, find the Calstars file, check the star fit and generate a report with the quality of the calibration. The report contains information about both the astrometry and the photometry calibration. Graphs will be saved in the given directory of the night. Arguments: config: [Config instance] night_dir_path: [str] Full path to the directory of the night. Keyword arguments: match_radius: [float] Match radius for star matching between image and catalog stars (px). platepar: [Platepar instance] Use this platepar instead of finding one in the folder. show_graphs: [bool] Show the graphs on the screen. False by default. Return: None """ # Find the CALSTARS file in the given folder calstars_file = None for calstars_file in os.listdir(night_dir_path): if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file): break if calstars_file is None: print('CALSTARS file could not be found in the given directory!') return None # Load the calstars file star_list = readCALSTARS(night_dir_path, calstars_file) ### Load recalibrated platepars, if they exist ### # Find recalibrated platepars file per FF file platepars_recalibrated_file = None for file_name in os.listdir(night_dir_path): if file_name == config.platepars_recalibrated_name: platepars_recalibrated_file = file_name break # Load all recalibrated platepars if the file is available recalibrated_platepars = None if platepars_recalibrated_file: with open(os.path.join(night_dir_path, platepars_recalibrated_file)) as f: recalibrated_platepars = json.load(f) print('Loaded recalibrated platepars JSON file for the calibration report...') ### ### ### Load the platepar file ### # Find the platepar file in the given directory if it was not given if platepar is None: # Find the platepar file platepar_file = None for file_name in os.listdir(night_dir_path): if file_name == config.platepar_name: platepar_file = file_name break if platepar_file is None: print('The platepar cannot be found in the night directory!') return None # Load the platepar file platepar = Platepar() platepar.read(os.path.join(night_dir_path, platepar_file)) ### ### night_name = os.path.split(night_dir_path.strip(os.sep))[1] # Go one mag deeper than in the config lim_mag = config.catalog_mag_limit + 1 # Load catalog stars (load one magnitude deeper) catalog_stars, mag_band_str, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(\ config.star_catalog_path, config.star_catalog_file, lim_mag=lim_mag, \ mag_band_ratios=config.star_catalog_band_ratios) ### Take only those CALSTARS entires for which FF files exist in the folder ### # Get a list of FF files in the folder\ ff_list = [] for file_name in os.listdir(night_dir_path): if validFFName(file_name): ff_list.append(file_name) # Filter out calstars entries, generate a star dictionary where the keys are JDs of FFs star_dict = {} ff_dict = {} for entry in star_list: ff_name, star_data = entry # Check if the FF from CALSTARS exists in the folder if ff_name not in ff_list: continue dt = getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True) jd = date2JD(*dt) # Add the time and the stars to the dict star_dict[jd] = star_data ff_dict[jd] = ff_name ### ### # If there are no FF files in the directory, don't generate a report if len(star_dict) == 0: print('No FF files from the CALSTARS file in the directory!') return None # If the recalibrated platepars file exists, take the one with the most stars max_jd = 0 if recalibrated_platepars is not None: max_stars = 0 for ff_name_temp in recalibrated_platepars: # Compute the Julian date of the FF middle dt = getMiddleTimeFF(ff_name_temp, config.fps, ret_milliseconds=True) jd = date2JD(*dt) # Check that this file exists in CALSTARS and the list of FF files if (jd not in star_dict) or (jd not in ff_dict): continue # Check if the number of stars on this FF file is larger than the before if len(star_dict[jd]) > max_stars: max_jd = jd max_stars = len(star_dict[jd]) # Set a flag to indicate if using recalibrated platepars has failed if max_jd == 0: using_recalib_platepars = False else: print('Using recalibrated platepars, file:', ff_dict[max_jd]) using_recalib_platepars = True # Select the platepar where the FF file has the most stars platepar_dict = recalibrated_platepars[ff_dict[max_jd]] platepar = Platepar() platepar.loadFromDict(platepar_dict) filtered_star_dict = {max_jd: star_dict[max_jd]} # Match stars on the image with the stars in the catalog n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \ filtered_star_dict, match_radius, ret_nmatch=True, lim_mag=lim_mag) max_matched_stars = n_matched # Otherwise take the optimal FF file for evaluation if (recalibrated_platepars is None) or (not using_recalib_platepars): # If there are more than a set number of FF files to evaluate, choose only the ones with most stars on # the image if len(star_dict) > config.calstars_files_N: # Find JDs of FF files with most stars on them top_nstars_indices = np.argsort([len(x) for x in star_dict.values()])[::-1][:config.calstars_files_N \ - 1] filtered_star_dict = {} for i in top_nstars_indices: filtered_star_dict[list(star_dict.keys())[i]] = list(star_dict.values())[i] star_dict = filtered_star_dict # Match stars on the image with the stars in the catalog n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \ star_dict, match_radius, ret_nmatch=True, lim_mag=lim_mag) # If no recalibrated platepars where found, find the image with the largest number of matched stars if (not using_recalib_platepars) or (max_jd == 0): max_jd = 0 max_matched_stars = 0 for jd in matched_stars: _, _, distances = matched_stars[jd] if len(distances) > max_matched_stars: max_jd = jd max_matched_stars = len(distances) # If there are no matched stars, use the image with the largest number of detected stars if max_matched_stars <= 2: max_jd = max(star_dict, key=lambda x: len(star_dict[x])) distances = [np.inf] # Take the FF file with the largest number of matched stars ff_name = ff_dict[max_jd] # Load the FF file ff = readFF(night_dir_path, ff_name) img_h, img_w = ff.avepixel.shape dpi = 200 plt.figure(figsize=(ff.avepixel.shape[1]/dpi, ff.avepixel.shape[0]/dpi), dpi=dpi) # Take the average pixel img = ff.avepixel # Slightly adjust the levels img = Image.adjustLevels(img, np.percentile(img, 1.0), 1.2, np.percentile(img, 99.99)) plt.imshow(img, cmap='gray', interpolation='nearest') legend_handles = [] # Plot detected stars for img_star in star_dict[max_jd]: y, x, _, _ = img_star rect_side = 5*match_radius square_patch = plt.Rectangle((x - rect_side/2, y - rect_side/2), rect_side, rect_side, color='g', \ fill=False, label='Image stars') plt.gca().add_artist(square_patch) legend_handles.append(square_patch) # If there are matched stars, plot them if max_matched_stars > 2: # Take the solution with the largest number of matched stars image_stars, matched_catalog_stars, distances = matched_stars[max_jd] # Plot matched stars for img_star in image_stars: x, y, _, _ = img_star circle_patch = plt.Circle((y, x), radius=3*match_radius, color='y', fill=False, \ label='Matched stars') plt.gca().add_artist(circle_patch) legend_handles.append(circle_patch) ### Plot match residuals ### # Compute preducted positions of matched image stars from the catalog x_predicted, y_predicted = raDecToXYPP(matched_catalog_stars[:, 0], \ matched_catalog_stars[:, 1], max_jd, platepar) img_y, img_x, _, _ = image_stars.T delta_x = x_predicted - img_x delta_y = y_predicted - img_y # Compute image residual and angle of the error res_angle = np.arctan2(delta_y, delta_x) res_distance = np.sqrt(delta_x**2 + delta_y**2) # Calculate coordinates of the beginning of the residual line res_x_beg = img_x + 3*match_radius*np.cos(res_angle) res_y_beg = img_y + 3*match_radius*np.sin(res_angle) # Calculate coordinates of the end of the residual line res_x_end = img_x + 100*np.cos(res_angle)*res_distance res_y_end = img_y + 100*np.sin(res_angle)*res_distance # Plot the 100x residuals for i in range(len(x_predicted)): res_plot = plt.plot([res_x_beg[i], res_x_end[i]], [res_y_beg[i], res_y_end[i]], color='orange', \ lw=0.5, label='100x residuals') legend_handles.append(res_plot[0]) ### ### else: distances = [np.inf] # If there are no matched stars, plot large text in the middle of the screen plt.text(img_w/2, img_h/2, "NO MATCHED STARS!", color='r', alpha=0.5, fontsize=20, ha='center', va='center') ### Plot positions of catalog stars to the limiting magnitude of the faintest matched star + 1 mag ### # Find the faintest magnitude among matched stars if max_matched_stars > 2: faintest_mag = np.max(matched_catalog_stars[:, 2]) + 1 else: # If there are no matched stars, use the limiting magnitude from config faintest_mag = config.catalog_mag_limit + 1 # Estimate RA,dec of the centre of the FOV _, RA_c, dec_c, _ = xyToRaDecPP([jd2Date(max_jd)], [platepar.X_res/2], [platepar.Y_res/2], [1], platepar) RA_c = RA_c[0] dec_c = dec_c[0] fov_radius = np.hypot(*computeFOVSize(platepar)) # Get stars from the catalog around the defined center in a given radius _, extracted_catalog = subsetCatalog(catalog_stars, RA_c, dec_c, fov_radius, faintest_mag) ra_catalog, dec_catalog, mag_catalog = extracted_catalog.T # Compute image positions of all catalog stars that should be on the image x_catalog, y_catalog = raDecToXYPP(ra_catalog, dec_catalog, max_jd, platepar) # Filter all catalog stars outside the image temp_arr = np.c_[x_catalog, y_catalog, mag_catalog] temp_arr = temp_arr[temp_arr[:, 0] >= 0] temp_arr = temp_arr[temp_arr[:, 0] <= ff.avepixel.shape[1]] temp_arr = temp_arr[temp_arr[:, 1] >= 0] temp_arr = temp_arr[temp_arr[:, 1] <= ff.avepixel.shape[0]] x_catalog, y_catalog, mag_catalog = temp_arr.T # Plot catalog stars on the image cat_stars_handle = plt.scatter(x_catalog, y_catalog, c='none', marker='D', lw=1.0, alpha=0.4, \ s=((4.0 + (faintest_mag - mag_catalog))/3.0)**(2*2.512), edgecolor='r', label='Catalog stars') legend_handles.append(cat_stars_handle) ### ### # Add info text info_text = ff_dict[max_jd] + '\n' \ + "Matched stars: {:d}/{:d}\n".format(max_matched_stars, len(star_dict[max_jd])) \ + "Median distance: {:.2f} px\n".format(np.median(distances)) \ + "Catalog limiting magnitude: {:.1f}".format(lim_mag) plt.text(10, 10, info_text, bbox=dict(facecolor='black', alpha=0.5), va='top', ha='left', fontsize=4, \ color='w') legend = plt.legend(handles=legend_handles, prop={'size': 4}, loc='upper right') legend.get_frame().set_facecolor('k') legend.get_frame().set_edgecolor('k') for txt in legend.get_texts(): txt.set_color('w') plt.axis('off') plt.gca().get_xaxis().set_visible(False) plt.gca().get_yaxis().set_visible(False) plt.xlim([0, ff.avepixel.shape[1]]) plt.ylim([ff.avepixel.shape[0], 0]) # Remove the margins plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0) plt.savefig(os.path.join(night_dir_path, night_name + '_calib_report_astrometry.jpg'), \ bbox_inches='tight', pad_inches=0, dpi=dpi) if show_graphs: plt.show() else: plt.clf() plt.close() if max_matched_stars > 2: ### Plot the photometry ### plt.figure(dpi=dpi) # Take only those stars which are inside the 3/4 of the shorter image axis from the center photom_selection_radius = np.min([img_h, img_w])/3 filter_indices = ((image_stars[:, 0] - img_h/2)**2 + (image_stars[:, 1] \ - img_w/2)**2) <= photom_selection_radius**2 star_intensities = image_stars[filter_indices, 2] catalog_mags = matched_catalog_stars[filter_indices, 2] # Plot intensities of image stars #star_intensities = image_stars[:, 2] plt.scatter(-2.5*np.log10(star_intensities), catalog_mags, s=5, c='r') # Fit the photometry on automated star intensities photom_offset, fit_stddev, _ = photometryFit(np.log10(star_intensities), catalog_mags) # Plot photometric offset from the platepar x_min, x_max = plt.gca().get_xlim() y_min, y_max = plt.gca().get_ylim() x_min_w = x_min - 3 x_max_w = x_max + 3 y_min_w = y_min - 3 y_max_w = y_max + 3 photometry_info = 'Platepar: {:+.2f}LSP {:+.2f} +/- {:.2f} \nGamma = {:.2f}'.format(platepar.mag_0, \ platepar.mag_lev, platepar.mag_lev_stddev, platepar.gamma) # Plot the photometry calibration from the platepar logsum_arr = np.linspace(x_min_w, x_max_w, 10) plt.plot(logsum_arr, logsum_arr + platepar.mag_lev, label=photometry_info, linestyle='--', \ color='k', alpha=0.5) # Plot the fitted photometry calibration fit_info = "Fit: {:+.2f}LSP {:+.2f} +/- {:.2f}".format(-2.5, photom_offset, fit_stddev) plt.plot(logsum_arr, logsum_arr + photom_offset, label=fit_info, linestyle='--', color='red', alpha=0.5) plt.legend() plt.ylabel("Catalog magnitude ({:s})".format(mag_band_str)) plt.xlabel("Uncalibrated magnitude") # Set wider axis limits plt.xlim(x_min_w, x_max_w) plt.ylim(y_min_w, y_max_w) plt.gca().invert_yaxis() plt.gca().invert_xaxis() plt.grid() plt.savefig(os.path.join(night_dir_path, night_name + '_calib_report_photometry.png'), dpi=150) if show_graphs: plt.show() else: plt.clf() plt.close()
# Load the config file config = cr.loadConfigFromDirectory(cml_args.config, cml_args.dir_path) if not os.path.exists(cml_args.dir_path): print('{:s} directory does not exist!'.format(cml_args.dir_path)) # Load all FF files in the given directory for file_name in os.listdir(cml_args.dir_path): # Check if the file is an FF file if validFFName(file_name): # Read the FF file ff = readFF(cml_args.dir_path, file_name) # Skip the file if it is corruped if ff is None: continue # Use the fireball thresholding if cml_args.fireball: k1 = config.k1 j1 = config.j1 # Meteor detection else: k1 = config.k1_det j1 = config.j1_det
def trackStack(dir_path, config, border=5, background_compensation=True, hide_plot=False): """ Generate a stack with aligned stars, so the sky appears static. The folder should have a platepars_all_recalibrated.json file. Arguments: dir_path: [str] Path to the directory with image files. config: [Config instance] Keyword arguments: border: [int] Border around the image to exclude (px). background_compensation: [bool] Normalize the background by applying a median filter to avepixel and use it as a flat field. Slows down the procedure and may sometimes introduce artifacts. True by default. """ # Load recalibrated platepars, if they exist ### # Find recalibrated platepars file per FF file platepars_recalibrated_file = None for file_name in os.listdir(dir_path): if file_name == config.platepars_recalibrated_name: platepars_recalibrated_file = file_name break # Load all recalibrated platepars if the file is available recalibrated_platepars = None if platepars_recalibrated_file is not None: with open(os.path.join(dir_path, platepars_recalibrated_file)) as f: recalibrated_platepars = json.load(f) print( 'Loaded recalibrated platepars JSON file for the calibration report...' ) # ### # If the recalib platepars is not found, stop if recalibrated_platepars is None: print("The {:s} file was not found!".format( config.platepars_recalibrated_name)) return False # Get a list of FF files in the folder ff_list = [] for file_name in os.listdir(dir_path): if validFFName(file_name): ff_list.append(file_name) # Take the platepar with the middle time as the reference one ff_found_list = [] jd_list = [] for ff_name_temp in recalibrated_platepars: if ff_name_temp in ff_list: # Compute the Julian date of the FF middle dt = getMiddleTimeFF(ff_name_temp, config.fps, ret_milliseconds=True) jd = date2JD(*dt) jd_list.append(jd) ff_found_list.append(ff_name_temp) if len(jd_list) < 2: print("Not more than 1 FF image!") return False # Take the FF file with the middle JD jd_list = np.array(jd_list) jd_middle = np.mean(jd_list) jd_mean_index = np.argmin(np.abs(jd_list - jd_middle)) ff_mid = ff_found_list[jd_mean_index] # Load the middle platepar as the reference one pp_ref = Platepar() pp_ref.loadFromDict(recalibrated_platepars[ff_mid], use_flat=config.use_flat) # Try loading the mask mask_path = None if os.path.exists(os.path.join(dir_path, config.mask_file)): mask_path = os.path.join(dir_path, config.mask_file) # Try loading the default mask elif os.path.exists(config.mask_file): mask_path = os.path.abspath(config.mask_file) # Load the mask if given mask = None if mask_path is not None: mask = loadMask(mask_path) print("Loaded mask:", mask_path) # If the shape of the mask doesn't fit, init an empty mask if mask is not None: if (mask.img.shape[0] != pp_ref.Y_res) or (mask.img.shape[1] != pp_ref.X_res): print("Mask is of wrong shape!") mask = None if mask is None: mask = MaskStructure(255 + np.zeros( (pp_ref.Y_res, pp_ref.X_res), dtype=np.uint8)) # Compute the middle RA/Dec of the reference platepar _, ra_temp, dec_temp, _ = xyToRaDecPP([jd2Date(jd_middle)], [pp_ref.X_res / 2], [pp_ref.Y_res / 2], [1], pp_ref, extinction_correction=False) ra_mid, dec_mid = ra_temp[0], dec_temp[0] # Go through all FF files and find RA/Dec of image corners to find the size of the stack image ### # List of corners x_corns = [0, pp_ref.X_res, 0, pp_ref.X_res] y_corns = [0, 0, pp_ref.Y_res, pp_ref.Y_res] ra_list = [] dec_list = [] for ff_temp in ff_found_list: # Load the recalibrated platepar pp_temp = Platepar() pp_temp.loadFromDict(recalibrated_platepars[ff_temp], use_flat=config.use_flat) for x_c, y_c in zip(x_corns, y_corns): _, ra_temp, dec_temp, _ = xyToRaDecPP( [getMiddleTimeFF(ff_temp, config.fps, ret_milliseconds=True)], [x_c], [y_c], [1], pp_ref, extinction_correction=False) ra_c, dec_c = ra_temp[0], dec_temp[0] ra_list.append(ra_c) dec_list.append(dec_c) # Compute the angular separation from the middle equatorial coordinates of the reference image to all # RA/Dec corner coordinates ang_sep_list = [] for ra_c, dec_c in zip(ra_list, dec_list): ang_sep = np.degrees( angularSeparation(np.radians(ra_mid), np.radians(dec_mid), np.radians(ra_c), np.radians(dec_c))) ang_sep_list.append(ang_sep) # Find the maximum angular separation and compute the image size using the plate scale # The image size will be resampled to 1/2 of the original size to avoid interpolation scale = 0.5 ang_sep_max = np.max(ang_sep_list) img_size = int(scale * 2 * ang_sep_max * pp_ref.F_scale) # # Create the stack platepar with no distortion and a large image size pp_stack = copy.deepcopy(pp_ref) pp_stack.resetDistortionParameters() pp_stack.X_res = img_size pp_stack.Y_res = img_size pp_stack.F_scale *= scale pp_stack.refraction = False # Init the image avg_stack_sum = np.zeros((img_size, img_size), dtype=float) avg_stack_count = np.zeros((img_size, img_size), dtype=int) max_deaveraged = np.zeros((img_size, img_size), dtype=np.uint8) # Load individual FFs and map them to the stack for i, ff_name in enumerate(ff_found_list): print("Stacking {:s}, {:.1f}% done".format( ff_name, 100 * i / len(ff_found_list))) # Read the FF file ff = readFF(dir_path, ff_name) # Load the recalibrated platepar pp_temp = Platepar() pp_temp.loadFromDict(recalibrated_platepars[ff_name], use_flat=config.use_flat) # Make a list of X and Y image coordinates x_coords, y_coords = np.meshgrid( np.arange(border, pp_ref.X_res - border), np.arange(border, pp_ref.Y_res - border)) x_coords = x_coords.ravel() y_coords = y_coords.ravel() # Map image pixels to sky jd_arr, ra_coords, dec_coords, _ = xyToRaDecPP( len(x_coords) * [getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)], x_coords, y_coords, len(x_coords) * [1], pp_temp, extinction_correction=False) # Map sky coordinates to stack image coordinates stack_x, stack_y = raDecToXYPP(ra_coords, dec_coords, jd_middle, pp_stack) # Round pixel coordinates stack_x = np.round(stack_x, decimals=0).astype(int) stack_y = np.round(stack_y, decimals=0).astype(int) # Cut the image to limits filter_arr = (stack_x > 0) & (stack_x < img_size) & (stack_y > 0) & ( stack_y < img_size) x_coords = x_coords[filter_arr].astype(int) y_coords = y_coords[filter_arr].astype(int) stack_x = stack_x[filter_arr] stack_y = stack_y[filter_arr] # Apply the mask to maxpixel and avepixel maxpixel = copy.deepcopy(ff.maxpixel) maxpixel[mask.img == 0] = 0 avepixel = copy.deepcopy(ff.avepixel) avepixel[mask.img == 0] = 0 # Compute deaveraged maxpixel max_deavg = maxpixel - avepixel # Normalize the backgroud brightness by applying a large-kernel median filter to avepixel if background_compensation: # # Apply a median filter to the avepixel to get an estimate of the background brightness # avepixel_median = scipy.ndimage.median_filter(ff.avepixel, size=101) avepixel_median = cv2.medianBlur(ff.avepixel, 301) # Make sure to avoid zero division avepixel_median[avepixel_median < 1] = 1 # Normalize the avepixel by subtracting out the background brightness avepixel = avepixel.astype(float) avepixel /= avepixel_median avepixel *= 50 # Normalize to a good background value, which is usually 50 avepixel = np.clip(avepixel, 0, 255) avepixel = avepixel.astype(np.uint8) # plt.imshow(avepixel, cmap='gray', vmin=0, vmax=255) # plt.show() # Add the average pixel to the sum avg_stack_sum[stack_y, stack_x] += avepixel[y_coords, x_coords] # Increment the counter image where the avepixel is not zero ones_img = np.ones_like(avepixel) ones_img[avepixel == 0] = 0 avg_stack_count[stack_y, stack_x] += ones_img[y_coords, x_coords] # Set pixel values to the stack, only take the max values max_deaveraged[stack_y, stack_x] = np.max(np.dstack( [max_deaveraged[stack_y, stack_x], max_deavg[y_coords, x_coords]]), axis=2) # Compute the blended avepixel background stack_img = avg_stack_sum stack_img[avg_stack_count > 0] /= avg_stack_count[avg_stack_count > 0] stack_img += max_deaveraged stack_img = np.clip(stack_img, 0, 255) stack_img = stack_img.astype(np.uint8) # Crop image non_empty_columns = np.where(stack_img.max(axis=0) > 0)[0] non_empty_rows = np.where(stack_img.max(axis=1) > 0)[0] crop_box = (np.min(non_empty_rows), np.max(non_empty_rows), np.min(non_empty_columns), np.max(non_empty_columns)) stack_img = stack_img[crop_box[0]:crop_box[1] + 1, crop_box[2]:crop_box[3] + 1] # Plot and save the stack ### dpi = 200 plt.figure(figsize=(stack_img.shape[1] / dpi, stack_img.shape[0] / dpi), dpi=dpi) plt.imshow(stack_img, cmap='gray', vmin=0, vmax=256, interpolation='nearest') plt.axis('off') plt.gca().get_xaxis().set_visible(False) plt.gca().get_yaxis().set_visible(False) plt.xlim([0, stack_img.shape[1]]) plt.ylim([stack_img.shape[0], 0]) # Remove the margins (top and right are set to 0.9999, as setting them to 1.0 makes the image blank in # some matplotlib versions) plt.subplots_adjust(left=0, bottom=0, right=0.9999, top=0.9999, wspace=0, hspace=0) filenam = os.path.join(dir_path, os.path.basename(dir_path) + "_track_stack.jpg") plt.savefig(filenam, bbox_inches='tight', pad_inches=0, dpi=dpi) # if hide_plot is False: plt.show()
def __init__(self, dir_path, img_type): # Take FF files if the the image type was not given if img_type is None: # Get all FF files in the given folder self.filenames = [os.path.abspath(os.path.join(dir_path, filename)) for filename \ in os.listdir(dir_path) if validFFName(filename)] else: # Get all images of the given extension self.filenames = [os.path.abspath(os.path.join(dir_path, filename)) for filename \ in os.listdir(dir_path) if filename.lower().endswith(img_type.lower())] # If no files were given, take if (self.filenames is None) or (len(self.filenames) == 0): print('No files in the directory that match the pattern!') sys.exit() self.files = self.filenames # Load an image for filename in self.filenames: if validFFName(os.path.basename(filename)): self.im8u = readFF(*os.path.split(filename)).maxpixel else: self.im8u = cv2.imread(filename, cv2.IMREAD_COLOR) break if VERBOSE: print(self.im8u.shape) self.HEIGHT = self.im8u.shape[0] self.WIDTH = self.im8u.shape[1] if VERBOSE: print("Width =", self.WIDTH, "Height = ", self.HEIGHT) if self.WIDTH > 2600: self.scale = 0.25 elif self.WIDTH > 1280: self.scale = 0.5 else: self.scale = 1.0 if len(self.im8u.shape) == 3: self.im8u_grey = cv2.cvtColor(self.im8u, cv2.COLOR_BGR2GRAY) self.last_im8u = cv2.cvtColor(self.im8u, cv2.COLOR_BGR2GRAY) else: self.im8u_grey = np.copy(self.im8u) self.last_im8u = np.copy(self.im8u) self.diff = np.copy(self.im8u) self.prev_image = np.copy(self.im8u) self.short_max_im8u = np.copy(self.im8u) self.long_max_im8u = np.copy(self.im8u) self.short_coadd = np.copy(self.im8u) self.short_coadd_scaled = np.copy(self.im8u) self.trigger_list = [] self.flip = False self.contrast = False self.clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8)) # Set the font for overlay self.font = cv2.FONT_HERSHEY_SIMPLEX self.index = 0 self.pause = False self.short_max_im8u.fill(0) cv2.imshow('CheckNight', cv2.resize(self.short_max_im8u, (0,0), fx=self.scale, fy=self.scale)) cv2.moveWindow("CheckNight", 0, 0)
def generateTimelapse(dir_path, nodel): t1 = datetime.datetime.utcnow() # Load the font for labeling try: font = ImageFont.truetype("/usr/share/fonts/dejavu/DejaVuSans.ttf", 18) except: font = ImageFont.load_default() # Create temporary directory dir_tmp_path = os.path.join(dir_path, "temp_img_dir") if os.path.exists(dir_tmp_path): shutil.rmtree(dir_tmp_path) print("Deleted directory : " + dir_tmp_path) mkdirP(dir_tmp_path) print("Created directory : " + dir_tmp_path) print("Preparing files for the timelapse...") c = 0 ff_list = [ ff_name for ff_name in sorted(os.listdir(dir_path)) if validFFName(ff_name) ] for file_name in ff_list: # Read the FF file ff = readFF(dir_path, file_name) # Skip the file if it could not be read if ff is None: continue # Get the timestamp from the FF name timestamp = filenameToDatetime(file_name).strftime("%Y-%m-%d %H:%M:%S") # Get id cam from the file name # e.g. FF499_20170626_020520_353_0005120.bin # or FF_CA0001_20170626_020520_353_0005120.fits file_split = file_name.split('_') # Check the number of list elements, and the new fits format has one more underscore i = 0 if len(file_split[0]) == 2: i = 1 camid = file_split[i] # Make a filename for the image, continuous count %04d img_file_name = 'temp_{:04d}.jpg'.format(c) img = ff.maxpixel # Draw text to image font = cv2.FONT_HERSHEY_SIMPLEX text = camid + " " + timestamp + " UTC" cv2.putText(img, text, (10, ff.nrows - 6), font, 0.4, (255, 255, 255), 1, cv2.LINE_AA) # Save the labelled image to disk cv2.imwrite(os.path.join(dir_tmp_path, img_file_name), img, [cv2.IMWRITE_JPEG_QUALITY, 100]) c = c + 1 # Print elapsed time if c % 30 == 0: print("{:>5d}/{:>5d}, Elapsed: {:s}".format(c + 1, len(ff_list), \ str(datetime.datetime.utcnow() - t1)), end="\r") sys.stdout.flush() # If running on Linux, use avconv if platform.system() == 'Linux': # If avconv is not found, try using ffmpeg. In case of using ffmpeg, # use parameter -nostdin to avoid it being stuck waiting for user input software_name = "avconv" nostdin = "" print("Checking if avconv is available...") if os.system(software_name + " --help > /dev/null"): software_name = "ffmpeg" nostdin = " -nostdin " # Construct the command for avconv mp4_path = os.path.join(dir_path, os.path.basename(dir_path) + ".mp4") temp_img_path = os.path.basename( dir_tmp_path) + os.sep + "temp_%04d.jpg" com = "cd " + dir_path + ";" \ + software_name + nostdin + " -v quiet -r "+ str(fps) +" -y -i " + temp_img_path \ + " -vcodec libx264 -pix_fmt yuv420p -crf 25 -movflags faststart -g 15 -vf \"hqdn3d=4:3:6:4.5,lutyuv=y=gammaval(0.77)\" " \ + mp4_path print("Creating timelapse using {:s}...".format(software_name)) print(com) subprocess.call([com], shell=True) # If running on Windows, use ffmpeg.exe elif platform.system() == 'Windows': # ffmpeg.exe path root = os.path.dirname(__file__) ffmpeg_path = os.path.join(root, "ffmpeg.exe") # Construct the ecommand for ffmpeg mp4_path = os.path.basename(dir_path) + ".mp4" temp_img_path = os.path.join(os.path.basename(dir_tmp_path), "temp_%04d.jpg") com = ffmpeg_path + " -v quiet -r " + str( fps ) + " -i " + temp_img_path + " -c:v libx264 -pix_fmt yuv420p -an -crf 25 -g 15 -vf \"hqdn3d=4:3:6:4.5,lutyuv=y=gammaval(0.77)\" -movflags faststart -y " + mp4_path print("Creating timelapse using ffmpeg...") print(com) subprocess.call(com, shell=True, cwd=dir_path) else: print( "generateTimelapse only works on Linux or Windows the video could not be encoded" ) #Delete temporary directory and files inside if os.path.exists(dir_tmp_path) and not nodel: shutil.rmtree(dir_tmp_path) print("Deleted temporary directory : " + dir_tmp_path) print("Total time:", datetime.datetime.utcnow() - t1)
def makeFlat(dir_path, config, nostars=False, use_images=False): """ Makes a flat field from the files in the given folder. CALSTARS file is needed to estimate the quality of every image by counting the number of detected stars. Arguments: dir_path: [str] Path to the directory which contains the FF files and a CALSTARS file. config: [config object] Keyword arguments: nostars: [bool] If True, all files will be taken regardless of if they have stars on them or not. use_images: [bool] Use image files instead of FF files. False by default. Return: [2d ndarray] Flat field image as a numpy array. If the flat generation failed, None will be returned. """ # If only images are used, then don't look for a CALSTARS file if use_images: nostars = True # Load the calstars file if it should be used if not nostars: # Find the CALSTARS file in the given folder calstars_file = None for calstars_file in os.listdir(dir_path): if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file): break if calstars_file is None: print('CALSTARS file could not be found in the given directory!') return None # Load the calstars file calstars_list = CALSTARS.readCALSTARS(dir_path, calstars_file) # Convert the list to a dictionary calstars = {ff_file: star_data for ff_file, star_data in calstars_list} print('CALSTARS file: ' + calstars_file + ' loaded!') # A list of FF files which have any stars on them calstars_ff_files = [line[0] for line in calstars_list] else: calstars = {} # Use image files if use_images: # Find the file type with the highest file frequency in the given folder file_extensions = [] for file_name in os.listdir(dir_path): file_ext = file_name.split('.')[-1] if file_ext.lower() in ['jpg', 'png', 'bmp']: file_extensions.append(file_ext) # Get only the most frequent file type file_freqs = np.unique(file_extensions, return_counts=True) most_freq_type = file_freqs[0][0] print('Using image type:', most_freq_type) # Take only files of that file type ff_list = [file_name for file_name in sorted(os.listdir(dir_path)) \ if file_name.lower().endswith(most_freq_type)] # Use FF files else: ff_list = [] # Get a list of FF files in the folder for file_name in os.listdir(dir_path): if validFFName(file_name) and ((file_name in calstars_ff_files) or nostars): ff_list.append(file_name) # Check that there are any FF files in the folder if not ff_list: print('No valid FF files in the selected folder!') return None ff_list_good = [] ff_times = [] # Take only those FF files with enough stars on them for ff_name in ff_list: if (ff_name in calstars) or nostars: # Disable requiring minimum number of stars if specified if not nostars: # Get the number of stars detected on the FF image ff_nstars = len(calstars[ff_name]) else: ff_nstars = 0 # Check if the number of stars on the image is over the detection threshold if (ff_nstars > config.ff_min_stars) or nostars: # Add the FF file to the list of FF files to be used to make a flat ff_list_good.append(ff_name) # If images are used, don't compute the time if use_images: ff_time = 0 else: # Calculate the time of the FF files ff_time = date2JD(*getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)) ff_times.append(ff_time) # Check that there are enough good FF files in the folder if (len(ff_times) < config.flat_min_imgs) and (not nostars): print('Not enough FF files have enough stars on them!') return None # Make sure the files cover at least 2 hours if (not (max(ff_times) - min(ff_times))*24 > 2) and (not nostars): print('Good FF files cover less than 2 hours!') return None # Sample FF files if there are more than 200 max_ff_flat = 200 if len(ff_list_good) > max_ff_flat: ff_list_good = sorted(random.sample(ff_list_good, max_ff_flat)) print('Using {:d} files for flat...'.format(len(ff_list_good))) c = 0 img_list = [] median_list = [] # Median combine all good FF files for i in range(len(ff_list_good)): # Load 10 files at the time and median combine them, which conserves memory if c < 10: # Use images if use_images: img = scipy.ndimage.imread(os.path.join(dir_path, ff_list_good[i]), -1) # Use FF files else: ff = readFF(dir_path, ff_list_good[i]) # Skip the file if it is corruped if ff is None: continue img = ff.avepixel img_list.append(img) c += 1 else: img_list = np.array(img_list) # Median combine the loaded 10 (or less) images ff_median = np.median(img_list, axis=0) median_list.append(ff_median) img_list = [] c = 0 # If there are more than 1 calculated median image, combine them if len(median_list) > 1: # Median combine all median images median_list = np.array(median_list) ff_median = np.median(median_list, axis=0) else: if len(median_list) > 0: ff_median = median_list[0] else: ff_median = np.median(np.array(img_list), axis=0) # Stretch flat to 0-255 ff_median = ff_median/np.max(ff_median)*255 # Convert the flat to 8 bits ff_median = ff_median.astype(np.uint8) return ff_median
def novaAstrometryNetSolve(ff_file_path=None, img=None, x_data=None, y_data=None, fov_w_range=None, api_key=None): """ Find an astrometric solution of X, Y image coordinates of stars detected on an image using the nova.astrometry.net service. Keyword arguments: ff_file_path: [str] Path to the FF file to load. img: [ndarray] Numpy array containing image data. x_data: [list] A list of star x image coordiantes. y_data: [list] A list of star y image coordiantes fov_w_range: [2 element tuple] A tuple of scale_lower and scale_upper, i.e. the estimate of the width of the FOV in degrees. api_key: [str] nova.astrometry.net user API key. None by default, in which case the default API key will be used. Return: (ra, dec, orientation, scale, fov_w, fov_h): [tuple of floats] All in degrees, scale in px/deg. """ # Read the FF file, if given if ff_file_path is not None: # Read the FF file ff = readFF(*os.path.split(cml_args.file_path[0])) img = ff.avepixel else: file_handle = None # Convert an image to a file handle if img is not None: # Save the avepixel as a memory file file_handle = BytesIO() pil_img = Image.fromarray(img) # Save image to memory as JPG pil_img.save(file_handle, format='JPEG') img_data = file_handle.getvalue() # Upload the image to imgur image_url = imgurUpload('skyfit_image.jpg', image_data=img_data) c = Client() # Log in to nova.astrometry.net if api_key is None: api_key = API_KEY c.login(api_key) # Add keyword arguments kwargs = {} kwargs['publicly_visible'] = 'n' kwargs['crpix_center'] = True kwargs['tweak_order'] = 3 # Add the scale to keyword arguments, if given if fov_w_range is not None: scale_lower, scale_upper = fov_w_range kwargs['scale_lower'] = scale_lower kwargs['scale_upper'] = scale_upper # Upload image or the list of stars if file_handle is not None: upres = c.url_upload(image_url, **kwargs) elif x_data is not None: upres = c.upload(x=x_data, y=y_data, **kwargs) else: print('No input given to the funtion!') if upres is None: print('Upload failed!') return None stat = upres['status'] if stat != 'success': print('Upload failed: status', stat) print(upres) return False # Submission ID sub_id = upres['subid'] # Wait until the plate is solved solution_tries = 20 tries = 0 while True: # Limit the number of checking if the fiels is solved, so the script does not get stuck if tries > solution_tries: return None stat = c.sub_status(sub_id, justdict=True) print('Got status:', stat) jobs = stat.get('jobs', []) if len(jobs): for j in jobs: if j is not None: break if j is not None: print('Selecting job id', j) solved_id = j break time.sleep(5) tries += 1 # Get results get_results_tries = 10 get_solution_tries = 30 results_tries = 0 solution_tries = 0 while True: # Limit the number of tries of getting the results, so the script does not get stuck if results_tries > get_results_tries: print('Too many tries in getting the results!') return None if solution_tries > get_solution_tries: print('Waiting too long for the solution!') return None # Get the job status stat = c.job_status(solved_id, justdict=True) # Check if the solution is done if stat.get('status','') in ['success']: # Get the calibration result = c.send_request('jobs/%s/calibration' % solved_id) print(result) break elif stat.get('status','') in ['failure']: print('Failed to find a solution!') return None # Wait until the job is solved elif stat.get('status','') in ['solving']: print('Solving... Try {:d}/{:d}'.format(solution_tries, get_solution_tries)) time.sleep(5) solution_tries += 1 continue # Print other error messages else: time.sleep(5) print('Got job status:', stat) results_tries += 1 # RA/Dec of centre ra = result['ra'] dec = result['dec'] # Orientation +E of N orientation = result['orientation'] # Image scale in px/deg scale = 3600/result['pixscale'] # FOV in deg fov_w = result['width_arcsec']/3600 fov_h = result['height_arcsec']/3600 return ra, dec, orientation, scale, fov_w, fov_h
print('MeteorDetection') # Load the config file config = cr.loadConfigFromDirectory(cml_args.config, cml_args.dir_path) if not os.path.exists(cml_args.dir_path): print('{:s} directory does not exist!'.format(cml_args.dir_path)) # Load all FF files in the given directory for file_name in os.listdir(cml_args.dir_path): # Check if the file is an FF file if validFFName(file_name): # Read the FF file ff = readFF(cml_args.dir_path, file_name) # Skip the file if it is corruped if ff is None: continue # Use the fireball thresholding if cml_args.fireball: k1 = config.k1 j1 = config.j1 # Meteor detection else: k1 = config.k1_det j1 = config.j1_det
def FFtoFrames(file_path, out_dir, file_format, deinterlace_mode, first_frame=0, last_frame=255): ######################### # Load the configuration file config = cr.parse(".config") # Read the deinterlace # -1 - no deinterlace # 0 - odd first # 1 - even first if deinterlace_mode not in (-1, 0, 1): print('Unknown deinterlace mode:', deinterlace_mode) sys.exit() # Check if the file exists if not os.path.isfile(file_path): print('The file {:s} does not exist!'.format(file_path)) sys.exit() # Check if the output directory exists, make it if it doesn't if not os.path.exists(out_dir): print('Making directory: out_dir') mkdirP(out_dir) # Open the FF file dir_path, file_name = os.path.split(file_path) ff = readFF(dir_path, file_name) # Take the FPS from the FF file, if available if hasattr(ff, 'fps'): fps = ff.fps # Take the FPS from the config file, if it was not given as an argument if fps is None: fps = config.fps # Try to read the number of frames from the FF file itself if ff.nframes > 0: nframes = ff.nframes else: nframes = 256 # Construct a file name for saving if file_format == 'pngm': # If the METAL type PNG file is given, make the file name 'dump' file_name_saving = 'dump' else: file_name_saving = file_name.replace('.fits', '').replace('.bin', '') frame_name_time_list = [] # Get the initial time of the FF file ff_dt = filenameToDatetime(file_name) # Go through all frames for i in range(first_frame, last_frame + 1): # Reconstruct individual frames frame = reconstructFrame(ff, i, avepixel=True) # Deinterlace the frame if necessary, odd first if deinterlace_mode == 0: frame_odd = deinterlaceOdd(frame) frame_name, frame_dt = saveFrame(frame_odd, i, out_dir, file_name_saving, file_format, ff_dt, fps, half_frame=0) frame_name_time_list.append([frame_name, frame_dt]) frame_even = deinterlaceEven(frame) frame_name, frame_dt = saveFrame(frame_even, i, out_dir, file_name_saving, file_format, ff_dt, fps, half_frame=1) frame_name_time_list.append([frame_name, frame_dt]) # Even first elif deinterlace_mode == 1: frame_even = deinterlaceEven(frame) frame_name, frame_dt = saveFrame(frame_even, i, out_dir, file_name_saving, file_format, ff_dt, fps, half_frame=0) frame_name_time_list.append([frame_name, frame_dt]) frame_odd = deinterlaceOdd(frame) frame_name, frame_dt = saveFrame(frame_odd, i, out_dir, file_name_saving, file_format, ff_dt, fps, half_frame=1) frame_name_time_list.append([frame_name, frame_dt]) # No deinterlace else: frame_name, frame_dt = saveFrame(frame, i - first_frame, out_dir, file_name_saving, file_format, ff_dt, fps) frame_name_time_list.append([frame_name, frame_dt]) # If the frames are saved for METAL, the times have to be given in a separate file if file_format == 'pngm': with open(os.path.join(out_dir, 'frtime.txt'), 'w') as f: # Write all frames and times in a file for frame_name, frame_dt in frame_name_time_list: # 20180117:01:08:29.8342 f.write('{:s} {:s}\n'.format( frame_name, frame_dt.strftime("%Y%m%d:%H:%M:%S.%f"))) return frame_name_time_list
def makeFlat(dir_path, config): """ Makes a flat field from the files in the given folder. CALSTARS file is needed to estimate the quality of every image by counting the number of detected stars. Arguments: dir_path: [str] Path to the directory which contains the FF files and a CALSTARS file. config: [config object] Return: [2d ndarray] Flat field image as a numpy array. If the flat generation failed, None will be returned. """ # Find the CALSTARS file in the given folder calstars_file = None for calstars_file in os.listdir(dir_path): if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file): break if calstars_file is None: print('CALSTARS file could not be found in the given directory!') return None # Load the calstars file calstars_list = CALSTARS.readCALSTARS(dir_path, calstars_file) # Convert the list to a dictionary calstars = {ff_file: star_data for ff_file, star_data in calstars_list} print('CALSTARS file: ' + calstars_file + ' loaded!') # A list of FF files which have any stars on them calstars_ff_files = [line[0] for line in calstars_list] ff_list = [] # Get a list of FF files in the folder for file_name in os.listdir(dir_path): if validFFName(file_name) and (file_name in calstars_ff_files): ff_list.append(file_name) # Check that there are any FF files in the folder if not ff_list: print('No FF files in the selected folder!') return None ff_list_good = [] ff_times = [] # Take only those FF files with enough stars on them for ff_name in ff_list: if not validFFName(ff_name): continue if ff_name in calstars: # Get the number of stars detected on the FF image ff_nstars = len(calstars[ff_name]) # Check if the number of stars on the image is over the detection threshold if ff_nstars > config.ff_min_stars: # Add the FF file to the list of FF files to be used to make a flat ff_list_good.append(ff_name) # Calculate the time of the FF files ff_time = date2JD(*getMiddleTimeFF( ff_name, config.fps, ret_milliseconds=True)) ff_times.append(ff_time) # Check that there are enough good FF files in the folder if len(ff_times) < config.flat_min_imgs: print('Not enough FF files have enough stars on them!') return None # Make sure the files cover at least 2 hours if not (max(ff_times) - min(ff_times)) * 24 > 2: print('Good FF files cover less than 2 hours!') return None # Sample FF files if there are more than 200 max_ff_flat = 200 if len(ff_list_good) > max_ff_flat: ff_list_good = sorted(random.sample(ff_list_good, max_ff_flat)) print('Using {:d} files for flat...'.format(len(ff_list_good))) c = 0 ff_avg_list = [] median_list = [] # Median combine all good FF files for i in range(len(ff_list_good)): # Load 10 files at the time and median combine them, which conserves memory if c < 10: ff = readFF(dir_path, ff_list_good[i]) ff_avg_list.append(ff.avepixel) c += 1 else: ff_avg_list = np.array(ff_avg_list) # Median combine the loaded 10 (or less) images ff_median = np.median(ff_avg_list, axis=0) median_list.append(ff_median) ff_avg_list = [] c = 0 # If there are more than 1 calculated median image, combine them if len(median_list) > 1: # Median combine all median images median_list = np.array(median_list) ff_median = np.median(median_list, axis=0) else: ff_median = median_list[0] # Stretch flat to 0-255 ff_median = ff_median / np.max(ff_median) * 255 # Convert the flat to 8 bits ff_median = ff_median.astype(np.uint8) return ff_median
strip_width = 10 # Number of meteor profiles to plot n_profiles = 20 # Difference in Y coordinates between every profile vertical_step_offset = 70 # Force sigma for fitting the Gaussian PSF (disable with -1) force_sigma = 1.3 dir_path, ff_name = os.path.split(cml_args.ff_file[0]) # Load the FF file ff = readFF(dir_path, ff_name) # Load the FTPdetectinfo file meteor_list = readFTPdetectinfo(*os.path.split(cml_args.ftpdetectinfo[0])) # Find the FF file among the detections for entry in meteor_list: ftp_ff_name, cam_code, meteor_No, n_segments, fps, hnr, mle, binn, px_fm, rho, phi, \ meteor_meas = entry # Take only the FF file with the detection if ff_name == ftp_ff_name:
def stackFFs(dir_path, file_format, deinterlace=False, subavg=False, filter_bright=False, flat_path=None, file_list=None, mask=None): """ Stack FF files in the given folder. Arguments: dir_path: [str] Path to the directory with FF files. file_format: [str] Image format for the stack. E.g. jpg, png, bmp Keyword arguments: deinterlace: [bool] True if the image shoud be deinterlaced prior to stacking. False by default. subavg: [bool] Whether the average pixel image should be subtracted form the max pixel image. False by default. filter_bright: [bool] Whether images with bright backgrounds (after average subtraction) should be skipped. False by defualt. flat_path: [str] Path to the flat calibration file. None by default. Will only be used if subavg is False. file_list: [list] A list of file for stacking. False by default, in which case all FF files in the given directory will be used. mask: [MaskStructure] Mask to apply to the stack. None by default. Return: stack_path, merge_img: - stack_path: [str] Path of the save stack. - merge_img: [ndarray] Numpy array of the stacked image. """ # Load the flat if it was given flat = None if flat_path != '': # Try finding the default flat if flat_path is None: flat_path = dir_path flat_file = 'flat.bmp' else: flat_path, flat_file = os.path.split(flat_path) flat_full_path = os.path.join(flat_path, flat_file) if os.path.isfile(flat_full_path): # Load the flat flat = loadFlat(flat_path, flat_file) print('Loaded flat:', flat_full_path) first_img = True n_stacked = 0 total_ff_files = 0 merge_img = None # If the list of files was not given, take all files in the given folder if file_list is None: file_list = sorted(os.listdir(dir_path)) # List all FF files in the current dir for ff_name in file_list: if validFFName(ff_name): # Load FF file ff = readFF(dir_path, ff_name) # Skip the file if it is corruped if ff is None: continue total_ff_files += 1 maxpixel = ff.maxpixel avepixel = ff.avepixel # Dinterlace the images if deinterlace: maxpixel = deinterlaceBlend(maxpixel) avepixel = deinterlaceBlend(avepixel) # If the flat was given, apply it to the image, only if no subtraction is done if (flat is not None) and not subavg: maxpixel = applyFlat(maxpixel, flat) avepixel = applyFlat(avepixel, flat) # Reject the image if the median subtracted image is too bright. This usually means that there # are clouds on the image which can ruin the stack if filter_bright: img = maxpixel - avepixel # Compute surface brightness median = np.median(img) # Compute top detection pixels top_brightness = np.percentile(img, 99.9) # Reject all images where the median brightness is high # Preserve images with very bright detections if (median > 10) and (top_brightness < (2**(8 * img.itemsize) - 10)): print('Skipping: ', ff_name, 'median:', median, 'top brightness:', top_brightness) continue # Subtract the average from maxpixel if subavg: img = maxpixel - avepixel else: img = maxpixel if first_img: merge_img = np.copy(img) first_img = False continue print('Stacking: ', ff_name) # Blend images 'if lighter' merge_img = blendLighten(merge_img, img) n_stacked += 1 # If the number of stacked image is less than 20% of the given images, stack without filtering if filter_bright and (n_stacked < 0.2 * total_ff_files): return stackFFs(dir_path, file_format, deinterlace=deinterlace, subavg=subavg, filter_bright=False, flat_path=flat_path, file_list=file_list) # If no images were stacked, do nothing if n_stacked == 0: return None, None # Extract the name of the night directory which contains the FF files night_dir = os.path.basename(dir_path) stack_path = os.path.join( dir_path, night_dir + '_stack_{:d}_meteors.'.format(n_stacked) + file_format) print("Saving stack to:", stack_path) # Stretch the levels merge_img = adjustLevels(merge_img, np.percentile(merge_img, 0.5), 1.3, np.percentile(merge_img, 99.9)) # Apply the mask, if given if mask is not None: merge_img = MaskImage.applyMask(merge_img, mask) # Save the blended image saveImage(stack_path, merge_img) return stack_path, merge_img
help='Path to directory with FF files.') arg_parser.add_argument('file_format', nargs=1, metavar='FILE_FORMAT', type=str, \ help='File format of the image, e.g. jpg or png.') # Parse the command line arguments cml_args = arg_parser.parse_args() ######################### dir_path = cml_args.dir_path[0] # Go through all files in the given folder for file_name in os.listdir(dir_path): # Check if the file is an FF file if validFFName(file_name): # Read the FF file ff = readFF(dir_path, file_name) # Make a filename for the image img_file_name = file_name.replace('fits', '') + cml_args.file_format[0] print('Saving: ', img_file_name) # Save the maxpixel to disk scipy.misc.imsave(os.path.join(dir_path, img_file_name), ff.maxpixel)
def generateCalibrationReport(config, night_dir_path, match_radius=2.0, platepar=None, show_graphs=False): """ Given the folder of the night, find the Calstars file, check the star fit and generate a report with the quality of the calibration. The report contains information about both the astrometry and the photometry calibration. Graphs will be saved in the given directory of the night. Arguments: config: [Config instance] night_dir_path: [str] Full path to the directory of the night. Keyword arguments: match_radius: [float] Match radius for star matching between image and catalog stars (px). platepar: [Platepar instance] Use this platepar instead of finding one in the folder. show_graphs: [bool] Show the graphs on the screen. False by default. Return: None """ # Find the CALSTARS file in the given folder calstars_file = None for calstars_file in os.listdir(night_dir_path): if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file): break if calstars_file is None: print('CALSTARS file could not be found in the given directory!') return None # Load the calstars file star_list = readCALSTARS(night_dir_path, calstars_file) ### Load recalibrated platepars, if they exist ### # Find recalibrated platepars file per FF file platepars_recalibrated_file = None for file_name in os.listdir(night_dir_path): if file_name == config.platepars_recalibrated_name: platepars_recalibrated_file = file_name break # Load all recalibrated platepars if the file is available recalibrated_platepars = None if platepars_recalibrated_file: with open(os.path.join(night_dir_path, platepars_recalibrated_file)) as f: recalibrated_platepars = json.load(f) print( 'Loaded recalibrated platepars JSON file for the calibration report...' ) ### ### ### Load the platepar file ### # Find the platepar file in the given directory if it was not given if platepar is None: # Find the platepar file platepar_file = None for file_name in os.listdir(night_dir_path): if file_name == config.platepar_name: platepar_file = file_name break if platepar_file is None: print('The platepar cannot be found in the night directory!') return None # Load the platepar file platepar = Platepar() platepar.read(os.path.join(night_dir_path, platepar_file), use_flat=config.use_flat) ### ### night_name = os.path.split(night_dir_path.strip(os.sep))[1] # Go one mag deeper than in the config lim_mag = config.catalog_mag_limit + 1 # Load catalog stars (load one magnitude deeper) catalog_stars, mag_band_str, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(\ config.star_catalog_path, config.star_catalog_file, lim_mag=lim_mag, \ mag_band_ratios=config.star_catalog_band_ratios) ### Take only those CALSTARS entires for which FF files exist in the folder ### # Get a list of FF files in the folder ff_list = [] for file_name in os.listdir(night_dir_path): if validFFName(file_name): ff_list.append(file_name) # Filter out calstars entries, generate a star dictionary where the keys are JDs of FFs star_dict = {} ff_dict = {} for entry in star_list: ff_name, star_data = entry # Check if the FF from CALSTARS exists in the folder if ff_name not in ff_list: continue dt = getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True) jd = date2JD(*dt) # Add the time and the stars to the dict star_dict[jd] = star_data ff_dict[jd] = ff_name ### ### # If there are no FF files in the directory, don't generate a report if len(star_dict) == 0: print('No FF files from the CALSTARS file in the directory!') return None # If the recalibrated platepars file exists, take the one with the most stars max_jd = 0 using_recalib_platepars = False if recalibrated_platepars is not None: max_stars = 0 for ff_name_temp in recalibrated_platepars: # Compute the Julian date of the FF middle dt = getMiddleTimeFF(ff_name_temp, config.fps, ret_milliseconds=True) jd = date2JD(*dt) # Check that this file exists in CALSTARS and the list of FF files if (jd not in star_dict) or (jd not in ff_dict): continue # Check if the number of stars on this FF file is larger than the before if len(star_dict[jd]) > max_stars: max_jd = jd max_stars = len(star_dict[jd]) # Set a flag to indicate if using recalibrated platepars has failed if max_jd == 0: using_recalib_platepars = False else: print('Using recalibrated platepars, file:', ff_dict[max_jd]) using_recalib_platepars = True # Select the platepar where the FF file has the most stars platepar_dict = recalibrated_platepars[ff_dict[max_jd]] platepar = Platepar() platepar.loadFromDict(platepar_dict, use_flat=config.use_flat) filtered_star_dict = {max_jd: star_dict[max_jd]} # Match stars on the image with the stars in the catalog n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \ filtered_star_dict, match_radius, ret_nmatch=True, lim_mag=lim_mag) max_matched_stars = n_matched # Otherwise take the optimal FF file for evaluation if (recalibrated_platepars is None) or (not using_recalib_platepars): # If there are more than a set number of FF files to evaluate, choose only the ones with most stars on # the image if len(star_dict) > config.calstars_files_N: # Find JDs of FF files with most stars on them top_nstars_indices = np.argsort([len(x) for x in star_dict.values()])[::-1][:config.calstars_files_N \ - 1] filtered_star_dict = {} for i in top_nstars_indices: filtered_star_dict[list(star_dict.keys())[i]] = list( star_dict.values())[i] star_dict = filtered_star_dict # Match stars on the image with the stars in the catalog n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \ star_dict, match_radius, ret_nmatch=True, lim_mag=lim_mag) # If no recalibrated platepars where found, find the image with the largest number of matched stars if (not using_recalib_platepars) or (max_jd == 0): max_jd = 0 max_matched_stars = 0 for jd in matched_stars: _, _, distances = matched_stars[jd] if len(distances) > max_matched_stars: max_jd = jd max_matched_stars = len(distances) # If there are no matched stars, use the image with the largest number of detected stars if max_matched_stars <= 2: max_jd = max(star_dict, key=lambda x: len(star_dict[x])) distances = [np.inf] # Take the FF file with the largest number of matched stars ff_name = ff_dict[max_jd] # Load the FF file ff = readFF(night_dir_path, ff_name) img_h, img_w = ff.avepixel.shape dpi = 200 plt.figure(figsize=(ff.avepixel.shape[1] / dpi, ff.avepixel.shape[0] / dpi), dpi=dpi) # Take the average pixel img = ff.avepixel # Slightly adjust the levels img = Image.adjustLevels(img, np.percentile(img, 1.0), 1.3, np.percentile(img, 99.99)) plt.imshow(img, cmap='gray', interpolation='nearest') legend_handles = [] # Plot detected stars for img_star in star_dict[max_jd]: y, x, _, _ = img_star rect_side = 5 * match_radius square_patch = plt.Rectangle((x - rect_side/2, y - rect_side/2), rect_side, rect_side, color='g', \ fill=False, label='Image stars') plt.gca().add_artist(square_patch) legend_handles.append(square_patch) # If there are matched stars, plot them if max_matched_stars > 2: # Take the solution with the largest number of matched stars image_stars, matched_catalog_stars, distances = matched_stars[max_jd] # Plot matched stars for img_star in image_stars: x, y, _, _ = img_star circle_patch = plt.Circle((y, x), radius=3*match_radius, color='y', fill=False, \ label='Matched stars') plt.gca().add_artist(circle_patch) legend_handles.append(circle_patch) ### Plot match residuals ### # Compute preducted positions of matched image stars from the catalog x_predicted, y_predicted = raDecToXYPP(matched_catalog_stars[:, 0], \ matched_catalog_stars[:, 1], max_jd, platepar) img_y, img_x, _, _ = image_stars.T delta_x = x_predicted - img_x delta_y = y_predicted - img_y # Compute image residual and angle of the error res_angle = np.arctan2(delta_y, delta_x) res_distance = np.sqrt(delta_x**2 + delta_y**2) # Calculate coordinates of the beginning of the residual line res_x_beg = img_x + 3 * match_radius * np.cos(res_angle) res_y_beg = img_y + 3 * match_radius * np.sin(res_angle) # Calculate coordinates of the end of the residual line res_x_end = img_x + 100 * np.cos(res_angle) * res_distance res_y_end = img_y + 100 * np.sin(res_angle) * res_distance # Plot the 100x residuals for i in range(len(x_predicted)): res_plot = plt.plot([res_x_beg[i], res_x_end[i]], [res_y_beg[i], res_y_end[i]], color='orange', \ lw=0.5, label='100x residuals') legend_handles.append(res_plot[0]) ### ### else: distances = [np.inf] # If there are no matched stars, plot large text in the middle of the screen plt.text(img_w / 2, img_h / 2, "NO MATCHED STARS!", color='r', alpha=0.5, fontsize=20, ha='center', va='center') ### Plot positions of catalog stars to the limiting magnitude of the faintest matched star + 1 mag ### # Find the faintest magnitude among matched stars if max_matched_stars > 2: faintest_mag = np.max(matched_catalog_stars[:, 2]) + 1 else: # If there are no matched stars, use the limiting magnitude from config faintest_mag = config.catalog_mag_limit + 1 # Estimate RA,dec of the centre of the FOV _, RA_c, dec_c, _ = xyToRaDecPP([jd2Date(max_jd)], [platepar.X_res / 2], [platepar.Y_res / 2], [1], platepar) RA_c = RA_c[0] dec_c = dec_c[0] fov_radius = np.hypot(*computeFOVSize(platepar)) # Get stars from the catalog around the defined center in a given radius _, extracted_catalog = subsetCatalog(catalog_stars, RA_c, dec_c, fov_radius, faintest_mag) ra_catalog, dec_catalog, mag_catalog = extracted_catalog.T # Compute image positions of all catalog stars that should be on the image x_catalog, y_catalog = raDecToXYPP(ra_catalog, dec_catalog, max_jd, platepar) # Filter all catalog stars outside the image temp_arr = np.c_[x_catalog, y_catalog, mag_catalog] temp_arr = temp_arr[temp_arr[:, 0] >= 0] temp_arr = temp_arr[temp_arr[:, 0] <= ff.avepixel.shape[1]] temp_arr = temp_arr[temp_arr[:, 1] >= 0] temp_arr = temp_arr[temp_arr[:, 1] <= ff.avepixel.shape[0]] x_catalog, y_catalog, mag_catalog = temp_arr.T # Plot catalog stars on the image cat_stars_handle = plt.scatter(x_catalog, y_catalog, c='none', marker='D', lw=1.0, alpha=0.4, \ s=((4.0 + (faintest_mag - mag_catalog))/3.0)**(2*2.512), edgecolor='r', label='Catalog stars') legend_handles.append(cat_stars_handle) ### ### # Add info text in the corner info_text = ff_dict[max_jd] + '\n' \ + "Matched stars within {:.1f} px radius: {:d}/{:d} \n".format(match_radius, max_matched_stars, \ len(star_dict[max_jd])) \ + "Median distance = {:.2f} px\n".format(np.median(distances)) \ + "Catalog lim mag = {:.1f}".format(lim_mag) plt.text(10, 10, info_text, bbox=dict(facecolor='black', alpha=0.5), va='top', ha='left', fontsize=4, \ color='w', family='monospace') legend = plt.legend(handles=legend_handles, prop={'size': 4}, loc='upper right') legend.get_frame().set_facecolor('k') legend.get_frame().set_edgecolor('k') for txt in legend.get_texts(): txt.set_color('w') ### Add FOV info (centre, size) ### # Mark FOV centre plt.scatter(platepar.X_res / 2, platepar.Y_res / 2, marker='+', s=20, c='r', zorder=4) # Compute FOV centre alt/az azim_centre, alt_centre = raDec2AltAz(max_jd, platepar.lon, platepar.lat, RA_c, dec_c) # Compute FOV size fov_h, fov_v = computeFOVSize(platepar) # Compute the rotation wrt. horizon rot_horizon = rotationWrtHorizon(platepar) fov_centre_text = "Azim = {:6.2f}$\\degree$\n".format(azim_centre) \ + "Alt = {:6.2f}$\\degree$\n".format(alt_centre) \ + "Rot h = {:6.2f}$\\degree$\n".format(rot_horizon) \ + "FOV h = {:6.2f}$\\degree$\n".format(fov_h) \ + "FOV v = {:6.2f}$\\degree$".format(fov_v) \ plt.text(10, platepar.Y_res - 10, fov_centre_text, bbox=dict(facecolor='black', alpha=0.5), \ va='bottom', ha='left', fontsize=4, color='w', family='monospace') ### ### # Plot RA/Dec gridlines # addEquatorialGrid(plt, platepar, max_jd) plt.axis('off') plt.gca().get_xaxis().set_visible(False) plt.gca().get_yaxis().set_visible(False) plt.xlim([0, ff.avepixel.shape[1]]) plt.ylim([ff.avepixel.shape[0], 0]) # Remove the margins plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0) plt.savefig(os.path.join(night_dir_path, night_name + '_calib_report_astrometry.jpg'), \ bbox_inches='tight', pad_inches=0, dpi=dpi) if show_graphs: plt.show() else: plt.clf() plt.close() if max_matched_stars > 2: ### PHOTOMETRY FIT ### # If a flat is used, set the vignetting coeff to 0 if config.use_flat: platepar.vignetting_coeff = 0.0 # Extact intensities and mangitudes star_intensities = image_stars[:, 2] catalog_mags = matched_catalog_stars[:, 2] # Compute radius of every star from image centre radius_arr = np.hypot(image_stars[:, 0] - img_h / 2, image_stars[:, 1] - img_w / 2) # Fit the photometry on automated star intensities (use the fixed vignetting coeff, use robust fit) photom_params, fit_stddev, fit_resid, star_intensities, radius_arr, catalog_mags = \ photometryFitRobust(star_intensities, radius_arr, catalog_mags, \ fixed_vignetting=platepar.vignetting_coeff) photom_offset, _ = photom_params ### ### ### PLOT PHOTOMETRY ### # Note: An almost identical code exists in RMS.Astrometry.SkyFit in the PlateTool.photometry function dpi = 130 fig_p, (ax_p, ax_r) = plt.subplots(nrows=2, facecolor=None, figsize=(6.0, 7.0), dpi=dpi, \ gridspec_kw={'height_ratios':[2, 1]}) # Plot raw star intensities ax_p.scatter(-2.5 * np.log10(star_intensities), catalog_mags, s=5, c='r', alpha=0.5, label="Raw") # If a flat is used, disregard the vignetting if not config.use_flat: # Plot intensities of image stars corrected for vignetting lsp_corr_arr = np.log10(correctVignetting(star_intensities, radius_arr, \ platepar.vignetting_coeff)) ax_p.scatter(-2.5*lsp_corr_arr, catalog_mags, s=5, c='b', alpha=0.5, \ label="Corrected for vignetting") # Plot photometric offset from the platepar x_min, x_max = ax_p.get_xlim() y_min, y_max = ax_p.get_ylim() x_min_w = x_min - 3 x_max_w = x_max + 3 y_min_w = y_min - 3 y_max_w = y_max + 3 photometry_info = "Platepar: {:+.1f}*LSP + {:.2f} +/- {:.2f}".format(platepar.mag_0, \ platepar.mag_lev, platepar.mag_lev_stddev) \ + "\nVignetting coeff = {:.5f}".format(platepar.vignetting_coeff) \ + "\nGamma = {:.2f}".format(platepar.gamma) # Plot the photometry calibration from the platepar logsum_arr = np.linspace(x_min_w, x_max_w, 10) ax_p.plot(logsum_arr, logsum_arr + platepar.mag_lev, label=photometry_info, linestyle='--', \ color='k', alpha=0.5) # Plot the fitted photometry calibration fit_info = "Fit: {:+.1f}*LSP + {:.2f} +/- {:.2f}".format( -2.5, photom_offset, fit_stddev) ax_p.plot(logsum_arr, logsum_arr + photom_offset, label=fit_info, linestyle='--', color='b', alpha=0.75) ax_p.legend() ax_p.set_ylabel("Catalog magnitude ({:s})".format(mag_band_str)) ax_p.set_xlabel("Uncalibrated magnitude") # Set wider axis limits ax_p.set_xlim(x_min_w, x_max_w) ax_p.set_ylim(y_min_w, y_max_w) ax_p.invert_yaxis() ax_p.invert_xaxis() ax_p.grid() ### Plot photometry vs radius ### img_diagonal = np.hypot(img_h / 2, img_w / 2) # Plot photometry residuals (including vignetting) ax_r.scatter(radius_arr, fit_resid, c='b', alpha=0.75, s=5, zorder=3) # Plot a zero line ax_r.plot(np.linspace(0, img_diagonal, 10), np.zeros(10), linestyle='dashed', alpha=0.5, \ color='k') # Plot only when no flat is used if not config.use_flat: # Plot radius from centre vs. fit residual fit_resids_novignetting = catalog_mags - photomLine((np.array(star_intensities), \ np.array(radius_arr)), photom_offset, 0.0) ax_r.scatter(radius_arr, fit_resids_novignetting, s=5, c='r', alpha=0.5, zorder=3) px_sum_tmp = 1000 radius_arr_tmp = np.linspace(0, img_diagonal, 50) # Plot vignetting loss curve vignetting_loss = 2.5*np.log10(px_sum_tmp) \ - 2.5*np.log10(correctVignetting(px_sum_tmp, radius_arr_tmp, \ platepar.vignetting_coeff)) ax_r.plot(radius_arr_tmp, vignetting_loss, linestyle='dotted', alpha=0.5, color='k') ax_r.grid() ax_r.set_ylabel("Fit residuals (mag)") ax_r.set_xlabel("Radius from centre (px)") ax_r.set_xlim(0, img_diagonal) ### ### plt.tight_layout() plt.savefig(os.path.join(night_dir_path, night_name + '_calib_report_photometry.png'), dpi=150) if show_graphs: plt.show() else: plt.clf() plt.close()
# Find the matching FTPdetectinfo file in the directory for ff_name in sorted(os.listdir(dir_path)): # Reject all non-FF files if not validFFName(ff_name): continue # Reject all FF files which do not match the name in the FTPdetecinfo if ff_name != ftp_ff_name: continue print('Correcting for saturation:', ff_name) # Load the FF file ff = readFF(dir_path, ff_name) # Apply the flat to avepixel if flat: avepixel = applyFlat(ff.avepixel, flat) else: avepixel = ff.avepixel # Compute angular velocity first_centroid = meteor_meas[0] last_centroid = meteor_meas[-1] frame1, x1, y1 = first_centroid[:3] frame2, x2, y2 = last_centroid[:3] px_fm = np.sqrt((x2 - x1)**2 +