def plotFieldsums(dir_path, config): """ Plots a graph of all intensity sums from FS*.bin files in the given directory. Arguments: dir_path: [str] Path to the directory which containes the FS*.bin files. config: [Config structure] Configuration. Return: None """ time_data = [] intensity_data_peak = [] intensity_data_avg = [] # Get all fieldsum files in the directory for file_name in sorted(os.listdir(dir_path)): # Check if it is the fieldsum file if ('FS' in file_name) and ('_fieldsum.bin' in file_name): # Read the field sums _, intensity_array = readFieldIntensitiesBin(dir_path, file_name) # Extract the date and time from the FF file dt = filenameToDatetime(file_name) # Take the peak intensity value intensity_data_peak.append(np.max(intensity_array)) # Take the average intensity value intensity_data_avg.append(np.mean(intensity_array)) time_data.append(dt) # If there are no fieldsums, do nothing if not time_data: return False ### Plot the raw intensity over time ### ########################################################################################################## # Plot peak intensitites plt.plot(time_data, intensity_data_peak, color='r', linewidth=0.5, zorder=3, label='Peak') # Plot average intensitites plt.plot(time_data, intensity_data_avg, color='k', linewidth=0.5, zorder=3, label='Average') plt.gca().set_yscale('log') plt.xlim(np.min(time_data), np.max(time_data)) plt.ylim(np.min(intensity_data_avg), np.max(intensity_data_peak)) plt.xlabel('Time') plt.ylabel('ADU') # Rotate x ticks so they do not overlap plt.xticks(rotation=30) plt.grid(color='0.9', which='both') plt.title('Peak field sums for ' + os.path.basename(dir_path)) plt.tight_layout() plt.legend() plt.savefig(os.path.join(dir_path, str(config.stationID) + '_' + os.path.basename(dir_path) \ + '_fieldsums.png'), dpi=300) plt.clf() plt.close() ########################################################################################################## ### Plot intensities without the average value ########################################################################################################## intensity_data_peak = np.array(intensity_data_peak) intensity_data_avg = np.array(intensity_data_avg) # Calculate the difference between the peak values and the average values per every FF file intensity_data_noavg = intensity_data_peak - intensity_data_avg plt.plot(time_data, intensity_data_noavg, color='k', linewidth=0.5, zorder=3) plt.gca().set_yscale('log') plt.xlim(np.min(time_data), np.max(time_data)) plt.xlabel('Time') plt.ylabel('Peak ADU - average') # Rotate x ticks so they do not overlap plt.xticks(rotation=30) plt.grid(color='0.9', which='both') plt.title('Deaveraged field sums for ' + os.path.basename(dir_path)) plt.tight_layout() plt.savefig(os.path.join(dir_path, str(config.stationID) + '_' + os.path.basename(dir_path) \ + '_fieldsums_noavg.png'), dpi=300) plt.clf() plt.close()
def applyPlateparToCentroids(ff_name, fps, meteor_meas, platepar, add_calstatus=False): """ Given the meteor centroids and a platepar file, compute meteor astrometry and photometry (RA/Dec, alt/az, mag). Arguments: ff_name: [str] Name of the FF file with the meteor. fps: [float] Frames per second of the video. meteor_meas: [list] A list of [calib_status, frame_n, x, y, ra, dec, azim, elev, inten, mag]. platepar: [Platepar instance] Platepar which will be used for astrometry and photometry. Keyword arguments: add_calstatus: [bool] Add a column with calibration status at the beginning. False by default. Return: meteor_picks: [ndarray] A numpy 2D array of: [frames, X_data, Y_data, RA_data, dec_data, az_data, alt_data, level_data, magnitudes] """ meteor_meas = np.array(meteor_meas) # Add a line which is indicating the calibration status if add_calstatus: meteor_meas = np.c_[np.ones((meteor_meas.shape[0], 1)), meteor_meas] # Remove all entries where levels are equal to or smaller than 0, unless all are zero level_data = meteor_meas[:, 8] if np.any(level_data): meteor_meas = meteor_meas[level_data > 0, :] # Extract frame number, x, y, intensity frames = meteor_meas[:, 1] X_data = meteor_meas[:, 2] Y_data = meteor_meas[:, 3] level_data = meteor_meas[:, 8] # Get the beginning time of the FF file time_beg = filenameToDatetime(ff_name) # Calculate time data of every point time_data = [] for frame_n in frames: t = time_beg + datetime.timedelta(seconds=frame_n / fps) time_data.append([ t.year, t.month, t.day, t.hour, t.minute, t.second, int(t.microsecond / 1000) ]) # Convert image cooredinates to RA and Dec, and do the photometry JD_data, RA_data, dec_data, magnitudes = xyToRaDecPP(np.array(time_data), X_data, Y_data, \ level_data, platepar) # Compute azimuth and altitude of centroids az_data = np.zeros_like(RA_data) alt_data = np.zeros_like(RA_data) for i in range(len(az_data)): jd = JD_data[i] ra_tmp = RA_data[i] dec_tmp = dec_data[i] # Alt and az are kept in the J2000 epoch, which is the CAMS standard! az_tmp, alt_tmp = trueRaDec2ApparentAltAz(ra_tmp, dec_tmp, jd, platepar.lat, platepar.lon) az_data[i] = az_tmp alt_data[i] = alt_tmp # print(ff_name, cam_code, meteor_No, fps) # print(X_data, Y_data) # print(RA_data, dec_data) # print('------------------------------------------') # Construct the meteor measurements array meteor_picks = np.c_[frames, X_data, Y_data, RA_data, dec_data, az_data, alt_data, level_data, \ magnitudes] return meteor_picks
def generateTimelapse(dir_path, nodel): t1 = datetime.datetime.utcnow() # Load the font for labeling try: font = ImageFont.truetype("/usr/share/fonts/dejavu/DejaVuSans.ttf", 18) except: font = ImageFont.load_default() # Create temporary directory dir_tmp_path = os.path.join(dir_path, "temp_img_dir") if os.path.exists(dir_tmp_path): shutil.rmtree(dir_tmp_path) print("Deleted directory : " + dir_tmp_path) mkdirP(dir_tmp_path) print("Created directory : " + dir_tmp_path) print("Preparing files for the timelapse...") c = 0 ff_list = [ ff_name for ff_name in sorted(os.listdir(dir_path)) if validFFName(ff_name) ] for file_name in ff_list: # Read the FF file ff = readFF(dir_path, file_name) # Skip the file if it could not be read if ff is None: continue # Get the timestamp from the FF name timestamp = filenameToDatetime(file_name).strftime("%Y-%m-%d %H:%M:%S") # Get id cam from the file name # e.g. FF499_20170626_020520_353_0005120.bin # or FF_CA0001_20170626_020520_353_0005120.fits file_split = file_name.split('_') # Check the number of list elements, and the new fits format has one more underscore i = 0 if len(file_split[0]) == 2: i = 1 camid = file_split[i] # Make a filename for the image, continuous count %04d img_file_name = 'temp_{:04d}.jpg'.format(c) img = ff.maxpixel # Draw text to image font = cv2.FONT_HERSHEY_SIMPLEX text = camid + " " + timestamp + " UTC" cv2.putText(img, text, (10, ff.nrows - 6), font, 0.4, (255, 255, 255), 1, cv2.LINE_AA) # Save the labelled image to disk cv2.imwrite(os.path.join(dir_tmp_path, img_file_name), img, [cv2.IMWRITE_JPEG_QUALITY, 100]) c = c + 1 # Print elapsed time if c % 30 == 0: print("{:>5d}/{:>5d}, Elapsed: {:s}".format(c + 1, len(ff_list), \ str(datetime.datetime.utcnow() - t1)), end="\r") sys.stdout.flush() # If running on Linux, use avconv if platform.system() == 'Linux': # If avconv is not found, try using ffmpeg. In case of using ffmpeg, # use parameter -nostdin to avoid it being stuck waiting for user input software_name = "avconv" nostdin = "" print("Checking if avconv is available...") if os.system(software_name + " --help > /dev/null"): software_name = "ffmpeg" nostdin = " -nostdin " # Construct the command for avconv mp4_path = os.path.join(dir_path, os.path.basename(dir_path) + ".mp4") temp_img_path = os.path.basename( dir_tmp_path) + os.sep + "temp_%04d.jpg" com = "cd " + dir_path + ";" \ + software_name + nostdin + " -v quiet -r "+ str(fps) +" -y -i " + temp_img_path \ + " -vcodec libx264 -pix_fmt yuv420p -crf 25 -movflags faststart -g 15 -vf \"hqdn3d=4:3:6:4.5,lutyuv=y=gammaval(0.77)\" " \ + mp4_path print("Creating timelapse using {:s}...".format(software_name)) print(com) subprocess.call([com], shell=True) # If running on Windows, use ffmpeg.exe elif platform.system() == 'Windows': # ffmpeg.exe path root = os.path.dirname(__file__) ffmpeg_path = os.path.join(root, "ffmpeg.exe") # Construct the ecommand for ffmpeg mp4_path = os.path.basename(dir_path) + ".mp4" temp_img_path = os.path.join(os.path.basename(dir_tmp_path), "temp_%04d.jpg") com = ffmpeg_path + " -v quiet -r " + str( fps ) + " -i " + temp_img_path + " -c:v libx264 -pix_fmt yuv420p -an -crf 25 -g 15 -vf \"hqdn3d=4:3:6:4.5,lutyuv=y=gammaval(0.77)\" -movflags faststart -y " + mp4_path print("Creating timelapse using ffmpeg...") print(com) subprocess.call(com, shell=True, cwd=dir_path) else: print( "generateTimelapse only works on Linux or Windows the video could not be encoded" ) #Delete temporary directory and files inside if os.path.exists(dir_tmp_path) and not nodel: shutil.rmtree(dir_tmp_path) print("Deleted temporary directory : " + dir_tmp_path) print("Total time:", datetime.datetime.utcnow() - t1)
def FFtoFrames(file_path, out_dir, file_format, deinterlace_mode, first_frame=0, last_frame=255): ######################### # Load the configuration file config = cr.parse(".config") # Read the deinterlace # -1 - no deinterlace # 0 - odd first # 1 - even first if deinterlace_mode not in (-1, 0, 1): print('Unknown deinterlace mode:', deinterlace_mode) sys.exit() # Check if the file exists if not os.path.isfile(file_path): print('The file {:s} does not exist!'.format(file_path)) sys.exit() # Check if the output directory exists, make it if it doesn't if not os.path.exists(out_dir): print('Making directory: out_dir') mkdirP(out_dir) # Open the FF file dir_path, file_name = os.path.split(file_path) ff = readFF(dir_path, file_name) # Take the FPS from the FF file, if available if hasattr(ff, 'fps'): fps = ff.fps # Take the FPS from the config file, if it was not given as an argument if fps is None: fps = config.fps # Try to read the number of frames from the FF file itself if ff.nframes > 0: nframes = ff.nframes else: nframes = 256 # Construct a file name for saving if file_format == 'pngm': # If the METAL type PNG file is given, make the file name 'dump' file_name_saving = 'dump' else: file_name_saving = file_name.replace('.fits', '').replace('.bin', '') frame_name_time_list = [] # Get the initial time of the FF file ff_dt = filenameToDatetime(file_name) # Go through all frames for i in range(first_frame, last_frame + 1): # Reconstruct individual frames frame = reconstructFrame(ff, i, avepixel=True) # Deinterlace the frame if necessary, odd first if deinterlace_mode == 0: frame_odd = deinterlaceOdd(frame) frame_name, frame_dt = saveFrame(frame_odd, i, out_dir, file_name_saving, file_format, ff_dt, fps, half_frame=0) frame_name_time_list.append([frame_name, frame_dt]) frame_even = deinterlaceEven(frame) frame_name, frame_dt = saveFrame(frame_even, i, out_dir, file_name_saving, file_format, ff_dt, fps, half_frame=1) frame_name_time_list.append([frame_name, frame_dt]) # Even first elif deinterlace_mode == 1: frame_even = deinterlaceEven(frame) frame_name, frame_dt = saveFrame(frame_even, i, out_dir, file_name_saving, file_format, ff_dt, fps, half_frame=0) frame_name_time_list.append([frame_name, frame_dt]) frame_odd = deinterlaceOdd(frame) frame_name, frame_dt = saveFrame(frame_odd, i, out_dir, file_name_saving, file_format, ff_dt, fps, half_frame=1) frame_name_time_list.append([frame_name, frame_dt]) # No deinterlace else: frame_name, frame_dt = saveFrame(frame, i - first_frame, out_dir, file_name_saving, file_format, ff_dt, fps) frame_name_time_list.append([frame_name, frame_dt]) # If the frames are saved for METAL, the times have to be given in a separate file if file_format == 'pngm': with open(os.path.join(out_dir, 'frtime.txt'), 'w') as f: # Write all frames and times in a file for frame_name, frame_dt in frame_name_time_list: # 20180117:01:08:29.8342 f.write('{:s} {:s}\n'.format( frame_name, frame_dt.strftime("%Y%m%d:%H:%M:%S.%f"))) return frame_name_time_list
def applyAstrometryFTPdetectinfo(dir_path, ftp_detectinfo_file, platepar_file, UT_corr=0): """ Use the given platepar to calculate the celestial coordinates of detected meteors from a FTPdetectinfo file and save the updates values. Arguments: dir_path: [str] Path to the night. ftp_detectinfo_file: [str] Name of the FTPdetectinfo file. platepar_file: [str] Name of the platepar file. Keyword arguments: UT_corr: [float] Difference of time from UTC in hours. Return: None """ # Save a copy of the uncalibrated FTPdetectinfo ftp_detectinfo_copy = "".join( ftp_detectinfo_file.split('.')[:-1]) + "_uncalibrated.txt" # Back up the original FTPdetectinfo, only if a backup does not exist already if not os.path.isfile(os.path.join(dir_path, ftp_detectinfo_copy)): shutil.copy2(os.path.join(dir_path, ftp_detectinfo_file), os.path.join(dir_path, ftp_detectinfo_copy)) # Load the platepar platepar = Platepar() platepar.read(os.path.join(dir_path, platepar_file)) # Load the FTPdetectinfo file meteor_data = readFTPdetectinfo(dir_path, ftp_detectinfo_file) # List for final meteor data meteor_list = [] # Go through every meteor for meteor in meteor_data: ff_name, cam_code, meteor_No, n_segments, fps, hnr, mle, binn, px_fm, rho, phi, meteor_meas = meteor meteor_meas = np.array(meteor_meas) # Extract frame number, x, y, intensity frames = meteor_meas[:, 1] X_data = meteor_meas[:, 2] Y_data = meteor_meas[:, 3] level_data = meteor_meas[:, 8] # Get the beginning time of the FF file time_beg = filenameToDatetime(ff_name) # Calculate time data of every point time_data = [] for frame_n in frames: t = time_beg + datetime.timedelta(seconds=frame_n / fps) time_data.append([ t.year, t.month, t.day, t.hour, t.minute, t.second, int(t.microsecond / 1000) ]) # Convert image cooredinates to RA and Dec, and do the photometry JD_data, RA_data, dec_data, magnitudes = XY2CorrectedRADecPP(np.array(time_data), X_data, Y_data, \ level_data, platepar) # Compute azimuth and altitude of centroids az_data = np.zeros_like(RA_data) alt_data = np.zeros_like(RA_data) for i in range(len(az_data)): jd = JD_data[i] ra_tmp = RA_data[i] dec_tmp = dec_data[i] az_tmp, alt_tmp = raDec2AltAz(jd, platepar.lon, platepar.lat, ra_tmp, dec_tmp) az_data[i] = az_tmp alt_data[i] = alt_tmp # print(ff_name, cam_code, meteor_No, fps) # print(X_data, Y_data) # print(RA_data, dec_data) # print('------------------------------------------') # Construct the meteor measurements array meteor_picks = np.c_[frames, X_data, Y_data, RA_data, dec_data, az_data, alt_data, level_data, \ magnitudes] # Add the calculated values to the final list meteor_list.append([ff_name, meteor_No, rho, phi, meteor_picks]) # Calibration string to be written to the FTPdetectinfo file calib_str = 'Calibrated with RMS on: ' + str( datetime.datetime.utcnow()) + ' UTC' # If no meteors were detected, set dummpy parameters if len(meteor_list) == 0: cam_code = '' fps = 0 # Save the updated FTPdetectinfo writeFTPdetectinfo(meteor_list, dir_path, ftp_detectinfo_file, dir_path, cam_code, fps, calibration=calib_str, celestial_coords_given=True)
def plotFieldsums(dir_path, config): """ Plots a graph of all intensity sums from FS*.bin files in the given directory. Arguments: dir_path: [str] Path to the directory which containes the FS*.bin files. config: [Config structure] Configuration. Return: None """ time_data = [] intensity_data_peak = [] intensity_data_avg = [] # Get all fieldsum files in the directory for file_name in sorted(os.listdir(dir_path)): # Check if it is the fieldsum file if ('FS' in file_name) and ('_fieldsum.bin' in file_name): # Try reading the intensities sum, because the file might be corrupted try: # Read the field sums _, intensity_array = readFieldIntensitiesBin(dir_path, file_name) except TypeError: print('File {:s} is corrupted!'.format(file_name)) # Extract the date and time from the FF file dt = filenameToDatetime(file_name) # Take the peak intensity value intensity_data_peak.append(np.max(intensity_array)) # Take the average intensity value intensity_data_avg.append(np.mean(intensity_array)) time_data.append(dt) # If there are no fieldsums, do nothing if not time_data: return False ### Plot the raw intensity over time ### ########################################################################################################## plt.figure() # Plot peak intensitites plt.plot(time_data, intensity_data_peak, color='r', linewidth=0.5, zorder=3, label='Peak') # Plot average intensitites plt.plot(time_data, intensity_data_avg, color='k', linewidth=0.5, zorder=3, label='Average') plt.gca().set_yscale('log') plt.xlim(np.min(time_data), np.max(time_data)) plt.ylim(np.min(intensity_data_avg), np.max(intensity_data_peak)) plt.xlabel('Time') plt.ylabel('ADU') # Rotate x ticks so they do not overlap plt.xticks(rotation=30) plt.grid(color='0.9', which='both') plt.title('Peak field sums for ' + os.path.basename(dir_path)) plt.tight_layout() plt.legend() plt.savefig(os.path.join(dir_path, str(config.stationID) + '_' + os.path.basename(dir_path) \ + '_fieldsums.png'), dpi=300) plt.clf() plt.close() ########################################################################################################## ### Plot intensities without the average value ########################################################################################################## intensity_data_peak = np.array(intensity_data_peak) intensity_data_avg = np.array(intensity_data_avg) # Calculate the difference between the peak values and the average values per every FF file intensity_data_noavg = intensity_data_peak - intensity_data_avg plt.figure() plt.plot(time_data, intensity_data_noavg, color='k', linewidth=0.5, zorder=3) plt.gca().set_yscale('log') plt.xlim(np.min(time_data), np.max(time_data)) plt.xlabel('Time') plt.ylabel('Peak ADU - average') # Rotate x ticks so they do not overlap plt.xticks(rotation=30) plt.grid(color='0.9', which='both') plt.title('Deaveraged field sums for ' + os.path.basename(dir_path)) plt.tight_layout() plt.savefig(os.path.join(dir_path, str(config.stationID) + '_' + os.path.basename(dir_path) \ + '_fieldsums_noavg.png'), dpi=300) plt.clf() plt.close()
# Construct a file name for saving if file_format == 'pngm': # If the METAL type PNG file is given, make the file name 'dump' file_name_saving = 'dump' else: file_name_saving = file_name.replace('.fits', '').replace('.bin', '') frame_name_time_list = [] # Get the initial time of the FF file ff_dt = filenameToDatetime(file_name) # Go through all frames for i in range(nframes): # Reconstruct individual frames frame = reconstructFrame(ff, i, avepixel=True) # Deinterlace the frame if necessary, odd first if deinterlace_mode == 0: frame_odd = deinterlaceOdd(frame) frame_name, frame_dt = saveFrame(frame_odd, i, out_dir, file_name_saving, file_format, ff_dt, fps, half_frame=0) frame_name_time_list.append([frame_name, frame_dt]) frame_even = deinterlaceEven(frame)
def generateMP4s(dir_path, ftpfile_name): t1 = datetime.datetime.utcnow() # Load the font for labeling try: font = ImageFont.truetype("/usr/share/fonts/dejavu/DejaVuSans.ttf", 18) except: font = ImageFont.load_default() print("Preparing files for the timelapse...") # load the ftpfile so we know which frames we want meteor_list = FTPdetectinfo.readFTPdetectinfo(dir_path, ftpfile_name) for meteor in meteor_list: ff_name, _, _, n_segments, _, _, _, _, _, _, _, \ meteor_meas = meteor # determine which frames we want first_frame = int(meteor_meas[0][1]) - 30 last_frame = first_frame + 60 if first_frame < 0: first_frame = 0 if (n_segments > 1): lastseg = int(n_segments) - 1 last_frame = int(meteor_meas[lastseg][1]) + 30 #if last_frame > 255 : # last_frame = 255 if last_frame < first_frame + 60: last_frame = first_frame + 60 print(ff_name, ' frames ', first_frame, last_frame) # Read the FF file ff = readFF(dir_path, ff_name) # Skip the file if it could not be read if ff is None: continue # Create temporary directory dir_tmp_path = os.path.join(dir_path, "temp_img_dir") if os.path.exists(dir_tmp_path): shutil.rmtree(dir_tmp_path) print("Deleted directory : " + dir_tmp_path) mkdirP(dir_tmp_path) print("Created directory : " + dir_tmp_path) # extract the individual frames f2f.FFtoFrames(dir_path + '/' + ff_name, dir_tmp_path, 'jpg', -1, first_frame, last_frame) # Get the timestamp from the FF name timestamp = filenameToDatetime(ff_name).strftime("%Y-%m-%d %H:%M:%S") # Get id cam from the file name # e.g. FF499_20170626_020520_353_0005120.bin # or FF_CA0001_20170626_020520_353_0005120.fits file_split = ff_name.split('_') # Check the number of list elements, and the new fits format has one more underscore i = 0 if len(file_split[0]) == 2: i = 1 camid = file_split[i] # add datestamp to each frame jpg_list = [jpg_name for jpg_name in sorted(os.listdir(dir_tmp_path))] for img_file_name in jpg_list: img = cv2.imread(os.path.join(dir_tmp_path, img_file_name)) # Draw text to image font = cv2.FONT_HERSHEY_SIMPLEX text = camid + " " + timestamp + " UTC" cv2.putText(img, text, (10, ff.nrows - 6), font, 0.4, (255, 255, 255), 1, cv2.LINE_AA) # Save the labelled image to disk cv2.imwrite(os.path.join(dir_tmp_path, img_file_name), img, [cv2.IMWRITE_JPEG_QUALITY, 100]) ffbasename = os.path.splitext(ff_name)[0] mp4_path = ffbasename + ".mp4" temp_img_path = os.path.join(dir_tmp_path, ffbasename + "_%03d.jpg") # If running on Windows, use ffmpeg.exe if platform.system() == 'Windows': # ffmpeg.exe path root = os.path.dirname(__file__) ffmpeg_path = os.path.join(root, "ffmpeg.exe") # Construct the ecommand for ffmpeg com = ffmpeg_path + " -y -f image2 -pattern_type sequence -i " + temp_img_path + " " + mp4_path print("Creating timelapse using ffmpeg...") else: # If avconv is not found, try using ffmpeg software_name = "avconv" print("Checking if avconv is available...") if os.system(software_name + " --help > /dev/null"): software_name = "ffmpeg" # Construct the ecommand for ffmpeg com = software_name + " -y -f image2 -pattern_type sequence -i " + temp_img_path + " " + mp4_path print("Creating timelapse using ffmpeg...") else: print("Creating timelapse using avconv...") com = "cd " + dir_path + ";" \ + software_name + " -v quiet -r 30 -y -i " + temp_img_path \ + " -vcodec libx264 -pix_fmt yuv420p -crf 25 -movflags faststart -g 15 -vf \"hqdn3d=4:3:6:4.5,lutyuv=y=gammaval(0.97)\" " \ + mp4_path #print(com) subprocess.call(com, shell=True, cwd=dir_path) #Delete temporary directory and files inside if os.path.exists(dir_tmp_path): try: shutil.rmtree(dir_tmp_path) except: # may occasionally fail due to ffmpeg thread still terminating # so catch this and wait a bit time.sleep(2) shutil.rmtree(dir_tmp_path) print("Deleted temporary directory : " + dir_tmp_path) print("Total time:", datetime.datetime.utcnow() - t1)
print("Preparing files for the timelapse...") c = 0 ff_list = [ff_name for ff_name in sorted(os.listdir(dir_path)) if validFFName(ff_name)] for file_name in ff_list: # Read the FF file ff = readFF(dir_path, file_name) # Skip the file if it could not be read if ff is None: continue # Get the timestamp from the FF name timestamp = filenameToDatetime(file_name).strftime("%Y-%m-%d %H:%M:%S") # Get id cam from the file name # e.g. FF499_20170626_020520_353_0005120.bin # or FF_CA0001_20170626_020520_353_0005120.fits file_split = file_name.split('_') # Check the number of list elements, and the new fits format has one more underscore i = 0 if len(file_split[0]) == 2: i = 1 camid = file_split[i] # Make a filename for the image, continuous count %04d img_file_name = 'temp_{:04d}.jpg'.format(c)
def showerAssociation(config, ftpdetectinfo_list, shower_code=None, show_plot=False, save_plot=False, \ plot_activity=False): """ Do single station shower association based on radiant direction and height. Arguments: config: [Config instance] ftpdetectinfo_list: [list] A list of paths to FTPdetectinfo files. Keyword arguments: shower_code: [str] Only use this one shower for association (e.g. ETA, PER, SDA). None by default, in which case all active showers will be associated. show_plot: [bool] Show the plot on the screen. False by default. save_plot: [bool] Save the plot in the folder with FTPdetectinfos. False by default. plot_activity: [bool] Whether to plot the shower activity plot of not. False by default. Return: associations, shower_counts: [tuple] - associations: [dict] A dictionary where the FF name and the meteor ordinal number on the FF file are keys, and the associated Shower object are values. - shower_counts: [list] A list of shower code and shower count pairs. """ # Load the list of meteor showers shower_list = loadShowers(config.shower_path, config.shower_file_name) # Load FTPdetectinfos meteor_data = [] for ftpdetectinfo_path in ftpdetectinfo_list: if not os.path.isfile(ftpdetectinfo_path): print('No such file:', ftpdetectinfo_path) continue meteor_data += readFTPdetectinfo(*os.path.split(ftpdetectinfo_path)) if not len(meteor_data): return {}, [] # Dictionary which holds FF names as keys and meteor measurements + associated showers as values associations = {} for meteor in meteor_data: ff_name, cam_code, meteor_No, n_segments, fps, hnr, mle, binn, px_fm, rho, phi, meteor_meas = meteor # Skip very short meteors if len(meteor_meas) < 4: continue # Check if the data is calibrated if not meteor_meas[0][0]: print( 'Data is not calibrated! Meteors cannot be associated to showers!' ) break # Init container for meteor observation meteor_obj = MeteorSingleStation(cam_code, config.latitude, config.longitude, ff_name) # Infill the meteor structure for entry in meteor_meas: calib_status, frame_n, x, y, ra, dec, azim, elev, inten, mag = entry # Compute the Julian data of every point jd = datetime2JD( filenameToDatetime(ff_name) + datetime.timedelta(seconds=float(frame_n) / fps)) meteor_obj.addPoint(jd, ra, dec, mag) # Fit the great circle and compute the geometrical parameters meteor_obj.fitGC() # Skip all meteors with beginning heights below 15 deg if meteor_obj.beg_alt < 15: continue # Go through all showers in the list and find the best match best_match_shower = None best_match_dist = np.inf for shower_entry in shower_list: # Extract shower parameters shower = Shower(shower_entry) # If the shower code was given, only check this one shower if shower_code is not None: if shower.name.lower() != shower_code.lower(): continue ### Solar longitude filter # If the shower doesn't have a stated beginning or end, check if the meteor is within a preset # threshold solar longitude difference if np.any(np.isnan([shower.lasun_beg, shower.lasun_end])): shower.lasun_beg = (shower.lasun_max - config.shower_lasun_threshold) % 360 shower.lasun_end = (shower.lasun_max + config.shower_lasun_threshold) % 360 # Filter out all showers which are not active if not isAngleBetween(np.radians(shower.lasun_beg), np.radians(meteor_obj.lasun), np.radians(shower.lasun_end)): continue ### ### ### Radiant filter ### # Assume a fixed meteor height for an approximate apparent radiant meteor_fixed_ht = 100000 # 100 km shower.computeApparentRadiant(config.latitude, config.longitude, meteor_obj.jdt_ref, \ meteor_fixed_ht=meteor_fixed_ht) # Compute the angle between the meteor radiant and the great circle normal radiant_separation = meteor_obj.angularSeparationFromGC( shower.ra, shower.dec) # Make sure the meteor is within the radiant distance threshold if radiant_separation > config.shower_max_radiant_separation: continue # Compute angle between the meteor's beginning and end, and the shower radiant shower.radiant_vector = vectNorm( raDec2Vector(shower.ra, shower.dec)) begin_separation = np.degrees(angularSeparationVect(shower.radiant_vector, \ meteor_obj.meteor_begin_cartesian)) end_separation = np.degrees(angularSeparationVect(shower.radiant_vector, \ meteor_obj.meteor_end_cartesian)) # Make sure the beginning of the meteor is closer to the radiant than it's end if begin_separation > end_separation: continue ### ### ### Height filter ### # Estimate the limiting meteor height from the velocity (meters) filter_beg_ht = heightModel(shower.v_init, ht_type='beg') filter_end_ht = heightModel(shower.v_init, ht_type='end') ### Estimate the meteor beginning height with +/- 1 frame, otherwise some short meteor may get ### rejected meteor_obj_orig = copy.deepcopy(meteor_obj) # Shorter meteor_obj_m1 = copy.deepcopy(meteor_obj_orig) meteor_obj_m1.duration -= 1.0 / config.fps meteor_beg_ht_m1 = estimateMeteorHeight(config, meteor_obj_m1, shower) # Nominal meteor_beg_ht = estimateMeteorHeight(config, meteor_obj_orig, shower) # Longer meteor_obj_p1 = copy.deepcopy(meteor_obj_orig) meteor_obj_p1.duration += 1.0 / config.fps meteor_beg_ht_p1 = estimateMeteorHeight(config, meteor_obj_p1, shower) meteor_obj = meteor_obj_orig ### ### # If all heights (even those with +/- 1 frame) are outside the height range, reject the meteor if ((meteor_beg_ht_p1 < filter_end_ht) or (meteor_beg_ht_p1 > filter_beg_ht)) and \ ((meteor_beg_ht < filter_end_ht) or (meteor_beg_ht > filter_beg_ht)) and \ ((meteor_beg_ht_m1 < filter_end_ht) or (meteor_beg_ht_m1 > filter_beg_ht)): continue ### ### # Compute the radiant elevation above the horizon shower.azim, shower.elev = raDec2AltAz(shower.ra, shower.dec, meteor_obj.jdt_ref, \ config.latitude, config.longitude) # Take the shower that's closest to the great circle if there are multiple candidates if radiant_separation < best_match_dist: best_match_dist = radiant_separation best_match_shower = copy.deepcopy(shower) # If a shower is given and the match is not this shower, skip adding the meteor to the list # If no specific shower is give for association, add all meteors if ((shower_code is not None) and (best_match_shower is not None)) or (shower_code is None): # Store the associated shower associations[(ff_name, meteor_No)] = [meteor_obj, best_match_shower] # Find shower frequency and sort by count shower_name_list_temp = [] shower_list_temp = [] for key in associations: _, shower = associations[key] if shower is None: shower_name = '...' else: shower_name = shower.name shower_name_list_temp.append(shower_name) shower_list_temp.append(shower) _, unique_showers_indices = np.unique(shower_name_list_temp, return_index=True) unique_shower_names = np.array( shower_name_list_temp)[unique_showers_indices] unique_showers = np.array(shower_list_temp)[unique_showers_indices] shower_counts = [[shower_obj, shower_name_list_temp.count(shower_name)] for shower_obj, \ shower_name in zip(unique_showers, unique_shower_names)] shower_counts = sorted(shower_counts, key=lambda x: x[1], reverse=True) # Create a plot of showers if show_plot or save_plot: # Generate consistent colours colors_by_name = makeShowerColors(shower_list) def get_shower_color(shower): try: return colors_by_name[shower.name] if shower else "0.4" except KeyError: return 'gray' # Init the figure plt.figure() # Init subplots depending on if the activity plot is done as well if plot_activity: gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1]) ax_allsky = plt.subplot(gs[0], facecolor='black') ax_activity = plt.subplot(gs[1], facecolor='black') else: ax_allsky = plt.subplot(111, facecolor='black') # Init the all-sky plot allsky_plot = AllSkyPlot(ax_handle=ax_allsky) # Plot all meteors for key in associations: meteor_obj, shower = associations[key] ### Plot the observed meteor points ### color = get_shower_color(shower) allsky_plot.plot(meteor_obj.ra_array, meteor_obj.dec_array, color=color, linewidth=1, zorder=4) # Plot the peak of shower meteors a different color peak_color = 'blue' if shower is not None: peak_color = 'tomato' allsky_plot.scatter(meteor_obj.ra_array[-1], meteor_obj.dec_array[-1], c=peak_color, marker='+', \ s=5, zorder=5) ### ### ### Plot fitted great circle points ### # Find the GC phase angle of the beginning of the meteor gc_beg_phase = meteor_obj.findGCPhase( meteor_obj.ra_array[0], meteor_obj.dec_array[0])[0] % 360 # If the meteor belongs to a shower, find the GC phase which ends at the shower if shower is not None: gc_end_phase = meteor_obj.findGCPhase(shower.ra, shower.dec)[0] % 360 # Fix 0/360 wrap if abs(gc_end_phase - gc_beg_phase) > 180: if gc_end_phase > gc_beg_phase: gc_end_phase -= 360 else: gc_beg_phase -= 360 gc_alpha = 1.0 else: # If it's a sporadic, find the direction to which the meteor should extend gc_end_phase = meteor_obj.findGCPhase(meteor_obj.ra_array[-1], \ meteor_obj.dec_array[-1])[0]%360 # Find the correct direction if (gc_beg_phase - gc_end_phase) % 360 > (gc_end_phase - gc_beg_phase) % 360: gc_end_phase = gc_beg_phase - 170 else: gc_end_phase = gc_beg_phase + 170 gc_alpha = 0.7 # Store great circle beginning and end phase meteor_obj.gc_beg_phase = gc_beg_phase meteor_obj.gc_end_phase = gc_end_phase # Get phases 180 deg before the meteor phase_angles = np.linspace(gc_end_phase, gc_beg_phase, 100) % 360 # Compute RA/Dec of points on the great circle ra_gc, dec_gc = meteor_obj.sampleGC(phase_angles) # Cull all points below the horizon azim_gc, elev_gc = raDec2AltAz(ra_gc, dec_gc, meteor_obj.jdt_ref, config.latitude, \ config.longitude) temp_arr = np.c_[ra_gc, dec_gc] temp_arr = temp_arr[elev_gc > 0] ra_gc, dec_gc = temp_arr.T # Plot the great circle fitted on the radiant gc_color = get_shower_color(shower) allsky_plot.plot(ra_gc, dec_gc, linestyle='dotted', color=gc_color, alpha=gc_alpha, linewidth=1) # Plot the point closest to the shower radiant if shower is not None: allsky_plot.plot(ra_gc[0], dec_gc[0], color='r', marker='+', ms=5, mew=1) # Store shower radiant point meteor_obj.radiant_ra = ra_gc[0] meteor_obj.radiant_dec = dec_gc[0] ### ### ### Plot all showers ### # Find unique showers and their apparent radiants computed at highest radiant elevation # (otherwise the apparent radiants can be quite off) shower_dict = {} for key in associations: meteor_obj, shower = associations[key] if shower is None: continue # If the shower name is in dict, find the shower with the highest radiant elevation if shower.name in shower_dict: if shower.elev > shower_dict[shower.name].elev: shower_dict[shower.name] = shower else: shower_dict[shower.name] = shower # Plot the location of shower radiants for shower_name in shower_dict: shower = shower_dict[shower_name] heading_arr = np.linspace(0, 360, 50) # Compute coordinates on a circle around the given RA, Dec ra_circle, dec_circle = sphericalPointFromHeadingAndDistance(shower.ra, shower.dec, \ heading_arr, config.shower_max_radiant_separation) # Plot the shower circle allsky_plot.plot(ra_circle, dec_circle, color=colors_by_name[shower_name]) # Plot the shower name x_text, y_text = allsky_plot.raDec2XY(shower.ra, shower.dec) allsky_plot.ax.text(x_text, y_text, shower.name, color='w', size=8, va='center', \ ha='center', zorder=6) # Plot station name and solar longiutde range allsky_plot.ax.text(-180, 89, "{:s}".format(cam_code), color='w', family='monospace') # Get a list of JDs of meteors jd_list = [associations[key][0].jdt_ref for key in associations] if len(jd_list): # Get the range of solar longitudes jd_min = min(jd_list) sol_min = np.degrees(jd2SolLonSteyaert(jd_min)) jd_max = max(jd_list) sol_max = np.degrees(jd2SolLonSteyaert(jd_max)) # Plot the date and solar longitude range date_sol_beg = u"Beg: {:s} (sol = {:.2f}\u00b0)".format( jd2Date(jd_min, dt_obj=True).strftime("%Y%m%d %H:%M:%S"), sol_min) date_sol_end = u"End: {:s} (sol = {:.2f}\u00b0)".format( jd2Date(jd_max, dt_obj=True).strftime("%Y%m%d %H:%M:%S"), sol_max) allsky_plot.ax.text(-180, 85, date_sol_beg, color='w', family='monospace') allsky_plot.ax.text(-180, 81, date_sol_end, color='w', family='monospace') allsky_plot.ax.text(-180, 77, "-" * len(date_sol_end), color='w', family='monospace') # Plot shower counts for i, (shower, count) in enumerate(shower_counts): if shower is not None: shower_name = shower.name else: shower_name = "..." allsky_plot.ax.text(-180, 73 - i*4, "{:s}: {:d}".format(shower_name, count), color='w', \ family='monospace') ### ### # Plot yearly meteor shower activity if plot_activity: # Plot the activity diagram generateActivityDiagram(config, shower_list, ax_handle=ax_activity, \ sol_marker=[sol_min, sol_max], colors=colors_by_name) # Save plot and text file if save_plot: dir_path, ftpdetectinfo_name = os.path.split(ftpdetectinfo_path) ftpdetectinfo_base_name = ftpdetectinfo_name.replace( 'FTPdetectinfo_', '').replace('.txt', '') plot_name = ftpdetectinfo_base_name + '_radiants.png' # Increase figure size allsky_plot.fig.set_size_inches(18, 9, forward=True) allsky_plot.beautify() plt.savefig(os.path.join(dir_path, plot_name), dpi=100, facecolor='k') # Save the text file with shower info if len(jd_list): with open( os.path.join(dir_path, ftpdetectinfo_base_name + "_radiants.txt"), 'w') as f: # Print station code f.write("# RMS single station association\n") f.write("# \n") f.write("# Station: {:s}\n".format(cam_code)) # Print date range f.write( "# Beg | End \n" ) f.write( "# -----------------------------------------------------\n" ) f.write("# Date | {:24s} | {:24s} \n".format(jd2Date(jd_min, \ dt_obj=True).strftime("%Y%m%d %H:%M:%S.%f"), jd2Date(jd_max, \ dt_obj=True).strftime("%Y%m%d %H:%M:%S.%f"))) f.write("# Sol | {:>24.2f} | {:>24.2f} \n".format( sol_min, sol_max)) # Write shower counts f.write("# \n") f.write("# Shower counts:\n") f.write("# --------------\n") f.write("# Code, Count, IAU link\n") for i, (shower, count) in enumerate(shower_counts): if shower is not None: shower_name = shower.name # Create link to the IAU database of showers iau_link = "https://www.ta3.sk/IAUC22DB/MDC2007/Roje/pojedynczy_obiekt.php?kodstrumienia={:05d}".format( shower.iau_code) else: shower_name = "..." iau_link = "None" f.write("# {:>4s}, {:>5d}, {:s}\n".format( shower_name, count, iau_link)) f.write("# \n") f.write("# Meteor parameters:\n") f.write("# ------------------\n") f.write( "# Date And Time, Beg Julian date, La Sun, Shower, RA beg, Dec beg, RA end, Dec end, RA rad, Dec rad, GC theta0, GC phi0, GC beg phase, GC end phase, Mag\n" ) # Create a sorted list of meteor associations by time associations_list = [ associations[key] for key in associations ] associations_list = sorted(associations_list, key=lambda x: x[0].jdt_ref) # Write out meteor parameters for meteor_obj, shower in associations_list: # Find peak magnitude if np.any(meteor_obj.mag_array): peak_mag = "{:+.1f}".format( np.min(meteor_obj.mag_array)) else: peak_mag = "None" if shower is not None: f.write("{:24s}, {:20.12f}, {:>10.6f}, {:>6s}, {:6.2f}, {:+7.2f}, {:6.2f}, {:+7.2f}, {:6.2f}, {:+7.2f}, {:9.3f}, {:8.3f}, {:12.3f}, {:12.3f}, {:4s}\n".format(jd2Date(meteor_obj.jdt_ref, dt_obj=True).strftime("%Y%m%d %H:%M:%S.%f"), \ meteor_obj.jdt_ref, meteor_obj.lasun, shower.name, \ meteor_obj.ra_array[0]%360, meteor_obj.dec_array[0], \ meteor_obj.ra_array[-1]%360, meteor_obj.dec_array[-1], \ meteor_obj.radiant_ra%360, meteor_obj.radiant_dec, \ np.degrees(meteor_obj.theta0), np.degrees(meteor_obj.phi0), \ meteor_obj.gc_beg_phase, meteor_obj.gc_end_phase, peak_mag)) else: f.write("{:24s}, {:20.12f}, {:>10.6f}, {:>6s}, {:6.2f}, {:+7.2f}, {:6.2f}, {:+7.2f}, {:>6s}, {:>7s}, {:9.3f}, {:8.3f}, {:12.3f}, {:12.3f}, {:4s}\n".format(jd2Date(meteor_obj.jdt_ref, dt_obj=True).strftime("%Y%m%d %H:%M:%S.%f"), \ meteor_obj.jdt_ref, meteor_obj.lasun, '...', meteor_obj.ra_array[0]%360, \ meteor_obj.dec_array[0], meteor_obj.ra_array[-1]%360, \ meteor_obj.dec_array[-1], "None", "None", np.degrees(meteor_obj.theta0), \ np.degrees(meteor_obj.phi0), meteor_obj.gc_beg_phase, \ meteor_obj.gc_end_phase, peak_mag)) if show_plot: allsky_plot.show() else: plt.clf() plt.close() return associations, shower_counts
def add_fffits_metadata(ff_filename, config, platepars_recalibrated, fallback_platepar): """ Add FITS metadata and WCS to FF files generated by RMS Args: ff_filename (str): full or relative path to FF file config (RMS.Config): config instance platepars_recalibrated (dict): dictionary with recalibrated platepars fallback_platepar (RMS.Platepar): platepar with fitted stars Returns: None """ ff_basename = os.path.basename(ff_filename) platepar_recalibrated = Platepar() try: platepar_data = platepars_recalibrated[ff_basename] with open("platepar_tmp.cal", "w") as f: json.dump(platepar_data, f) platepar_recalibrated.read("platepar_tmp.cal") except (FileNotFoundError, KeyError): platepar_recalibrated = fallback_platepar logger.warning(f"Using non-recalibrated platepar for {ff_basename}") fftime = getMiddleTimeFF(ff_basename, config.fps) fit_xy = np.array(fallback_platepar.star_list)[:, 1:3] _, fit_ra, fit_dec, _ = xyToRaDecPP([fftime] * len(fit_xy), fit_xy[:, 0], fit_xy[:, 1], [1] * len(fit_xy), platepar_recalibrated, extinction_correction=False) x0 = platepar_recalibrated.X_res / 2 y0 = platepar_recalibrated.Y_res / 2 _, ra0, dec0, _ = xyToRaDecPP([fftime], [x0], [y0], [1], platepar_recalibrated, extinction_correction=False) w = fit_wcs(fit_xy[:, 0], fit_xy[:, 1], fit_ra, fit_dec, x0, y0, ra0[0], dec0[0], 5, projection="ZEA") hdu_list = fits.open(ff_filename, scale_back=True) obstime = Time(filenameToDatetime(ff_basename)) header_meta = {} header_meta["OBSERVER"] = config.stationID.strip() header_meta["INSTRUME"] = "Global Meteor Network" header_meta["MJD-OBS"] = obstime.mjd header_meta["DATE-OBS"] = obstime.fits header_meta["NFRAMES"] = 256 header_meta["EXPTIME"] = 256 / config.fps header_meta["SITELONG"] = round(config.longitude, 2) header_meta["SITELAT"] = round(config.latitude, 2) for hdu in hdu_list: if hdu.header[ "NAXIS"] == 0: # First header is not an image so should not get WCS new_header = Header() else: new_header = w.to_fits(relax=True)[0].header for key, value in header_meta.items(): new_header.append((key, value)) for key, value in new_header.items(): if key in hdu.header: continue hdu.header[key] = value hdu_list.writeto(ff_filename, overwrite=True)