def saveFTPdetectinfo(self):
""" Saves the picks to a FTPdetectinfo file in the same folder where the first given file is. """
# Compute the intensity sum done on the previous frame
self.computeIntensitySum()
# PNG mode
if self.png_mode:
dir_path = self.png_dir
ff_name_ftp = "FF_" + self.frame0_time.strftime(
"%Y%m%d_%H%M%S.%f") + '.bin'
# FF mode
else:
# Extract the save directory
if self.ff_file is not None:
dir_path, ff_name_ftp = os.path.split(self.ff_file)
elif self.fr_file is not None:
dir_path, ff_name_ftp = os.path.split(self.fr_file)
# Remove the file extension of the image file
ff_name_ftp = ff_name_ftp.replace('.bin', '').replace('.fits', '')
# Create the list of picks for saving
centroids = []
for pick in self.pick_list:
centroids.append([
pick.frame, pick.x_centroid, pick.y_centroid,
pick.intensity_sum
])
meteor_list = [[ff_name_ftp, 1, 0, 0, centroids]]
# Create a name for the FTPdetectinfo
ftpdetectinfo_name = "FTPdetectinfo_" + "_".join(
ff_name_ftp.split('_')[1:]) + '_manual.txt'
# Read the station code for the file name
station_id = ff_name_ftp.split('_')[1]
# Take the FPS from the FF file, if available
if self.ff is not None:
if hasattr(self.ff, 'fps'):
self.fps = self.ff.fps
if self.fps is None:
self.fps = self.config.fps
# Write the FTPdetect info
writeFTPdetectinfo(meteor_list, dir_path, ftpdetectinfo_name, '',
station_id, self.fps)
print('FTPdetecinfo written to:',
os.path.join(dir_path, ftpdetectinfo_name))
def applyAstrometryFTPdetectinfo(dir_path,
ftp_detectinfo_file,
platepar_file,
UT_corr=0,
platepar=None):
""" Use the given platepar to calculate the celestial coordinates of detected meteors from a FTPdetectinfo
file and save the updates values.
Arguments:
dir_path: [str] Path to the night.
ftp_detectinfo_file: [str] Name of the FTPdetectinfo file.
platepar_file: [str] Name of the platepar file.
Keyword arguments:
UT_corr: [float] Difference of time from UTC in hours.
platepar: [Platepar obj] Loaded platepar. None by default. If given, the platepar file won't be read,
but this platepar structure will be used instead.
Return:
None
"""
# If the FTPdetectinfo file does not exist, skip everything
if not os.path.isfile(os.path.join(dir_path, ftp_detectinfo_file)):
print('The given FTPdetectinfo file does not exist:',
os.path.join(dir_path, ftp_detectinfo_file))
print('The astrometry was not computed!')
return None
# Save a copy of the uncalibrated FTPdetectinfo
ftp_detectinfo_copy = "".join(
ftp_detectinfo_file.split('.')[:-1]) + "_uncalibrated.txt"
# Back up the original FTPdetectinfo, only if a backup does not exist already
if not os.path.isfile(os.path.join(dir_path, ftp_detectinfo_copy)):
shutil.copy2(os.path.join(dir_path, ftp_detectinfo_file),
os.path.join(dir_path, ftp_detectinfo_copy))
# Load platepar from file if not given
if platepar is None:
# Load the platepar
platepar = Platepar()
platepar.read(os.path.join(dir_path, platepar_file), use_flat=None)
# Load the FTPdetectinfo file
meteor_data = readFTPdetectinfo(dir_path, ftp_detectinfo_file)
# List for final meteor data
meteor_list = []
# Go through every meteor
for meteor in meteor_data:
ff_name, cam_code, meteor_No, n_segments, fps, hnr, mle, binn, px_fm, rho, phi, meteor_meas = meteor
# Apply the platepar to the given centroids
meteor_picks = applyPlateparToCentroids(ff_name, fps, meteor_meas,
platepar)
# Add the calculated values to the final list
meteor_list.append([ff_name, meteor_No, rho, phi, meteor_picks])
# Calibration string to be written to the FTPdetectinfo file
calib_str = 'Calibrated with RMS on: ' + str(
datetime.datetime.utcnow()) + ' UTC'
# If no meteors were detected, set dummpy parameters
if len(meteor_list) == 0:
cam_code = ''
fps = 0
# Save the updated FTPdetectinfo
writeFTPdetectinfo(meteor_list,
dir_path,
ftp_detectinfo_file,
dir_path,
cam_code,
fps,
calibration=calib_str,
celestial_coords_given=True)
def processNight(night_data_dir, config, detection_results=None, nodetect=False):
""" Given the directory with FF files, run detection and archiving.
Arguments:
night_data_dir: [str] Path to the directory with FF files.
config: [Config obj]
Keyword arguments:
detection_results: [list] An optional list of detection. If None (default), detection will be done
on the the files in the folder.
nodetect: [bool] True if detection should be skipped. False by default.
Return:
night_archive_dir: [str] Path to the night directory in ArchivedFiles.
archive_name: [str] Path to the archive.
detector: [QueuedPool instance] Handle to the detector.
"""
# Remove final slash in the night dir
if night_data_dir.endswith(os.sep):
night_data_dir = night_data_dir[:-1]
# Extract the name of the night
night_data_dir_name = os.path.basename(os.path.abspath(night_data_dir))
platepar = None
# If the detection should be run
if (not nodetect):
# If no detection was performed, run it
if detection_results is None:
# Run detection on the given directory
calstars_name, ftpdetectinfo_name, ff_detected, \
detector = detectStarsAndMeteorsDirectory(night_data_dir, config)
# Otherwise, save detection results
else:
# Save CALSTARS and FTPdetectinfo to disk
calstars_name, ftpdetectinfo_name, ff_detected = saveDetections(detection_results, \
night_data_dir, config)
# If the files were previously detected, there is no detector
detector = None
# Get the platepar file
platepar, platepar_path, platepar_fmt = getPlatepar(config, night_data_dir)
# Run calibration check and auto astrometry refinement
if (platepar is not None) and (calstars_name is not None):
# Read in the CALSTARS file
calstars_list = CALSTARS.readCALSTARS(night_data_dir, calstars_name)
# Run astrometry check and refinement
platepar, fit_status = autoCheckFit(config, platepar, calstars_list)
# If the fit was sucessful, apply the astrometry to detected meteors
if fit_status:
log.info('Astrometric calibration SUCCESSFUL!')
# Save the refined platepar to the night directory and as default
platepar.write(os.path.join(night_data_dir, config.platepar_name), fmt=platepar_fmt)
platepar.write(platepar_path, fmt=platepar_fmt)
else:
log.info('Astrometric calibration FAILED!, Using old platepar for calibration...')
# # Calculate astrometry for meteor detections
# applyAstrometryFTPdetectinfo(night_data_dir, ftpdetectinfo_name, platepar_path)
# If a flat is used, disable vignetting correction
if config.use_flat:
platepar.vignetting_coeff = 0.0
log.info("Recalibrating astrometry on FF files with detections...")
# Recalibrate astrometry on every FF file and apply the calibration to detections
recalibrateIndividualFFsAndApplyAstrometry(night_data_dir, os.path.join(night_data_dir, \
ftpdetectinfo_name), calstars_list, config, platepar)
log.info("Converting RMS format to UFOOrbit format...")
# Convert the FTPdetectinfo into UFOOrbit input file
FTPdetectinfo2UFOOrbitInput(night_data_dir, ftpdetectinfo_name, platepar_path)
# Generate a calibration report
log.info("Generating a calibration report...")
try:
generateCalibrationReport(config, night_data_dir, platepar=platepar)
except Exception as e:
log.debug('Generating calibration report failed with the message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
# Perform single station shower association
log.info("Performing single station shower association...")
try:
showerAssociation(config, [os.path.join(night_data_dir, ftpdetectinfo_name)], \
save_plot=True, plot_activity=True)
except Exception as e:
log.debug('Shower association failed with the message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
else:
ff_detected = []
detector = None
log.info('Plotting field sums...')
# Plot field sums
try:
plotFieldsums(night_data_dir, config)
except Exception as e:
log.debug('Plotting field sums failed with message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
# Archive all fieldsums to one archive
archiveFieldsums(night_data_dir)
# List for any extra files which will be copied to the night archive directory. Full paths have to be
# given
extra_files = []
log.info('Making a flat...')
# Make a new flat field image
try:
flat_img = makeFlat(night_data_dir, config)
except Exception as e:
log.debug('Making a flat failed with message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
flat_img = None
# If making flat was sucessfull, save it
if flat_img is not None:
# Save the flat in the night directory, to keep the operational flat updated
flat_path = os.path.join(night_data_dir, os.path.basename(config.flat_file))
saveImage(flat_path, flat_img)
log.info('Flat saved to: ' + flat_path)
# Copy the flat to the night's directory as well
extra_files.append(flat_path)
else:
log.info('Making flat image FAILED!')
### Add extra files to archive
# Add the config file to the archive too
extra_files.append(os.path.join(os.getcwd(), '.config'))
# Add the mask
if (not nodetect):
if os.path.exists(config.mask_file):
mask_path = os.path.abspath(config.mask_file)
extra_files.append(mask_path)
# Add the platepar to the archive if it exists
if (not nodetect):
if os.path.exists(platepar_path):
extra_files.append(platepar_path)
# Add the json file with recalibrated platepars to the archive
if (not nodetect):
recalibrated_platepars_path = os.path.join(night_data_dir, config.platepars_recalibrated_name)
if os.path.exists(recalibrated_platepars_path):
extra_files.append(recalibrated_platepars_path)
### ###
# If the detection should be run
if (not nodetect):
# Make a CAL file and a special CAMS FTPdetectinfo if full CAMS compatibility is desired
if (config.cams_code > 0) and (platepar is not None):
log.info('Generating a CAMS FTPdetectinfo file...')
# Write the CAL file to disk
cal_file_name = writeCAL(night_data_dir, config, platepar)
# Check if the CAL file was successfully generated
if cal_file_name is not None:
cams_code_formatted = "{:06d}".format(int(config.cams_code))
# Load the FTPdetectinfo
_, fps, meteor_list = readFTPdetectinfo(night_data_dir, ftpdetectinfo_name, \
ret_input_format=True)
# Replace the camera code with the CAMS code
for met in meteor_list:
# Replace the station name and the FF file format
ff_name = met[0]
ff_name = ff_name.replace('.fits', '.bin')
ff_name = ff_name.replace(config.stationID, cams_code_formatted)
met[0] = ff_name
# Write the CAMS compatible FTPdetectinfo file
writeFTPdetectinfo(meteor_list, night_data_dir, \
ftpdetectinfo_name.replace(config.stationID, cams_code_formatted),\
night_data_dir, cams_code_formatted, fps, calibration=cal_file_name, \
celestial_coords_given=(platepar is not None))
night_archive_dir = os.path.join(os.path.abspath(config.data_dir), config.archived_dir,
night_data_dir_name)
log.info('Archiving detections to ' + night_archive_dir)
# Archive the detections
archive_name = archiveDetections(night_data_dir, night_archive_dir, ff_detected, config, \
extra_files=extra_files)
return night_archive_dir, archive_name, detector
print("{:5.1f}, {:7.2f}, {:8.2f}, {:10.1f}".format(
frame_n, mag, unsaturated_mag, bg_val))
# Compute the intensity from unsaturated magnitude
unsaturated_inten = round(10**((photom_offset - mag) / 2.5), 0)
corrected_meteor_meas.append([
frame_n, x, y, ra, dec, azim, elev, unsaturated_inten,
unsaturated_mag
])
if not corrected_meteor_meas:
corrected_meteor_meas = meteor_meas
corrected_meteor_list.append(
[ftp_ff_name, meteor_No, rho, phi, corrected_meteor_meas])
# Calibration string to be written to the FTPdetectinfo file
calib_str = "RMS - Saturation corrected on {:s} UTC".format(
str(datetime.datetime.utcnow()))
# Write a corrected FTPdetectinfo file
corrected_ftpdetectinfo_name = ftpdetectinfo_name.strip(
'.txt') + '_saturation_corrected.txt'
print('Saving to:', os.path.join(dir_path, corrected_ftpdetectinfo_name))
writeFTPdetectinfo(corrected_meteor_list, dir_path, corrected_ftpdetectinfo_name, dir_path, cam_code, \
fps, calibration=calib_str, celestial_coords_given=True)
def applyAstrometryFTPdetectinfo(dir_path,
ftp_detectinfo_file,
platepar_file,
UT_corr=0):
""" Use the given platepar to calculate the celestial coordinates of detected meteors from a FTPdetectinfo
file and save the updates values.
Arguments:
dir_path: [str] Path to the night.
ftp_detectinfo_file: [str] Name of the FTPdetectinfo file.
platepar_file: [str] Name of the platepar file.
Keyword arguments:
UT_corr: [float] Difference of time from UTC in hours.
Return:
None
"""
# Save a copy of the uncalibrated FTPdetectinfo
ftp_detectinfo_copy = "".join(
ftp_detectinfo_file.split('.')[:-1]) + "_uncalibrated.txt"
# Back up the original FTPdetectinfo, only if a backup does not exist already
if not os.path.isfile(os.path.join(dir_path, ftp_detectinfo_copy)):
shutil.copy2(os.path.join(dir_path, ftp_detectinfo_file),
os.path.join(dir_path, ftp_detectinfo_copy))
# Load the platepar
platepar = Platepar()
platepar.read(os.path.join(dir_path, platepar_file))
# Load the FTPdetectinfo file
meteor_data = readFTPdetectinfo(dir_path, ftp_detectinfo_file)
# List for final meteor data
meteor_list = []
# Go through every meteor
for meteor in meteor_data:
ff_name, cam_code, meteor_No, n_segments, fps, hnr, mle, binn, px_fm, rho, phi, meteor_meas = meteor
meteor_meas = np.array(meteor_meas)
# Extract frame number, x, y, intensity
frames = meteor_meas[:, 1]
X_data = meteor_meas[:, 2]
Y_data = meteor_meas[:, 3]
level_data = meteor_meas[:, 8]
# Get the beginning time of the FF file
time_beg = filenameToDatetime(ff_name)
# Calculate time data of every point
time_data = []
for frame_n in frames:
t = time_beg + datetime.timedelta(seconds=frame_n / fps)
time_data.append([
t.year, t.month, t.day, t.hour, t.minute, t.second,
int(t.microsecond / 1000)
])
# Convert image cooredinates to RA and Dec, and do the photometry
JD_data, RA_data, dec_data, magnitudes = XY2CorrectedRADecPP(np.array(time_data), X_data, Y_data, \
level_data, platepar)
# Compute azimuth and altitude of centroids
az_data = np.zeros_like(RA_data)
alt_data = np.zeros_like(RA_data)
for i in range(len(az_data)):
jd = JD_data[i]
ra_tmp = RA_data[i]
dec_tmp = dec_data[i]
az_tmp, alt_tmp = raDec2AltAz(jd, platepar.lon, platepar.lat,
ra_tmp, dec_tmp)
az_data[i] = az_tmp
alt_data[i] = alt_tmp
# print(ff_name, cam_code, meteor_No, fps)
# print(X_data, Y_data)
# print(RA_data, dec_data)
# print('------------------------------------------')
# Construct the meteor measurements array
meteor_picks = np.c_[frames, X_data, Y_data, RA_data, dec_data, az_data, alt_data, level_data, \
magnitudes]
# Add the calculated values to the final list
meteor_list.append([ff_name, meteor_No, rho, phi, meteor_picks])
# Calibration string to be written to the FTPdetectinfo file
calib_str = 'Calibrated with RMS on: ' + str(
datetime.datetime.utcnow()) + ' UTC'
# If no meteors were detected, set dummpy parameters
if len(meteor_list) == 0:
cam_code = ''
fps = 0
# Save the updated FTPdetectinfo
writeFTPdetectinfo(meteor_list,
dir_path,
ftp_detectinfo_file,
dir_path,
cam_code,
fps,
calibration=calib_str,
celestial_coords_given=True)
# Compute the intensity from unsaturated magnitude
unsaturated_inten = round(10**((photom_offset - mag)/2.5), 0)
corrected_meteor_meas.append([frame_n, x, y, ra, dec, azim, elev, unsaturated_inten,
unsaturated_mag])
if not corrected_meteor_meas:
corrected_meteor_meas = meteor_meas
corrected_meteor_list.append([ftp_ff_name, meteor_No, rho, phi, corrected_meteor_meas])
# Calibration string to be written to the FTPdetectinfo file
calib_str = "RMS - Saturation corrected on {:s} UTC".format(str(datetime.datetime.utcnow()))
# Write a corrected FTPdetectinfo file
corrected_ftpdetectinfo_name = ftpdetectinfo_name.strip('.txt') + '_saturation_corrected.txt'
print('Saving to:', os.path.join(dir_path, corrected_ftpdetectinfo_name))
writeFTPdetectinfo(corrected_meteor_list, dir_path, corrected_ftpdetectinfo_name, dir_path, cam_code, \
fps, calibration=calib_str, celestial_coords_given=True)
def processNight(night_data_dir, config, detection_results=None, nodetect=False):
""" Given the directory with FF files, run detection and archiving.
Arguments:
night_data_dir: [str] Path to the directory with FF files.
config: [Config obj]
Keyword arguments:
detection_results: [list] An optional list of detection. If None (default), detection will be done
on the the files in the folder.
nodetect: [bool] True if detection should be skipped. False by default.
Return:
night_archive_dir: [str] Path to the night directory in ArchivedFiles.
archive_name: [str] Path to the archive.
detector: [QueuedPool instance] Handle to the detector.
"""
# Remove final slash in the night dir
if night_data_dir.endswith(os.sep):
night_data_dir = night_data_dir[:-1]
# Extract the name of the night
night_data_dir_name = os.path.basename(night_data_dir)
# If the detection should be run
if (not nodetect):
# If no detection was performed, run it
if detection_results is None:
# Run detection on the given directory
calstars_name, ftpdetectinfo_name, ff_detected, \
detector = detectStarsAndMeteorsDirectory(night_data_dir, config)
# Otherwise, save detection results
else:
# Save CALSTARS and FTPdetectinfo to disk
calstars_name, ftpdetectinfo_name, ff_detected = saveDetections(detection_results, \
night_data_dir, config)
# If the files were previously detected, there is no detector
detector = None
# Get the platepar file
platepar, platepar_path, platepar_fmt = getPlatepar(config, night_data_dir)
# Run calibration check and auto astrometry refinement
if platepar is not None:
# Read in the CALSTARS file
calstars_list = CALSTARS.readCALSTARS(night_data_dir, calstars_name)
# Run astrometry check and refinement
platepar, fit_status = autoCheckFit(config, platepar, calstars_list)
# If the fit was sucessful, apply the astrometry to detected meteors
if fit_status:
log.info('Astrometric calibration SUCCESSFUL!')
# Save the refined platepar to the night directory and as default
platepar.write(os.path.join(night_data_dir, config.platepar_name), fmt=platepar_fmt)
platepar.write(platepar_path, fmt=platepar_fmt)
else:
log.info('Astrometric calibration FAILED!, Using old platepar for calibration...')
# # Calculate astrometry for meteor detections
# applyAstrometryFTPdetectinfo(night_data_dir, ftpdetectinfo_name, platepar_path)
log.info("Recalibrating astrometry on FF files with detections...")
# Recalibrate astrometry on every FF file and apply the calibration to detections
recalibrateIndividualFFsAndApplyAstrometry(night_data_dir, os.path.join(night_data_dir, \
ftpdetectinfo_name), calstars_list, config, platepar)
log.info("Converting RMS format to UFOOrbit format...")
# Convert the FTPdetectinfo into UFOOrbit input file
FTPdetectinfo2UFOOrbitInput(night_data_dir, ftpdetectinfo_name, platepar_path)
# Generate a calibration report
log.info("Generating a calibration report...")
try:
generateCalibrationReport(config, night_data_dir, platepar=platepar)
except Exception as e:
log.debug('Generating calibration report failed with message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
else:
ff_detected = []
detector = None
log.info('Plotting field sums...')
# Plot field sums
try:
plotFieldsums(night_data_dir, config)
except Exception as e:
log.debug('Plotting field sums failed with message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
# Archive all fieldsums to one archive
archiveFieldsums(night_data_dir)
# List for any extra files which will be copied to the night archive directory. Full paths have to be
# given
extra_files = []
log.info('Making a flat...')
# Make a new flat field image
try:
flat_img = makeFlat(night_data_dir, config)
except Exception as e:
log.debug('Making a flat failed with message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
flat_img = None
# If making flat was sucessfull, save it
if flat_img is not None:
# Save the flat in the night directory, to keep the operational flat updated
flat_path = os.path.join(night_data_dir, os.path.basename(config.flat_file))
scipy.misc.imsave(flat_path, flat_img)
log.info('Flat saved to: ' + flat_path)
# Copy the flat to the night's directory as well
extra_files.append(flat_path)
else:
log.info('Making flat image FAILED!')
### Add extra files to archive
# Add the config file to the archive too
extra_files.append(os.path.join(os.getcwd(), '.config'))
# Add the platepar to the archive if it exists
if (not nodetect):
if os.path.exists(platepar_path):
extra_files.append(platepar_path)
# Add the json file with recalibrated platepars to the archive
if (not nodetect):
recalibrated_platepars_path = os.path.join(night_data_dir, config.platepars_recalibrated_name)
if os.path.exists(recalibrated_platepars_path):
extra_files.append(recalibrated_platepars_path)
### ###
# If the detection should be run
if (not nodetect):
# Make a CAL file and a special CAMS FTpdetectinfo if full CAMS compatibility is desired
if config.cams_code > 0:
log.info('Generating a CAMS FTPdetectinfo file...')
# Write the CAL file to disk
cal_file_name = writeCAL(night_data_dir, config, platepar)
cams_code_formatted = "{:06d}".format(int(config.cams_code))
# Load the FTPdetectinfo
_, fps, meteor_list = readFTPdetectinfo(night_data_dir, ftpdetectinfo_name, \
ret_input_format=True)
# Replace the camera code with the CAMS code
for met in meteor_list:
# Replace the station name and the FF file format
ff_name = met[0]
ff_name = ff_name.replace('.fits', '.bin')
ff_name = ff_name.replace(config.stationID, cams_code_formatted)
met[0] = ff_name
# Replace the station name
met[1] = cams_code_formatted
# Write the CAMS compatible FTPdetectinfo file
writeFTPdetectinfo(meteor_list, night_data_dir, \
ftpdetectinfo_name.replace(config.stationID, cams_code_formatted),\
night_data_dir, cams_code_formatted, fps, calibration=cal_file_name, \
celestial_coords_given=(platepar is not None))
night_archive_dir = os.path.join(os.path.abspath(config.data_dir), config.archived_dir,
night_data_dir_name)
log.info('Archiving detections to ' + night_archive_dir)
# Archive the detections
archive_name = archiveDetections(night_data_dir, night_archive_dir, ff_detected, config, \
extra_files=extra_files)
return night_archive_dir, archive_name, detector
