def applyRecalibrate(ftpdetectinfo_path, config, generate_plot=True):
""" Recalibrate FF files with detections and apply the recalibrated platepar to those detections.
Arguments:
ftpdetectinfo_path: [str] Name of the FTPdetectinfo file.
config: [Config instance]
Keyword arguments:
generate_plot: [bool] Generate the calibration variation plot. True by default.
Return:
recalibrated_platepars: [dict] A dictionary where the keys are FF file names and values are
recalibrated platepar instances for every FF file.
"""
# Extract parent directory
dir_path = os.path.dirname(ftpdetectinfo_path)
# Get a list of files in the night folder
file_list = sorted(os.listdir(dir_path))
# Find and load the platepar file
if config.platepar_name in file_list:
# Load the platepar
platepar = Platepar.Platepar()
platepar.read(os.path.join(dir_path, config.platepar_name),
use_flat=config.use_flat)
else:
print('Cannot find the platepar file in the night directory: ',
config.platepar_name)
sys.exit()
# Find the CALSTARS file in the given folder
calstars_file = None
for calstars_file in file_list:
if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file):
break
if calstars_file is None:
print('CALSTARS file could not be found in the given directory!')
sys.exit()
# Load the calstars file
calstars_list = CALSTARS.readCALSTARS(dir_path, calstars_file)
print('CALSTARS file: ' + calstars_file + ' loaded!')
# Recalibrate and apply astrometry on every FF file with detections individually
recalibrated_platepars = recalibrateIndividualFFsAndApplyAstrometry(dir_path, ftpdetectinfo_path, \
calstars_list, config, platepar, generate_plot=generate_plot)
### Generate the updated UFOorbit file ###
Utils.RMS2UFO.FTPdetectinfo2UFOOrbitInput(dir_path, os.path.basename(ftpdetectinfo_path), None, \
platepar_dict=recalibrated_platepars)
### ###
return recalibrated_platepars
def runCapture(config,
duration=None,
video_file=None,
nodetect=False,
detect_end=False,
upload_manager=None):
""" Run capture and compression for the given time.given
Arguments:
config: [config object] Configuration read from the .config file
Keyword arguments:
duration: [float] Time in seconds to capture. None by default.
video_file: [str] Path to the video file, if it was given as the video source. None by default.
nodetect: [bool] If True, detection will not be performed. False by defualt.
detect_end: [bool] If True, detection will be performed at the end of the night, when capture
finishes. False by default.
upload_manager: [UploadManager object] A handle to the UploadManager, which handles uploading files to
the central server. None by default.
"""
global STOP_CAPTURE
# Create a directory for captured files
night_data_dir_name = str(
config.stationID) + '_' + datetime.datetime.utcnow().strftime(
'%Y%m%d_%H%M%S_%f')
# Full path to the data directory
night_data_dir = os.path.join(os.path.abspath(config.data_dir),
config.captured_dir, night_data_dir_name)
# Make a directory for the night
mkdirP(night_data_dir)
log.info('Data directory: ' + night_data_dir)
# Load the default flat field image if it is available
flat_struct = None
if config.use_flat:
# Check if the flat exists
if os.path.exists(os.path.join(os.getcwd(), config.flat_file)):
flat_struct = Image.loadFlat(os.getcwd(), config.flat_file)
log.info('Loaded flat field image: ' +
os.path.join(os.getcwd(), config.flat_file))
# Get the platepar file
platepar, platepar_path, platepar_fmt = getPlatepar(config)
log.info('Initializing frame buffers...')
### For some reason, the RPi 3 does not like memory chunks which size is the multipier of its L2
### cache size (512 kB). When such a memory chunk is provided, the compression becomes 10x slower
### then usual. We are applying a dirty fix here where we just add an extra image row and column
### if such a memory chunk will be created. The compression is performed, and the image is cropped
### back to its original dimensions.
array_pad = 0
# Check if the image dimensions are divisible by RPi3 L2 cache size and add padding
if (256 * config.width * config.height) % (512 * 1024) == 0:
array_pad = 1
# Init arrays for parallel compression on 2 cores
sharedArrayBase = multiprocessing.Array(
ctypes.c_uint8,
256 * (config.width + array_pad) * (config.height + array_pad))
sharedArray = np.ctypeslib.as_array(sharedArrayBase.get_obj())
sharedArray = sharedArray.reshape(256, (config.height + array_pad),
(config.width + array_pad))
startTime = multiprocessing.Value('d', 0.0)
sharedArrayBase2 = multiprocessing.Array(
ctypes.c_uint8,
256 * (config.width + array_pad) * (config.height + array_pad))
sharedArray2 = np.ctypeslib.as_array(sharedArrayBase2.get_obj())
sharedArray2 = sharedArray2.reshape(256, (config.height + array_pad),
(config.width + array_pad))
startTime2 = multiprocessing.Value('d', 0.0)
log.info('Initializing frame buffers done!')
# Check if the detection should be performed or not
if nodetect:
detector = None
else:
if detect_end:
# Delay detection until the end of the night
delay_detection = duration
else:
# Delay the detection for 2 minutes after capture start
delay_detection = 120
# Initialize the detector
detector = QueuedPool(detectStarsAndMeteors,
cores=1,
log=log,
delay_start=delay_detection)
detector.startPool()
# Initialize buffered capture
bc = BufferedCapture(sharedArray,
startTime,
sharedArray2,
startTime2,
config,
video_file=video_file)
# Initialize the live image viewer
live_view = LiveViewer(window_name='Maxpixel')
# Initialize compression
compressor = Compressor(night_data_dir,
sharedArray,
startTime,
sharedArray2,
startTime2,
config,
detector=detector,
live_view=live_view,
flat_struct=flat_struct)
# Start buffered capture
bc.startCapture()
# Start the compression
compressor.start()
# Capture until Ctrl+C is pressed
wait(duration)
# If capture was manually stopped, end capture
if STOP_CAPTURE:
log.info('Ending capture...')
# Stop the capture
log.debug('Stopping capture...')
bc.stopCapture()
log.debug('Capture stopped')
dropped_frames = bc.dropped_frames
log.info('Total number of dropped frames: ' + str(dropped_frames))
# Stop the compressor
log.debug('Stopping compression...')
detector, live_view = compressor.stop()
log.debug('Compression stopped')
# Stop the live viewer
log.debug('Stopping live viewer...')
live_view.stop()
del live_view
log.debug('Live view stopped')
# Init data lists
star_list = []
meteor_list = []
ff_detected = []
# If detection should be performed
if not nodetect:
log.info('Finishing up the detection, ' +
str(detector.input_queue.qsize()) + ' files to process...')
# Reset the Ctrl+C to KeyboardInterrupt
resetSIGINT()
try:
# If there are some more files to process, process them on more cores
if detector.input_queue.qsize() > 0:
# Let the detector use all cores, but leave 1 free
available_cores = multiprocessing.cpu_count() - 1
if available_cores > 1:
log.info('Running the detection on {:d} cores...'.format(
available_cores))
# Start the detector
detector.updateCoreNumber(cores=available_cores)
log.info('Waiting for the detection to finish...')
# Wait for the detector to finish and close it
detector.closePool()
log.info('Detection finished!')
except KeyboardInterrupt:
log.info('Ctrl + C pressed, exiting...')
if upload_manager is not None:
# Stop the upload manager
if upload_manager.is_alive():
log.debug('Closing upload manager...')
upload_manager.stop()
del upload_manager
# Terminate the detector
if detector is not None:
del detector
sys.exit()
# Set the Ctrl+C back to 'soft' program kill
setSIGINT()
### SAVE DETECTIONS TO FILE
log.info('Collecting results...')
# Get the detection results from the queue
detection_results = detector.getResults()
# Remove all 'None' results, which were errors
detection_results = [
res for res in detection_results if res is not None
]
# Count the number of detected meteors
meteors_num = 0
for _, _, meteor_data in detection_results:
for meteor in meteor_data:
meteors_num += 1
log.info('TOTAL: ' + str(meteors_num) + ' detected meteors.')
# Save the detections to a file
for ff_name, star_data, meteor_data in detection_results:
x2, y2, background, intensity = star_data
# Skip if no stars were found
if not x2:
continue
# Construct the table of the star parameters
star_data = zip(x2, y2, background, intensity)
# Add star info to the star list
star_list.append([ff_name, star_data])
# Handle the detected meteors
meteor_No = 1
for meteor in meteor_data:
rho, theta, centroids = meteor
# Append to the results list
meteor_list.append([ff_name, meteor_No, rho, theta, centroids])
meteor_No += 1
# Add the FF file to the archive list if a meteor was detected on it
if meteor_data:
ff_detected.append(ff_name)
# Generate the name for the CALSTARS file
calstars_name = 'CALSTARS_' + "{:s}".format(str(config.stationID)) + '_' \
+ os.path.basename(night_data_dir) + '.txt'
# Write detected stars to the CALSTARS file
CALSTARS.writeCALSTARS(star_list, night_data_dir, calstars_name, config.stationID, config.height, \
config.width)
# Generate FTPdetectinfo file name
ftpdetectinfo_name = 'FTPdetectinfo_' + os.path.basename(
night_data_dir) + '.txt'
# Write FTPdetectinfo file
FTPdetectinfo.writeFTPdetectinfo(meteor_list, night_data_dir, ftpdetectinfo_name, night_data_dir, \
config.stationID, config.fps)
# Get the platepar file
platepar, platepar_path, platepar_fmt = getPlatepar(config)
# Run calibration check and auto astrometry refinement
if platepar is not None:
# Read in the CALSTARS file
calstars_list = CALSTARS.readCALSTARS(night_data_dir,
calstars_name)
# Run astrometry check and refinement
platepar, fit_status = autoCheckFit(config, platepar,
calstars_list)
# If the fit was sucessful, apply the astrometry to detected meteors
if fit_status:
log.info('Astrometric calibration SUCCESSFUL!')
# Save the refined platepar to the night directory and as default
platepar.write(os.path.join(night_data_dir,
config.platepar_name),
fmt=platepar_fmt)
platepar.write(platepar_path, fmt=platepar_fmt)
else:
log.info(
'Astrometric calibration FAILED!, Using old platepar for calibration...'
)
# Calculate astrometry for meteor detections
applyAstrometryFTPdetectinfo(night_data_dir, ftpdetectinfo_name,
platepar_path)
log.info('Plotting field sums...')
# Plot field sums to a graph
plotFieldsums(night_data_dir, config)
# Archive all fieldsums to one archive
archiveFieldsums(night_data_dir)
# List for any extra files which will be copied to the night archive directory. Full paths have to be
# given
extra_files = []
log.info('Making a flat...')
# Make a new flat field
flat_img = makeFlat(night_data_dir, config)
# If making flat was sucessfull, save it
if flat_img is not None:
# Save the flat in the root directory, to keep the operational flat updated
scipy.misc.imsave(config.flat_file, flat_img)
flat_path = os.path.join(os.getcwd(), config.flat_file)
log.info('Flat saved to: ' + flat_path)
# Copy the flat to the night's directory as well
extra_files.append(flat_path)
else:
log.info('Making flat image FAILED!')
### Add extra files to archive
# Add the platepar to the archive if it exists
if os.path.exists(platepar_path):
extra_files.append(platepar_path)
# Add the config file to the archive too
extra_files.append(os.path.join(os.getcwd(), '.config'))
### ###
night_archive_dir = os.path.join(os.path.abspath(config.data_dir),
config.archived_dir, night_data_dir_name)
log.info('Archiving detections to ' + night_archive_dir)
# Archive the detections
archive_name = archiveDetections(night_data_dir, night_archive_dir, ff_detected, config, \
extra_files=extra_files)
# Put the archive up for upload
if upload_manager is not None:
log.info('Adding file on upload list: ' + archive_name)
upload_manager.addFiles([archive_name])
# If capture was manually stopped, end program
if STOP_CAPTURE:
log.info('Ending program')
# Stop the upload manager
if upload_manager is not None:
if upload_manager.is_alive():
upload_manager.stop()
log.info('Closing upload manager...')
sys.exit()
def sensorCharacterization(config, dir_path):
""" Characterize the standard deviation of the background and the FWHM of stars on every image. """
# Find the CALSTARS file in the given folder that has FWHM information
found_good_calstars = False
for cal_file in os.listdir(dir_path):
if ('CALSTARS' in cal_file) and ('.txt' in cal_file) and (not found_good_calstars):
# Load the calstars file
calstars_list = CALSTARS.readCALSTARS(dir_path, cal_file)
if len(calstars_list) > 0:
# Check that at least one image has good FWHM measurements
for ff_name, star_data in calstars_list:
if len(star_data) > 1:
star_data = np.array(star_data)
# Check if the calstars file have FWHM information
fwhm = star_data[:, 4]
# Check that FWHM values have been computed well
if np.all(fwhm > 1):
found_good_calstars = True
print('CALSTARS file: ' + cal_file + ' loaded!')
break
# If the FWHM information is not present, run the star extraction
if not found_good_calstars:
print()
print("No FWHM information found in existing CALSTARS files!")
print()
print("Rerunning star detection...")
print()
found_good_calstars = False
# Run star extraction again, and now FWHM will be computed
calstars_list = extractStarsAndSave(config, dir_path)
if len(calstars_list) == 0:
found_good_calstars = False
# Check for a minimum of detected stars
for ff_name, star_data in calstars_list:
if len(star_data) >= config.ff_min_stars:
found_good_calstars = True
break
# If no good calstars exist, stop computing the flux
if not found_good_calstars:
print("No stars were detected in the data!")
return False
# Dictionary which holds information about FWHM and standard deviation of the image background
sensor_data = {}
# Compute median FWHM per FF file
for ff_name, star_data in calstars_list:
# Check that the FF file exists in the data directory
if ff_name not in os.listdir(dir_path):
continue
star_data = np.array(star_data)
# Compute the median star FWHM
fwhm_median = np.median(star_data[:, 4])
# Load the FF file and compute the standard deviation of the background
ff = FFfile.read(dir_path, ff_name)
# Compute the median stddev of the background
stddev_median = np.median(ff.stdpixel)
# Store the values to the dictionary
sensor_data[ff_name] = [fwhm_median, stddev_median]
print("{:s}, {:5.2f}, {:5.2f}".format(ff_name, fwhm_median, stddev_median))
return sensor_data
def processNight(night_data_dir, config, detection_results=None, nodetect=False):
""" Given the directory with FF files, run detection and archiving.
Arguments:
night_data_dir: [str] Path to the directory with FF files.
config: [Config obj]
Keyword arguments:
detection_results: [list] An optional list of detection. If None (default), detection will be done
on the the files in the folder.
nodetect: [bool] True if detection should be skipped. False by default.
Return:
night_archive_dir: [str] Path to the night directory in ArchivedFiles.
archive_name: [str] Path to the archive.
detector: [QueuedPool instance] Handle to the detector.
"""
# Remove final slash in the night dir
if night_data_dir.endswith(os.sep):
night_data_dir = night_data_dir[:-1]
# Extract the name of the night
night_data_dir_name = os.path.basename(os.path.abspath(night_data_dir))
platepar = None
# If the detection should be run
if (not nodetect):
# If no detection was performed, run it
if detection_results is None:
# Run detection on the given directory
calstars_name, ftpdetectinfo_name, ff_detected, \
detector = detectStarsAndMeteorsDirectory(night_data_dir, config)
# Otherwise, save detection results
else:
# Save CALSTARS and FTPdetectinfo to disk
calstars_name, ftpdetectinfo_name, ff_detected = saveDetections(detection_results, \
night_data_dir, config)
# If the files were previously detected, there is no detector
detector = None
# Get the platepar file
platepar, platepar_path, platepar_fmt = getPlatepar(config, night_data_dir)
# Run calibration check and auto astrometry refinement
if (platepar is not None) and (calstars_name is not None):
# Read in the CALSTARS file
calstars_list = CALSTARS.readCALSTARS(night_data_dir, calstars_name)
# Run astrometry check and refinement
platepar, fit_status = autoCheckFit(config, platepar, calstars_list)
# If the fit was sucessful, apply the astrometry to detected meteors
if fit_status:
log.info('Astrometric calibration SUCCESSFUL!')
# Save the refined platepar to the night directory and as default
platepar.write(os.path.join(night_data_dir, config.platepar_name), fmt=platepar_fmt)
platepar.write(platepar_path, fmt=platepar_fmt)
else:
log.info('Astrometric calibration FAILED!, Using old platepar for calibration...')
# # Calculate astrometry for meteor detections
# applyAstrometryFTPdetectinfo(night_data_dir, ftpdetectinfo_name, platepar_path)
# If a flat is used, disable vignetting correction
if config.use_flat:
platepar.vignetting_coeff = 0.0
log.info("Recalibrating astrometry on FF files with detections...")
# Recalibrate astrometry on every FF file and apply the calibration to detections
recalibrateIndividualFFsAndApplyAstrometry(night_data_dir, os.path.join(night_data_dir, \
ftpdetectinfo_name), calstars_list, config, platepar)
log.info("Converting RMS format to UFOOrbit format...")
# Convert the FTPdetectinfo into UFOOrbit input file
FTPdetectinfo2UFOOrbitInput(night_data_dir, ftpdetectinfo_name, platepar_path)
# Generate a calibration report
log.info("Generating a calibration report...")
try:
generateCalibrationReport(config, night_data_dir, platepar=platepar)
except Exception as e:
log.debug('Generating calibration report failed with the message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
# Perform single station shower association
log.info("Performing single station shower association...")
try:
showerAssociation(config, [os.path.join(night_data_dir, ftpdetectinfo_name)], \
save_plot=True, plot_activity=True)
except Exception as e:
log.debug('Shower association failed with the message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
else:
ff_detected = []
detector = None
log.info('Plotting field sums...')
# Plot field sums
try:
plotFieldsums(night_data_dir, config)
except Exception as e:
log.debug('Plotting field sums failed with message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
# Archive all fieldsums to one archive
archiveFieldsums(night_data_dir)
# List for any extra files which will be copied to the night archive directory. Full paths have to be
# given
extra_files = []
log.info('Making a flat...')
# Make a new flat field image
try:
flat_img = makeFlat(night_data_dir, config)
except Exception as e:
log.debug('Making a flat failed with message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
flat_img = None
# If making flat was sucessfull, save it
if flat_img is not None:
# Save the flat in the night directory, to keep the operational flat updated
flat_path = os.path.join(night_data_dir, os.path.basename(config.flat_file))
saveImage(flat_path, flat_img)
log.info('Flat saved to: ' + flat_path)
# Copy the flat to the night's directory as well
extra_files.append(flat_path)
else:
log.info('Making flat image FAILED!')
### Add extra files to archive
# Add the config file to the archive too
extra_files.append(os.path.join(os.getcwd(), '.config'))
# Add the mask
if (not nodetect):
if os.path.exists(config.mask_file):
mask_path = os.path.abspath(config.mask_file)
extra_files.append(mask_path)
# Add the platepar to the archive if it exists
if (not nodetect):
if os.path.exists(platepar_path):
extra_files.append(platepar_path)
# Add the json file with recalibrated platepars to the archive
if (not nodetect):
recalibrated_platepars_path = os.path.join(night_data_dir, config.platepars_recalibrated_name)
if os.path.exists(recalibrated_platepars_path):
extra_files.append(recalibrated_platepars_path)
### ###
# If the detection should be run
if (not nodetect):
# Make a CAL file and a special CAMS FTPdetectinfo if full CAMS compatibility is desired
if (config.cams_code > 0) and (platepar is not None):
log.info('Generating a CAMS FTPdetectinfo file...')
# Write the CAL file to disk
cal_file_name = writeCAL(night_data_dir, config, platepar)
# Check if the CAL file was successfully generated
if cal_file_name is not None:
cams_code_formatted = "{:06d}".format(int(config.cams_code))
# Load the FTPdetectinfo
_, fps, meteor_list = readFTPdetectinfo(night_data_dir, ftpdetectinfo_name, \
ret_input_format=True)
# Replace the camera code with the CAMS code
for met in meteor_list:
# Replace the station name and the FF file format
ff_name = met[0]
ff_name = ff_name.replace('.fits', '.bin')
ff_name = ff_name.replace(config.stationID, cams_code_formatted)
met[0] = ff_name
# Write the CAMS compatible FTPdetectinfo file
writeFTPdetectinfo(meteor_list, night_data_dir, \
ftpdetectinfo_name.replace(config.stationID, cams_code_formatted),\
night_data_dir, cams_code_formatted, fps, calibration=cal_file_name, \
celestial_coords_given=(platepar is not None))
night_archive_dir = os.path.join(os.path.abspath(config.data_dir), config.archived_dir,
night_data_dir_name)
log.info('Archiving detections to ' + night_archive_dir)
# Archive the detections
archive_name = archiveDetections(night_data_dir, night_archive_dir, ff_detected, config, \
extra_files=extra_files)
return night_archive_dir, archive_name, detector
def makeFlat(dir_path, config):
""" Makes a flat field from the files in the given folder. CALSTARS file is needed to estimate the
quality of every image by counting the number of detected stars.
Arguments:
dir_path: [str] Path to the directory which contains the FF files and a CALSTARS file.
config: [config object]
Return:
[2d ndarray] Flat field image as a numpy array. If the flat generation failed, None will be returned.
"""
# Find the CALSTARS file in the given folder
calstars_file = None
for calstars_file in os.listdir(dir_path):
if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file):
break
if calstars_file is None:
print('CALSTARS file could not be found in the given directory!')
return None
# Load the calstars file
calstars_list = CALSTARS.readCALSTARS(dir_path, calstars_file)
# Convert the list to a dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
print('CALSTARS file: ' + calstars_file + ' loaded!')
# A list of FF files which have any stars on them
calstars_ff_files = [line[0] for line in calstars_list]
ff_list = []
# Get a list of FF files in the folder
for file_name in os.listdir(dir_path):
if validFFName(file_name) and (file_name in calstars_ff_files):
ff_list.append(file_name)
# Check that there are any FF files in the folder
if not ff_list:
print('No FF files in the selected folder!')
return None
ff_list_good = []
ff_times = []
# Take only those FF files with enough stars on them
for ff_name in ff_list:
if not validFFName(ff_name):
continue
if ff_name in calstars:
# Get the number of stars detected on the FF image
ff_nstars = len(calstars[ff_name])
# Check if the number of stars on the image is over the detection threshold
if ff_nstars > config.ff_min_stars:
# Add the FF file to the list of FF files to be used to make a flat
ff_list_good.append(ff_name)
# Calculate the time of the FF files
ff_time = date2JD(*getMiddleTimeFF(
ff_name, config.fps, ret_milliseconds=True))
ff_times.append(ff_time)
# Check that there are enough good FF files in the folder
if len(ff_times) < config.flat_min_imgs:
print('Not enough FF files have enough stars on them!')
return None
# Make sure the files cover at least 2 hours
if not (max(ff_times) - min(ff_times)) * 24 > 2:
print('Good FF files cover less than 2 hours!')
return None
# Sample FF files if there are more than 200
max_ff_flat = 200
if len(ff_list_good) > max_ff_flat:
ff_list_good = sorted(random.sample(ff_list_good, max_ff_flat))
print('Using {:d} files for flat...'.format(len(ff_list_good)))
c = 0
ff_avg_list = []
median_list = []
# Median combine all good FF files
for i in range(len(ff_list_good)):
# Load 10 files at the time and median combine them, which conserves memory
if c < 10:
ff = readFF(dir_path, ff_list_good[i])
ff_avg_list.append(ff.avepixel)
c += 1
else:
ff_avg_list = np.array(ff_avg_list)
# Median combine the loaded 10 (or less) images
ff_median = np.median(ff_avg_list, axis=0)
median_list.append(ff_median)
ff_avg_list = []
c = 0
# If there are more than 1 calculated median image, combine them
if len(median_list) > 1:
# Median combine all median images
median_list = np.array(median_list)
ff_median = np.median(median_list, axis=0)
else:
ff_median = median_list[0]
# Stretch flat to 0-255
ff_median = ff_median / np.max(ff_median) * 255
# Convert the flat to 8 bits
ff_median = ff_median.astype(np.uint8)
return ff_median
print('Cannot find the platepar file in the night directory: ', config.platepar_name)
sys.exit()
# Find the CALSTARS file in the given folder
calstars_file = None
for calstars_file in file_list:
if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file):
break
if calstars_file is None:
print('CALSTARS file could not be found in the given directory!')
sys.exit()
# Load the calstars file
calstars_list = CALSTARS.readCALSTARS(dir_path, calstars_file)
calstars_dict = {ff_file: star_data for ff_file, star_data in calstars_list}
print('CALSTARS file: ' + calstars_file + ' loaded!')
# Extract star list from CALSTARS file from FF file with most stars
max_len_ff = max(calstars_dict, key=lambda k: len(calstars_dict[k]))
# Take only X, Y (change order so X is first)
calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
# Get the time of the FF file
calstars_time = getMiddleTimeFF(max_len_ff, config.fps, ret_milliseconds=True)
def makeFlat(dir_path, config, nostars=False, use_images=False):
""" Makes a flat field from the files in the given folder. CALSTARS file is needed to estimate the
quality of every image by counting the number of detected stars.
Arguments:
dir_path: [str] Path to the directory which contains the FF files and a CALSTARS file.
config: [config object]
Keyword arguments:
nostars: [bool] If True, all files will be taken regardless of if they have stars on them or not.
use_images: [bool] Use image files instead of FF files. False by default.
Return:
[2d ndarray] Flat field image as a numpy array. If the flat generation failed, None will be returned.
"""
# If only images are used, then don't look for a CALSTARS file
if use_images:
nostars = True
# Load the calstars file if it should be used
if not nostars:
# Find the CALSTARS file in the given folder
calstars_file = None
for calstars_file in os.listdir(dir_path):
if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file):
break
if calstars_file is None:
print('CALSTARS file could not be found in the given directory!')
return None
# Load the calstars file
calstars_list = CALSTARS.readCALSTARS(dir_path, calstars_file)
# Convert the list to a dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
print('CALSTARS file: ' + calstars_file + ' loaded!')
# A list of FF files which have any stars on them
calstars_ff_files = [line[0] for line in calstars_list]
else:
calstars = {}
# Use image files
if use_images:
# Find the file type with the highest file frequency in the given folder
file_extensions = []
for file_name in os.listdir(dir_path):
file_ext = file_name.split('.')[-1]
if file_ext.lower() in ['jpg', 'png', 'bmp']:
file_extensions.append(file_ext)
# Get only the most frequent file type
file_freqs = np.unique(file_extensions, return_counts=True)
most_freq_type = file_freqs[0][0]
print('Using image type:', most_freq_type)
# Take only files of that file type
ff_list = [file_name for file_name in sorted(os.listdir(dir_path)) \
if file_name.lower().endswith(most_freq_type)]
# Use FF files
else:
ff_list = []
# Get a list of FF files in the folder
for file_name in os.listdir(dir_path):
if validFFName(file_name) and ((file_name in calstars_ff_files) or nostars):
ff_list.append(file_name)
# Check that there are any FF files in the folder
if not ff_list:
print('No valid FF files in the selected folder!')
return None
ff_list_good = []
ff_times = []
# Take only those FF files with enough stars on them
for ff_name in ff_list:
if (ff_name in calstars) or nostars:
# Disable requiring minimum number of stars if specified
if not nostars:
# Get the number of stars detected on the FF image
ff_nstars = len(calstars[ff_name])
else:
ff_nstars = 0
# Check if the number of stars on the image is over the detection threshold
if (ff_nstars > config.ff_min_stars) or nostars:
# Add the FF file to the list of FF files to be used to make a flat
ff_list_good.append(ff_name)
# If images are used, don't compute the time
if use_images:
ff_time = 0
else:
# Calculate the time of the FF files
ff_time = date2JD(*getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True))
ff_times.append(ff_time)
# Check that there are enough good FF files in the folder
if (len(ff_times) < config.flat_min_imgs) and (not nostars):
print('Not enough FF files have enough stars on them!')
return None
# Make sure the files cover at least 2 hours
if (not (max(ff_times) - min(ff_times))*24 > 2) and (not nostars):
print('Good FF files cover less than 2 hours!')
return None
# Sample FF files if there are more than 200
max_ff_flat = 200
if len(ff_list_good) > max_ff_flat:
ff_list_good = sorted(random.sample(ff_list_good, max_ff_flat))
print('Using {:d} files for flat...'.format(len(ff_list_good)))
c = 0
img_list = []
median_list = []
# Median combine all good FF files
for i in range(len(ff_list_good)):
# Load 10 files at the time and median combine them, which conserves memory
if c < 10:
# Use images
if use_images:
img = scipy.ndimage.imread(os.path.join(dir_path, ff_list_good[i]), -1)
# Use FF files
else:
ff = readFF(dir_path, ff_list_good[i])
# Skip the file if it is corruped
if ff is None:
continue
img = ff.avepixel
img_list.append(img)
c += 1
else:
img_list = np.array(img_list)
# Median combine the loaded 10 (or less) images
ff_median = np.median(img_list, axis=0)
median_list.append(ff_median)
img_list = []
c = 0
# If there are more than 1 calculated median image, combine them
if len(median_list) > 1:
# Median combine all median images
median_list = np.array(median_list)
ff_median = np.median(median_list, axis=0)
else:
if len(median_list) > 0:
ff_median = median_list[0]
else:
ff_median = np.median(np.array(img_list), axis=0)
# Stretch flat to 0-255
ff_median = ff_median/np.max(ff_median)*255
# Convert the flat to 8 bits
ff_median = ff_median.astype(np.uint8)
return ff_median
print('Cannot find the platepar file in the night directory: ',
config.platepar_name)
sys.exit()
# Find the CALSTARS file in the given folder
calstars_file = None
for calstars_file in file_list:
if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file):
break
if calstars_file is None:
print('CALSTARS file could not be found in the given directory!')
sys.exit()
# Load the calstars file
calstars_list = CALSTARS.readCALSTARS(dir_path, calstars_file)
calstars_dict = {
ff_file: star_data
for ff_file, star_data in calstars_list
}
print('CALSTARS file: ' + calstars_file + ' loaded!')
# Extract star list from CALSTARS file from FF file with most stars
max_len_ff = max(calstars_dict, key=lambda k: len(calstars_dict[k]))
# Take only X, Y (change order so X is first)
calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2]
calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]
# Get the time of the FF file
def makeFlat(dir_path, config, nostars=False, use_images=False):
""" Makes a flat field from the files in the given folder. CALSTARS file is needed to estimate the
quality of every image by counting the number of detected stars.
Arguments:
dir_path: [str] Path to the directory which contains the FF files and a CALSTARS file.
config: [config object]
Keyword arguments:
nostars: [bool] If True, all files will be taken regardless of if they have stars on them or not.
use_images: [bool] Use image files instead of FF files. False by default.
Return:
[2d ndarray] Flat field image as a numpy array. If the flat generation failed, None will be returned.
"""
# If only images are used, then don't look for a CALSTARS file
if use_images:
nostars = True
# Load the calstars file if it should be used
if not nostars:
# Find the CALSTARS file in the given folder
calstars_file = None
for calstars_file in os.listdir(dir_path):
if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file):
break
if calstars_file is None:
print('CALSTARS file could not be found in the given directory!')
return None
# Load the calstars file
calstars_list = CALSTARS.readCALSTARS(dir_path, calstars_file)
# Convert the list to a dictionary
calstars = {ff_file: star_data for ff_file, star_data in calstars_list}
print('CALSTARS file: ' + calstars_file + ' loaded!')
# A list of FF files which have any stars on them
calstars_ff_files = [line[0] for line in calstars_list]
else:
calstars = {}
calstars_ff_files = []
# Use image files
if use_images:
# Find the file type with the highest file frequency in the given folder
file_extensions = []
for file_name in os.listdir(dir_path):
file_ext = file_name.split('.')[-1]
if file_ext.lower() in ['jpg', 'png', 'bmp']:
file_extensions.append(file_ext)
# Get only the most frequent file type
file_freqs = np.unique(file_extensions, return_counts=True)
most_freq_type = file_freqs[0][0]
print('Using image type:', most_freq_type)
# Take only files of that file type
ff_list = [file_name for file_name in sorted(os.listdir(dir_path)) \
if file_name.lower().endswith(most_freq_type)]
# Use FF files
else:
ff_list = []
# Get a list of FF files in the folder
for file_name in os.listdir(dir_path):
if validFFName(file_name) and ((file_name in calstars_ff_files)
or nostars):
ff_list.append(file_name)
# Check that there are any FF files in the folder
if not ff_list:
print('No valid FF files in the selected folder!')
return None
ff_list_good = []
ff_times = []
# Take only those FF files with enough stars on them
for ff_name in ff_list:
if (ff_name in calstars) or nostars:
# Disable requiring minimum number of stars if specified
if not nostars:
# Get the number of stars detected on the FF image
ff_nstars = len(calstars[ff_name])
else:
ff_nstars = 0
# Check if the number of stars on the image is over the detection threshold
if (ff_nstars > config.ff_min_stars) or nostars:
# Add the FF file to the list of FF files to be used to make a flat
ff_list_good.append(ff_name)
# If images are used, don't compute the time
if use_images:
ff_time = 0
else:
# Calculate the time of the FF files
ff_time = date2JD(*getMiddleTimeFF(
ff_name, config.fps, ret_milliseconds=True))
ff_times.append(ff_time)
# Check that there are enough good FF files in the folder
if (len(ff_times) < config.flat_min_imgs) and (not nostars):
print('Not enough FF files have enough stars on them!')
return None
# Make sure the files cover at least 2 hours
if (not (max(ff_times) - min(ff_times)) * 24 > 2) and (not nostars):
print('Good FF files cover less than 2 hours!')
return None
# Sample FF files if there are more than 200
max_ff_flat = 200
if len(ff_list_good) > max_ff_flat:
ff_list_good = sorted(random.sample(ff_list_good, max_ff_flat))
print('Using {:d} files for flat...'.format(len(ff_list_good)))
c = 0
img_list = []
median_list = []
# Median combine all good FF files
for i in range(len(ff_list_good)):
# Load 10 files at the time and median combine them, which conserves memory
if c < 10:
# Use images
if use_images:
img = loadImage(os.path.join(dir_path, ff_list_good[i]), -1)
# Use FF files
else:
ff = readFF(dir_path, ff_list_good[i])
# Skip the file if it is corruped
if ff is None:
continue
img = ff.avepixel
img_list.append(img)
c += 1
else:
img_list = np.array(img_list)
# Median combine the loaded 10 (or less) images
ff_median = np.median(img_list, axis=0)
median_list.append(ff_median)
img_list = []
c = 0
# If there are more than 1 calculated median image, combine them
if len(median_list) > 1:
# Median combine all median images
median_list = np.array(median_list)
ff_median = np.median(median_list, axis=0)
else:
if len(median_list) > 0:
ff_median = median_list[0]
else:
ff_median = np.median(np.array(img_list), axis=0)
# Stretch flat to 0-255
ff_median = ff_median / np.max(ff_median) * 255
# Convert the flat to 8 bits
ff_median = ff_median.astype(np.uint8)
return ff_median
def processNight(night_data_dir, config, detection_results=None, nodetect=False):
""" Given the directory with FF files, run detection and archiving.
Arguments:
night_data_dir: [str] Path to the directory with FF files.
config: [Config obj]
Keyword arguments:
detection_results: [list] An optional list of detection. If None (default), detection will be done
on the the files in the folder.
nodetect: [bool] True if detection should be skipped. False by default.
Return:
night_archive_dir: [str] Path to the night directory in ArchivedFiles.
archive_name: [str] Path to the archive.
detector: [QueuedPool instance] Handle to the detector.
"""
# Remove final slash in the night dir
if night_data_dir.endswith(os.sep):
night_data_dir = night_data_dir[:-1]
# Extract the name of the night
night_data_dir_name = os.path.basename(night_data_dir)
# If the detection should be run
if (not nodetect):
# If no detection was performed, run it
if detection_results is None:
# Run detection on the given directory
calstars_name, ftpdetectinfo_name, ff_detected, \
detector = detectStarsAndMeteorsDirectory(night_data_dir, config)
# Otherwise, save detection results
else:
# Save CALSTARS and FTPdetectinfo to disk
calstars_name, ftpdetectinfo_name, ff_detected = saveDetections(detection_results, \
night_data_dir, config)
# If the files were previously detected, there is no detector
detector = None
# Get the platepar file
platepar, platepar_path, platepar_fmt = getPlatepar(config, night_data_dir)
# Run calibration check and auto astrometry refinement
if platepar is not None:
# Read in the CALSTARS file
calstars_list = CALSTARS.readCALSTARS(night_data_dir, calstars_name)
# Run astrometry check and refinement
platepar, fit_status = autoCheckFit(config, platepar, calstars_list)
# If the fit was sucessful, apply the astrometry to detected meteors
if fit_status:
log.info('Astrometric calibration SUCCESSFUL!')
# Save the refined platepar to the night directory and as default
platepar.write(os.path.join(night_data_dir, config.platepar_name), fmt=platepar_fmt)
platepar.write(platepar_path, fmt=platepar_fmt)
else:
log.info('Astrometric calibration FAILED!, Using old platepar for calibration...')
# # Calculate astrometry for meteor detections
# applyAstrometryFTPdetectinfo(night_data_dir, ftpdetectinfo_name, platepar_path)
log.info("Recalibrating astrometry on FF files with detections...")
# Recalibrate astrometry on every FF file and apply the calibration to detections
recalibrateIndividualFFsAndApplyAstrometry(night_data_dir, os.path.join(night_data_dir, \
ftpdetectinfo_name), calstars_list, config, platepar)
log.info("Converting RMS format to UFOOrbit format...")
# Convert the FTPdetectinfo into UFOOrbit input file
FTPdetectinfo2UFOOrbitInput(night_data_dir, ftpdetectinfo_name, platepar_path)
# Generate a calibration report
log.info("Generating a calibration report...")
try:
generateCalibrationReport(config, night_data_dir, platepar=platepar)
except Exception as e:
log.debug('Generating calibration report failed with message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
else:
ff_detected = []
detector = None
log.info('Plotting field sums...')
# Plot field sums
try:
plotFieldsums(night_data_dir, config)
except Exception as e:
log.debug('Plotting field sums failed with message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
# Archive all fieldsums to one archive
archiveFieldsums(night_data_dir)
# List for any extra files which will be copied to the night archive directory. Full paths have to be
# given
extra_files = []
log.info('Making a flat...')
# Make a new flat field image
try:
flat_img = makeFlat(night_data_dir, config)
except Exception as e:
log.debug('Making a flat failed with message:\n' + repr(e))
log.debug(repr(traceback.format_exception(*sys.exc_info())))
flat_img = None
# If making flat was sucessfull, save it
if flat_img is not None:
# Save the flat in the night directory, to keep the operational flat updated
flat_path = os.path.join(night_data_dir, os.path.basename(config.flat_file))
scipy.misc.imsave(flat_path, flat_img)
log.info('Flat saved to: ' + flat_path)
# Copy the flat to the night's directory as well
extra_files.append(flat_path)
else:
log.info('Making flat image FAILED!')
### Add extra files to archive
# Add the config file to the archive too
extra_files.append(os.path.join(os.getcwd(), '.config'))
# Add the platepar to the archive if it exists
if (not nodetect):
if os.path.exists(platepar_path):
extra_files.append(platepar_path)
# Add the json file with recalibrated platepars to the archive
if (not nodetect):
recalibrated_platepars_path = os.path.join(night_data_dir, config.platepars_recalibrated_name)
if os.path.exists(recalibrated_platepars_path):
extra_files.append(recalibrated_platepars_path)
### ###
# If the detection should be run
if (not nodetect):
# Make a CAL file and a special CAMS FTpdetectinfo if full CAMS compatibility is desired
if config.cams_code > 0:
log.info('Generating a CAMS FTPdetectinfo file...')
# Write the CAL file to disk
cal_file_name = writeCAL(night_data_dir, config, platepar)
cams_code_formatted = "{:06d}".format(int(config.cams_code))
# Load the FTPdetectinfo
_, fps, meteor_list = readFTPdetectinfo(night_data_dir, ftpdetectinfo_name, \
ret_input_format=True)
# Replace the camera code with the CAMS code
for met in meteor_list:
# Replace the station name and the FF file format
ff_name = met[0]
ff_name = ff_name.replace('.fits', '.bin')
ff_name = ff_name.replace(config.stationID, cams_code_formatted)
met[0] = ff_name
# Replace the station name
met[1] = cams_code_formatted
# Write the CAMS compatible FTPdetectinfo file
writeFTPdetectinfo(meteor_list, night_data_dir, \
ftpdetectinfo_name.replace(config.stationID, cams_code_formatted),\
night_data_dir, cams_code_formatted, fps, calibration=cal_file_name, \
celestial_coords_given=(platepar is not None))
night_archive_dir = os.path.join(os.path.abspath(config.data_dir), config.archived_dir,
night_data_dir_name)
log.info('Archiving detections to ' + night_archive_dir)
# Archive the detections
archive_name = archiveDetections(night_data_dir, night_archive_dir, ff_detected, config, \
extra_files=extra_files)
return night_archive_dir, archive_name, detector