def runCapture(config,
duration=None,
video_file=None,
nodetect=False,
detect_end=False,
upload_manager=None):
""" Run capture and compression for the given time.given
Arguments:
config: [config object] Configuration read from the .config file
Keyword arguments:
duration: [float] Time in seconds to capture. None by default.
video_file: [str] Path to the video file, if it was given as the video source. None by default.
nodetect: [bool] If True, detection will not be performed. False by defualt.
detect_end: [bool] If True, detection will be performed at the end of the night, when capture
finishes. False by default.
upload_manager: [UploadManager object] A handle to the UploadManager, which handles uploading files to
the central server. None by default.
"""
global STOP_CAPTURE
# Create a directory for captured files
night_data_dir_name = str(
config.stationID) + '_' + datetime.datetime.utcnow().strftime(
'%Y%m%d_%H%M%S_%f')
# Full path to the data directory
night_data_dir = os.path.join(os.path.abspath(config.data_dir),
config.captured_dir, night_data_dir_name)
# Make a directory for the night
mkdirP(night_data_dir)
log.info('Data directory: ' + night_data_dir)
# Load the default flat field image if it is available
flat_struct = None
if config.use_flat:
# Check if the flat exists
if os.path.exists(os.path.join(os.getcwd(), config.flat_file)):
flat_struct = Image.loadFlat(os.getcwd(), config.flat_file)
log.info('Loaded flat field image: ' +
os.path.join(os.getcwd(), config.flat_file))
# Get the platepar file
platepar, platepar_path, platepar_fmt = getPlatepar(config)
log.info('Initializing frame buffers...')
### For some reason, the RPi 3 does not like memory chunks which size is the multipier of its L2
### cache size (512 kB). When such a memory chunk is provided, the compression becomes 10x slower
### then usual. We are applying a dirty fix here where we just add an extra image row and column
### if such a memory chunk will be created. The compression is performed, and the image is cropped
### back to its original dimensions.
array_pad = 0
# Check if the image dimensions are divisible by RPi3 L2 cache size and add padding
if (256 * config.width * config.height) % (512 * 1024) == 0:
array_pad = 1
# Init arrays for parallel compression on 2 cores
sharedArrayBase = multiprocessing.Array(
ctypes.c_uint8,
256 * (config.width + array_pad) * (config.height + array_pad))
sharedArray = np.ctypeslib.as_array(sharedArrayBase.get_obj())
sharedArray = sharedArray.reshape(256, (config.height + array_pad),
(config.width + array_pad))
startTime = multiprocessing.Value('d', 0.0)
sharedArrayBase2 = multiprocessing.Array(
ctypes.c_uint8,
256 * (config.width + array_pad) * (config.height + array_pad))
sharedArray2 = np.ctypeslib.as_array(sharedArrayBase2.get_obj())
sharedArray2 = sharedArray2.reshape(256, (config.height + array_pad),
(config.width + array_pad))
startTime2 = multiprocessing.Value('d', 0.0)
log.info('Initializing frame buffers done!')
# Check if the detection should be performed or not
if nodetect:
detector = None
else:
if detect_end:
# Delay detection until the end of the night
delay_detection = duration
else:
# Delay the detection for 2 minutes after capture start
delay_detection = 120
# Initialize the detector
detector = QueuedPool(detectStarsAndMeteors,
cores=1,
log=log,
delay_start=delay_detection)
detector.startPool()
# Initialize buffered capture
bc = BufferedCapture(sharedArray,
startTime,
sharedArray2,
startTime2,
config,
video_file=video_file)
# Initialize the live image viewer
live_view = LiveViewer(window_name='Maxpixel')
# Initialize compression
compressor = Compressor(night_data_dir,
sharedArray,
startTime,
sharedArray2,
startTime2,
config,
detector=detector,
live_view=live_view,
flat_struct=flat_struct)
# Start buffered capture
bc.startCapture()
# Start the compression
compressor.start()
# Capture until Ctrl+C is pressed
wait(duration)
# If capture was manually stopped, end capture
if STOP_CAPTURE:
log.info('Ending capture...')
# Stop the capture
log.debug('Stopping capture...')
bc.stopCapture()
log.debug('Capture stopped')
dropped_frames = bc.dropped_frames
log.info('Total number of dropped frames: ' + str(dropped_frames))
# Stop the compressor
log.debug('Stopping compression...')
detector, live_view = compressor.stop()
log.debug('Compression stopped')
# Stop the live viewer
log.debug('Stopping live viewer...')
live_view.stop()
del live_view
log.debug('Live view stopped')
# Init data lists
star_list = []
meteor_list = []
ff_detected = []
# If detection should be performed
if not nodetect:
log.info('Finishing up the detection, ' +
str(detector.input_queue.qsize()) + ' files to process...')
# Reset the Ctrl+C to KeyboardInterrupt
resetSIGINT()
try:
# If there are some more files to process, process them on more cores
if detector.input_queue.qsize() > 0:
# Let the detector use all cores, but leave 1 free
available_cores = multiprocessing.cpu_count() - 1
if available_cores > 1:
log.info('Running the detection on {:d} cores...'.format(
available_cores))
# Start the detector
detector.updateCoreNumber(cores=available_cores)
log.info('Waiting for the detection to finish...')
# Wait for the detector to finish and close it
detector.closePool()
log.info('Detection finished!')
except KeyboardInterrupt:
log.info('Ctrl + C pressed, exiting...')
if upload_manager is not None:
# Stop the upload manager
if upload_manager.is_alive():
log.debug('Closing upload manager...')
upload_manager.stop()
del upload_manager
# Terminate the detector
if detector is not None:
del detector
sys.exit()
# Set the Ctrl+C back to 'soft' program kill
setSIGINT()
### SAVE DETECTIONS TO FILE
log.info('Collecting results...')
# Get the detection results from the queue
detection_results = detector.getResults()
# Remove all 'None' results, which were errors
detection_results = [
res for res in detection_results if res is not None
]
# Count the number of detected meteors
meteors_num = 0
for _, _, meteor_data in detection_results:
for meteor in meteor_data:
meteors_num += 1
log.info('TOTAL: ' + str(meteors_num) + ' detected meteors.')
# Save the detections to a file
for ff_name, star_data, meteor_data in detection_results:
x2, y2, background, intensity = star_data
# Skip if no stars were found
if not x2:
continue
# Construct the table of the star parameters
star_data = zip(x2, y2, background, intensity)
# Add star info to the star list
star_list.append([ff_name, star_data])
# Handle the detected meteors
meteor_No = 1
for meteor in meteor_data:
rho, theta, centroids = meteor
# Append to the results list
meteor_list.append([ff_name, meteor_No, rho, theta, centroids])
meteor_No += 1
# Add the FF file to the archive list if a meteor was detected on it
if meteor_data:
ff_detected.append(ff_name)
# Generate the name for the CALSTARS file
calstars_name = 'CALSTARS_' + "{:s}".format(str(config.stationID)) + '_' \
+ os.path.basename(night_data_dir) + '.txt'
# Write detected stars to the CALSTARS file
CALSTARS.writeCALSTARS(star_list, night_data_dir, calstars_name, config.stationID, config.height, \
config.width)
# Generate FTPdetectinfo file name
ftpdetectinfo_name = 'FTPdetectinfo_' + os.path.basename(
night_data_dir) + '.txt'
# Write FTPdetectinfo file
FTPdetectinfo.writeFTPdetectinfo(meteor_list, night_data_dir, ftpdetectinfo_name, night_data_dir, \
config.stationID, config.fps)
# Get the platepar file
platepar, platepar_path, platepar_fmt = getPlatepar(config)
# Run calibration check and auto astrometry refinement
if platepar is not None:
# Read in the CALSTARS file
calstars_list = CALSTARS.readCALSTARS(night_data_dir,
calstars_name)
# Run astrometry check and refinement
platepar, fit_status = autoCheckFit(config, platepar,
calstars_list)
# If the fit was sucessful, apply the astrometry to detected meteors
if fit_status:
log.info('Astrometric calibration SUCCESSFUL!')
# Save the refined platepar to the night directory and as default
platepar.write(os.path.join(night_data_dir,
config.platepar_name),
fmt=platepar_fmt)
platepar.write(platepar_path, fmt=platepar_fmt)
else:
log.info(
'Astrometric calibration FAILED!, Using old platepar for calibration...'
)
# Calculate astrometry for meteor detections
applyAstrometryFTPdetectinfo(night_data_dir, ftpdetectinfo_name,
platepar_path)
log.info('Plotting field sums...')
# Plot field sums to a graph
plotFieldsums(night_data_dir, config)
# Archive all fieldsums to one archive
archiveFieldsums(night_data_dir)
# List for any extra files which will be copied to the night archive directory. Full paths have to be
# given
extra_files = []
log.info('Making a flat...')
# Make a new flat field
flat_img = makeFlat(night_data_dir, config)
# If making flat was sucessfull, save it
if flat_img is not None:
# Save the flat in the root directory, to keep the operational flat updated
scipy.misc.imsave(config.flat_file, flat_img)
flat_path = os.path.join(os.getcwd(), config.flat_file)
log.info('Flat saved to: ' + flat_path)
# Copy the flat to the night's directory as well
extra_files.append(flat_path)
else:
log.info('Making flat image FAILED!')
### Add extra files to archive
# Add the platepar to the archive if it exists
if os.path.exists(platepar_path):
extra_files.append(platepar_path)
# Add the config file to the archive too
extra_files.append(os.path.join(os.getcwd(), '.config'))
### ###
night_archive_dir = os.path.join(os.path.abspath(config.data_dir),
config.archived_dir, night_data_dir_name)
log.info('Archiving detections to ' + night_archive_dir)
# Archive the detections
archive_name = archiveDetections(night_data_dir, night_archive_dir, ff_detected, config, \
extra_files=extra_files)
# Put the archive up for upload
if upload_manager is not None:
log.info('Adding file on upload list: ' + archive_name)
upload_manager.addFiles([archive_name])
# If capture was manually stopped, end program
if STOP_CAPTURE:
log.info('Ending program')
# Stop the upload manager
if upload_manager is not None:
if upload_manager.is_alive():
upload_manager.stop()
log.info('Closing upload manager...')
sys.exit()
# Check if there are any file in the directory
if(len(ff_list) == None):
print("No files found!")
sys.exit()
# Try loading a flat field image
flat_struct = None
if config.use_flat:
# Check if there is flat in the data directory
if os.path.exists(os.path.join(ff_dir, config.flat_file)):
flat_struct = Image.loadFlat(ff_dir, config.flat_file)
# Try loading the default flat
elif os.path.exists(config.flat_file):
flat_struct = Image.loadFlat(os.getcwd(), config.flat_file)
# Initialize the detector
detector = QueuedPool(detectStarsAndMeteors, cores=-1, log=log)
# Give detector jobs
for ff_name in ff_list:
detector.addJob([ff_dir, ff_name, config, flat_struct])
# Start the detection
def loadImageCalibration(dir_path, config, dtype=None, byteswap=False):
""" Load the mask, dark and flat.
Arguments:
dir_path: [str] Path to the directory with calibration.
config: [ConfigStruct]
Keyword arguments:
dtype: [object] Numpy array dtype for the image. None by default, if which case it will be determined
from the input image.
byteswap: [bool] If the dark and flat should be byteswapped. False by default, and should be True for
UWO PNGs.
Return:
mask, dark, flat_struct: [tuple of ndarrays]
"""
mask_path = None
mask = None
# Try loading the mask
if os.path.exists(os.path.join(dir_path, config.mask_file)):
mask_path = os.path.join(dir_path, config.mask_file)
# Try loading the default mask
elif os.path.exists(config.mask_file):
mask_path = os.path.abspath(config.mask_file)
# Load the mask if given
if mask_path:
mask = MaskImage.loadMask(mask_path)
if mask is not None:
print('Loaded mask:', mask_path)
log.info('Loaded mask: {:s}'.format(mask_path))
# Try loading the dark frame
dark = None
if config.use_dark:
dark_path = None
# Check if dark is in the data directory
if os.path.exists(os.path.join(dir_path, config.dark_file)):
dark_path = os.path.join(dir_path, config.dark_file)
# Try loading the default dark
elif os.path.exists(config.dark_file):
dark_path = os.path.abspath(config.dark_file)
if dark_path is not None:
# Load the dark
dark = Image.loadDark(*os.path.split(dark_path),
dtype=dtype,
byteswap=byteswap)
if dark is not None:
print('Loaded dark:', dark_path)
log.info('Loaded dark: {:s}'.format(dark_path))
# Try loading a flat field image
flat_struct = None
if config.use_flat:
flat_path = None
# Check if there is flat in the data directory
if os.path.exists(os.path.join(dir_path, config.flat_file)):
flat_path = os.path.join(dir_path, config.flat_file)
# Try loading the default flat
elif os.path.exists(config.flat_file):
flat_path = os.path.abspath(config.flat_file)
if flat_path is not None:
# Load the flat
flat_struct = Image.loadFlat(*os.path.split(flat_path),
dtype=dtype,
byteswap=byteswap)
if flat_struct is not None:
print('Loaded flat:', flat_path)
log.info('Loaded flat: {:s}'.format(flat_path))
return mask, dark, flat_struct
def loadImageCalibration(dir_path, config, dtype=None, byteswap=False):
""" Load the mask, dark and flat.
Arguments:
dir_path: [str] Path to the directory with calibration.
config: [ConfigStruct]
Keyword arguments:
dtype: [object] Numpy array dtype for the image. None by default, if which case it will be determined
from the input image.
byteswap: [bool] If the dark and flat should be byteswapped. False by default, and should be True for
UWO PNGs.
Return:
mask, dark, flat_struct: [tuple of ndarrays]
"""
mask_path = None
mask = None
# Try loading the mask
if os.path.exists(os.path.join(dir_path, config.mask_file)):
mask_path = os.path.join(dir_path, config.mask_file)
# Try loading the default mask
elif os.path.exists(config.mask_file):
mask_path = os.path.abspath(config.mask_file)
# Load the mask if given
if mask_path:
mask = MaskImage.loadMask(mask_path)
if mask is not None:
print('Loaded mask:', mask_path)
log.info('Loaded mask: {:s}'.format(mask_path))
# Try loading the dark frame
dark = None
if config.use_dark:
dark_path = None
# Check if dark is in the data directory
if os.path.exists(os.path.join(dir_path, config.dark_file)):
dark_path = os.path.join(dir_path, config.dark_file)
# Try loading the default dark
elif os.path.exists(config.dark_file):
dark_path = os.path.abspath(config.dark_file)
if dark_path is not None:
# Load the dark
dark = Image.loadDark(*os.path.split(dark_path), dtype=dtype, byteswap=byteswap)
if dark is not None:
print('Loaded dark:', dark_path)
log.info('Loaded dark: {:s}'.format(dark_path))
# Try loading a flat field image
flat_struct = None
if config.use_flat:
flat_path = None
# Check if there is flat in the data directory
if os.path.exists(os.path.join(dir_path, config.flat_file)):
flat_path = os.path.join(dir_path, config.flat_file)
# Try loading the default flat
elif os.path.exists(config.flat_file):
flat_path = os.path.abspath(config.flat_file)
if flat_path is not None:
# Load the flat
flat_struct = Image.loadFlat(*os.path.split(flat_path), dtype=dtype, byteswap=byteswap)
if flat_struct is not None:
print('Loaded flat:', flat_path)
log.info('Loaded flat: {:s}'.format(flat_path))
return mask, dark, flat_struct
# Get paths to every FF bin file in a directory
ff_list = [ff for ff in os.listdir(dir_path) if FFfile.validFFName(ff)]
# Check if there are any file in the directory
if (len(ff_list) == None):
print("No files found!")
sys.exit()
# Try loading a flat field image
flat_struct = None
if config.use_flat:
# Check if there is flat in the data directory
if os.path.exists(os.path.join(dir_path, config.flat_file)):
flat_struct = Image.loadFlat(dir_path, config.flat_file)
# Try loading the default flat
elif os.path.exists(config.flat_file):
flat_struct = Image.loadFlat(os.getcwd(), config.flat_file)
# Init results list
results_list = []
# Open a file for results
results_path = os.path.abspath(dir_path) + os.sep
results_name = results_path.split(os.sep)[-2]
results_file = open(results_path + results_name + '_results.txt', 'w')
total_meteors = 0
# Check if there are any file in the directory
if(len(ff_list) == None):
print("No files found!")
sys.exit()
# Try loading a flat field image
flat_struct = None
if config.use_flat:
# Check if there is flat in the data directory
if os.path.exists(os.path.join(ff_dir, config.flat_file)):
flat_struct = Image.loadFlat(ff_dir, config.flat_file)
# Try loading the default flat
elif os.path.exists(config.flat_file):
flat_struct = Image.loadFlat(os.getcwd(), config.flat_file)
extraction_list = []
# Go through all files in the directory and add them to the detection list
for ff_name in sorted(ff_list):
# Check if the given file is a valid FF file
if not FFfile.validFFName(ff_name):