def getFilesAndUpload(logger, nm_config, main_data_dir, log_file_fd):
"""The argument 'log_file_fd' is assumed to be passed in
open state, and need not be closed during the call."""
logger.info('Starting the NM upload manager ...')
# create the upload manager for the local files
upload_manager = UploadManager(nm_config)
upload_manager.start()
# Get the files from the main_data_dir for NM upload
all_files = os.listdir(main_data_dir)
files_to_upload = []
png_files = findFiles(main_data_dir, all_files, "*.png")
logger.info("Adding %d png files to queue ..." % len(png_files))
files_to_upload.extend(png_files)
jpg_files = findFiles(main_data_dir, all_files, "*.jpg")
logger.info("Adding %d jpg files to queue ..." % len(jpg_files))
files_to_upload.extend(jpg_files)
ftp_files = findFiles(main_data_dir, all_files, \
"FTP*_[0-9][0-9][0-9][0-9][0-9][0-9].txt")
logger.info("Adding %d ftp files to queue ..." % len(ftp_files))
files_to_upload.extend(ftp_files)
txt_files = findFiles(main_data_dir, all_files, "*_radiants.txt")
logger.info("Adding %d txt files to queue ..." % len(txt_files))
files_to_upload.extend(txt_files)
csv_files = findFiles(main_data_dir, all_files, "*.csv")
logger.info("Adding %d csv files to queue ..." % len(csv_files))
files_to_upload.extend(csv_files)
csv_dir = os.path.join(nm_config.data_dir, 'csv/')
logger.info("csv_dir set to %s" % csv_dir)
fits_count_file = findFiles(csv_dir, os.listdir(csv_dir), \
"*fits_counts.txt")
logger.info("Adding %d fits_count.txt files to queue ..." \
% len(fits_count_file) )
files_to_upload.extend(fits_count_file)
#write all the files at oce to the upload manager
upload_manager.addFiles(files_to_upload)
# Get and print the contents of ~/RMS_data/NM_FILES_TO_UPLOAD.inf
queue_file_name = os.path.expanduser('~/RMS_data/NM_FILES_TO_UPLOAD.inf')
with open(queue_file_name, 'r') as queue_file:
logger.info('Contents of %s:' % queue_file_name)
upload_queue = queue_file.read()
logger.info('%s' % upload_queue)
# Begin the upload!
logger.info('Beginning upload ...')
upload_manager.uploadData()
logger.info('Upload complete ...')
upload_manager.stop()
if not deleteOldObservations(config.data_dir,
config.captured_dir,
config.archived_dir,
config,
duration=duration):
log.error(
'No more disk space can be freed up! Stopping capture...')
sys.exit()
upload_manager = None
if config.upload_enabled:
# Init the upload manager
log.info('Starting the upload manager...')
upload_manager = UploadManager(config)
upload_manager.start()
log.info("Running for " + str(duration / 60 / 60) + ' hours...')
# Run the capture for the given number of hours
runCapture(config, duration=duration, nodetect=cml_args.nodetect, upload_manager=upload_manager, \
detect_end=cml_args.detectend)
if upload_manager is not None:
# Stop the upload manager
if upload_manager.is_alive():
log.info('Closing upload manager...')
upload_manager.stop()
del upload_manager
log = logging.getLogger("logger")
######
# Process the night
_, archive_name, detector = processNight(cml_args.dir_path[0], config)
# Upload the archive, if upload is enabled
if config.upload_enabled:
# Init the upload manager
print('Starting the upload manager...')
upload_manager = UploadManager(config)
upload_manager.start()
# Add file for upload
print('Adding file to upload list: ' + archive_name)
upload_manager.addFiles([archive_name])
# Stop the upload manager
if upload_manager.is_alive():
upload_manager.stop()
print('Closing upload manager...')
# Delete detection backup files
if detector is not None:
detector.deleteBackupFiles()
"~/source/RMS/.config")
print(
"Running FlushNMqueue.py, 08-Aug, 2021, byte count 1452: flushing the NM upload queue..."
)
# Create the config object for New Mexico Meteor Array purposes
nm_config = copy.copy(config)
nm_config.stationID = 'pi'
nm_config.hostname = '10.8.0.61'
nm_config.remote_dir = '/home/pi/RMS_Station_data'
nm_config.upload_queue_file = 'NM_FILES_TO_UPLOAD.inf'
initLogging(config, "NM_UPLOAD_")
# Get the logger handle.
log = logging.getLogger("logger.FlushNMqueue")
# Upload files to the NM Server
# create the upload manager for the local files
upload_manager = UploadManager(nm_config)
upload_manager.start()
# Begin the upload!
upload_manager.uploadData()
upload_manager.stop()
log.info("FlushNMqueue has finished!")
####################################
def getFilesAndUpload(logger, nm_config, main_data_dir, log_file_fd):
"""The argument 'log_file_fd' is assumed to be passed in
open state, and need not be closed during the call."""
logger.info('Starting the NM upload manager ...')
# create the upload manager for the local files
upload_manager = UploadManager(nm_config)
upload_manager.start()
# Get the files from the main_data_dir for NM upload
all_files = os.listdir(main_data_dir)
png_files = findFiles(main_data_dir, all_files, "*.png")
print("Adding %d png files to queue ..." % len(png_files),
file=log_file_fd)
upload_manager.addFiles(png_files)
jpg_files = findFiles(main_data_dir, all_files, "*.jpg")
print("Adding %d jpg files to queue ..." % len(jpg_files),
file=log_file_fd)
upload_manager.addFiles(jpg_files)
ftp_files = findFiles(main_data_dir, all_files, \
"FTP*_[0-9][0-9][0-9][0-9][0-9][0-9].txt")
print("Adding %d ftp files to queue ..." % ftp_files.__len__())
upload_manager.addFiles(ftp_files)
txt_files = findFiles(main_data_dir, all_files, "*_radiants.txt")
print("Adding %d txt files to queue ..." % len(txt_files),
file=log_file_fd)
upload_manager.addFiles(txt_files)
csv_files = findFiles(main_data_dir, all_files, "*.csv")
print("Adding %d csv files to queue ..." % len(csv_files),
file=log_file_fd)
upload_manager.addFiles(csv_files)
csv_dir = os.path.join(nm_config.data_dir, 'csv/')
print("csv_dir set to %s" % csv_dir, file=log_file_fd)
fits_count_file = findFiles(csv_dir, os.listdir(csv_dir), \
"*fits_counts.txt")
print("Adding %d fits_count.txt files to queue ..." % len(fits_count_file), \
file=log_file_fd)
upload_manager.addFiles(fits_count_file)
# Begin the upload!
upload_manager.uploadData()
upload_manager.stop()
log = logging.getLogger("logger")
######
# Process the night
_, archive_name, detector = processNight(cml_args.dir_path[0], config)
# Upload the archive, if upload is enabled
if config.upload_enabled:
# Init the upload manager
print('Starting the upload manager...')
upload_manager = UploadManager(config)
upload_manager.start()
# Add file for upload
print('Adding file to upload list: ' + archive_name)
upload_manager.addFiles([archive_name])
# Stop the upload manager
if upload_manager.is_alive():
upload_manager.stop()
print('Closing upload manager...')
# Delete detection backup files
if detector is not None:
detector.deleteBackupFiles()