def _log_summary(self, elapsed_time):
create_units_text = text_utils.create_units_text
log = self._logger.info
# Number of audio files processed.
units = create_units_text(self._file_count, 'audio file')
timing_text = command_utils.get_timing_text(elapsed_time,
self._file_count, 'files')
log('Processed {} {}{}.'.format(self._file_count, units, timing_text))
# Number of file path parse errors.
count = self._file_count - self._parsed_count
if count != 0:
units = create_units_text(count, 'audio file path')
log(('{} {} could not be parsed. See error messages above '
'for details.').format(count, units))
# Number of excluded files.
count = self._parsed_count - self._eligible_count
if count != 0:
units = create_units_text(count, 'audio file was',
'audio files were')
log('{} {} excluded by date.'.format(count, units))
# Number of failed imports.
count = self._eligible_count - self._imported_count
if count != 0:
units = create_units_text(count, 'attempted clip import')
log('{} {} failed. See error messages above for details.'.format(
count, units))
# Number of successful imports.
units = create_units_text(self._imported_count, 'clip')
log('{} {} were imported.'.format(self._imported_count, units))
def _log_summary(self, elapsed_time):
create_units_text = text_utils.create_units_text
log = self._logger.info
# Number of audio files processed.
units = create_units_text(self._file_count, 'audio file')
timing_text = command_utils.get_timing_text(
elapsed_time, self._file_count, 'files')
log('Processed {} {}{}.'.format(self._file_count, units, timing_text))
# Number of file path parse errors.
count = self._file_count - self._parsed_count
if count != 0:
units = create_units_text(count, 'audio file path')
log(('{} {} could not be parsed. See error messages above '
'for details.').format(count, units))
# Number of excluded files.
count = self._parsed_count - self._eligible_count
if count != 0:
units = create_units_text(
count, 'audio file was', 'audio files were')
log('{} {} excluded by date.'.format(count, units))
# Number of failed imports.
count = self._eligible_count - self._imported_count
if count != 0:
units = create_units_text(count, 'attempted clip import')
log('{} {} failed. See error messages above for details.'.format(
count, units))
# Number of successful imports.
units = create_units_text(self._imported_count, 'clip')
log('{} {} were imported.'.format(self._imported_count, units))
def _execute_create_clips_action(self, args):
clips = args['clips']
num_clips = len(clips)
self._logger.info('Creating {} clips...'.format(num_clips))
start_time = time.time()
clip_num = 0
for clip in clips:
self._create_clip(clip)
clip_num += 1
if clip_num % _LOGGING_PERIOD == 0:
self._logger.info('Created {} clips...'.format(clip_num))
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, num_clips, 'clips')
self._logger.info(
'Created {} clips{}.'.format(num_clips, timing_text))
def _execute_create_clips_action(self, args):
clips = args['clips']
num_clips = len(clips)
self._logger.info('Creating {} clips...'.format(num_clips))
start_time = time.time()
clip_num = 0
for clip in clips:
self._create_clip(clip)
clip_num += 1
if clip_num % _LOGGING_PERIOD == 0:
self._logger.info('Created {} clips...'.format(clip_num))
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, num_clips, 'clips')
self._logger.info(
'Created {} clips{}.'.format(num_clips, timing_text))
def _add_clip_start_indices(self, recordings):
start_time = time.time()
total_clips = 0
total_clips_found = 0
for recording in recordings:
files = recording.files.all().order_by('file_num')
if files.count() == 0:
# archive has no files for this recording
self._logger.warning(
f'Archive contains no audio files for recording '
f'"{str(recording)}". No clips of this recording '
f'will be processed.')
else:
# archive has files for this recording
self._recording_reader = self._create_recording_reader(files)
for channel in recording.channels.all():
for detector in self._detectors:
try:
num_clips, num_clips_found = \
self._add_channel_clip_start_indices(
channel, detector)
except Exception as e:
self._logger.error(
f'Processing of clips for recording channel '
f'"{str(channel)}" failed with an exception.\n'
f'The exception message was:\n'
f' {str(e)}\n'
f'No clips of the channel were modified.\n'
f'See below for exception traceback.')
raise
total_clips += num_clips
total_clips_found += num_clips_found
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, total_clips, 'clips')
self._logger.info(
f'Added start indices for {total_clips_found} of {total_clips} '
f'processed clips{timing_text}.')
self._log_archive_status()
def _add_clip_start_indices(self):
start_time = time.time()
self._min_start_time_change = 100
self._max_start_time_change = -100
total_clips = 0
total_clips_found = 0
for recording in self._recordings:
for channel in recording.channels.all():
for detector in self._detectors:
try:
num_clips, num_clips_found = \
self._add_channel_clip_start_indices(
channel, detector)
except Exception as e:
self._logger.error(
f'Processing of clips for recording channel '
f'"{str(channel)}" failed with an exception.\n'
f'The exception message was:\n'
f' {str(e)}\n'
f'No clips of the channel were modified.\n'
f'See below for exception traceback.')
raise
total_clips += num_clips
total_clips_found += num_clips_found
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(elapsed_time, total_clips,
'clips')
self._logger.info(
f'Added start indices for {total_clips_found} of {total_clips} '
f'processed clips{timing_text}.')
self._logger.info(
f'Range of start time changes was '
f'({self._min_start_time_change}, {self._max_start_time_change}).')
self._log_archive_status()
def _adjust_clips(self):
start_time = time.time()
value_tuples = self._create_clip_query_values_iterator()
total_adjusted_count = 0
total_count = 0
for detector, station, mic_output, date in value_tuples:
clips = model_utils.get_clips(station,
mic_output,
detector,
date,
self._query_annotation_name,
self._query_annotation_value,
order=False)
adjusted_count = 0
count = 0
for clip in clips:
if self._adjust_clip(clip):
adjusted_count += 1
total_adjusted_count += 1
count += 1
total_count += 1
# Log clip count for this detector/station/mic_output/date.
count_text = text_utils.create_count_text(count, 'clip')
_logger.info(
('Adjusted {} of {} for detector "{}", station "{}", mic '
'output "{}", and date {}.').format(adjusted_count,
count_text, detector.name,
station.name,
mic_output.name, date))
# Log total clips and processing rate.
count_text = text_utils.create_count_text(total_count, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(elapsed_time, total_count,
'clips')
_logger.info('Adjusted {} of a total of {}{}.'.format(
total_adjusted_count, count_text, timing_text))
def _delete_clip_audio_files(self):
start_time = time.time()
value_tuples = self._create_clip_query_values_iterator()
total_num_clips = 0
total_num_deleted_files = 0
for station, mic_output, date, detector in value_tuples:
clips = model_utils.get_clips(
station=station,
mic_output=mic_output,
date=date,
detector=detector,
annotation_name=self._annotation_name,
annotation_value=self._annotation_value,
tag_name=self._tag_name,
order=False)
num_clips = len(clips)
num_deleted_files = 0
for clip in clips:
if self._delete_clip_audio_file_if_needed(clip):
num_deleted_files += 1
# Log file deletions for this detector/station/mic_output/date.
count_text = text_utils.create_count_text(num_clips, 'clip')
_logger.info(
f'Deleted audio files for {num_deleted_files} of '
f'{count_text} for station "{station.name}", '
f'mic output "{mic_output.name}", date {date}, '
f'and detector "{detector.name}".')
total_num_clips += num_clips
total_num_deleted_files += num_deleted_files
# Log total file deletions and deletion rate.
count_text = text_utils.create_count_text(total_num_clips, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, total_num_clips, 'clips')
_logger.info(f'Processed a total of {count_text}{timing_text}.')
def _classify_clips(clips, classifier):
start_time = time.time()
if hasattr(classifier, 'annotate_clips'):
classify = _classify_clip_batches
else:
classify = _classify_clips_individually
num_clips_classified = classify(clips, classifier)
elapsed_time = time.time() - start_time
num_clips = len(clips)
timing_text = command_utils.get_timing_text(elapsed_time, num_clips,
'clips')
_logger.info(('Classified {} of {} visited clips{}.').format(
num_clips_classified, num_clips, timing_text))
def _create_clip_audio_files(self):
start_time = time.time()
value_tuples = self._create_clip_query_values_iterator()
total_num_clips = 0
total_num_created_files = 0
for detector, station, mic_output, date in value_tuples:
clips = model_utils.get_clips(station,
mic_output,
detector,
date,
self._annotation_name,
self._annotation_value,
order=False)
num_clips = len(clips)
num_created_files = 0
for clip in clips:
if self._create_clip_audio_file_if_needed(clip):
num_created_files += 1
# Log file creations for this detector/station/mic_output/date.
count_text = text_utils.create_count_text(num_clips, 'clip')
_logger.info(
('Created audio files for {} of {} for detector "{}", '
'station "{}", mic output "{}", and date {}.').format(
num_created_files, count_text, detector.name,
station.name, mic_output.name, date))
total_num_clips += num_clips
total_num_created_files += num_created_files
# Log total file creations and creation rate.
count_text = text_utils.create_count_text(total_num_clips, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(elapsed_time,
total_num_clips, 'clips')
_logger.info('Processed a total of {}{}.'.format(
count_text, timing_text))
def _refresh_recording_file_paths(self, recording_file_paths):
start_time = time.time()
file_count = RecordingFile.objects.count()
count_text = text_utils.create_count_text(file_count, 'recording file')
self._logger.info('Command will visit {}.'.format(count_text))
updated_count = 0
for i, file_ in enumerate(RecordingFile.objects.all()):
visited_count = i + 1
if visited_count % _LOGGING_PERIOD == 0:
self._logger.info(
' Visited {} files...'.format(visited_count))
old_path = file_.path
file_name = _get_file_name(old_path)
new_path = recording_file_paths.get(file_name)
if new_path is None:
self._logger.warning(
(' Could not find recording file "{}" in recording '
'directories.').format(file_name))
elif new_path != old_path:
# self._logger.info(
# ' Update "{}" to "{}"...'.format(
# old_path, new_path))
file_.path = new_path
file_.save()
updated_count += 1
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(elapsed_time,
visited_count, 'files')
self._logger.info(
('Updated paths for {} of {} visited recording files{}.').format(
updated_count, visited_count, timing_text))
def _adjust_clips(self):
start_time = time.time()
value_tuples = self._create_clip_query_values_iterator()
total_adjusted_count = 0
total_count = 0
for detector, station, mic_output, date in value_tuples:
clips = model_utils.get_clips(
station, mic_output, detector, date,
self._query_annotation_name, self._query_annotation_value,
order=False)
adjusted_count = 0
count = 0
for clip in clips:
if self._adjust_clip(clip):
adjusted_count += 1
total_adjusted_count += 1
count += 1
total_count += 1
# Log clip count for this detector/station/mic_output/date.
count_text = text_utils.create_count_text(count, 'clip')
_logger.info((
'Adjusted {} of {} for detector "{}", station "{}", mic '
'output "{}", and date {}.').format(
adjusted_count, count_text, detector.name, station.name,
mic_output.name, date))
# Log total clips and processing rate.
count_text = text_utils.create_count_text(total_count, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, total_count, 'clips')
_logger.info('Adjusted {} of a total of {}{}.'.format(
total_adjusted_count, count_text, timing_text))
def _classify_clips(clips, classifier):
start_time = time.time()
if hasattr(classifier, 'annotate_clips'):
classify = _classify_clip_batches
else:
classify = _classify_clips_individually
num_clips_classified = classify(clips, classifier)
elapsed_time = time.time() - start_time
num_clips = len(clips)
timing_text = command_utils.get_timing_text(
elapsed_time, num_clips, 'clips')
_logger.info((
'Classified {} of {} visited clips{}.').format(
num_clips_classified, num_clips, timing_text))
def _create_clip_audio_files(self):
start_time = time.time()
value_tuples = self._create_clip_query_values_iterator()
total_num_clips = 0
total_num_created_files = 0
for detector, station, mic_output, date in value_tuples:
clips = model_utils.get_clips(
station, mic_output, detector, date, self._annotation_name,
self._annotation_value, order=False)
num_clips = len(clips)
num_created_files = 0
for clip in clips:
if self._create_clip_audio_file_if_needed(clip):
num_created_files += 1
# Log file creations for this detector/station/mic_output/date.
count_text = text_utils.create_count_text(num_clips, 'clip')
_logger.info((
'Created audio files for {} of {} for detector "{}", '
'station "{}", mic output "{}", and date {}.').format(
num_created_files, count_text, detector.name,
station.name, mic_output.name, date))
total_num_clips += num_clips
total_num_created_files += num_created_files
# Log total file creations and creation rate.
count_text = text_utils.create_count_text(total_num_clips, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, total_num_clips, 'clips')
_logger.info(
'Processed a total of {}{}.'.format(count_text, timing_text))
def _delete_clips(self, retain_indices):
start_time = time.time()
retaining_clips = len(retain_indices) == 0
value_tuples = self._create_clip_query_values_iterator()
index = 0
total_retained_count = 0
for detector, station, mic_output, date in value_tuples:
# Get clips for this detector, station, mic_output, and date
clips = model_utils.get_clips(
station, mic_output, detector, date, self._annotation_name,
self._annotation_value, order=False)
# Figure out which clips should be deleted.
count = 0
retained_count = 0
clips_to_delete = []
for clip in clips:
if index not in retain_indices:
clips_to_delete.append(clip)
else:
retained_count += 1
count += 1
index += 1
# Delete clips.
try:
self._delete_clip_batch(clips_to_delete)
except Exception as e:
batch_text = \
_get_batch_text(detector, station, mic_output, date)
command_utils.log_and_reraise_fatal_exception(
e, 'Deletion of clips for {}'.format(batch_text))
# Log deletions.
if retaining_clips:
prefix = 'Deleted'
else:
deleted_count = count - retained_count
prefix = 'Deleted {} and retained {} of'.format(
deleted_count, retained_count)
count_text = text_utils.create_count_text(count, 'clip')
batch_text = _get_batch_text(detector, station, mic_output, date)
_logger.info(
'{} {} for {}.'.format(prefix, count_text, batch_text))
total_retained_count += retained_count
# Log total deletions and deletion rate.
if total_retained_count == 0:
prefix = 'Deleted'
else:
deleted_count = index - total_retained_count
prefix = 'Deleted {} and retained {} of'.format(
deleted_count, total_retained_count)
count_text = text_utils.create_count_text(index, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, index, 'clips')
_logger.info('{} a total of {}{}.'.format(
prefix, count_text, timing_text))
def _delete_clips(self, retain_indices):
start_time = time.time()
retaining_clips = len(retain_indices) == 0
value_tuples = self._create_clip_query_values_iterator()
index = 0
total_retained_count = 0
for station, mic_output, date, detector in value_tuples:
# Get clips for this station, mic_output, date, and detector
clips = model_utils.get_clips(
station=station,
mic_output=mic_output,
date=date,
detector=detector,
annotation_name=self._annotation_name,
annotation_value=self._annotation_value,
tag_name=self._tag_name,
order=False)
# Figure out which clips should be deleted.
count = 0
retained_count = 0
clips_to_delete = []
for clip in clips:
if index not in retain_indices:
clips_to_delete.append(clip)
else:
retained_count += 1
count += 1
index += 1
# Delete clips.
try:
self._delete_clip_batch(clips_to_delete)
except Exception as e:
batch_text = \
_get_batch_text(station, mic_output, date, detector)
command_utils.log_and_reraise_fatal_exception(
e, f'Deletion of clips for {batch_text}')
# Log deletions.
if retaining_clips:
prefix = 'Deleted'
else:
deleted_count = count - retained_count
prefix = (
f'Deleted {deleted_count} and retained '
f'{retained_count} of')
count_text = text_utils.create_count_text(count, 'clip')
batch_text = _get_batch_text(station, mic_output, date, detector)
_logger.info(f'{prefix} {count_text} for {batch_text}.')
total_retained_count += retained_count
# Log total deletions and deletion rate.
if total_retained_count == 0:
prefix = 'Deleted'
else:
deleted_count = index - total_retained_count
prefix = (
f'Deleted {deleted_count} and retained '
f'{total_retained_count} of')
count_text = text_utils.create_count_text(index, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, index, 'clips')
_logger.info(f'{prefix} a total of {count_text}{timing_text}.')
