def _log_summary(self, elapsed_time): create_units_text = text_utils.create_units_text log = self._logger.info # Number of audio files processed. units = create_units_text(self._file_count, 'audio file') timing_text = command_utils.get_timing_text(elapsed_time, self._file_count, 'files') log('Processed {} {}{}.'.format(self._file_count, units, timing_text)) # Number of file path parse errors. count = self._file_count - self._parsed_count if count != 0: units = create_units_text(count, 'audio file path') log(('{} {} could not be parsed. See error messages above ' 'for details.').format(count, units)) # Number of excluded files. count = self._parsed_count - self._eligible_count if count != 0: units = create_units_text(count, 'audio file was', 'audio files were') log('{} {} excluded by date.'.format(count, units)) # Number of failed imports. count = self._eligible_count - self._imported_count if count != 0: units = create_units_text(count, 'attempted clip import') log('{} {} failed. See error messages above for details.'.format( count, units)) # Number of successful imports. units = create_units_text(self._imported_count, 'clip') log('{} {} were imported.'.format(self._imported_count, units))
def _log_summary(self, elapsed_time): create_units_text = text_utils.create_units_text log = self._logger.info # Number of audio files processed. units = create_units_text(self._file_count, 'audio file') timing_text = command_utils.get_timing_text( elapsed_time, self._file_count, 'files') log('Processed {} {}{}.'.format(self._file_count, units, timing_text)) # Number of file path parse errors. count = self._file_count - self._parsed_count if count != 0: units = create_units_text(count, 'audio file path') log(('{} {} could not be parsed. See error messages above ' 'for details.').format(count, units)) # Number of excluded files. count = self._parsed_count - self._eligible_count if count != 0: units = create_units_text( count, 'audio file was', 'audio files were') log('{} {} excluded by date.'.format(count, units)) # Number of failed imports. count = self._eligible_count - self._imported_count if count != 0: units = create_units_text(count, 'attempted clip import') log('{} {} failed. See error messages above for details.'.format( count, units)) # Number of successful imports. units = create_units_text(self._imported_count, 'clip') log('{} {} were imported.'.format(self._imported_count, units))
def _execute_create_clips_action(self, args): clips = args['clips'] num_clips = len(clips) self._logger.info('Creating {} clips...'.format(num_clips)) start_time = time.time() clip_num = 0 for clip in clips: self._create_clip(clip) clip_num += 1 if clip_num % _LOGGING_PERIOD == 0: self._logger.info('Created {} clips...'.format(clip_num)) elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text( elapsed_time, num_clips, 'clips') self._logger.info( 'Created {} clips{}.'.format(num_clips, timing_text))
def _add_clip_start_indices(self, recordings): start_time = time.time() total_clips = 0 total_clips_found = 0 for recording in recordings: files = recording.files.all().order_by('file_num') if files.count() == 0: # archive has no files for this recording self._logger.warning( f'Archive contains no audio files for recording ' f'"{str(recording)}". No clips of this recording ' f'will be processed.') else: # archive has files for this recording self._recording_reader = self._create_recording_reader(files) for channel in recording.channels.all(): for detector in self._detectors: try: num_clips, num_clips_found = \ self._add_channel_clip_start_indices( channel, detector) except Exception as e: self._logger.error( f'Processing of clips for recording channel ' f'"{str(channel)}" failed with an exception.\n' f'The exception message was:\n' f' {str(e)}\n' f'No clips of the channel were modified.\n' f'See below for exception traceback.') raise total_clips += num_clips total_clips_found += num_clips_found elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text( elapsed_time, total_clips, 'clips') self._logger.info( f'Added start indices for {total_clips_found} of {total_clips} ' f'processed clips{timing_text}.') self._log_archive_status()
def _add_clip_start_indices(self): start_time = time.time() self._min_start_time_change = 100 self._max_start_time_change = -100 total_clips = 0 total_clips_found = 0 for recording in self._recordings: for channel in recording.channels.all(): for detector in self._detectors: try: num_clips, num_clips_found = \ self._add_channel_clip_start_indices( channel, detector) except Exception as e: self._logger.error( f'Processing of clips for recording channel ' f'"{str(channel)}" failed with an exception.\n' f'The exception message was:\n' f' {str(e)}\n' f'No clips of the channel were modified.\n' f'See below for exception traceback.') raise total_clips += num_clips total_clips_found += num_clips_found elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text(elapsed_time, total_clips, 'clips') self._logger.info( f'Added start indices for {total_clips_found} of {total_clips} ' f'processed clips{timing_text}.') self._logger.info( f'Range of start time changes was ' f'({self._min_start_time_change}, {self._max_start_time_change}).') self._log_archive_status()
def _adjust_clips(self): start_time = time.time() value_tuples = self._create_clip_query_values_iterator() total_adjusted_count = 0 total_count = 0 for detector, station, mic_output, date in value_tuples: clips = model_utils.get_clips(station, mic_output, detector, date, self._query_annotation_name, self._query_annotation_value, order=False) adjusted_count = 0 count = 0 for clip in clips: if self._adjust_clip(clip): adjusted_count += 1 total_adjusted_count += 1 count += 1 total_count += 1 # Log clip count for this detector/station/mic_output/date. count_text = text_utils.create_count_text(count, 'clip') _logger.info( ('Adjusted {} of {} for detector "{}", station "{}", mic ' 'output "{}", and date {}.').format(adjusted_count, count_text, detector.name, station.name, mic_output.name, date)) # Log total clips and processing rate. count_text = text_utils.create_count_text(total_count, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text(elapsed_time, total_count, 'clips') _logger.info('Adjusted {} of a total of {}{}.'.format( total_adjusted_count, count_text, timing_text))
def _delete_clip_audio_files(self): start_time = time.time() value_tuples = self._create_clip_query_values_iterator() total_num_clips = 0 total_num_deleted_files = 0 for station, mic_output, date, detector in value_tuples: clips = model_utils.get_clips( station=station, mic_output=mic_output, date=date, detector=detector, annotation_name=self._annotation_name, annotation_value=self._annotation_value, tag_name=self._tag_name, order=False) num_clips = len(clips) num_deleted_files = 0 for clip in clips: if self._delete_clip_audio_file_if_needed(clip): num_deleted_files += 1 # Log file deletions for this detector/station/mic_output/date. count_text = text_utils.create_count_text(num_clips, 'clip') _logger.info( f'Deleted audio files for {num_deleted_files} of ' f'{count_text} for station "{station.name}", ' f'mic output "{mic_output.name}", date {date}, ' f'and detector "{detector.name}".') total_num_clips += num_clips total_num_deleted_files += num_deleted_files # Log total file deletions and deletion rate. count_text = text_utils.create_count_text(total_num_clips, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text( elapsed_time, total_num_clips, 'clips') _logger.info(f'Processed a total of {count_text}{timing_text}.')
def _classify_clips(clips, classifier): start_time = time.time() if hasattr(classifier, 'annotate_clips'): classify = _classify_clip_batches else: classify = _classify_clips_individually num_clips_classified = classify(clips, classifier) elapsed_time = time.time() - start_time num_clips = len(clips) timing_text = command_utils.get_timing_text(elapsed_time, num_clips, 'clips') _logger.info(('Classified {} of {} visited clips{}.').format( num_clips_classified, num_clips, timing_text))
def _create_clip_audio_files(self): start_time = time.time() value_tuples = self._create_clip_query_values_iterator() total_num_clips = 0 total_num_created_files = 0 for detector, station, mic_output, date in value_tuples: clips = model_utils.get_clips(station, mic_output, detector, date, self._annotation_name, self._annotation_value, order=False) num_clips = len(clips) num_created_files = 0 for clip in clips: if self._create_clip_audio_file_if_needed(clip): num_created_files += 1 # Log file creations for this detector/station/mic_output/date. count_text = text_utils.create_count_text(num_clips, 'clip') _logger.info( ('Created audio files for {} of {} for detector "{}", ' 'station "{}", mic output "{}", and date {}.').format( num_created_files, count_text, detector.name, station.name, mic_output.name, date)) total_num_clips += num_clips total_num_created_files += num_created_files # Log total file creations and creation rate. count_text = text_utils.create_count_text(total_num_clips, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text(elapsed_time, total_num_clips, 'clips') _logger.info('Processed a total of {}{}.'.format( count_text, timing_text))
def _refresh_recording_file_paths(self, recording_file_paths): start_time = time.time() file_count = RecordingFile.objects.count() count_text = text_utils.create_count_text(file_count, 'recording file') self._logger.info('Command will visit {}.'.format(count_text)) updated_count = 0 for i, file_ in enumerate(RecordingFile.objects.all()): visited_count = i + 1 if visited_count % _LOGGING_PERIOD == 0: self._logger.info( ' Visited {} files...'.format(visited_count)) old_path = file_.path file_name = _get_file_name(old_path) new_path = recording_file_paths.get(file_name) if new_path is None: self._logger.warning( (' Could not find recording file "{}" in recording ' 'directories.').format(file_name)) elif new_path != old_path: # self._logger.info( # ' Update "{}" to "{}"...'.format( # old_path, new_path)) file_.path = new_path file_.save() updated_count += 1 elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text(elapsed_time, visited_count, 'files') self._logger.info( ('Updated paths for {} of {} visited recording files{}.').format( updated_count, visited_count, timing_text))
def _adjust_clips(self): start_time = time.time() value_tuples = self._create_clip_query_values_iterator() total_adjusted_count = 0 total_count = 0 for detector, station, mic_output, date in value_tuples: clips = model_utils.get_clips( station, mic_output, detector, date, self._query_annotation_name, self._query_annotation_value, order=False) adjusted_count = 0 count = 0 for clip in clips: if self._adjust_clip(clip): adjusted_count += 1 total_adjusted_count += 1 count += 1 total_count += 1 # Log clip count for this detector/station/mic_output/date. count_text = text_utils.create_count_text(count, 'clip') _logger.info(( 'Adjusted {} of {} for detector "{}", station "{}", mic ' 'output "{}", and date {}.').format( adjusted_count, count_text, detector.name, station.name, mic_output.name, date)) # Log total clips and processing rate. count_text = text_utils.create_count_text(total_count, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text( elapsed_time, total_count, 'clips') _logger.info('Adjusted {} of a total of {}{}.'.format( total_adjusted_count, count_text, timing_text))
def _classify_clips(clips, classifier): start_time = time.time() if hasattr(classifier, 'annotate_clips'): classify = _classify_clip_batches else: classify = _classify_clips_individually num_clips_classified = classify(clips, classifier) elapsed_time = time.time() - start_time num_clips = len(clips) timing_text = command_utils.get_timing_text( elapsed_time, num_clips, 'clips') _logger.info(( 'Classified {} of {} visited clips{}.').format( num_clips_classified, num_clips, timing_text))
def _create_clip_audio_files(self): start_time = time.time() value_tuples = self._create_clip_query_values_iterator() total_num_clips = 0 total_num_created_files = 0 for detector, station, mic_output, date in value_tuples: clips = model_utils.get_clips( station, mic_output, detector, date, self._annotation_name, self._annotation_value, order=False) num_clips = len(clips) num_created_files = 0 for clip in clips: if self._create_clip_audio_file_if_needed(clip): num_created_files += 1 # Log file creations for this detector/station/mic_output/date. count_text = text_utils.create_count_text(num_clips, 'clip') _logger.info(( 'Created audio files for {} of {} for detector "{}", ' 'station "{}", mic output "{}", and date {}.').format( num_created_files, count_text, detector.name, station.name, mic_output.name, date)) total_num_clips += num_clips total_num_created_files += num_created_files # Log total file creations and creation rate. count_text = text_utils.create_count_text(total_num_clips, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text( elapsed_time, total_num_clips, 'clips') _logger.info( 'Processed a total of {}{}.'.format(count_text, timing_text))
def _delete_clips(self, retain_indices): start_time = time.time() retaining_clips = len(retain_indices) == 0 value_tuples = self._create_clip_query_values_iterator() index = 0 total_retained_count = 0 for detector, station, mic_output, date in value_tuples: # Get clips for this detector, station, mic_output, and date clips = model_utils.get_clips( station, mic_output, detector, date, self._annotation_name, self._annotation_value, order=False) # Figure out which clips should be deleted. count = 0 retained_count = 0 clips_to_delete = [] for clip in clips: if index not in retain_indices: clips_to_delete.append(clip) else: retained_count += 1 count += 1 index += 1 # Delete clips. try: self._delete_clip_batch(clips_to_delete) except Exception as e: batch_text = \ _get_batch_text(detector, station, mic_output, date) command_utils.log_and_reraise_fatal_exception( e, 'Deletion of clips for {}'.format(batch_text)) # Log deletions. if retaining_clips: prefix = 'Deleted' else: deleted_count = count - retained_count prefix = 'Deleted {} and retained {} of'.format( deleted_count, retained_count) count_text = text_utils.create_count_text(count, 'clip') batch_text = _get_batch_text(detector, station, mic_output, date) _logger.info( '{} {} for {}.'.format(prefix, count_text, batch_text)) total_retained_count += retained_count # Log total deletions and deletion rate. if total_retained_count == 0: prefix = 'Deleted' else: deleted_count = index - total_retained_count prefix = 'Deleted {} and retained {} of'.format( deleted_count, total_retained_count) count_text = text_utils.create_count_text(index, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text( elapsed_time, index, 'clips') _logger.info('{} a total of {}{}.'.format( prefix, count_text, timing_text))
def _delete_clips(self, retain_indices): start_time = time.time() retaining_clips = len(retain_indices) == 0 value_tuples = self._create_clip_query_values_iterator() index = 0 total_retained_count = 0 for station, mic_output, date, detector in value_tuples: # Get clips for this station, mic_output, date, and detector clips = model_utils.get_clips( station=station, mic_output=mic_output, date=date, detector=detector, annotation_name=self._annotation_name, annotation_value=self._annotation_value, tag_name=self._tag_name, order=False) # Figure out which clips should be deleted. count = 0 retained_count = 0 clips_to_delete = [] for clip in clips: if index not in retain_indices: clips_to_delete.append(clip) else: retained_count += 1 count += 1 index += 1 # Delete clips. try: self._delete_clip_batch(clips_to_delete) except Exception as e: batch_text = \ _get_batch_text(station, mic_output, date, detector) command_utils.log_and_reraise_fatal_exception( e, f'Deletion of clips for {batch_text}') # Log deletions. if retaining_clips: prefix = 'Deleted' else: deleted_count = count - retained_count prefix = ( f'Deleted {deleted_count} and retained ' f'{retained_count} of') count_text = text_utils.create_count_text(count, 'clip') batch_text = _get_batch_text(station, mic_output, date, detector) _logger.info(f'{prefix} {count_text} for {batch_text}.') total_retained_count += retained_count # Log total deletions and deletion rate. if total_retained_count == 0: prefix = 'Deleted' else: deleted_count = index - total_retained_count prefix = ( f'Deleted {deleted_count} and retained ' f'{total_retained_count} of') count_text = text_utils.create_count_text(index, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text( elapsed_time, index, 'clips') _logger.info(f'{prefix} a total of {count_text}{timing_text}.')