def _transfer_classifications_aux(self, station, mic_output, date): source_clips = model_utils.get_clips( station=station, mic_output=mic_output, date=date, detector=self._source_detector, annotation_name=self._annotation_name, annotation_value=self._annotation_value) target_clips = model_utils.get_clips( station=station, mic_output=mic_output, date=date, detector=self._target_detector, annotation_name=self._annotation_name) matches = self._match_clips_with_calls(source_clips, target_clips) _logger.info( f'{self._source_detector.name} -> {self._target_detector.name} / ' f'{station.name} / {mic_output.name} / {date} / ' f'{source_clips.count()} {target_clips.count()} {len(matches)}') if len(matches) > 0: source_clips = list(source_clips.all()) target_clips = list(target_clips.all()) # self._show_matches(matches, source_clips, target_clips) for i, j in matches: self._transfer_classification(source_clips[i], target_clips[j])
def _transfer_classifications_aux(self, station, mic_output, date): source_clips = model_utils.get_clips( station, mic_output, self._source_detector, date, self._annotation_name, self._annotation_value) target_clips = model_utils.get_clips( station, mic_output, self._target_detector, date, self._annotation_name, None) matches = self._match_clips_with_calls(source_clips, target_clips) _logger.info('{} -> {} / {} / {} / {} / {} {} {}'.format( self._source_detector.name, self._target_detector.name, station.name, mic_output.name, date, source_clips.count(), target_clips.count(), len(matches))) if len(matches) > 0: source_clips = list(source_clips.all()) target_clips = list(target_clips.all()) # self._show_matches(matches, source_clips, target_clips) for i, j in matches: self._transfer_classification(source_clips[i], target_clips[j])
def _get_bird_counts(station, mic_output, detector, start_date, end_date, annotation_name, annotation_values, count_suppression_interval): counts = {} for annotation_value in annotation_values: if _is_call_type(annotation_value): for date in _date_range(start_date, end_date): clips = model_utils.get_clips( station=station, mic_output=mic_output, date=date, detector=detector, annotation_name=annotation_name, annotation_value=annotation_value) times = sorted(c.start_time for c in clips) count = clip_count_utils.get_bird_count( times, count_suppression_interval) counts[(date, annotation_value)] = count return counts
def _get_clips(station, mic_output, date, detector, tag_name): try: return model_utils.get_clips(station=station, mic_output=mic_output, date=date, detector=detector, tag_name=tag_name) except Exception as e: command_utils.log_and_reraise_fatal_exception(e, 'Clip query')
def _count_clips(self): value_tuples = self._create_clip_query_values_iterator() count = 0 for detector, station, mic_output, date in value_tuples: clips = model_utils.get_clips( station, mic_output, detector, date, self._annotation_name, self._annotation_value, order=False) count += clips.count() return count
def _adjust_clips(self): start_time = time.time() value_tuples = self._create_clip_query_values_iterator() total_adjusted_count = 0 total_count = 0 for detector, station, mic_output, date in value_tuples: clips = model_utils.get_clips(station, mic_output, detector, date, self._query_annotation_name, self._query_annotation_value, order=False) adjusted_count = 0 count = 0 for clip in clips: if self._adjust_clip(clip): adjusted_count += 1 total_adjusted_count += 1 count += 1 total_count += 1 # Log clip count for this detector/station/mic_output/date. count_text = text_utils.create_count_text(count, 'clip') _logger.info( ('Adjusted {} of {} for detector "{}", station "{}", mic ' 'output "{}", and date {}.').format(adjusted_count, count_text, detector.name, station.name, mic_output.name, date)) # Log total clips and processing rate. count_text = text_utils.create_count_text(total_count, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text(elapsed_time, total_count, 'clips') _logger.info('Adjusted {} of a total of {}{}.'.format( total_adjusted_count, count_text, timing_text))
def _delete_clip_audio_files(self): start_time = time.time() value_tuples = self._create_clip_query_values_iterator() total_num_clips = 0 total_num_deleted_files = 0 for station, mic_output, date, detector in value_tuples: clips = model_utils.get_clips( station=station, mic_output=mic_output, date=date, detector=detector, annotation_name=self._annotation_name, annotation_value=self._annotation_value, tag_name=self._tag_name, order=False) num_clips = len(clips) num_deleted_files = 0 for clip in clips: if self._delete_clip_audio_file_if_needed(clip): num_deleted_files += 1 # Log file deletions for this detector/station/mic_output/date. count_text = text_utils.create_count_text(num_clips, 'clip') _logger.info( f'Deleted audio files for {num_deleted_files} of ' f'{count_text} for station "{station.name}", ' f'mic output "{mic_output.name}", date {date}, ' f'and detector "{detector.name}".') total_num_clips += num_clips total_num_deleted_files += num_deleted_files # Log total file deletions and deletion rate. count_text = text_utils.create_count_text(total_num_clips, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text( elapsed_time, total_num_clips, 'clips') _logger.info(f'Processed a total of {count_text}{timing_text}.')
def _create_clip_audio_files(self): start_time = time.time() value_tuples = self._create_clip_query_values_iterator() total_num_clips = 0 total_num_created_files = 0 for detector, station, mic_output, date in value_tuples: clips = model_utils.get_clips(station, mic_output, detector, date, self._annotation_name, self._annotation_value, order=False) num_clips = len(clips) num_created_files = 0 for clip in clips: if self._create_clip_audio_file_if_needed(clip): num_created_files += 1 # Log file creations for this detector/station/mic_output/date. count_text = text_utils.create_count_text(num_clips, 'clip') _logger.info( ('Created audio files for {} of {} for detector "{}", ' 'station "{}", mic output "{}", and date {}.').format( num_created_files, count_text, detector.name, station.name, mic_output.name, date)) total_num_clips += num_clips total_num_created_files += num_created_files # Log total file creations and creation rate. count_text = text_utils.create_count_text(total_num_clips, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text(elapsed_time, total_num_clips, 'clips') _logger.info('Processed a total of {}{}.'.format( count_text, timing_text))
def _adjust_clips(self): start_time = time.time() value_tuples = self._create_clip_query_values_iterator() total_adjusted_count = 0 total_count = 0 for detector, station, mic_output, date in value_tuples: clips = model_utils.get_clips( station, mic_output, detector, date, self._query_annotation_name, self._query_annotation_value, order=False) adjusted_count = 0 count = 0 for clip in clips: if self._adjust_clip(clip): adjusted_count += 1 total_adjusted_count += 1 count += 1 total_count += 1 # Log clip count for this detector/station/mic_output/date. count_text = text_utils.create_count_text(count, 'clip') _logger.info(( 'Adjusted {} of {} for detector "{}", station "{}", mic ' 'output "{}", and date {}.').format( adjusted_count, count_text, detector.name, station.name, mic_output.name, date)) # Log total clips and processing rate. count_text = text_utils.create_count_text(total_count, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text( elapsed_time, total_count, 'clips') _logger.info('Adjusted {} of a total of {}{}.'.format( total_adjusted_count, count_text, timing_text))
def _create_clip_audio_files(self): start_time = time.time() value_tuples = self._create_clip_query_values_iterator() total_num_clips = 0 total_num_created_files = 0 for detector, station, mic_output, date in value_tuples: clips = model_utils.get_clips( station, mic_output, detector, date, self._annotation_name, self._annotation_value, order=False) num_clips = len(clips) num_created_files = 0 for clip in clips: if self._create_clip_audio_file_if_needed(clip): num_created_files += 1 # Log file creations for this detector/station/mic_output/date. count_text = text_utils.create_count_text(num_clips, 'clip') _logger.info(( 'Created audio files for {} of {} for detector "{}", ' 'station "{}", mic output "{}", and date {}.').format( num_created_files, count_text, detector.name, station.name, mic_output.name, date)) total_num_clips += num_clips total_num_created_files += num_created_files # Log total file creations and creation rate. count_text = text_utils.create_count_text(total_num_clips, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text( elapsed_time, total_num_clips, 'clips') _logger.info( 'Processed a total of {}{}.'.format(count_text, timing_text))
def annotate_old_bird_calls(): center_index_annotation_info = \ AnnotationInfo.objects.get(name=CENTER_INDEX_ANNOTATION_NAME) center_freq_annotation_info = \ AnnotationInfo.objects.get(name=CENTER_FREQ_ANNOTATION_NAME) classification_annotation_info = \ AnnotationInfo.objects.get(name=CLASSIFICATION_ANNOTATION_NAME) user = User.objects.get(username='******') sm_pairs = model_utils.get_station_mic_output_pairs_list() ground_truth_detector = Processor.objects.get( name=GROUND_TRUTH_DETECTOR_NAME) rows = [] for detector_name, annotation_value in DETECTOR_DATA: short_detector_name = detector_name.split()[2] old_bird_detector = Processor.objects.get(name=detector_name) window = utils.OLD_BIRD_CLIP_CALL_CENTER_WINDOWS[short_detector_name] for station, mic_output in sm_pairs: station_num = int(station.name.split()[1]) print('{} {}...'.format(short_detector_name, station_num)) ground_truth_clips = list(model_utils.get_clips( station, mic_output, ground_truth_detector, None, CLASSIFICATION_ANNOTATION_NAME, annotation_value)) ground_truth_call_center_indices = \ [c.start_index + c.length // 2 for c in ground_truth_clips] ground_truth_call_count = len(ground_truth_clips) old_bird_clips = list(model_utils.get_clips( station, mic_output, old_bird_detector)) old_bird_clip_count = len(old_bird_clips) clips = [(c.start_index, c.length) for c in old_bird_clips] matches = utils.match_clips_with_calls( clips, ground_truth_call_center_indices, window) old_bird_call_count = len(matches) rows.append([ short_detector_name, station_num, ground_truth_call_count, old_bird_call_count, old_bird_clip_count]) if ANNOTATE: # Clear any existing annotations. for clip in old_bird_clips: model_utils.delete_clip_annotation( clip, classification_annotation_info, creating_user=user) # Create new annotations. for i, j in matches: old_bird_clip = old_bird_clips[i] call_center_index = ground_truth_call_center_indices[j] ground_truth_clip = ground_truth_clips[j] # Annotate Old Bird clip call center index. model_utils.annotate_clip( old_bird_clip, center_index_annotation_info, str(call_center_index), creating_user=user) # Get ground truth clip call center frequency. annotations = \ model_utils.get_clip_annotations(ground_truth_clip) call_center_freq = annotations[CENTER_FREQ_ANNOTATION_NAME] # Annotate Old Bird clip call center frequency. model_utils.annotate_clip( old_bird_clip, center_freq_annotation_info, call_center_freq, creating_user=user) model_utils.annotate_clip( old_bird_clip, classification_annotation_info, annotation_value, creating_user=user) return rows
def annotate_detected_calls(): center_index_annotation_info = \ AnnotationInfo.objects.get(name=CENTER_INDEX_ANNOTATION_NAME) center_freq_annotation_info = \ AnnotationInfo.objects.get(name=CENTER_FREQ_ANNOTATION_NAME) classification_annotation_info = \ AnnotationInfo.objects.get(name=CLASSIFICATION_ANNOTATION_NAME) user = User.objects.get(username='******') sm_pairs = model_utils.get_station_mic_output_pairs_list() ground_truth_detector = Processor.objects.get( name=GROUND_TRUTH_DETECTOR_NAME) rows = [] for detector_name, annotation_value in DETECTOR_DATA: short_detector_name = detector_name.split()[2] old_bird_detector = Processor.objects.get(name=detector_name) window = utils.CALL_CENTER_WINDOWS[short_detector_name] for station, mic_output in sm_pairs: station_num = int(station.name.split()[1]) print('{} {}...'.format(short_detector_name, station_num)) ground_truth_clips = list( model_utils.get_clips(station, mic_output, ground_truth_detector, None, CLASSIFICATION_ANNOTATION_NAME, annotation_value)) ground_truth_call_center_indices = \ [c.start_index + c.length // 2 for c in ground_truth_clips] ground_truth_call_count = len(ground_truth_clips) old_bird_clips = list( model_utils.get_clips(station, mic_output, old_bird_detector)) old_bird_clip_count = len(old_bird_clips) clips = [(c.start_index, c.length) for c in old_bird_clips] matches = utils.match_clips_with_calls( clips, ground_truth_call_center_indices, window) old_bird_call_count = len(matches) rows.append([ short_detector_name, station_num, ground_truth_call_count, old_bird_call_count, old_bird_clip_count ]) if ANNOTATE: # Clear any existing annotations. for clip in old_bird_clips: model_utils.delete_clip_annotation( clip, classification_annotation_info, creating_user=user) # Create new annotations. for i, j in matches: old_bird_clip = old_bird_clips[i] call_center_index = ground_truth_call_center_indices[j] ground_truth_clip = ground_truth_clips[j] # Annotate Old Bird clip call center index. model_utils.annotate_clip(old_bird_clip, center_index_annotation_info, str(call_center_index), creating_user=user) # Get ground truth clip call center frequency. annotations = \ model_utils.get_clip_annotations(ground_truth_clip) call_center_freq = annotations[CENTER_FREQ_ANNOTATION_NAME] # Annotate Old Bird clip call center frequency. model_utils.annotate_clip(old_bird_clip, center_freq_annotation_info, call_center_freq, creating_user=user) model_utils.annotate_clip(old_bird_clip, classification_annotation_info, annotation_value, creating_user=user) return rows
def _delete_clips(self, retain_indices): start_time = time.time() retaining_clips = len(retain_indices) == 0 value_tuples = self._create_clip_query_values_iterator() index = 0 total_retained_count = 0 for detector, station, mic_output, date in value_tuples: # Get clips for this detector, station, mic_output, and date clips = model_utils.get_clips( station, mic_output, detector, date, self._annotation_name, self._annotation_value, order=False) # Figure out which clips should be deleted. count = 0 retained_count = 0 clips_to_delete = [] for clip in clips: if index not in retain_indices: clips_to_delete.append(clip) else: retained_count += 1 count += 1 index += 1 # Delete clips. try: self._delete_clip_batch(clips_to_delete) except Exception as e: batch_text = \ _get_batch_text(detector, station, mic_output, date) command_utils.log_and_reraise_fatal_exception( e, 'Deletion of clips for {}'.format(batch_text)) # Log deletions. if retaining_clips: prefix = 'Deleted' else: deleted_count = count - retained_count prefix = 'Deleted {} and retained {} of'.format( deleted_count, retained_count) count_text = text_utils.create_count_text(count, 'clip') batch_text = _get_batch_text(detector, station, mic_output, date) _logger.info( '{} {} for {}.'.format(prefix, count_text, batch_text)) total_retained_count += retained_count # Log total deletions and deletion rate. if total_retained_count == 0: prefix = 'Deleted' else: deleted_count = index - total_retained_count prefix = 'Deleted {} and retained {} of'.format( deleted_count, total_retained_count) count_text = text_utils.create_count_text(index, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text( elapsed_time, index, 'clips') _logger.info('{} a total of {}{}.'.format( prefix, count_text, timing_text))
def _delete_clips(self, retain_indices): start_time = time.time() retaining_clips = len(retain_indices) == 0 value_tuples = self._create_clip_query_values_iterator() index = 0 total_retained_count = 0 for station, mic_output, date, detector in value_tuples: # Get clips for this station, mic_output, date, and detector clips = model_utils.get_clips( station=station, mic_output=mic_output, date=date, detector=detector, annotation_name=self._annotation_name, annotation_value=self._annotation_value, tag_name=self._tag_name, order=False) # Figure out which clips should be deleted. count = 0 retained_count = 0 clips_to_delete = [] for clip in clips: if index not in retain_indices: clips_to_delete.append(clip) else: retained_count += 1 count += 1 index += 1 # Delete clips. try: self._delete_clip_batch(clips_to_delete) except Exception as e: batch_text = \ _get_batch_text(station, mic_output, date, detector) command_utils.log_and_reraise_fatal_exception( e, f'Deletion of clips for {batch_text}') # Log deletions. if retaining_clips: prefix = 'Deleted' else: deleted_count = count - retained_count prefix = ( f'Deleted {deleted_count} and retained ' f'{retained_count} of') count_text = text_utils.create_count_text(count, 'clip') batch_text = _get_batch_text(station, mic_output, date, detector) _logger.info(f'{prefix} {count_text} for {batch_text}.') total_retained_count += retained_count # Log total deletions and deletion rate. if total_retained_count == 0: prefix = 'Deleted' else: deleted_count = index - total_retained_count prefix = ( f'Deleted {deleted_count} and retained ' f'{total_retained_count} of') count_text = text_utils.create_count_text(index, 'clip') elapsed_time = time.time() - start_time timing_text = command_utils.get_timing_text( elapsed_time, index, 'clips') _logger.info(f'{prefix} a total of {count_text}{timing_text}.')
def _get_clips(*args): try: return model_utils.get_clips(*args) except Exception as e: command_utils.log_and_reraise_fatal_exception(e, 'Clip query')