def execute(self, job_info):
self._exporter.begin_exports()
value_tuples = self._create_clip_query_values_iterator()
for station, mic_output, date, detector in value_tuples:
clips = _get_clips(
station, mic_output, date, detector, self._annotation_name,
self._annotation_value, self._tag_name)
count = clips.count()
count_text = text_utils.create_count_text(count, 'clip')
_logger.info(
f'Exporter will visit {count_text} for station '
f'"{station.name}", mic output "{mic_output.name}", '
f'date {date}, and detector {detector.name}.')
try:
_export_clips(clips, self._exporter)
except Exception:
_logger.error(
'Clip export failed. See below for exception traceback.')
raise
self._exporter.end_exports()
return True
def execute(self, job_info):
self._exporter.begin_exports()
annotation_name, annotation_value = \
model_utils.get_clip_query_annotation_data(
'Classification', self._classification)
value_tuples = self._create_clip_query_values_iterator()
for detector, station, mic_output, date in value_tuples:
clips = _get_clips(station, mic_output, detector, date,
annotation_name, annotation_value)
count = clips.count()
count_text = text_utils.create_count_text(count, 'clip')
_logger.info(
('Exporter will visit {} for detector "{}", station "{}", '
'mic output "{}", and date {}.').format(
count_text, detector.name, station.name, mic_output.name,
date))
try:
_export_clips(clips, self._exporter)
except Exception:
_logger.error(
'Clip export failed. See below for exception traceback.')
raise
self._exporter.end_exports()
return True
def execute(self, job_info):
self._exporter.begin_exports()
value_tuples = self._create_clip_query_values_iterator()
for detector, station, mic_output, date in value_tuples:
clips = _get_clips(station, mic_output, detector, date)
count = clips.count()
count_text = text_utils.create_count_text(count, 'clip')
_logger.info((
'Exporter will visit {} for detector "{}", station "{}", '
'mic output "{}", and date {}.').format(
count_text, detector.name, station.name, mic_output.name,
date))
try:
_export_clips(clips, self._exporter)
except Exception:
_logger.error(
'Clip export failed. See below for exception traceback.')
raise
self._exporter.end_exports()
return True
def _log_detection_performance(
self, num_detectors, num_channels, interval_duration,
processing_time):
format_ = text_utils.format_number
dur = format_(interval_duration)
time = format_(processing_time)
detectors_text = text_utils.create_count_text(
num_detectors, 'detector')
message = (
' Ran {} on {} seconds of {}-channel '
'audio in {} seconds').format(
detectors_text, dur, num_channels, time)
if processing_time != 0:
total_duration = num_detectors * num_channels * interval_duration
speedup = format_(total_duration / processing_time)
message += ', {} times faster than real time.'.format(speedup)
else:
message += '.'
self._logger.info(message)
def execute(self, job_info):
classifier = self._create_classifier(job_info.job_id)
classifier.begin_annotations()
value_tuples = self._create_clip_query_values_iterator()
for detector, station, mic_output, date in value_tuples:
clips = _get_clips(station, mic_output, detector, date)
count = clips.count()
count_text = text_utils.create_count_text(count, 'clip')
_logger.info(
('Classifier will visit {} for detector "{}", station "{}", '
'mic output "{}", and date {}.').format(
count_text, detector.name, station.name, mic_output.name,
date))
try:
_classify_clips(clips, classifier)
except Exception:
_logger.error(
'Clip classification failed. See below for exception '
'traceback.')
raise
classifier.end_annotations()
return True
def execute(self, job_info):
classifier = self._create_classifier(job_info.job_id)
classifier.begin_annotations()
value_tuples = self._create_clip_query_values_iterator()
tag_name = model_utils.get_clip_query_tag_name(self._tag_name)
for station, mic_output, date, detector in value_tuples:
clips = _get_clips(station, mic_output, date, detector, tag_name)
count = clips.count()
count_text = text_utils.create_count_text(count, 'clip')
_logger.info(f'Classifier will visit {count_text} for station '
f'"{station.name}", mic output "{mic_output.name}", '
f'date {date}, and detector "{detector.name}".')
try:
_classify_clips(clips, classifier)
except Exception:
_logger.error(
'Clip classification failed. See below for exception '
'traceback.')
raise
classifier.end_annotations()
return True
def _adjust_clips(self):
start_time = time.time()
value_tuples = self._create_clip_query_values_iterator()
total_adjusted_count = 0
total_count = 0
for detector, station, mic_output, date in value_tuples:
clips = model_utils.get_clips(station,
mic_output,
detector,
date,
self._query_annotation_name,
self._query_annotation_value,
order=False)
adjusted_count = 0
count = 0
for clip in clips:
if self._adjust_clip(clip):
adjusted_count += 1
total_adjusted_count += 1
count += 1
total_count += 1
# Log clip count for this detector/station/mic_output/date.
count_text = text_utils.create_count_text(count, 'clip')
_logger.info(
('Adjusted {} of {} for detector "{}", station "{}", mic '
'output "{}", and date {}.').format(adjusted_count,
count_text, detector.name,
station.name,
mic_output.name, date))
# Log total clips and processing rate.
count_text = text_utils.create_count_text(total_count, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(elapsed_time, total_count,
'clips')
_logger.info('Adjusted {} of a total of {}{}.'.format(
total_adjusted_count, count_text, timing_text))
def _delete_clip_audio_files(self):
start_time = time.time()
value_tuples = self._create_clip_query_values_iterator()
total_num_clips = 0
total_num_deleted_files = 0
for station, mic_output, date, detector in value_tuples:
clips = model_utils.get_clips(
station=station,
mic_output=mic_output,
date=date,
detector=detector,
annotation_name=self._annotation_name,
annotation_value=self._annotation_value,
tag_name=self._tag_name,
order=False)
num_clips = len(clips)
num_deleted_files = 0
for clip in clips:
if self._delete_clip_audio_file_if_needed(clip):
num_deleted_files += 1
# Log file deletions for this detector/station/mic_output/date.
count_text = text_utils.create_count_text(num_clips, 'clip')
_logger.info(
f'Deleted audio files for {num_deleted_files} of '
f'{count_text} for station "{station.name}", '
f'mic output "{mic_output.name}", date {date}, '
f'and detector "{detector.name}".')
total_num_clips += num_clips
total_num_deleted_files += num_deleted_files
# Log total file deletions and deletion rate.
count_text = text_utils.create_count_text(total_num_clips, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, total_num_clips, 'clips')
_logger.info(f'Processed a total of {count_text}{timing_text}.')
def _get_recording_lists(self):
try:
# Get iterator for all recordings of specified station-nights.
recordings = itertools.chain.from_iterable(
self._get_station_recordings(
name, self._start_date, self._end_date)
for name in self._station_names)
# Get mapping from station-nights to recording lists.
recording_lists = defaultdict(list)
for recording in recordings:
station = recording.station
night = station.get_night(recording.start_time)
recording_lists[(station.name, night)].append(recording)
total_num_station_nights = len(recording_lists)
station_nights_text = text_utils.create_count_text(
total_num_station_nights, 'station-night')
if self._process_random_station_nights:
# will process recordings for randomly selected subset
# of station-nights
recording_lists = self._select_recording_lists(recording_lists)
start_num = self._start_station_night_index + 1
end_num = self._end_station_night_index
self._logger.info((
'This command will process recordings for '
'station-nights {} to {} of a shuffled sequence '
'of {}.').format(start_num, end_num, station_nights_text))
else:
# will process recordings for all station-nights
self._logger.info(
'This command will process recordings for {}.'.format(
station_nights_text))
# Sort recordings for each station-night by start time.
for recordings in recording_lists.values():
recordings.sort(key=lambda r: r.start_time)
return recording_lists
except Exception as e:
self._logger.error((
'Collection of recordings to process failed with '
'an exception.\n'
'The exception message was:\n'
' {}\n'
'The archive was not modified.\n'
'See below for exception traceback.').format(str(e)))
raise
def _get_recording_lists(self):
try:
# Get iterator for all recordings of specified station-nights.
recordings = itertools.chain.from_iterable(
self._get_station_recordings(name, self._start_date,
self._end_date)
for name in self._station_names)
# Get mapping from station-nights to recording lists.
recording_lists = defaultdict(list)
for recording in recordings:
station = recording.station
night = station.get_night(recording.start_time)
recording_lists[(station.name, night)].append(recording)
total_num_station_nights = len(recording_lists)
station_nights_text = text_utils.create_count_text(
total_num_station_nights, 'station-night')
if self._process_random_station_nights:
# will process recordings for randomly selected subset
# of station-nights
recording_lists = self._select_recording_lists(recording_lists)
start_num = self._start_station_night_index + 1
end_num = self._end_station_night_index
self._logger.info(
('This command will process recordings for '
'station-nights {} to {} of a shuffled sequence '
'of {}.').format(start_num, end_num, station_nights_text))
else:
# will process recordings for all station-nights
self._logger.info(
'This command will process recordings for {}.'.format(
station_nights_text))
# Sort recordings for each station-night by start time.
for recordings in recording_lists.values():
recordings.sort(key=lambda r: r.start_time)
return recording_lists
except Exception as e:
self._logger.error(
('Collection of recordings to process failed with '
'an exception.\n'
'The exception message was:\n'
' {}\n'
'The archive was not modified.\n'
'See below for exception traceback.').format(str(e)))
raise
def _log_clips_not_found(self, num_clips):
indices_text = 'index' if num_clips == 1 else 'indices'
count_text = text_utils.create_count_text(num_clips, 'clip')
self._logger.info(
f' Could not find start {indices_text} of {count_text} '
f'in recording channel.')
def _create_clip_audio_files(self):
start_time = time.time()
value_tuples = self._create_clip_query_values_iterator()
total_num_clips = 0
total_num_created_files = 0
for detector, station, mic_output, date in value_tuples:
clips = model_utils.get_clips(station,
mic_output,
detector,
date,
self._annotation_name,
self._annotation_value,
order=False)
num_clips = len(clips)
num_created_files = 0
for clip in clips:
if self._create_clip_audio_file_if_needed(clip):
num_created_files += 1
# Log file creations for this detector/station/mic_output/date.
count_text = text_utils.create_count_text(num_clips, 'clip')
_logger.info(
('Created audio files for {} of {} for detector "{}", '
'station "{}", mic output "{}", and date {}.').format(
num_created_files, count_text, detector.name,
station.name, mic_output.name, date))
total_num_clips += num_clips
total_num_created_files += num_created_files
# Log total file creations and creation rate.
count_text = text_utils.create_count_text(total_num_clips, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(elapsed_time,
total_num_clips, 'clips')
_logger.info('Processed a total of {}{}.'.format(
count_text, timing_text))
def _adjust_clips(self):
start_time = time.time()
value_tuples = self._create_clip_query_values_iterator()
total_adjusted_count = 0
total_count = 0
for detector, station, mic_output, date in value_tuples:
clips = model_utils.get_clips(
station, mic_output, detector, date,
self._query_annotation_name, self._query_annotation_value,
order=False)
adjusted_count = 0
count = 0
for clip in clips:
if self._adjust_clip(clip):
adjusted_count += 1
total_adjusted_count += 1
count += 1
total_count += 1
# Log clip count for this detector/station/mic_output/date.
count_text = text_utils.create_count_text(count, 'clip')
_logger.info((
'Adjusted {} of {} for detector "{}", station "{}", mic '
'output "{}", and date {}.').format(
adjusted_count, count_text, detector.name, station.name,
mic_output.name, date))
# Log total clips and processing rate.
count_text = text_utils.create_count_text(total_count, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, total_count, 'clips')
_logger.info('Adjusted {} of a total of {}{}.'.format(
total_adjusted_count, count_text, timing_text))
def complete_processing(self, threshold=None):
# Create remaining clips.
self._create_clips(threshold)
clips_text = text_utils.create_count_text(self._num_clips, 'clip')
if self._defer_clip_creation:
self._write_deferred_clips_file()
self._logger.info((
' Processed {} from detector "{}".').format(
clips_text, self._detector_model.name))
elif self._num_database_failures == 0 and self._num_file_failures == 0:
self._logger.info((
' Created {} from detector "{}".').format(
clips_text, self._detector_model.name))
else:
db_failures_text = text_utils.create_count_text(
self._num_database_failures, 'clip creation failure')
if self._create_clip_files:
num_file_failures = \
self._num_database_failures + self._num_file_failures
file_failures_text = ' and ' + text_utils.create_count_text(
num_file_failures, 'audio file creation failure')
else:
file_failures_text = ''
self._logger.info(
' Processed {} from detector "{}" with {}{}.'.format(
clips_text, self._detector_model.name,
db_failures_text, file_failures_text))
def complete_processing(self, threshold=None):
# Create remaining clips.
self._create_clips(threshold)
clips_text = text_utils.create_count_text(self._num_clips, 'clip')
if self._defer_clip_creation:
self._write_deferred_clips_file()
self._logger.info(
(' Processed {} from detector "{}".').format(
clips_text, self._detector_model.name))
elif self._num_database_failures == 0 and self._num_file_failures == 0:
self._logger.info(
(' Created {} from detector "{}".').format(
clips_text, self._detector_model.name))
else:
db_failures_text = text_utils.create_count_text(
self._num_database_failures, 'clip creation failure')
if self._create_clip_files:
num_file_failures = \
self._num_database_failures + self._num_file_failures
file_failures_text = ' and ' + text_utils.create_count_text(
num_file_failures, 'audio file creation failure')
else:
file_failures_text = ''
self._logger.info(
' Processed {} from detector "{}" with {}{}.'.format(
clips_text, self._detector_model.name, db_failures_text,
file_failures_text))
def _create_clip_audio_files(self):
start_time = time.time()
value_tuples = self._create_clip_query_values_iterator()
total_num_clips = 0
total_num_created_files = 0
for detector, station, mic_output, date in value_tuples:
clips = model_utils.get_clips(
station, mic_output, detector, date, self._annotation_name,
self._annotation_value, order=False)
num_clips = len(clips)
num_created_files = 0
for clip in clips:
if self._create_clip_audio_file_if_needed(clip):
num_created_files += 1
# Log file creations for this detector/station/mic_output/date.
count_text = text_utils.create_count_text(num_clips, 'clip')
_logger.info((
'Created audio files for {} of {} for detector "{}", '
'station "{}", mic output "{}", and date {}.').format(
num_created_files, count_text, detector.name,
station.name, mic_output.name, date))
total_num_clips += num_clips
total_num_created_files += num_created_files
# Log total file creations and creation rate.
count_text = text_utils.create_count_text(total_num_clips, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, total_num_clips, 'clips')
_logger.info(
'Processed a total of {}{}.'.format(count_text, timing_text))
def _refresh_recording_file_paths(self, recording_file_paths):
start_time = time.time()
file_count = RecordingFile.objects.count()
count_text = text_utils.create_count_text(file_count, 'recording file')
self._logger.info('Command will visit {}.'.format(count_text))
updated_count = 0
for i, file_ in enumerate(RecordingFile.objects.all()):
visited_count = i + 1
if visited_count % _LOGGING_PERIOD == 0:
self._logger.info(
' Visited {} files...'.format(visited_count))
old_path = file_.path
file_name = _get_file_name(old_path)
new_path = recording_file_paths.get(file_name)
if new_path is None:
self._logger.warning(
(' Could not find recording file "{}" in recording '
'directories.').format(file_name))
elif new_path != old_path:
# self._logger.info(
# ' Update "{}" to "{}"...'.format(
# old_path, new_path))
file_.path = new_path
file_.save()
updated_count += 1
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(elapsed_time,
visited_count, 'files')
self._logger.info(
('Updated paths for {} of {} visited recording files{}.').format(
updated_count, visited_count, timing_text))
def _log_detection_performance(self, num_detectors, num_channels,
interval_duration, processing_time):
format_ = text_utils.format_number
dur = format_(interval_duration)
time = format_(processing_time)
detectors_text = text_utils.create_count_text(num_detectors,
'detector')
message = (' Ran {} on {} seconds of {}-channel '
'audio in {} seconds').format(detectors_text, dur,
num_channels, time)
if processing_time != 0:
total_duration = num_detectors * num_channels * interval_duration
speedup = format_(total_duration / processing_time)
message += ', {} times faster than real time.'.format(speedup)
else:
message += '.'
self._logger.info(message)
def _delete_clips(self, retain_indices):
start_time = time.time()
retaining_clips = len(retain_indices) == 0
value_tuples = self._create_clip_query_values_iterator()
index = 0
total_retained_count = 0
for station, mic_output, date, detector in value_tuples:
# Get clips for this station, mic_output, date, and detector
clips = model_utils.get_clips(
station=station,
mic_output=mic_output,
date=date,
detector=detector,
annotation_name=self._annotation_name,
annotation_value=self._annotation_value,
tag_name=self._tag_name,
order=False)
# Figure out which clips should be deleted.
count = 0
retained_count = 0
clips_to_delete = []
for clip in clips:
if index not in retain_indices:
clips_to_delete.append(clip)
else:
retained_count += 1
count += 1
index += 1
# Delete clips.
try:
self._delete_clip_batch(clips_to_delete)
except Exception as e:
batch_text = \
_get_batch_text(station, mic_output, date, detector)
command_utils.log_and_reraise_fatal_exception(
e, f'Deletion of clips for {batch_text}')
# Log deletions.
if retaining_clips:
prefix = 'Deleted'
else:
deleted_count = count - retained_count
prefix = (
f'Deleted {deleted_count} and retained '
f'{retained_count} of')
count_text = text_utils.create_count_text(count, 'clip')
batch_text = _get_batch_text(station, mic_output, date, detector)
_logger.info(f'{prefix} {count_text} for {batch_text}.')
total_retained_count += retained_count
# Log total deletions and deletion rate.
if total_retained_count == 0:
prefix = 'Deleted'
else:
deleted_count = index - total_retained_count
prefix = (
f'Deleted {deleted_count} and retained '
f'{total_retained_count} of')
count_text = text_utils.create_count_text(index, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, index, 'clips')
_logger.info(f'{prefix} a total of {count_text}{timing_text}.')
def _delete_clips(self, retain_indices):
start_time = time.time()
retaining_clips = len(retain_indices) == 0
value_tuples = self._create_clip_query_values_iterator()
index = 0
total_retained_count = 0
for detector, station, mic_output, date in value_tuples:
# Get clips for this detector, station, mic_output, and date
clips = model_utils.get_clips(
station, mic_output, detector, date, self._annotation_name,
self._annotation_value, order=False)
# Figure out which clips should be deleted.
count = 0
retained_count = 0
clips_to_delete = []
for clip in clips:
if index not in retain_indices:
clips_to_delete.append(clip)
else:
retained_count += 1
count += 1
index += 1
# Delete clips.
try:
self._delete_clip_batch(clips_to_delete)
except Exception as e:
batch_text = \
_get_batch_text(detector, station, mic_output, date)
command_utils.log_and_reraise_fatal_exception(
e, 'Deletion of clips for {}'.format(batch_text))
# Log deletions.
if retaining_clips:
prefix = 'Deleted'
else:
deleted_count = count - retained_count
prefix = 'Deleted {} and retained {} of'.format(
deleted_count, retained_count)
count_text = text_utils.create_count_text(count, 'clip')
batch_text = _get_batch_text(detector, station, mic_output, date)
_logger.info(
'{} {} for {}.'.format(prefix, count_text, batch_text))
total_retained_count += retained_count
# Log total deletions and deletion rate.
if total_retained_count == 0:
prefix = 'Deleted'
else:
deleted_count = index - total_retained_count
prefix = 'Deleted {} and retained {} of'.format(
deleted_count, total_retained_count)
count_text = text_utils.create_count_text(index, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, index, 'clips')
_logger.info('{} a total of {}{}.'.format(
prefix, count_text, timing_text))