def _is_consecutive_file(self, f):
duration = signal_utils.get_duration(self.length, self.sample_rate)
end_time = self.start_time + datetime.timedelta(seconds=duration)
delta = abs((f.start_time - end_time).total_seconds())
threshold = max(1, duration * self.tolerance / 3600)
return f.station == self.station and \
f.recorder == self.recorder and \
f.recorder_channel_nums == self.recorder_channel_nums and \
f.mic_outputs == self.mic_outputs and \
f.num_channels == self.num_channels and \
f.sample_rate == self.sample_rate and \
delta <= threshold
def _is_consecutive_file(self, f):
duration = signal_utils.get_duration(self.length, self.sample_rate)
end_time = self.start_time + datetime.timedelta(seconds=duration)
delta = abs((f.start_time - end_time).total_seconds())
threshold = max(1, duration * self.tolerance / 3600)
return f.station == self.station and \
f.recorder == self.recorder and \
f.recorder_channel_nums == self.recorder_channel_nums and \
f.mic_outputs == self.mic_outputs and \
f.num_channels == self.num_channels and \
f.sample_rate == self.sample_rate and \
delta <= threshold
def _notify_listener_of_clips(self, peak_indices, peak_scores,
input_length, threshold):
# print('Clips:')
start_offset = self._input_chunk_start_index + self._clip_start_offset
peak_indices *= self._hop_size
for i, score in zip(peak_indices, peak_scores):
# Convert classification index to input index, accounting for
# any difference between classification sample rate and input
# rate.
f = self._input_sample_rate / self._purported_input_sample_rate
classification_sample_rate = f * self._classifier_sample_rate
t = signal_utils.get_duration(i, classification_sample_rate)
i = signal_utils.seconds_to_frames(t, self._input_sample_rate)
clip_start_index = i + start_offset
clip_end_index = clip_start_index + self._clip_length
chunk_end_index = self._input_chunk_start_index + input_length
if clip_start_index < 0:
logging.warning(
'Rejected clip that started before beginning of '
'recording.')
elif clip_end_index > chunk_end_index:
# clip might extend past end of recording, since it extends
# past the end of this chunk (we do not know whether or
# not the current chunk is the last)
logging.warning(
'Rejected clip that ended after end of recording chunk.')
else:
# all clip samples are in the recording interval extending
# from the beginning of the recording to the end of the
# current chunk
# print(
# ' {} {}'.format(clip_start_index, self._clip_length))
annotations = {'Detector Score': 100 * score}
self._listener.process_clip(clip_start_index,
self._clip_length, threshold,
annotations)
def _notify_listener_of_clips(
self, peak_indices, peak_scores, input_length, threshold):
# print('Clips:')
start_offset = self._input_chunk_start_index + self._clip_start_offset
peak_indices *= self._hop_size
for i, score in zip(peak_indices, peak_scores):
# Convert classification index to input index, accounting
# for difference between classifier sample rate and input
# sample rate.
t = signal_utils.get_duration(i, self._classifier_sample_rate)
i = signal_utils.seconds_to_frames(t, self._input_sample_rate)
clip_start_index = i + start_offset
clip_end_index = clip_start_index + self._clip_length
chunk_end_index = self._input_chunk_start_index + input_length
if clip_start_index < 0:
logging.warning(
'Rejected clip that started before beginning of '
'recording.')
elif clip_end_index > chunk_end_index:
# clip might extend past end of recording, since it extends
# past the end of this chunk (we do not know whether or
# not the current chunk is the last)
logging.warning(
'Rejected clip that ended after end of recording chunk.')
else:
# all clip samples are in the recording interval extending
# from the beginning of the recording to the end of the
# current chunk
# print(
# ' {} {}'.format(clip_start_index, self._clip_length))
annotations = {'Detector Score': 100 * score}
self._listener.process_clip(
clip_start_index, self._clip_length, threshold,
annotations)
def _create_clip(self, clip_info):
(recording_channel_id, start_index, length, creation_time,
creating_job_id, creating_processor_id, annotations) = clip_info
channel, station, mic_output, sample_rate, start_time = \
self._get_recording_channel_info(recording_channel_id)
start_offset = signal_utils.get_duration(start_index, sample_rate)
start_time += datetime.timedelta(seconds=start_offset)
end_time = signal_utils.get_end_time(start_time, length, sample_rate)
job = self._get_job(creating_job_id)
processor = self._get_processor(creating_processor_id)
clip = Clip.objects.create(
station=station,
mic_output=mic_output,
recording_channel=channel,
start_index=start_index,
length=length,
sample_rate=sample_rate,
start_time=start_time,
end_time=end_time,
date=station.get_night(start_time),
creation_time=creation_time,
creating_user=None,
creating_job=job,
creating_processor=processor
)
if annotations is not None:
for name, value in annotations.items():
annotation_info = self._get_annotation_info(name)
model_utils.annotate_clip(
clip, annotation_info, str(value),
creation_time=creation_time, creating_user=None,
creating_job=self._job, creating_processor=processor)
def _create_clip(self, clip_info):
(recording_channel_id, start_index, length, creation_time,
creating_job_id, creating_processor_id, annotations) = clip_info
channel, station, mic_output, sample_rate, start_time = \
self._get_recording_channel_info(recording_channel_id)
start_offset = signal_utils.get_duration(start_index, sample_rate)
start_time += datetime.timedelta(seconds=start_offset)
end_time = signal_utils.get_end_time(start_time, length, sample_rate)
job = self._get_job(creating_job_id)
processor = self._get_processor(creating_processor_id)
clip = Clip.objects.create(
station=station,
mic_output=mic_output,
recording_channel=channel,
start_index=start_index,
length=length,
sample_rate=sample_rate,
start_time=start_time,
end_time=end_time,
date=station.get_night(start_time),
creation_time=creation_time,
creating_user=None,
creating_job=job,
creating_processor=processor
)
if annotations is not None:
for name, value in annotations.items():
annotation_info = self._get_annotation_info(name)
model_utils.annotate_clip(
clip, annotation_info, str(value),
creation_time=creation_time, creating_user=None,
creating_job=self._job, creating_processor=processor)
def _add_channel_clip_start_indices(self, channel, detector):
recording = channel.recording
recording_start_time = recording.start_time
recording_length = recording.length
sample_rate = recording.sample_rate
create_count_text = text_utils.create_count_text
with archive_lock.atomic():
with transaction.atomic():
clips = Clip.objects.filter(recording_channel=channel,
creating_processor=detector,
start_index=None)
num_clips = clips.count()
num_clips_found = 0
if num_clips != 0:
count_text = create_count_text(num_clips, 'clip')
self._logger.info(
f'Processing {count_text} for recording channel '
f'"{str(channel)}" and detector "{detector.name}"...')
start_time = recording_start_time
duration = datetime.timedelta(seconds=recording_length /
sample_rate)
end_time = start_time + duration
# self._logger.info(
# f' Recording has start time {str(start_time)} '
# f'and end time {end_time}.')
for clip in clips:
result = self._find_clip_in_recording(clip, channel)
if not isinstance(result, str):
# found clip
# Get result parts. Note that the clip channel
# can change when the clip is found, since in
# some cases clips were attributed to the wrong
# recordings when the clips were imported. In
# one scenario, for example, a clip that was
# actually toward the beginning of the second
# of two contiguous recordings of a night was
# incorrectly assigned to the end of the first
# recording, since according to the purported
# start times and sample rates of the recordings
# the end of the first recording overlapped
# the start of the second recording in time.
samples, found_channel, start_index = result
# Get clip start time.
start_seconds = start_index / sample_rate
delta = datetime.timedelta(seconds=start_seconds)
if found_channel == channel:
start_time = recording_start_time + delta
else:
start_time = \
found_channel.recording.start_time + delta
# Get change in clip start time.
start_time_change = \
(start_time - clip.start_time).total_seconds()
if start_time_change < self._min_start_time_change:
self._min_start_time_change = start_time_change
if start_time_change > self._max_start_time_change:
self._max_start_time_change = start_time_change
# Get clip length. The Old Bird detectors
# sometimes append zeros to a clip that were
# not in the recording that the clip refers
# to. We ignore the appended zeros.
length = len(samples)
duration = signal_utils.get_duration(
length, sample_rate)
# Get clip end time.
end_time = signal_utils.get_end_time(
start_time, length, sample_rate)
clip.channel = found_channel
clip.start_index = start_index
clip.length = length
clip.start_time = start_time
clip.end_time = end_time
if not self._dry_run:
clip.save()
num_clips_found += 1
if num_clips_found != num_clips:
self._log_clips_not_found(num_clips - num_clips_found)
return num_clips, num_clips_found
def duration(self):
return signal_utils.get_duration(self.length, self.sample_rate)
def start_time(self):
offset = signal_utils.get_duration(self.start_index, self.sample_rate)
return self.recording.start_time + datetime.timedelta(seconds=offset)
def _create_clips(self, threshold):
if not _CREATE_CLIPS:
return
# TODO: Find out exactly what database queries are
# executed during detection (ideally, record the sequence
# of queries) to see if database interaction could be
# made more efficient, for example with a cache.
recording_channel = self._recording_channel
detector_model = self._detector_model
start_offset = self._file_start_index + self._interval_start_index
creation_time = time_utils.get_utc_now()
create_clip_files = self._create_clip_files
if self._defer_clip_creation:
for start_index, length, annotations in self._clips:
start_index += start_offset
clip = [
recording_channel.id, start_index, length, creation_time,
self._job.id, detector_model.id, annotations
]
self._deferred_clips.append(clip)
else:
# database writes not deferred
station = self._recording.station
sample_rate = self._recording.sample_rate
mic_output = recording_channel.mic_output
if create_clip_files:
clips = []
# Create database records for current batch of clips in one
# database transaction.
# trans_start_time = time.time()
try:
with archive_lock.atomic(), transaction.atomic():
for start_index, length, annotations in self._clips:
# Get clip start time as a `datetime`.
start_index += start_offset
start_delta = datetime.timedelta(seconds=start_index /
sample_rate)
start_time = \
self._recording.start_time + start_delta
end_time = signal_utils.get_end_time(
start_time, length, sample_rate)
try:
# It would be nice to use Django's
# `bulk_create` here, but unfortunately that
# won't automatically set clip IDs for us
# except (as of this writing) if we're using
# PostgreSQL.
clip = Clip.objects.create(
station=station,
mic_output=mic_output,
recording_channel=recording_channel,
start_index=start_index,
length=length,
sample_rate=sample_rate,
start_time=start_time,
end_time=end_time,
date=station.get_night(start_time),
creation_time=creation_time,
creating_user=None,
creating_job=self._job,
creating_processor=detector_model)
if create_clip_files:
# Save clip so we can create clip file
# outside of transaction.
clips.append(clip)
if annotations is not None:
for name, value in annotations.items():
annotation_info = \
self._get_annotation_info(name)
model_utils.annotate_clip(
clip,
annotation_info,
str(value),
creation_time=creation_time,
creating_user=None,
creating_job=self._job,
creating_processor=detector_model)
except Exception as e:
# Note that it's important not to perform any
# database queries here. If the database raised
# the exception, we have to wait until we're
# outside of the transaction to query the
# database again.
raise _ClipCreationError(e)
# trans_end_time = time.time()
# self._num_transactions += 1
# self._total_transactions_duration += \
# trans_end_time - trans_start_time
except _ClipCreationError as e:
duration = signal_utils.get_duration(length, sample_rate)
clip_string = Clip.get_string(station.name, mic_output.name,
detector_model.name, start_time,
duration)
batch_size = len(self._clips)
self._num_database_failures += batch_size
if batch_size == 1:
prefix = 'Clip'
else:
prefix = f'All {batch_size} clips in this batch'
self._logger.error(
f' Attempt to create clip {clip_string} '
f'failed with message: {str(e.wrapped_exception)}. '
f'{prefix} will be ignored.')
else:
# clip creation succeeded
if create_clip_files:
for clip in clips:
try:
self._clip_manager.create_audio_file(clip)
except Exception as e:
self._num_file_failures += 1
self._logger.error(
(' Attempt to create audio file '
'for clip {} failed with message: {} Clip '
'database record was still created.').format(
str(clip), str(e)))
self._clips = []
def duration(self):
return signal_utils.get_duration(self.length, self.sample_rate)
def start_time(self):
offset = signal_utils.get_duration(self.start_index, self.sample_rate)
return self.recording.start_time + datetime.timedelta(seconds=offset)
def _create_clips(self, threshold):
if not _CREATE_CLIPS:
return
# TODO: Find out exactly what database queries are
# executed during detection (ideally, record the sequence
# of queries) to see if database interaction could be
# made more efficient, for example with a cache.
recording_channel = self._recording_channel
detector_model = self._detector_model
start_offset = self._file_start_index + self._interval_start_index
creation_time = time_utils.get_utc_now()
create_clip_files = self._create_clip_files
if self._defer_clip_creation:
for start_index, length, annotations in self._clips:
start_index += start_offset
clip = [
recording_channel.id, start_index, length, creation_time,
self._job.id, detector_model.id, annotations]
self._deferred_clips.append(clip)
else:
# database writes not deferred
station = self._recording.station
sample_rate = self._recording.sample_rate
mic_output = recording_channel.mic_output
if create_clip_files:
clips = []
# Create database records for current batch of clips in one
# database transaction.
# trans_start_time = time.time()
try:
with archive_lock.atomic(), transaction.atomic():
for start_index, length, annotations in self._clips:
try:
# Get clip start time as a `datetime`.
start_index += start_offset
start_delta = datetime.timedelta(
seconds=start_index / sample_rate)
start_time = \
self._recording.start_time + start_delta
end_time = signal_utils.get_end_time(
start_time, length, sample_rate)
# It would be nice to use Django's
# `bulk_create` here, but unfortunately that
# won't automatically set clip IDs for us
# except (as of this writing) if we're using
# PostgreSQL.
clip = Clip.objects.create(
station=station,
mic_output=mic_output,
recording_channel=recording_channel,
start_index=start_index,
length=length,
sample_rate=sample_rate,
start_time=start_time,
end_time=end_time,
date=station.get_night(start_time),
creation_time=creation_time,
creating_user=None,
creating_job=self._job,
creating_processor=detector_model
)
if create_clip_files:
# Save clip so we can create clip file
# outside of transaction.
clips.append(clip)
if annotations is not None:
for name, value in annotations.items():
annotation_info = \
self._get_annotation_info(name)
model_utils.annotate_clip(
clip, annotation_info, str(value),
creation_time=creation_time,
creating_user=None,
creating_job=self._job,
creating_processor=detector_model)
except Exception as e:
duration = signal_utils.get_duration(
length, sample_rate)
clip_string = Clip.get_string(
station.name, mic_output.name,
detector_model.name, start_time, duration)
raise _ClipCreationError(clip_string, e)
# trans_end_time = time.time()
# self._num_transactions += 1
# self._total_transactions_duration += \
# trans_end_time - trans_start_time
except _ClipCreationError as e:
batch_size = len(self._clips)
self._num_database_failures += batch_size
if batch_size == 1:
prefix = 'Clip'
else:
prefix = 'All {} clips in this batch'.format(
batch_size)
self._logger.error((
' Attempt to create clip {} failed with '
'message: {} {} will be ignored.').format(
clip_string, str(e.wrapped_exception), prefix))
else:
# clip creation succeeded
if create_clip_files:
for clip in clips:
try:
self._clip_manager.create_audio_file(clip)
except Exception as e:
self._num_file_failures += 1
self._logger.error((
' Attempt to create audio file '
'for clip {} failed with message: {} Clip '
'database record was still created.').format(
str(clip), str(e)))
self._clips = []