def _annotate(self, clip, annotation_value):
model_utils.annotate_clip(
clip, self._annotation_info, annotation_value,
creating_user=self._creating_user,
creating_job=self._creating_job,
creating_processor=self._creating_processor)
def _set_clip_score(self, clip, score):
value = '{:.3f}'.format(score)
model_utils.annotate_clip(
clip, self._score_annotation_info, value,
creating_user=self._creating_user,
creating_job=self._creating_job,
creating_processor=self._creating_processor)
def _transfer_classification(self, source_clip, target_clip):
# Get source clip classification.
annotations = model_utils.get_clip_annotations(source_clip)
classification = annotations.get(self._annotation_name)
# Classify target clip.
model_utils.annotate_clip(
target_clip, self._annotation_info, classification,
creating_job=self._job)
def _transfer_classification(self, source_clip, target_clip):
# Get source clip classification.
annotations = model_utils.get_clip_annotations(source_clip)
classification = annotations.get(self._annotation_name)
# Classify target clip.
model_utils.annotate_clip(
target_clip, self._annotation_info, classification,
creating_job=self._job)
def _set_clip_score(self, clip, score):
value = '{:.3f}'.format(score)
model_utils.annotate_clip(clip,
self._score_annotation_info,
value,
creating_user=self._creating_user,
creating_job=self._creating_job,
creating_processor=self._creating_processor)
def _import_clip(self, file_path, info):
length, sample_rate = _get_audio_file_info(file_path)
start_time = info.start_time
end_time = signal_utils.get_end_time(start_time, length, sample_rate)
mic_output = self._get_mic_output(info.station.name)
recording_channel = self._get_recording_channel(
info.station, info.date, sample_rate)
recording = recording_channel.recording
_assert_recording_contains_clip(recording, start_time, end_time)
creation_time = time_utils.get_utc_now()
clip = Clip.objects.create(station=info.station,
mic_output=mic_output,
recording_channel=recording_channel,
start_index=None,
length=length,
sample_rate=sample_rate,
start_time=start_time,
end_time=end_time,
date=info.date,
creation_time=creation_time,
creating_job=self._job,
creating_processor=info.detector)
_copy_clip_audio_file(file_path, clip)
if info.classification is not None:
creation_time = time_utils.get_utc_now()
# We assume that any classification performed before the
# import was by the user who started the import.
creating_user = self._job.creating_user
model_utils.annotate_clip(clip,
self._annotation_info,
info.classification,
creation_time=creation_time,
creating_user=creating_user)
def _import_clip(self, file_path, info):
length, sample_rate = _get_audio_file_info(file_path)
start_time = info.start_time
end_time = signal_utils.get_end_time(start_time, length, sample_rate)
mic_output = self._get_mic_output(info.station.name)
recording_channel = self._get_recording_channel(
info.station, info.date, sample_rate)
recording = recording_channel.recording
_assert_recording_contains_clip(recording, start_time, end_time)
creation_time = time_utils.get_utc_now()
clip = Clip.objects.create(
station=info.station,
mic_output=mic_output,
recording_channel=recording_channel,
start_index=None,
length=length,
sample_rate=sample_rate,
start_time=start_time,
end_time=end_time,
date=info.date,
creation_time=creation_time,
creating_job=self._job,
creating_processor=info.detector)
_copy_clip_audio_file(file_path, clip)
if info.classification is not None:
creation_time = time_utils.get_utc_now()
# We assume that any classification performed before the
# import was by the user who started the import.
creating_user = self._job.creating_user
model_utils.annotate_clip(
clip, self._annotation_info, info.classification,
creation_time=creation_time, creating_user=creating_user)
def _create_clip(self, clip_info):
(recording_channel_id, start_index, length, creation_time,
creating_job_id, creating_processor_id, annotations) = clip_info
channel, station, mic_output, sample_rate, start_time = \
self._get_recording_channel_info(recording_channel_id)
start_offset = signal_utils.get_duration(start_index, sample_rate)
start_time += datetime.timedelta(seconds=start_offset)
end_time = signal_utils.get_end_time(start_time, length, sample_rate)
job = self._get_job(creating_job_id)
processor = self._get_processor(creating_processor_id)
clip = Clip.objects.create(
station=station,
mic_output=mic_output,
recording_channel=channel,
start_index=start_index,
length=length,
sample_rate=sample_rate,
start_time=start_time,
end_time=end_time,
date=station.get_night(start_time),
creation_time=creation_time,
creating_user=None,
creating_job=job,
creating_processor=processor
)
if annotations is not None:
for name, value in annotations.items():
annotation_info = self._get_annotation_info(name)
model_utils.annotate_clip(
clip, annotation_info, str(value),
creation_time=creation_time, creating_user=None,
creating_job=self._job, creating_processor=processor)
def _create_clip(self, clip_info):
(recording_channel_id, start_index, length, creation_time,
creating_job_id, creating_processor_id, annotations) = clip_info
channel, station, mic_output, sample_rate, start_time = \
self._get_recording_channel_info(recording_channel_id)
start_offset = signal_utils.get_duration(start_index, sample_rate)
start_time += datetime.timedelta(seconds=start_offset)
end_time = signal_utils.get_end_time(start_time, length, sample_rate)
job = self._get_job(creating_job_id)
processor = self._get_processor(creating_processor_id)
clip = Clip.objects.create(
station=station,
mic_output=mic_output,
recording_channel=channel,
start_index=start_index,
length=length,
sample_rate=sample_rate,
start_time=start_time,
end_time=end_time,
date=station.get_night(start_time),
creation_time=creation_time,
creating_user=None,
creating_job=job,
creating_processor=processor
)
if annotations is not None:
for name, value in annotations.items():
annotation_info = self._get_annotation_info(name)
model_utils.annotate_clip(
clip, annotation_info, str(value),
creation_time=creation_time, creating_user=None,
creating_job=self._job, creating_processor=processor)
def _annotate_clip(self, clip, annotation_name, index,
inference_sample_rate):
# If needed, modify index to account for difference between
# clip and inference sample rates.
if clip.sample_rate != inference_sample_rate:
factor = clip.sample_rate / inference_sample_rate
index = int(round(index * factor))
# Make index a recording index rather than a clip index.
index += clip.start_index
annotation_info = self._annotation_infos[annotation_name]
annotation_value = str(index)
model_utils.annotate_clip(clip,
annotation_info,
annotation_value,
creating_user=self._creating_user,
creating_job=self._creating_job,
creating_processor=self._creating_processor)
def _populate_database(num_clips, query=None, query_period=None):
station = Station.objects.create(
name='Test Station', time_zone='US/Eastern')
model = DeviceModel.objects.create(
name='Test Recorder Model', type='Recorder', manufacturer='Nagra',
model='X')
device = Device.objects.create(
name='Test Device', model=model, serial_number='0')
station_recorder = StationDevice.objects.create(
station=station, device=device, start_time=_dt(2017, 3, 1),
end_time=_dt(2017, 4, 1))
sample_rate = 22050.
clip_duration = 1.
clip_length = int(round(clip_duration * sample_rate))
num_channels = 1
recording_length = num_clips * clip_length
recording_start_time = _dt(2017, 3, 1)
recording_duration = recording_length / sample_rate
recording_end_time = \
recording_start_time + \
datetime.timedelta(seconds=recording_duration)
# Use same creation time for all objects that have one.
creation_time = _dt(2017, 4, 1)
recording = Recording.objects.create(
station_recorder=station_recorder, num_channels=num_channels,
length=recording_length, sample_rate=sample_rate,
start_time=recording_start_time, end_time=recording_end_time,
creation_time=creation_time)
classification = AnnotationInfo.objects.create(
name='Classification', type='String', creation_time=creation_time)
other = AnnotationInfo.objects.create(
name='Other', type='String', creation_time=creation_time)
another = AnnotationInfo.objects.create(
name='Another', type='String', creation_time=creation_time)
for i in range(num_clips):
if query_period is not None and i != 0 and i % query_period == 0:
query(i)
clip_start_index = i * clip_length
offset = clip_start_index / sample_rate
clip_start_time = \
recording_start_time + datetime.timedelta(seconds=offset)
clip_duration = clip_length / sample_rate
clip_end_time = \
clip_start_time + datetime.timedelta(seconds=clip_duration)
clip = Clip.objects.create(
recording=recording, channel_num=0,
start_index=clip_start_index, length=clip_length,
start_time=clip_start_time, end_time=clip_end_time,
creation_time=creation_time)
info = classification if i % 2 == 0 else other
model_utils.annotate_clip(clip, info, str(i), creation_time)
if i < num_clips / 2:
model_utils.annotate_clip(clip, another, str(i), creation_time)
def _populate_database_from_yaml(s):
d = yaml_utils.load(s)
num_stations = d.get('num_stations', 1)
stations = []
for i in range(num_stations):
name = 'Station {}'.format(i + 1)
station = Station.objects.create(name=name, time_zone='US/Eastern')
stations.append(station)
model = DeviceModel.objects.create(
name='Recorder Model', type='Recorder', manufacturer='Nagra',
model='X')
device = Device.objects.create(
name='Recorder', model=model, serial_number='0')
clips = d['clips']
num_clips = len(clips)
clip_length = _CLIP_DURATION * _SAMPLE_RATE
recording_length = num_clips * clip_length
recording_duration = recording_length / _SAMPLE_RATE
recording_end_time = \
_RECORDING_START_TIME + datetime.timedelta(seconds=recording_duration)
station_recorder = StationDevice.objects.create(
station=station, device=device, start_time=_RECORDING_START_TIME,
end_time=recording_end_time)
creation_time = _RECORDING_START_TIME
recording = Recording.objects.create(
station_recorder=station_recorder, num_channels=_NUM_CHANNELS,
length=recording_length, sample_rate=_SAMPLE_RATE,
start_time=_RECORDING_START_TIME, end_time=recording_end_time,
creation_time=creation_time)
annotation_names = _get_annotation_names(clips)
annotation_infos = dict(
(name, _create_annotation_info(name, creation_time))
for name in annotation_names)
for i, clip_d in enumerate(clips):
clip_start_index = i * clip_length
offset = clip_start_index / _SAMPLE_RATE
clip_start_time = \
_RECORDING_START_TIME + datetime.timedelta(seconds=offset)
clip_duration = clip_length / _SAMPLE_RATE
clip_end_time = \
clip_start_time + datetime.timedelta(seconds=clip_duration)
clip = Clip.objects.create(
recording=recording, channel_num=0,
start_index=clip_start_index, length=clip_length,
start_time=clip_start_time, end_time=clip_end_time,
creation_time=creation_time)
for name, value in clip_d['annotations'].items():
info = annotation_infos[name]
model_utils.annotate_clip(clip, info, value, creation_time)
def _create_clips(self, threshold):
if not _CREATE_CLIPS:
return
# TODO: Find out exactly what database queries are
# executed during detection (ideally, record the sequence
# of queries) to see if database interaction could be
# made more efficient, for example with a cache.
recording_channel = self._recording_channel
detector_model = self._detector_model
start_offset = self._file_start_index + self._interval_start_index
creation_time = time_utils.get_utc_now()
create_clip_files = self._create_clip_files
if self._defer_clip_creation:
for start_index, length, annotations in self._clips:
start_index += start_offset
clip = [
recording_channel.id, start_index, length, creation_time,
self._job.id, detector_model.id, annotations
]
self._deferred_clips.append(clip)
else:
# database writes not deferred
station = self._recording.station
sample_rate = self._recording.sample_rate
mic_output = recording_channel.mic_output
if create_clip_files:
clips = []
# Create database records for current batch of clips in one
# database transaction.
# trans_start_time = time.time()
try:
with archive_lock.atomic(), transaction.atomic():
for start_index, length, annotations in self._clips:
# Get clip start time as a `datetime`.
start_index += start_offset
start_delta = datetime.timedelta(seconds=start_index /
sample_rate)
start_time = \
self._recording.start_time + start_delta
end_time = signal_utils.get_end_time(
start_time, length, sample_rate)
try:
# It would be nice to use Django's
# `bulk_create` here, but unfortunately that
# won't automatically set clip IDs for us
# except (as of this writing) if we're using
# PostgreSQL.
clip = Clip.objects.create(
station=station,
mic_output=mic_output,
recording_channel=recording_channel,
start_index=start_index,
length=length,
sample_rate=sample_rate,
start_time=start_time,
end_time=end_time,
date=station.get_night(start_time),
creation_time=creation_time,
creating_user=None,
creating_job=self._job,
creating_processor=detector_model)
if create_clip_files:
# Save clip so we can create clip file
# outside of transaction.
clips.append(clip)
if annotations is not None:
for name, value in annotations.items():
annotation_info = \
self._get_annotation_info(name)
model_utils.annotate_clip(
clip,
annotation_info,
str(value),
creation_time=creation_time,
creating_user=None,
creating_job=self._job,
creating_processor=detector_model)
except Exception as e:
# Note that it's important not to perform any
# database queries here. If the database raised
# the exception, we have to wait until we're
# outside of the transaction to query the
# database again.
raise _ClipCreationError(e)
# trans_end_time = time.time()
# self._num_transactions += 1
# self._total_transactions_duration += \
# trans_end_time - trans_start_time
except _ClipCreationError as e:
duration = signal_utils.get_duration(length, sample_rate)
clip_string = Clip.get_string(station.name, mic_output.name,
detector_model.name, start_time,
duration)
batch_size = len(self._clips)
self._num_database_failures += batch_size
if batch_size == 1:
prefix = 'Clip'
else:
prefix = f'All {batch_size} clips in this batch'
self._logger.error(
f' Attempt to create clip {clip_string} '
f'failed with message: {str(e.wrapped_exception)}. '
f'{prefix} will be ignored.')
else:
# clip creation succeeded
if create_clip_files:
for clip in clips:
try:
self._clip_manager.create_audio_file(clip)
except Exception as e:
self._num_file_failures += 1
self._logger.error(
(' Attempt to create audio file '
'for clip {} failed with message: {} Clip '
'database record was still created.').format(
str(clip), str(e)))
self._clips = []
def _create_clips(self, threshold):
if not _CREATE_CLIPS:
return
# TODO: Find out exactly what database queries are
# executed during detection (ideally, record the sequence
# of queries) to see if database interaction could be
# made more efficient, for example with a cache.
recording_channel = self._recording_channel
detector_model = self._detector_model
start_offset = self._file_start_index + self._interval_start_index
creation_time = time_utils.get_utc_now()
create_clip_files = self._create_clip_files
if self._defer_clip_creation:
for start_index, length, annotations in self._clips:
start_index += start_offset
clip = [
recording_channel.id, start_index, length, creation_time,
self._job.id, detector_model.id, annotations]
self._deferred_clips.append(clip)
else:
# database writes not deferred
station = self._recording.station
sample_rate = self._recording.sample_rate
mic_output = recording_channel.mic_output
if create_clip_files:
clips = []
# Create database records for current batch of clips in one
# database transaction.
# trans_start_time = time.time()
try:
with archive_lock.atomic(), transaction.atomic():
for start_index, length, annotations in self._clips:
try:
# Get clip start time as a `datetime`.
start_index += start_offset
start_delta = datetime.timedelta(
seconds=start_index / sample_rate)
start_time = \
self._recording.start_time + start_delta
end_time = signal_utils.get_end_time(
start_time, length, sample_rate)
# It would be nice to use Django's
# `bulk_create` here, but unfortunately that
# won't automatically set clip IDs for us
# except (as of this writing) if we're using
# PostgreSQL.
clip = Clip.objects.create(
station=station,
mic_output=mic_output,
recording_channel=recording_channel,
start_index=start_index,
length=length,
sample_rate=sample_rate,
start_time=start_time,
end_time=end_time,
date=station.get_night(start_time),
creation_time=creation_time,
creating_user=None,
creating_job=self._job,
creating_processor=detector_model
)
if create_clip_files:
# Save clip so we can create clip file
# outside of transaction.
clips.append(clip)
if annotations is not None:
for name, value in annotations.items():
annotation_info = \
self._get_annotation_info(name)
model_utils.annotate_clip(
clip, annotation_info, str(value),
creation_time=creation_time,
creating_user=None,
creating_job=self._job,
creating_processor=detector_model)
except Exception as e:
duration = signal_utils.get_duration(
length, sample_rate)
clip_string = Clip.get_string(
station.name, mic_output.name,
detector_model.name, start_time, duration)
raise _ClipCreationError(clip_string, e)
# trans_end_time = time.time()
# self._num_transactions += 1
# self._total_transactions_duration += \
# trans_end_time - trans_start_time
except _ClipCreationError as e:
batch_size = len(self._clips)
self._num_database_failures += batch_size
if batch_size == 1:
prefix = 'Clip'
else:
prefix = 'All {} clips in this batch'.format(
batch_size)
self._logger.error((
' Attempt to create clip {} failed with '
'message: {} {} will be ignored.').format(
clip_string, str(e.wrapped_exception), prefix))
else:
# clip creation succeeded
if create_clip_files:
for clip in clips:
try:
self._clip_manager.create_audio_file(clip)
except Exception as e:
self._num_file_failures += 1
self._logger.error((
' Attempt to create audio file '
'for clip {} failed with message: {} Clip '
'database record was still created.').format(
str(clip), str(e)))
self._clips = []
def annotate_detected_calls():
center_index_annotation_info = \
AnnotationInfo.objects.get(name=CENTER_INDEX_ANNOTATION_NAME)
center_freq_annotation_info = \
AnnotationInfo.objects.get(name=CENTER_FREQ_ANNOTATION_NAME)
classification_annotation_info = \
AnnotationInfo.objects.get(name=CLASSIFICATION_ANNOTATION_NAME)
user = User.objects.get(username='******')
sm_pairs = model_utils.get_station_mic_output_pairs_list()
ground_truth_detector = Processor.objects.get(
name=GROUND_TRUTH_DETECTOR_NAME)
rows = []
for detector_name, annotation_value in DETECTOR_DATA:
short_detector_name = detector_name.split()[2]
old_bird_detector = Processor.objects.get(name=detector_name)
window = utils.CALL_CENTER_WINDOWS[short_detector_name]
for station, mic_output in sm_pairs:
station_num = int(station.name.split()[1])
print('{} {}...'.format(short_detector_name, station_num))
ground_truth_clips = list(
model_utils.get_clips(station, mic_output,
ground_truth_detector, None,
CLASSIFICATION_ANNOTATION_NAME,
annotation_value))
ground_truth_call_center_indices = \
[c.start_index + c.length // 2 for c in ground_truth_clips]
ground_truth_call_count = len(ground_truth_clips)
old_bird_clips = list(
model_utils.get_clips(station, mic_output, old_bird_detector))
old_bird_clip_count = len(old_bird_clips)
clips = [(c.start_index, c.length) for c in old_bird_clips]
matches = utils.match_clips_with_calls(
clips, ground_truth_call_center_indices, window)
old_bird_call_count = len(matches)
rows.append([
short_detector_name, station_num, ground_truth_call_count,
old_bird_call_count, old_bird_clip_count
])
if ANNOTATE:
# Clear any existing annotations.
for clip in old_bird_clips:
model_utils.delete_clip_annotation(
clip,
classification_annotation_info,
creating_user=user)
# Create new annotations.
for i, j in matches:
old_bird_clip = old_bird_clips[i]
call_center_index = ground_truth_call_center_indices[j]
ground_truth_clip = ground_truth_clips[j]
# Annotate Old Bird clip call center index.
model_utils.annotate_clip(old_bird_clip,
center_index_annotation_info,
str(call_center_index),
creating_user=user)
# Get ground truth clip call center frequency.
annotations = \
model_utils.get_clip_annotations(ground_truth_clip)
call_center_freq = annotations[CENTER_FREQ_ANNOTATION_NAME]
# Annotate Old Bird clip call center frequency.
model_utils.annotate_clip(old_bird_clip,
center_freq_annotation_info,
call_center_freq,
creating_user=user)
model_utils.annotate_clip(old_bird_clip,
classification_annotation_info,
annotation_value,
creating_user=user)
return rows
def _populate_database(num_clips, query=None, query_period=None):
station = Station.objects.create(name='Test Station',
time_zone='US/Eastern')
model = DeviceModel.objects.create(name='Test Recorder Model',
type='Recorder',
manufacturer='Nagra',
model='X')
device = Device.objects.create(name='Test Device',
model=model,
serial_number='0')
station_recorder = StationDevice.objects.create(station=station,
device=device,
start_time=_dt(2017, 3, 1),
end_time=_dt(2017, 4, 1))
sample_rate = 22050.
clip_duration = 1.
clip_length = int(round(clip_duration * sample_rate))
num_channels = 1
recording_length = num_clips * clip_length
recording_start_time = _dt(2017, 3, 1)
recording_duration = recording_length / sample_rate
recording_end_time = \
recording_start_time + \
datetime.timedelta(seconds=recording_duration)
# Use same creation time for all objects that have one.
creation_time = _dt(2017, 4, 1)
recording = Recording.objects.create(station_recorder=station_recorder,
num_channels=num_channels,
length=recording_length,
sample_rate=sample_rate,
start_time=recording_start_time,
end_time=recording_end_time,
creation_time=creation_time)
classification = AnnotationInfo.objects.create(name='Classification',
type='String',
creation_time=creation_time)
other = AnnotationInfo.objects.create(name='Other',
type='String',
creation_time=creation_time)
another = AnnotationInfo.objects.create(name='Another',
type='String',
creation_time=creation_time)
for i in range(num_clips):
if query_period is not None and i != 0 and i % query_period == 0:
query(i)
clip_start_index = i * clip_length
offset = clip_start_index / sample_rate
clip_start_time = \
recording_start_time + datetime.timedelta(seconds=offset)
clip_duration = clip_length / sample_rate
clip_end_time = \
clip_start_time + datetime.timedelta(seconds=clip_duration)
clip = Clip.objects.create(recording=recording,
channel_num=0,
start_index=clip_start_index,
length=clip_length,
start_time=clip_start_time,
end_time=clip_end_time,
creation_time=creation_time)
info = classification if i % 2 == 0 else other
model_utils.annotate_clip(clip, info, str(i), creation_time)
if i < num_clips / 2:
model_utils.annotate_clip(clip, another, str(i), creation_time)
def _populate_database_from_yaml(s):
d = yaml_utils.load(s)
num_stations = d.get('num_stations', 1)
stations = []
for i in range(num_stations):
name = 'Station {}'.format(i + 1)
station = Station.objects.create(name=name, time_zone='US/Eastern')
stations.append(station)
model = DeviceModel.objects.create(name='Recorder Model',
type='Recorder',
manufacturer='Nagra',
model='X')
device = Device.objects.create(name='Recorder',
model=model,
serial_number='0')
clips = d['clips']
num_clips = len(clips)
clip_length = _CLIP_DURATION * _SAMPLE_RATE
recording_length = num_clips * clip_length
recording_duration = recording_length / _SAMPLE_RATE
recording_end_time = \
_RECORDING_START_TIME + datetime.timedelta(seconds=recording_duration)
station_recorder = StationDevice.objects.create(
station=station,
device=device,
start_time=_RECORDING_START_TIME,
end_time=recording_end_time)
creation_time = _RECORDING_START_TIME
recording = Recording.objects.create(station_recorder=station_recorder,
num_channels=_NUM_CHANNELS,
length=recording_length,
sample_rate=_SAMPLE_RATE,
start_time=_RECORDING_START_TIME,
end_time=recording_end_time,
creation_time=creation_time)
annotation_names = _get_annotation_names(clips)
annotation_infos = dict(
(name, _create_annotation_info(name, creation_time))
for name in annotation_names)
for i, clip_d in enumerate(clips):
clip_start_index = i * clip_length
offset = clip_start_index / _SAMPLE_RATE
clip_start_time = \
_RECORDING_START_TIME + datetime.timedelta(seconds=offset)
clip_duration = clip_length / _SAMPLE_RATE
clip_end_time = \
clip_start_time + datetime.timedelta(seconds=clip_duration)
clip = Clip.objects.create(recording=recording,
channel_num=0,
start_index=clip_start_index,
length=clip_length,
start_time=clip_start_time,
end_time=clip_end_time,
creation_time=creation_time)
for name, value in clip_d['annotations'].items():
info = annotation_infos[name]
model_utils.annotate_clip(clip, info, value, creation_time)
def annotate_old_bird_calls():
center_index_annotation_info = \
AnnotationInfo.objects.get(name=CENTER_INDEX_ANNOTATION_NAME)
center_freq_annotation_info = \
AnnotationInfo.objects.get(name=CENTER_FREQ_ANNOTATION_NAME)
classification_annotation_info = \
AnnotationInfo.objects.get(name=CLASSIFICATION_ANNOTATION_NAME)
user = User.objects.get(username='******')
sm_pairs = model_utils.get_station_mic_output_pairs_list()
ground_truth_detector = Processor.objects.get(
name=GROUND_TRUTH_DETECTOR_NAME)
rows = []
for detector_name, annotation_value in DETECTOR_DATA:
short_detector_name = detector_name.split()[2]
old_bird_detector = Processor.objects.get(name=detector_name)
window = utils.OLD_BIRD_CLIP_CALL_CENTER_WINDOWS[short_detector_name]
for station, mic_output in sm_pairs:
station_num = int(station.name.split()[1])
print('{} {}...'.format(short_detector_name, station_num))
ground_truth_clips = list(model_utils.get_clips(
station, mic_output, ground_truth_detector, None,
CLASSIFICATION_ANNOTATION_NAME, annotation_value))
ground_truth_call_center_indices = \
[c.start_index + c.length // 2 for c in ground_truth_clips]
ground_truth_call_count = len(ground_truth_clips)
old_bird_clips = list(model_utils.get_clips(
station, mic_output, old_bird_detector))
old_bird_clip_count = len(old_bird_clips)
clips = [(c.start_index, c.length) for c in old_bird_clips]
matches = utils.match_clips_with_calls(
clips, ground_truth_call_center_indices, window)
old_bird_call_count = len(matches)
rows.append([
short_detector_name, station_num, ground_truth_call_count,
old_bird_call_count, old_bird_clip_count])
if ANNOTATE:
# Clear any existing annotations.
for clip in old_bird_clips:
model_utils.delete_clip_annotation(
clip, classification_annotation_info,
creating_user=user)
# Create new annotations.
for i, j in matches:
old_bird_clip = old_bird_clips[i]
call_center_index = ground_truth_call_center_indices[j]
ground_truth_clip = ground_truth_clips[j]
# Annotate Old Bird clip call center index.
model_utils.annotate_clip(
old_bird_clip, center_index_annotation_info,
str(call_center_index), creating_user=user)
# Get ground truth clip call center frequency.
annotations = \
model_utils.get_clip_annotations(ground_truth_clip)
call_center_freq = annotations[CENTER_FREQ_ANNOTATION_NAME]
# Annotate Old Bird clip call center frequency.
model_utils.annotate_clip(
old_bird_clip, center_freq_annotation_info,
call_center_freq, creating_user=user)
model_utils.annotate_clip(
old_bird_clip, classification_annotation_info,
annotation_value, creating_user=user)
return rows
def _add_clips_aux(clips, night, detectors, annotation_infos):
global _clip_count
annotation_info = \
_get_annotation_info('Classification', annotation_infos)
num_added = 0
num_rejected = 0
num_excluded = 0
for c in clips:
_clip_count += 1
if _clip_count % _PAUSE_CHECK_PERIOD == 0:
_pause_if_indicated()
if not _include_clip(c):
num_excluded += 1
continue
file_path = c.file_path
if _CLIP_FILES_AVAILABLE and not (os.path.exists(file_path)):
print(
'Could not find clip file "{}". Clip will be ignored.'.format(
file_path))
num_rejected += 1
continue
try:
channel = _get_clip_recording_channel(c)
except Exception:
print((
'Could not get recording channel for clip "{}". '
'Clip will be ignored').format(file_path))
num_rejected += 1
continue
try:
detector = _get_detector(c, detectors)
except ValueError:
print((
'Could not get detector "{}" for clip "{}". '
'Clip will be ignored.').format(c.detector_name, file_path))
num_rejected += 1
continue
# The code between here and the return statement used to be a
# single database transaction.
# with transaction.atomic():
recording = channel.recording
station = recording.station
mic_output = channel.mic_output
sample_rate = recording.sample_rate
if _CLIP_FILES_AVAILABLE:
try:
length = audio_file_utils.get_wave_file_info(file_path).length
except Exception as e:
print((
'Could not read audio file info for clip "{}". '
'Error message was: {}. '
'Clip will be ignored.').format(file_path, str(e)))
num_rejected += 1
continue
else:
length = c.duration * sample_rate
start_time = c.start_time
span = (length - 1) / sample_rate
end_time = start_time + datetime.timedelta(seconds=span)
creation_time = time_utils.get_utc_now()
clip = Clip(
station=station,
mic_output=mic_output,
recording_channel=channel,
start_index=None,
length=length,
sample_rate=sample_rate,
start_time=start_time,
end_time=end_time,
date=night,
creation_time=creation_time,
creating_processor=detector)
if _SHOW_CLIPS:
print('Clip', _clip_count, clip)
clip.save()
if _CLIP_FILES_AVAILABLE and _COPY_CLIP_FILES:
try:
_copy_clip_audio_file(file_path, clip)
except Exception as e:
print((
'Copy failed for clip file "{}". '
'Error message was: {}. '
'Clip will be ignored.').format(file_path, str(e)))
num_rejected += 1
continue
if c.clip_class_name is not None:
# TODO: When this script becomes an importer, add the
# creating job to the following.
model_utils.annotate_clip(
clip, annotation_info, c.clip_class_name)
num_added += 1
return (num_added, num_rejected, num_excluded)
