def create_file(
station_name, recorder_name, start_time, duration, num_channels=1,
sample_rate=22050, recorder_channel_nums=(0,),
mic_outputs=('Mic Output 0',)):
t = datetime.datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
start_time = time_utils.create_utc_datetime(
t.year, t.month, t.day, t.hour, t.minute, t.second)
h, m, s = duration.split(':')
delta = datetime.timedelta(hours=int(h), minutes=int(m), seconds=int(s))
duration = delta.total_seconds()
length = int(duration * sample_rate)
station = Bunch(name=station_name)
recorder = Bunch(name=recorder_name)
return Bunch(
station=station,
recorder=recorder,
recorder_channel_nums=recorder_channel_nums,
mic_outputs=mic_outputs,
num_channels=num_channels,
length=length,
sample_rate=sample_rate,
start_time=start_time)
class BunchTests(TestCase):
def setUp(self):
self.a = Bunch(one=1, two=2)
self.b = Bunch(three=3, four=4)
self.c = Bunch(self.a, self.b, five=5)
def test_initializer(self):
a = self.a
self.assertEqual(a.one, 1)
self.assertEqual(a.two, 2)
b = self.b
self.assertEqual(b.three, 3)
self.assertEqual(b.four, 4)
c = self.c
self.assertEqual(c.one, 1)
self.assertEqual(c.two, 2)
self.assertEqual(c.three, 3)
self.assertEqual(c.four, 4)
self.assertEqual(c.five, 5)
def test_eq(self):
a = Bunch(one=1, two=2)
self.assertEqual(a, self.a)
self.assertNotEqual(a, self.b)
def test_contains(self):
self.assertIn('one', self.a)
self.assertIn('two', self.a)
self.assertNotIn('three', self.a)
def test_iter(self):
keys = sorted(k for k in self.a)
self.assertEqual(keys, ['one', 'two'])
def test_get(self):
self.assertEqual(self.a.get('one'), 1)
self.assertEqual(self.a.get('two'), 2)
self.assertIsNone(self.a.get('three'))
def test_get_defaults(self):
self.assertEqual(self.a.get('seventeen'), None)
self.assertEqual(self.a.get('seventeen', 17), 17)
def get_recorder_microphone_infos(station, microphone_output):
"""
Gets a mapping from (recorder_id, microphone_output_id) pairs
to lists of (channel_num, start_time, end_time) bunches.
The bunches are ordered by channel numbers and start times.
"""
# Get recorders that were used at station.
recorders = station.devices.filter(model__type='Audio Recorder')
rm_infos = defaultdict(list)
for recorder in recorders:
key = (recorder.id, microphone_output.id)
# Get connections from microphone to recorder.
connections = DeviceConnection.objects.filter(
output=microphone_output,
input__device=recorder)
# Remember channel number and time interval of each connection.
for connection in connections:
info = Bunch(
channel_num=connection.input.channel_num,
start_time=connection.start_time,
end_time=connection.end_time)
rm_infos[key].append(info)
rm_infos[key].sort(key=_get_rm_info_sort_key)
return rm_infos
def test_init(self):
channel_count = 2
waveform_frame_count = 32
waveform_frame_rate = 16000
window_size = 16
hop_size = 8
dft_size = 16
samples = _get_waveform_samples(
channel_count, waveform_frame_count, window_size)
waveform = RamSignal(waveform_frame_rate, samples, False)
window = np.ones(window_size)
settings = Bunch(window=window, hop_size=hop_size, dft_size=dft_size)
gram = Spectrogram(waveform, settings)
gram_frame_count = 1 + (waveform_frame_count - window_size) // hop_size
gram_frame_rate = waveform_frame_rate / hop_size
offset = (window_size - 1) / 2 / waveform_frame_rate
time_axis = TimeAxis(gram_frame_count, gram_frame_rate, offset)
array_shape = ((dft_size // 2) + 1,)
samples = _get_gram_samples(waveform, window, hop_size, dft_size)
SignalTests.assert_signal(
gram, 'Spectrogram', time_axis, channel_count, array_shape,
'float64', samples)
def _parse_file_path(self, file_path):
parts = Path(file_path).parts
file_name = parts[-1]
dir_names = tuple(reversed(parts[:-1]))
# Get classification from file parent dir name if name indicates it.
key = dir_names[0].lower()
classification = self._classifications.get(key)
if classification is None:
self._logger.warning(
('Could not find classification for file "{}". Clip will '
'be unclassified.').format(file_path))
else:
dir_names = dir_names[1:]
station = self._get_station(dir_names)
# Get clip detector, start time, and date.
detector_name, local_start_time = _parse_file_name(file_name)
detector = self._get_detector(detector_name)
start_time = station.local_to_utc(local_start_time)
date = station.get_night(start_time)
return Bunch(station=station,
detector=detector,
start_time=start_time,
date=date,
classification=classification)
def detect_events(audio, config):
spectrogram = Spectrogram(audio, config.spectrogram_settings)
x, times = measurements.apply_measurement_to_spectra(
measurements.entropy,
spectrogram,
start_freq=config.start_freq,
end_freq=config.end_freq,
denoise=True,
block_size=1)
if len(times) < 2:
return []
else:
# spectrogram has at least two frames
period = times[1] - times[0]
min_event_length = _to_frames(config.min_event_duration, period)
max_event_length = _to_frames(config.max_event_duration, period)
min_event_separation = _to_frames(config.min_event_separation, period)
detector_config = Bunch(
typical_background_percentile=config.typical_background_percentile,
small_background_percentile=config.small_background_percentile,
bit_threshold_factor=config.bit_threshold_factor,
min_event_length=min_event_length,
max_event_length=max_event_length,
min_event_separation=min_event_separation,
min_event_density=config.min_event_density)
selections, _ = _detect(-x, detector_config)
return [_convert_selection(s, times[0], period) for s in selections]
def parse_csv_file_line(line):
(station_name, _, recorder_model, recorder_sn, microphone_sn, latitude,
longitude, elevation, station_name_alias) = line.split(',')
if recorder_model == 'SM2':
recorder_model = 'SM2+'
if recorder_sn == '':
recorder_sn = create_sn(recorder_model)
microphone_model = '21c'
if microphone_sn == '':
microphone_sn = create_sn(microphone_model)
return Bunch(station_name=station_name,
station_name_alias=station_name_alias,
description='',
time_zone='US/Mountain',
latitude=latitude,
longitude=longitude,
elevation=elevation,
recorder_model=recorder_model,
recorder_sn=recorder_sn,
microphone_model=microphone_model,
microphone_sn=microphone_sn)
def resample(audio, target_sample_rate):
"""
Resamples audio to a specified sample rate.
This function should only be used for relatively short audio segments,
say not longer than a second or so. It uses the `scipy.signal.resample`
method to perform the resampling, which computes a length-M DFT and a
length-N inverse DFT, where M and N are the input and output length,
respectively. M and N may not be powers of two, and they may even be
prime, which can make this function slow if M or N is too large.
"""
if audio.sample_rate == target_sample_rate:
# do not need to resample
return audio
else:
# need to resample
# We put this import here instead of at the top of this module
# so the module can be used in Python environments that don't
# include SciPy as long as this function is not called.
import scipy.signal as signal
ratio = target_sample_rate / audio.sample_rate
num_samples = int(round(len(audio.samples) * ratio))
samples = signal.resample(audio.samples, num_samples)
return Bunch(samples=samples, sample_rate=target_sample_rate)
def extract_call(audio, selection, config):
samples = audio.samples
sample_rate = audio.sample_rate
start_index, end_index = selection
center_index = (start_index + end_index - 1) // 2
duration = config.call_segment_duration
length = seconds_to_frames(duration, sample_rate)
start_index = center_index - length // 2
if start_index < 0:
return None
else:
# start index is at least zero
end_index = start_index + length
if end_index > len(samples):
return None
else:
return Bunch(
samples=samples[start_index:end_index],
sample_rate=sample_rate)
def create_channel_stat_bunch(stat_dicts):
return Bunch(
max=np.array(stat_dicts['max']),
min=np.array(stat_dicts['min']),
mean=np.array(stat_dicts['mean']),
mean_abs=np.array(stat_dicts['mean_abs']),
std=np.array(stat_dicts['std']),
ampligram=np.stack(stat_dicts['ampligram']))
def _get_input_device(info, default_index, host_api_names):
return Bunch(host_api_name=host_api_names[info['hostApi']],
index=info['index'],
default=info['index'] == default_index,
name=info['name'],
num_input_channels=info['maxInputChannels'],
default_sample_rate=info['defaultSampleRate'],
default_low_input_latency=info['defaultLowInputLatency'],
default_high_input_latency=info['defaultHighInputLatency'])
def _end_recording(self):
return Bunch(station=self.station,
recorder=self.recorder,
recorder_channel_nums=self.recorder_channel_nums,
mic_outputs=self.mic_outputs,
num_channels=self.num_channels,
length=self.length,
sample_rate=self.sample_rate,
start_time=self.start_time,
files=self.files)
def _test_spectrogram(self, num_channels, dft_size, hop_size, bin_num):
num_samples = dft_size * 2
samples = self._create_test_signal(num_channels, num_samples, dft_size,
bin_num)
audio = Bunch(samples=samples, sample_rate=_SAMPLE_RATE)
window = RectangularWindow(dft_size)
settings = Bunch(window=window,
hop_size=hop_size,
dft_size=dft_size,
reference_power=None)
spectrogram = Spectrogram(audio, settings)
spectra = spectrogram.spectra
expected = self._get_expected_spectra(num_channels, num_samples,
hop_size, dft_size, bin_num)
self.assertTrue(np.allclose(spectra, expected))
self.assertEqual(spectrogram.freq_spacing, _SAMPLE_RATE / dft_size)
def _get_absolute_path_info(self, file_path):
manager = recording_manager.instance
try:
_, rel_path = manager.get_relative_recording_file_path(file_path)
except ValueError:
self._handle_bad_recording_file_path(file_path, 'is not in',
manager)
return Bunch(absolute_path=file_path, relative_path=rel_path)
def _parse_file_name(file_name):
m = _FILE_NAME_REGEX.match(file_name)
if m is not None:
m = Bunch(**m.groupdict())
try:
start_time = time_utils.parse_date_time(
m.year, m.month, m.day, m.hour, m.minute, m.second)
except Exception as e:
raise ValueError(
'Could not get start time from file name: {}'.format(str(e)))
tenths = datetime.timedelta(microseconds=100000 * int(m.num))
start_time += tenths
return m.detector_name, start_time
else:
raise ValueError('Could not parse file name.')
def _create_clip(self, dataset):
attrs = dataset.attrs
return Bunch(id=attrs['id'],
waveform=dataset[:],
station=attrs['station'],
microphone=attrs['microphone'],
detector=attrs['detector'],
night=attrs['night'],
start_time=attrs['start_time'],
original_sample_rate=attrs['original_sample_rate'],
classification=attrs['classification'])
def _parse_file_name(file_name):
m = _FILE_NAME_REGEX.match(file_name)
if m is not None:
m = Bunch(**m.groupdict())
try:
start_time = time_utils.parse_date_time(m.year, m.month, m.day,
m.hour, m.minute, m.second)
except Exception as e:
raise ValueError(
'Could not get start time from file name: {}'.format(str(e)))
tenths = datetime.timedelta(microseconds=100000 * int(m.num))
start_time += tenths
return m.detector_name, start_time
else:
raise ValueError('Could not parse file name.')
def create_data_sets(features, targets, settings):
num_examples = len(features)
assert (len(targets) == num_examples)
train_size = settings.training_set_size
val_size = settings.validation_set_size
test_size = settings.test_set_size
assert (val_size + test_size < num_examples)
if train_size is None:
train_size = num_examples - val_size - test_size
assert (train_size + val_size + test_size <= num_examples)
# Shuffle examples.
permutation = numpy_utils.reproducible_permutation(num_examples)
features = features[permutation]
targets = targets[permutation]
test_start = num_examples - test_size
val_start = test_start - val_size
train_set = Bunch(name='training',
features=features[:val_start],
targets=targets[:val_start])
val_set = Bunch(name='validation',
features=features[val_start:test_start],
targets=targets[val_start:test_start])
test_set = Bunch(name='test',
features=features[test_start:],
targets=targets[test_start:])
return train_set, val_set, test_set
def initialize(archive_dir_path, archive_settings):
global archive_paths
archive_paths = Bunch(
archive_dir_path=archive_dir_path,
clip_dir_path=archive_dir_path / 'Clips',
deferred_action_dir_path=archive_dir_path / 'Deferred Actions',
job_log_dir_path=archive_dir_path / 'Logs' / 'Jobs',
preference_file_path=archive_dir_path / 'Preferences.yaml',
preset_dir_path=archive_dir_path / 'Presets',
recording_dir_paths=_create_recording_dir_paths(
archive_settings, archive_dir_path),
sqlite_database_file_path=archive_dir_path / 'Archive Database.sqlite')
def create_detector_settings_aux(
detector_type, parameter_name, parameter_value):
value_string = format_parameter_value(parameter_value)
name = '{} {}'.format(detector_type, value_string)
base_settings = DETECTOR_BASE_SETTINGS[detector_type]
kwargs = {
parameter_name: parameter_value,
}
settings = Bunch(base_settings, **kwargs)
return name, settings
def start_job(self, command_spec, user):
info = Bunch()
info.command_spec = command_spec
info.job_id = _create_job(command_spec, user)
info.archive_lock = archive_lock.get_lock()
info.stop_event = Event()
with self._lock:
self._job_infos[info.job_id] = info
info.process = Process(target=job_runner.run_job, args=(info, ))
info.process.start()
return info.job_id
def _generate_segments(audio, segment_length, hop_size, start_index=0):
samples = audio.samples
sample_rate = float(audio.sample_rate)
n = len(samples)
i = start_index
while i + segment_length <= n:
segment = Bunch(samples=samples[i:i + segment_length],
sample_rate=sample_rate,
start_time=i / sample_rate)
yield segment
i += hop_size
def _read_header(reader, check_format=True):
p = reader.getparams()
sample_size = p.sampwidth * 8
if check_format:
_check_wave_file_format(sample_size, p.comptype)
sample_rate = float(p.framerate)
return Bunch(
num_channels=p.nchannels,
length=p.nframes,
sample_size=sample_size,
sample_rate=sample_rate,
compression_type=p.comptype,
compression_name=p.compname)
def parse_example(data):
data = [int(d) for d in data]
clip_id = data[0]
call_start_index = data[1]
label = data[2]
scores = np.array(data[3:])
max_score = np.max(scores)
max_score_index = np.argmax(scores)
correct = max_score_index == label
return Bunch(clip_id=clip_id,
call_start_index=call_start_index,
label=label,
scores=scores,
max_score=max_score,
max_score_index=max_score_index,
correct=correct)
def start_job(self, command_spec, user):
info = Bunch()
info.command_spec = command_spec
info.job_id = _create_job(command_spec, user)
info.archive_lock = archive_lock.get_lock()
info.stop_event = Event()
with self._lock:
self._job_infos[info.job_id] = info
info.process = Process(target=job_runner.run_job, args=(info,))
info.process.start()
return info.job_id
def parse_file(self, file_path):
"""
Parses the specified recording file for recording information.
Some information is obtained from the file path, while other
information is obtained from within the file.
:Parameters:
file_path : `str`
the path of the file to parse.
:Returns:
a `Bunch` with the following attributes:
`station` - the `Station` of the recording.
`recorder` - the `Recorder` of the recording, or `None` if unknown.
`recorder_channel_nums` - sequence of recorder channel numbers
indexed by recording channel number, or `None` if unknown.
`num_channels` - the number of channels of the file.
`length` - the length of the file in sample frames.
`sample_rate` - the sample rate of the file in Hertz.
`start_time` - the UTC start time of the file.
`path` - the path of the file.
"""
station, recorder_channel_nums, start_time = \
self._parse_file_name(file_path)
num_channels, length, sample_rate = \
self._get_audio_file_info(file_path)
return Bunch(station=station,
recorder=None,
recorder_channel_nums=recorder_channel_nums,
num_channels=num_channels,
length=length,
sample_rate=sample_rate,
start_time=start_time,
path=file_path)
def extract_clip_segment(clip,
segment_duration,
segment_source,
source_duration=None):
source = _get_segment_source(clip, segment_source, source_duration)
if source is None:
return None
else:
source_start_index, source_length = source
sample_rate = clip.sample_rate
segment_length = signal_utils.seconds_to_frames(
segment_duration, sample_rate)
if source_length < segment_length:
# source not long enough to extract segment from
return None
else:
# Extract samples from source.
if source_length == segment_length:
offset = 0
else:
offset = random.randrange(source_length - segment_length)
start_index = source_start_index + offset
end_index = start_index + segment_length
samples = clip_manager.instance.get_samples(clip)
samples = samples[start_index:end_index]
return Bunch(samples=samples,
sample_rate=clip.sample_rate,
start_index=start_index)
def _create_station_devices(model_stations, device_type):
devices = []
model_names = sorted(model_stations.keys())
for model_name in model_names:
station_names = sorted(model_stations[model_name])
for serial_num, station_name in enumerate(station_names):
name = '{} {}'.format(model_name, serial_num)
device = Bunch(
name=name,
model_name=model_name,
serial_num=serial_num,
station_name=station_name,
type=device_type)
devices.append(device)
return devices
def get_audio(self, clip):
samples = self.get_samples(clip)
sample_rate = clip.sample_rate
return Bunch(samples=samples, sample_rate=sample_rate)
def _create_bunches(self, cls, rows):
return [Bunch(**dict(zip(cls._fields, r))) for r in rows]
"""
the sample rate of the original Old Bird detectors, in hertz.
The reimplemented detectors can operate on input of a variety of sample
rates. We use the sample rate of the original detectors only to convert
detector settings that were originally specified in units of sample
periods to units of seconds.
"""
_TSEEP_SETTINGS = Bunch(
filter_f0=6000, # hertz
filter_f1=10000, # hertz
filter_bw=100, # hertz
filter_duration=100 / _OLD_FS, # seconds
integration_time=2000 / _OLD_FS, # seconds
ratio_delay=.02, # seconds
ratio_threshold=2, # dimensionless
min_duration=.100, # seconds
max_duration=.400, # seconds
initial_padding=3000 / _OLD_FS, # seconds
suppressor_count_threshold=15, # clips
suppressor_period=20 # seconds
)
_THRUSH_SETTINGS = Bunch(
filter_f0=2800, # hertz
filter_f1=5000, # hertz
filter_bw=100, # hertz
filter_duration=100 / _OLD_FS, # seconds
integration_time=4000 / _OLD_FS, # seconds
ratio_delay=.02, # seconds
ratio_threshold=1.3, # dimensionless
def _create_recording_file_bunch(self, f):
path = self._get_absolute_file_path(f.path)
return Bunch(path=path, start_index=f.start_index, length=f.length)