def detect(self, samples):
if self._input_buffer is None:
self._input_buffer = SampleBuffer(samples.dtype)
self._input_buffer.write(samples)
self._process_input_chunks()
def test_write_read(self):
b = SampleBuffer(np.int64)
b.write(np.arange(0, 10))
b.write(np.arange(10, 20))
self._assert_buffer(b, 20, 0, 20)
self._assert_arrays_equal(b.read(7), np.arange(0, 7))
self._assert_buffer(b, 20, 7, 13)
self._assert_arrays_equal(b.read(7), np.arange(7, 14))
self._assert_buffer(b, 20, 14, 6)
self._assert_arrays_equal(b.read(6), np.arange(14, 20))
self._assert_buffer(b, 20, 20, 0)
def detect(self, samples):
if self._input_buffer is None:
self._input_buffer = SampleBuffer(samples.dtype)
self._input_buffer.write(samples)
self._process_input_chunks()
def test_write_read(self):
b = SampleBuffer(np.int64)
b.write(np.arange(0, 10))
b.write(np.arange(10, 20))
self._assert_buffer(b, 20, 0, 20)
self._assert_arrays_equal(b.read(7), np.arange(0, 7))
self._assert_buffer(b, 20, 7, 13)
self._assert_arrays_equal(b.read(7), np.arange(7, 14))
self._assert_buffer(b, 20, 14, 6)
self._assert_arrays_equal(b.read(6), np.arange(14, 20))
self._assert_buffer(b, 20, 20, 0)
def test_simple_write_read(self):
b = SampleBuffer(np.int64)
b.write(np.arange(0, 10))
b.write(np.arange(10, 20))
self._assert_arrays_equal(b.read(20), np.arange(0, 20))
def test_read_all(self):
b = SampleBuffer(np.int64)
b.write(np.arange(0, 10))
b.write(np.arange(10, 20))
self._assert_arrays_equal(b.read(), np.arange(0, 20))
self.assertEqual(b.read_index, 20)
def test_zero_length_read(self):
b = SampleBuffer(np.float64)
result = b.read(0)
self._assert_arrays_equal(result, np.array([], np.float64))
self._assert_buffer(b, 0, 0, 0)
def test_zero_length_write(self):
b = SampleBuffer(np.float64)
b.write(np.array([]))
self._assert_buffer(b, 0, 0, 0)
class _Detector:
"""
MPG Ranch NFC detector.
An instance of this class operates on a single audio channel. It has a
`detect` method that takes a NumPy array of samples. The method can be
called repeatedly with consecutive sample arrays. The `complete_detection`
method should be called after the final call to the `detect` method.
During detection, each time the detector detects a clip it notifies
a listener by invoking the listener's `process_clip` method. The
`process_clip` method must accept two arguments, the start index and
length of the detected clip.
See the `_TSEEP_SETTINGS` and `_THRUSH_SETTINGS` objects above for
settings that make a `_Detector` detect higher-frequency and
lower-frequency NFCs, respectively, using the MPG Ranch tseep and
thrush coarse classifiers. The `TseepDetector` and `ThrushDetector`
classes of this module subclass the `_Detector` class with fixed
settings, namely `_TSEEP_SETTINGS` and `_THRUSH_SETTINGS`, respectively.
"""
def __init__(
self, settings, input_sample_rate, listener,
extra_thresholds=None):
open_mp_utils.work_around_multiple_copies_issue()
# Suppress TensorFlow INFO and DEBUG log messages.
tf.logging.set_verbosity(tf.logging.WARN)
self._settings = settings
self._input_sample_rate = input_sample_rate
self._listener = listener
s2f = signal_utils.seconds_to_frames
s = self._settings
fs = self._input_sample_rate
self._input_buffer = None
self._input_chunk_size = s2f(s.input_chunk_size, fs)
self._thresholds = self._get_thresholds(extra_thresholds)
self._clip_start_offset = -s2f(s.initial_clip_padding, fs)
self._clip_length = s2f(s.clip_duration, fs)
self._input_chunk_start_index = 0
self._classifier_settings = self._load_classifier_settings()
self._estimator = self._create_estimator()
s = self._classifier_settings
fs = s.waveform_sample_rate
self._classifier_sample_rate = fs
self._classifier_waveform_length = s2f(s.waveform_duration, fs)
fraction = self._settings.hop_size / 100
self._hop_size = s2f(fraction * s.waveform_duration, fs)
if _SCORE_OUTPUT_ENABLED:
file_path = _SCORE_FILE_PATH_FORMAT.format(settings.clip_type)
self._score_file_writer = DetectionScoreFileWriter(
file_path, self._input_sample_rate, _SCORE_SCALE_FACTOR,
self._hop_size, _SCORE_OUTPUT_START_OFFSET,
_SCORE_OUTPUT_DURATION)
# settings = self._classifier_settings.__dict__
# names = sorted(settings.keys())
# for name in names:
# print('{}: {}'.format(name, settings[name]))
@property
def settings(self):
return self._settings
@property
def input_sample_rate(self):
return self._input_sample_rate
@property
def listener(self):
return self._listener
def _get_thresholds(self, extra_thresholds):
thresholds = set([self._settings.threshold])
if extra_thresholds is not None:
thresholds |= set(extra_thresholds)
return sorted(thresholds)
def _load_classifier_settings(self):
s = self._settings
path = classifier_utils.get_settings_file_path(s.clip_type)
logging.info('Loading classifier settings from "{}"...'.format(path))
return Settings.create_from_yaml_file(path)
def _create_estimator(self):
s = self._settings
path = classifier_utils.get_tensorflow_model_dir_path(s.clip_type)
logging.info((
'Creating TensorFlow estimator from saved model in directory '
'"{}"...').format(path))
return tf.contrib.estimator.SavedModelEstimator(str(path))
def _create_dataset(self):
s = self._classifier_settings
return dataset_utils.create_spectrogram_dataset_from_waveforms_array(
self._waveforms, dataset_utils.DATASET_MODE_INFERENCE, s,
batch_size=64, feature_name=s.model_input_name)
def detect(self, samples):
if self._input_buffer is None:
self._input_buffer = SampleBuffer(samples.dtype)
self._input_buffer.write(samples)
self._process_input_chunks()
def _process_input_chunks(self, process_all_samples=False):
# Process as many chunks of input samples of size
# `self._input_chunk_size` as possible.
while len(self._input_buffer) >= self._input_chunk_size:
chunk = self._input_buffer.read(self._input_chunk_size)
self._process_input_chunk(chunk)
# If indicated, process any remaining input samples as one chunk.
# The size of the chunk will differ from `self._input_chunk_size`.
if process_all_samples and len(self._input_buffer) != 0:
chunk = self._input_buffer.read()
self._process_input_chunk(chunk)
def _process_input_chunk(self, samples):
input_length = len(samples)
if self._classifier_sample_rate != self._input_sample_rate:
samples = resampy.resample(
samples, self._input_sample_rate, self._classifier_sample_rate,
filter='kaiser_fast')
self._waveforms = _get_analysis_records(
samples, self._classifier_waveform_length, self._hop_size)
# print('Scoring chunk waveforms...')
# start_time = time.time()
scores = classifier_utils.score_dataset_examples(
self._estimator, self._create_dataset)
# elapsed_time = time.time() - start_time
# num_waveforms = self._waveforms.shape[0]
# rate = num_waveforms / elapsed_time
# print((
# 'Scored {} waveforms in {:.1f} seconds, a rate of {:.1f} '
# 'waveforms per second.').format(
# num_waveforms, elapsed_time, rate))
if _SCORE_OUTPUT_ENABLED:
self._score_file_writer.write(samples, scores)
for threshold in self._thresholds:
peak_indices = signal_utils.find_peaks(scores, threshold)
peak_scores = scores[peak_indices]
self._notify_listener_of_clips(
peak_indices, peak_scores, input_length, threshold)
self._input_chunk_start_index += input_length
def _notify_listener_of_clips(
self, peak_indices, peak_scores, input_length, threshold):
# print('Clips:')
start_offset = self._input_chunk_start_index + self._clip_start_offset
peak_indices *= self._hop_size
for i, score in zip(peak_indices, peak_scores):
# Convert classification index to input index, accounting
# for difference between classifier sample rate and input
# sample rate.
t = signal_utils.get_duration(i, self._classifier_sample_rate)
i = signal_utils.seconds_to_frames(t, self._input_sample_rate)
clip_start_index = i + start_offset
clip_end_index = clip_start_index + self._clip_length
chunk_end_index = self._input_chunk_start_index + input_length
if clip_start_index < 0:
logging.warning(
'Rejected clip that started before beginning of '
'recording.')
elif clip_end_index > chunk_end_index:
# clip might extend past end of recording, since it extends
# past the end of this chunk (we do not know whether or
# not the current chunk is the last)
logging.warning(
'Rejected clip that ended after end of recording chunk.')
else:
# all clip samples are in the recording interval extending
# from the beginning of the recording to the end of the
# current chunk
# print(
# ' {} {}'.format(clip_start_index, self._clip_length))
annotations = {'Detector Score': 100 * score}
self._listener.process_clip(
clip_start_index, self._clip_length, threshold,
annotations)
def complete_detection(self):
"""
Completes detection after the `detect` method has been called
for all input.
"""
self._process_input_chunks(process_all_samples=True)
self._listener.complete_processing()
if _SCORE_OUTPUT_ENABLED:
self._score_file_writer.close()
def test_zero_length_read(self):
b = SampleBuffer(np.float64)
result = b.read(0)
self._assert_arrays_equal(result, np.array([], np.float64))
self._assert_buffer(b, 0, 0, 0)
def test_init(self):
b = SampleBuffer(np.int64)
self.assertEqual(b.dtype, np.int64)
self._assert_buffer(b, 0, 0, 0)
def test_simple_increments(self):
b = SampleBuffer(np.int64)
b.increment(10)
b.increment(20)
self._assert_buffer(b, 0, 30, 0)
def test_read_with_inc(self):
b = SampleBuffer(np.int64)
b.write(np.arange(0, 20))
self._assert_arrays_equal(b.read(10, 5), np.arange(0, 10))
self._assert_arrays_equal(b.read(10, 5), np.arange(5, 15))
self._assert_arrays_equal(b.read(10, 5), np.arange(10, 20))
def test_write_read_inc(self):
b = SampleBuffer(np.int64)
b.increment(5)
b.write(np.arange(0, 10))
self._assert_arrays_equal(b.read(2), np.arange(5, 7))
self._assert_buffer(b, 10, 7, 3)
b.write(np.arange(10, 20))
self._assert_arrays_equal(b.read(5), np.arange(7, 12))
self._assert_buffer(b, 20, 12, 8)
b.increment(5)
self._assert_arrays_equal(b.read(3), np.arange(17, 20))
self._assert_buffer(b, 20, 20, 0)
b.increment(5)
self._assert_buffer(b, 20, 25, 0)
def test_simple_increments(self):
b = SampleBuffer(np.int64)
b.increment(10)
b.increment(20)
self._assert_buffer(b, 0, 30, 0)
def test_simple_write_read(self):
b = SampleBuffer(np.int64)
b.write(np.arange(0, 10))
b.write(np.arange(10, 20))
self._assert_arrays_equal(b.read(20), np.arange(0, 20))
def test_read_all(self):
b = SampleBuffer(np.int64)
b.write(np.arange(0, 10))
b.write(np.arange(10, 20))
self._assert_arrays_equal(b.read(), np.arange(0, 20))
self.assertEqual(b.read_index, 20)
def test_write_read_inc(self):
b = SampleBuffer(np.int64)
b.increment(5)
b.write(np.arange(0, 10))
self._assert_arrays_equal(b.read(2), np.arange(5, 7))
self._assert_buffer(b, 10, 7, 3)
b.write(np.arange(10, 20))
self._assert_arrays_equal(b.read(5), np.arange(7, 12))
self._assert_buffer(b, 20, 12, 8)
b.increment(5)
self._assert_arrays_equal(b.read(3), np.arange(17, 20))
self._assert_buffer(b, 20, 20, 0)
b.increment(5)
self._assert_buffer(b, 20, 25, 0)
class _Detector:
"""
MPG Ranch NFC detector.
An instance of this class operates on a single audio channel. It has a
`detect` method that takes a NumPy array of samples. The method can be
called repeatedly with consecutive sample arrays. The `complete_detection`
method should be called after the final call to the `detect` method.
During detection, each time the detector detects a clip it notifies
a listener by invoking the listener's `process_clip` method. The
`process_clip` method must accept two arguments, the start index and
length of the detected clip.
See the `_TSEEP_SETTINGS` and `_THRUSH_SETTINGS` objects above for
settings that make a `_Detector` detect higher-frequency and
lower-frequency NFCs, respectively, using the MPG Ranch tseep and
thrush coarse classifiers. The `TseepDetector` and `ThrushDetector`
classes of this module subclass the `_Detector` class with fixed
settings, namely `_TSEEP_SETTINGS` and `_THRUSH_SETTINGS`, respectively.
"""
def __init__(self,
settings,
input_sample_rate,
listener,
extra_thresholds=None):
open_mp_utils.work_around_multiple_copies_issue()
# Suppress TensorFlow INFO and DEBUG log messages.
tf.logging.set_verbosity(tf.logging.WARN)
self._settings = settings
self._input_sample_rate = input_sample_rate
self._listener = listener
s2f = signal_utils.seconds_to_frames
s = self._settings
fs = self._input_sample_rate
self._input_buffer = None
self._input_chunk_size = s2f(s.input_chunk_size, fs)
self._thresholds = self._get_thresholds(extra_thresholds)
self._clip_start_offset = -s2f(s.initial_clip_padding, fs)
self._clip_length = s2f(s.clip_duration, fs)
self._input_chunk_start_index = 0
self._classifier_settings = self._load_classifier_settings()
self._estimator = self._create_estimator()
s = self._classifier_settings
fs = s.waveform_sample_rate
self._classifier_sample_rate = fs
self._classifier_waveform_length = s2f(s.waveform_duration, fs)
fraction = self._settings.hop_size / 100
self._hop_size = s2f(fraction * s.waveform_duration, fs)
if _SCORE_OUTPUT_ENABLED:
file_path = _SCORE_FILE_PATH_FORMAT.format(settings.clip_type)
self._score_file_writer = DetectionScoreFileWriter(
file_path, self._input_sample_rate, _SCORE_SCALE_FACTOR,
self._hop_size, _SCORE_OUTPUT_START_OFFSET,
_SCORE_OUTPUT_DURATION)
# settings = self._classifier_settings.__dict__
# names = sorted(settings.keys())
# for name in names:
# print('{}: {}'.format(name, settings[name]))
@property
def settings(self):
return self._settings
@property
def input_sample_rate(self):
return self._input_sample_rate
@property
def listener(self):
return self._listener
def _get_thresholds(self, extra_thresholds):
thresholds = set([self._settings.threshold])
if extra_thresholds is not None:
thresholds |= set(extra_thresholds)
return sorted(thresholds)
def _load_classifier_settings(self):
s = self._settings
path = classifier_utils.get_settings_file_path(s.clip_type)
logging.info('Loading classifier settings from "{}"...'.format(path))
return Settings.create_from_yaml_file(path)
def _create_estimator(self):
s = self._settings
path = classifier_utils.get_tensorflow_model_dir_path(s.clip_type)
logging.info(
('Creating TensorFlow estimator from saved model in directory '
'"{}"...').format(path))
return tf.contrib.estimator.SavedModelEstimator(str(path))
def _create_dataset(self):
s = self._classifier_settings
return dataset_utils.create_spectrogram_dataset_from_waveforms_array(
self._waveforms,
dataset_utils.DATASET_MODE_INFERENCE,
s,
batch_size=64,
feature_name=s.model_input_name)
def detect(self, samples):
if self._input_buffer is None:
self._input_buffer = SampleBuffer(samples.dtype)
self._input_buffer.write(samples)
self._process_input_chunks()
def _process_input_chunks(self, process_all_samples=False):
# Process as many chunks of input samples of size
# `self._input_chunk_size` as possible.
while len(self._input_buffer) >= self._input_chunk_size:
chunk = self._input_buffer.read(self._input_chunk_size)
self._process_input_chunk(chunk)
# If indicated, process any remaining input samples as one chunk.
# The size of the chunk will differ from `self._input_chunk_size`.
if process_all_samples and len(self._input_buffer) != 0:
chunk = self._input_buffer.read()
self._process_input_chunk(chunk)
def _process_input_chunk(self, samples):
input_length = len(samples)
if self._classifier_sample_rate != self._input_sample_rate:
# start_time = time.time()
samples = resampy.resample(samples,
self._input_sample_rate,
self._classifier_sample_rate,
filter='kaiser_fast')
# processing_time = time.time() - start_time
# input_duration = input_length / self._input_sample_rate
# rate = input_duration / processing_time
# print((
# 'Resampled {:.1f} seconds of input in {:.1f} seconds, '
# 'or {:.1f} times faster than real time.').format(
# input_duration, processing_time, rate))
self._waveforms = _get_analysis_records(
samples, self._classifier_waveform_length, self._hop_size)
# print('Scoring chunk waveforms...')
# start_time = time.time()
scores = classifier_utils.score_dataset_examples(
self._estimator, self._create_dataset)
# elapsed_time = time.time() - start_time
# num_waveforms = self._waveforms.shape[0]
# rate = num_waveforms / elapsed_time
# print((
# 'Scored {} waveforms in {:.1f} seconds, a rate of {:.1f} '
# 'waveforms per second.').format(
# num_waveforms, elapsed_time, rate))
if _SCORE_OUTPUT_ENABLED:
self._score_file_writer.write(samples, scores)
for threshold in self._thresholds:
peak_indices = signal_utils.find_peaks(scores, threshold)
peak_scores = scores[peak_indices]
self._notify_listener_of_clips(peak_indices, peak_scores,
input_length, threshold)
self._input_chunk_start_index += input_length
def _notify_listener_of_clips(self, peak_indices, peak_scores,
input_length, threshold):
# print('Clips:')
start_offset = self._input_chunk_start_index + self._clip_start_offset
peak_indices *= self._hop_size
for i, score in zip(peak_indices, peak_scores):
# Convert classification index to input index, accounting
# for difference between classifier sample rate and input
# sample rate.
t = signal_utils.get_duration(i, self._classifier_sample_rate)
i = signal_utils.seconds_to_frames(t, self._input_sample_rate)
clip_start_index = i + start_offset
clip_end_index = clip_start_index + self._clip_length
chunk_end_index = self._input_chunk_start_index + input_length
if clip_start_index < 0:
logging.warning(
'Rejected clip that started before beginning of '
'recording.')
elif clip_end_index > chunk_end_index:
# clip might extend past end of recording, since it extends
# past the end of this chunk (we do not know whether or
# not the current chunk is the last)
logging.warning(
'Rejected clip that ended after end of recording chunk.')
else:
# all clip samples are in the recording interval extending
# from the beginning of the recording to the end of the
# current chunk
# print(
# ' {} {}'.format(clip_start_index, self._clip_length))
annotations = {'Detector Score': 100 * score}
self._listener.process_clip(clip_start_index,
self._clip_length, threshold,
annotations)
def complete_detection(self):
"""
Completes detection after the `detect` method has been called
for all input.
"""
self._process_input_chunks(process_all_samples=True)
self._listener.complete_processing()
if _SCORE_OUTPUT_ENABLED:
self._score_file_writer.close()
def test_read_with_inc(self):
b = SampleBuffer(np.int64)
b.write(np.arange(0, 20))
self._assert_arrays_equal(b.read(10, 5), np.arange(0, 10))
self._assert_arrays_equal(b.read(10, 5), np.arange(5, 15))
self._assert_arrays_equal(b.read(10, 5), np.arange(10, 20))
def test_zero_length_write(self):
b = SampleBuffer(np.float64)
b.write(np.array([]))
self._assert_buffer(b, 0, 0, 0)