def take_action(self, parsed_args):
module_name, class_name = parsed_args.parser.rsplit(".", 1)
parser_class = getattr(importlib.import_module(module_name),
class_name)
if not issubclass(parser_class, Parser):
raise ValueError(
"specified parser does not inherit audeep.backend.parsers.Parser"
)
parser = parser_class(parsed_args.basedir)
if not parser.can_parse():
raise ValueError(
"specified parser is unable to parse data set at {}".format(
parsed_args.basedir))
lengths = []
sample_rates = []
channels = []
non_seekable_files = False
instance_metadata = parser.parse()
self.log.info("reading audio file information")
for index, metadata in enumerate(instance_metadata):
self.log.debug(
"processing %%s (%%%dd/%%d)" %
int(math.ceil(math.log10(len(instance_metadata)))),
metadata.path, index + 1, len(instance_metadata))
with SoundFile(str(metadata.path)) as sf:
sample_rates.append(sf.samplerate)
channels.append(sf.channels)
if sf.seekable():
lengths.append(sf.seek(0, SEEK_END) / sf.samplerate)
else:
non_seekable_files = True
if non_seekable_files:
self.log.warning(
"could not determine the length of some files - information may be inaccurate"
)
information = [
("number of audio files", parser.num_instances),
("number of labels", len(parser.label_map)),
("cross validation folds", parser.num_folds),
("minimum sample length", "%.2f s" % np.min(lengths)),
("maximum sample length", "%.2f s" % np.max(lengths)),
]
if len(np.unique(sample_rates)) > 1:
information.append(("sample rates", "%d Hz - %d Hz" %
(np.min(sample_rates), np.max(sample_rates))))
else:
information.append(("sample rate", "%d Hz" % sample_rates[0]))
if len(np.unique(channels)) > 1:
information.append(
("channels", "%d - %d" % (np.min(channels), np.max(channels))))
else:
information.append(("channels", channels[0]))
TableFormatter("lr").print(data=information,
header="data set information")
def take_action(self, parsed_args):
if (parsed_args.chunk_count is not None) ^ (parsed_args.chunk_length
is not None):
raise ValueError(
"--chunk-count can only be used with --chunk-length and vice-versa"
)
module_name, class_name = parsed_args.parser.rsplit(".", 1)
parser_class = getattr(importlib.import_module(module_name),
class_name)
if not issubclass(parser_class, Parser):
raise ValueError(
"specified parser does not inherit audeep.backend.parsers.Parser"
)
parser = parser_class(parsed_args.basedir)
preprocessor = self.get_preprocessor(parsed_args)
if not parser.can_parse():
raise ValueError(
"specified parser is unable to parse data set at {}".format(
parsed_args.basedir))
self.log.info("parsing data set at %s", parsed_args.basedir)
instance_metadata = parser.parse()
if parsed_args.pretend is not None:
metadata = instance_metadata[
parsed_args.pretend] # type: _InstanceMetadata
sxx, f, t = preprocessor.process(metadata.path)
# noinspection PyTypeChecker
spectrogram_info = [("data file", metadata.path)]
if parser.label_map is not None:
spectrogram_info.append(("label", "{} ({})".format(
metadata.label_nominal,
parser.label_map[metadata.label_nominal])))
else:
spectrogram_info.append(
("label", "{} ({})".format(metadata.label_nominal,
metadata.label_numeric)))
if metadata.cv_folds:
# noinspection PyTypeChecker
spectrogram_info.append(
("cross validation splits",
",".join([x.name for x in metadata.cv_folds])))
if metadata.partition is not None:
spectrogram_info.append(("partition", metadata.partition.name))
# noinspection PyTypeChecker
spectrogram_info.append(("number of chunks", len(sxx)))
spectrogram_info.append(
("spectrogram time steps", [x.shape[1] for x in sxx]))
spectrogram_info.append(
("spectrogram frequency bands", f.shape[0]))
TableFormatter().print(data=spectrogram_info)
fig = plt.figure()
sxx_full = np.concatenate(sxx, axis=1)
t_full = np.concatenate(t)
nxticks = sxx_full.shape[1] // 25
nyticks = 4
# spectrogram
ax = fig.add_subplot(2, 1, 1)
plt.title("Spectrogram")
ax.set_xticks(
np.arange(0, t_full.shape[0], t_full.shape[0] // nxticks))
ax.set_xticklabels(np.round(t_full[::t_full.shape[0] // nxticks]))
ax.set_xlabel("Time (s)")
ax.set_yticks(np.arange(0, len(f), len(f) // nyticks))
ax.set_yticklabels(np.round(f[::-len(f) // nyticks]))
ax.set_ylabel("Frequency (Hz)")
ax.imshow(sxx_full[::-1], cmap="magma")
# histogram
ax = fig.add_subplot(2, 1, 2)
plt.title("Amplitude Histogram")
ax.set_xlabel("Amplitude (dB)")
ax.set_ylabel("Probability")
range_min = parsed_args.clip_below + 0.01 if parsed_args.clip_below is not None else sxx_full.min(
)
range_max = parsed_args.clip_above - 0.01 if parsed_args.clip_above is not None else 0
ax.hist(sxx_full.flatten(),
range=(range_min, range_max),
bins=100,
normed=True,
histtype="stepfilled")
plt.tight_layout()
plt.show()
else:
num_instances = parser.num_instances * (
1 if parsed_args.chunk_count is None else
parsed_args.chunk_count)
data_set = None
index = 0
for file_index, metadata in enumerate(
instance_metadata): # type: Tuple[int, _InstanceMetadata]
self.log.info(
"processing %%s (%%%dd/%%d)" %
int(math.ceil(math.log10(len(instance_metadata)))),
metadata.path, file_index + 1, len(instance_metadata))
sxx, _, _ = preprocessor.process(metadata.path)
for chunk_nr, sxx_chunk in enumerate(sxx):
if data_set is None:
data_set = empty(num_instances=num_instances,
feature_dimensions=[
("time", sxx_chunk.shape[1]),
("freq", sxx_chunk.shape[0])
],
num_folds=parser.num_folds)
data_set.label_map = parser.label_map
instance = data_set[index]
instance.filename = metadata.filename
instance.chunk_nr = chunk_nr
instance.label_nominal = metadata.label_nominal
if data_set.label_map is None:
instance.label_numeric = metadata.label_numeric
instance.cv_folds = metadata.cv_folds
instance.partition = metadata.partition
instance.features = np.transpose(sxx_chunk)
index += 1
data_set.save(parsed_args.output)
def take_action(self, parsed_args):
if not parsed_args.input.exists():
raise IOError("unable to open data set at {}".format(
parsed_args.input))
data_set = load(parsed_args.input)
formatter = TableFormatter()
# print global information
global_information = [
("number of instances", data_set.num_instances),
("cross validation info", data_set.has_cv_info),
("partition info", data_set.has_partition_info),
("fully labeled", data_set.is_fully_labeled),
("feature dimensions", data_set.feature_dims),
]
print()
formatter.print(data=global_information,
header="global data set information")
print()
# print instance information
if parsed_args.instance is not None:
instance = data_set[parsed_args.instance]
instance_information = [
("data file", instance.filename),
("chunk number", instance.chunk_nr),
("label", "{} ({})".format(instance.label_nominal,
instance.label_numeric)),
("cross validation splits", ", ".join([
"None" if x is None else x.name for x in instance.cv_folds
]) or None),
("partition", None
if instance.partition is None else instance.partition.name),
("shape", instance.feature_shape),
]
formatter.print(data=instance_information,
header="instance {} information:".format(
parsed_args.instance))
print()
if parsed_args.detailed_folds and data_set.has_cv_info and data_set.is_fully_labeled:
formatter = TableFormatter(alignment="lrrrr")
inverse_label_map = dict(map(reversed, data_set.label_map.items()))
for fold in range(data_set.num_folds):
train_split = data_set.split(fold, Split.TRAIN)
valid_split = data_set.split(fold, Split.VALID)
labels, train_counts = np.unique(train_split.labels_numeric,
return_counts=True)
_, valid_counts = np.unique(valid_split.labels_numeric,
return_counts=True)
train_total = sum(train_counts)
valid_total = sum(valid_counts)
fold_information = []
for i in range(len(labels)):
train_count = train_counts[i]
valid_count = valid_counts[i]
train_relative = 100 * train_count / train_total
valid_relative = 100 * valid_count / valid_total
fold_information.append(
(inverse_label_map[labels[i]], train_count,
"%2.2f%%" % train_relative, valid_count,
"%2.2f%%" % valid_relative))
fold_information.append(
("total", train_total, "", valid_total, ""))
formatter.print(data=fold_information,
header="fold {} information:".format(fold + 1),
dividers=[len(labels) - 1])
print()