def get_learner_params(parameters_filename, parameter_set):
# Initialize ParameterContainer
params = ParameterContainer(
project_base=os.path.dirname(os.path.realpath(__file__)))
# Load default parameters from a file
params.load(filename=parameters_filename)
params['active_set'] = parameter_set
# Process parameters
params.process()
return DottedDict(params.get_path('learner')['parameters'])
def learner_batcher_params(parameter_set):
# Load default parameters from a file
default_parameters_filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'parameters',
'task1' + '.defaults.yaml')
# Initialize ParameterContainer
params = ParameterContainer(project_base=os.path.dirname(os.path.realpath(__file__)))
# Load default parameters from a file
params.load(filename=default_parameters_filename)
params['active_set'] = parameter_set
# Process parameters
params.process()
learner_params = DottedDict(params.get_path('learner')['parameters'])
return learner_params
def system_evaluation(self):
"""System evaluation stage.
Testing outputs are collected and evaluated.
Parameters
----------
Returns
-------
None
Raises
-------
IOError
Result file not found
"""
if not self.dataset.reference_data_present:
return ' No reference data available for dataset.'
else:
output = ''
if self.params.get_path(
'evaluator.scene_handling') == 'scene-dependent':
# reset groundtruth and prediction files
with open("groundtruth.txt", "w") as groundtruth_file:
groundtruth_file.write("")
groundtruth_file.close()
with open("prediction.txt", "w") as prediction_file:
prediction_file.write("")
prediction_file.close()
tagging_overall_metrics_per_scene = {}
event_overall_metrics_per_scene = {}
for scene_id, scene_label in enumerate(
self.dataset.scene_labels):
if scene_label not in event_overall_metrics_per_scene:
event_overall_metrics_per_scene[scene_label] = {}
segment_based_metric = sed_eval.sound_event.SegmentBasedMetrics(
event_label_list=self.dataset.event_labels(
scene_label=scene_label),
time_resolution=1.0,
)
event_based_metric = sed_eval.sound_event.EventBasedMetrics(
event_label_list=self.dataset.event_labels(
scene_label=scene_label),
evaluate_onset=True,
evaluate_offset=False,
t_collar=0.5,
percentage_of_length=0.5)
for fold in self._get_active_folds():
result_filename = self._get_result_filename(
fold=fold,
scene_label=scene_label,
path=self.params.get_path('path.recognizer'))
results = MetaDataContainer().load(
filename=result_filename)
for file_id, audio_filename in enumerate(
self.dataset.test(
fold, scene_label=scene_label).file_list):
# Subtask A (audio tagging)
# Subtask B (sound event detection)
# Select only row which are from current file and contains only detected event
current_file_results = []
for result_item in results.filter(
filename=posix_path(
self.dataset.absolute_to_relative(
audio_filename))):
if 'event_label' in result_item and result_item.event_label:
current_file_results.append(result_item)
meta = []
for meta_item in self.dataset.file_meta(
filename=posix_path(
self.dataset.absolute_to_relative(
audio_filename))):
if 'event_label' in meta_item and meta_item.event_label:
meta.append(meta_item)
for item in meta:
# Actual
item = str(item)
item1 = ""
if self.setup_label == 'Evaluation setup':
item1 = item.split('|')[0].lstrip()
else:
item1 = item.split('|')[0].split(
'audio/')[1].lstrip()
item2 = item.split('|')[2].lstrip()
item3 = item.split('|')[3].lstrip()
item4 = item.split('|')[4].lstrip()
with open('groundtruth.txt', 'a') as file1:
file1.write(
str(item1) + str("\t") + str(item2) +
str("\t") + str(item3) + str("\t") +
str(item4) + str('\n'))
file1.close()
for item in current_file_results:
# Predicted
item = str(item)
item1 = ""
if self.setup_label == 'Evaluation setup':
item1 = item.split('|')[0].lstrip()
else:
item1 = item.split('|')[0].split(
'audio/')[1].lstrip()
item2 = item.split('|')[2].lstrip()
item3 = item.split('|')[3].lstrip()
item4 = item.split('|')[4].lstrip()
with open('prediction.txt', 'a') as file2:
file2.write(
str(item1) + str("\t") + str(item2) +
str("\t") + str(item3) + str("\t") +
str(item4) + str('\n'))
file2.close()
segment_based_metric.evaluate(
reference_event_list=meta,
estimated_event_list=current_file_results)
event_based_metric.evaluate(
reference_event_list=meta,
estimated_event_list=current_file_results)
# from IPython import embed
# embed()
event_overall_metrics_per_scene[scene_label][
'segment_based_metrics'] = segment_based_metric.results(
)
event_overall_metrics_per_scene[scene_label][
'event_based_metrics'] = event_based_metric.results()
if self.params.get_path('evaluator.show_details', False):
output += " Scene [{scene}], Evaluation over {folds:d} folds\n".format(
scene=scene_label, folds=self.dataset.fold_count)
output += " \n"
output += segment_based_metric.result_report_overall()
output += segment_based_metric.result_report_class_wise(
)
event_overall_metrics_per_scene = DottedDict(
event_overall_metrics_per_scene)
output += " \n"
output += " Subtask B (event detection): Overall metrics \n"
output += " =============== \n"
output += " {event_label:<17s} | {segment_based_fscore:7s} | {segment_based_er:7s} | {event_based_fscore:7s} | {event_based_er:7s} | \n".format(
event_label='Event label',
segment_based_fscore='Seg. F1',
segment_based_er='Seg. ER',
event_based_fscore='Evt. F1',
event_based_er='Evt. ER',
)
output += " {event_label:<17s} + {segment_based_fscore:7s} + {segment_based_er:7s} + {event_based_fscore:7s} + {event_based_er:7s} + \n".format(
event_label='-' * 17,
segment_based_fscore='-' * 7,
segment_based_er='-' * 7,
event_based_fscore='-' * 7,
event_based_er='-' * 7,
)
avg = {
'segment_based_fscore': [],
'segment_based_er': [],
'event_based_fscore': [],
'event_based_er': [],
}
for scene_id, scene_label in enumerate(
self.dataset.scene_labels):
output += " {scene_label:<17s} | {segment_based_fscore:<7s} | {segment_based_er:<7s} | {event_based_fscore:<7s} | {event_based_er:<7s} | \n".format(
scene_label=scene_label,
segment_based_fscore="{:4.2f}".format(
event_overall_metrics_per_scene.get_path(
scene_label +
'.segment_based_metrics.overall.f_measure.f_measure'
) * 100),
segment_based_er="{:4.2f}".format(
event_overall_metrics_per_scene.get_path(
scene_label +
'.segment_based_metrics.overall.error_rate.error_rate'
)),
event_based_fscore="{:4.2f}".format(
event_overall_metrics_per_scene.get_path(
scene_label +
'.event_based_metrics.overall.f_measure.f_measure'
) * 100),
event_based_er="{:4.2f}".format(
event_overall_metrics_per_scene.get_path(
scene_label +
'.event_based_metrics.overall.error_rate.error_rate'
)),
)
avg['segment_based_fscore'].append(
event_overall_metrics_per_scene.get_path(
scene_label +
'.segment_based_metrics.overall.f_measure.f_measure'
) * 100)
avg['segment_based_er'].append(
event_overall_metrics_per_scene.get_path(
scene_label +
'.segment_based_metrics.overall.error_rate.error_rate'
))
avg['event_based_fscore'].append(
event_overall_metrics_per_scene.get_path(
scene_label +
'.event_based_metrics.overall.f_measure.f_measure')
* 100)
avg['event_based_er'].append(
event_overall_metrics_per_scene.get_path(
scene_label +
'.event_based_metrics.overall.error_rate.error_rate'
))
output += " {scene_label:<17s} + {segment_based_fscore:7s} + {segment_based_er:7s} + {event_based_fscore:7s} + {event_based_er:7s} + \n".format(
scene_label='-' * 17,
segment_based_fscore='-' * 7,
segment_based_er='-' * 7,
event_based_fscore='-' * 7,
event_based_er='-' * 7,
)
output += " {scene_label:<17s} | {segment_based_fscore:<7s} | {segment_based_er:<7s} | {event_based_fscore:<7s} | {event_based_er:<7s} | \n".format(
scene_label='Average',
segment_based_fscore="{:4.2f}".format(
numpy.mean(avg['segment_based_fscore'])),
segment_based_er="{:4.2f}".format(
numpy.mean(avg['segment_based_er'])),
event_based_fscore="{:4.2f}".format(
numpy.mean(avg['event_based_fscore'])),
event_based_er="{:4.2f}".format(
numpy.mean(avg['event_based_er'])),
)
output += " \n"
output += " Subtask A (tagging): Overall metrics \n"
output += " =============== \n"
# Insert audio tagging evaluation results here
GroundTruthDS = FileFormat('groundtruth.txt')
PredictedDS = FileFormat('prediction.txt')
output += GroundTruthDS.computeMetricsString(PredictedDS)
output += "\n"
elif self.params.get_path(
'evaluator.scene_handling') == 'scene-independent':
message = '{name}: Scene handling mode not implemented yet [{mode}]'.format(
name=self.__class__.__name__,
mode=self.params.get_path('evaluator.scene_handling'))
self.logger.exception(message)
raise ValueError(message)
else:
message = '{name}: Unknown scene handling mode [{mode}]'.format(
name=self.__class__.__name__,
mode=self.params.get_path('evaluator.scene_handling'))
self.logger.exception(message)
raise ValueError(message)
if self.params.get_path('evaluator.saving.enable'):
filename = self.params.get_path(
'evaluator.saving.filename').format(
dataset_name=self.dataset.storage_name,
parameter_set=self.params['active_set'],
parameter_hash=self.params['_hash'])
output_file = os.path.join(
self.params.get_path('path.evaluator'), filename)
output_data = {
'overall_metrics_per_scene':
event_overall_metrics_per_scene,
'average': {
'segment_based_fscore':
numpy.mean(avg['segment_based_fscore']),
'segment_based_er':
numpy.mean(avg['segment_based_er']),
'event_based_fscore':
numpy.mean(avg['event_based_fscore']),
'event_based_er':
numpy.mean(avg['event_based_er']),
},
'parameters': dict(self.params)
}
ParameterFile(output_data, filename=output_file).save()
with open("TaskB_metrics", "w") as file1:
file1.write(output)
file1.close()
return output
def create_model(learner_params, input_shape):
from keras.models import Sequential
model = Sequential()
model_params = copy.deepcopy(learner_params.get_path('model.config'))
if 'conv_kernel' in learner_params['model']:
kernel_params = learner_params['model']['conv_kernel']
kernel = getattr(importlib.import_module("keras.initializers"), kernel_params['class'])
kernel = kernel(**kernel_params['config'])
set_kernel = True
for layer_id, layer_setup in enumerate(model_params):
layer_setup = DottedDict(layer_setup)
try:
LayerClass = getattr(importlib.import_module("keras.layers"), layer_setup['class_name'])
except AttributeError:
message = '{name}: Invalid Keras layer type [{type}].'.format(
name=__class__.__name__,
type=layer_setup['class_name']
)
raise AttributeError(message)
if 'config' not in layer_setup:
layer_setup['config'] = {}
# Set layer input
if layer_id == 0 and layer_setup.get_path('config.input_shape') is None:
# Set input layer dimension for the first layer if not set
if layer_setup.get('class_name') == 'Dropout':
layer_setup['config']['input_shape'] = (input_shape,)
else:
layer_setup['config']['input_shape'] = input_shape
elif layer_setup.get_path('config.input_dim') == 'FEATURE_VECTOR_LENGTH':
layer_setup['config']['input_dim'] = input_shape
# Set layer output
if layer_setup.get_path('config.units') == 'CLASS_COUNT':
layer_setup['config']['units'] = len(class_labels)
# Set kernel initializer
if 'kernel_initializer' in layer_setup.get('config') and set_kernel:
layer_setup.get('config')['kernel_initializer'] = kernel
if layer_setup.get('config'):
model.add(LayerClass(**dict(layer_setup.get('config'))))
else:
model.add(LayerClass())
try:
OptimizerClass = getattr(importlib.import_module("keras.optimizers"),
learner_params.get_path('model.optimizer.type')
)
except AttributeError:
message = '{name}: Invalid Keras optimizer type [{type}].'.format(
name=__class__.__name__,
type=learner_params.get_path('model.optimizer.type')
)
raise AttributeError(message)
model.compile(
loss=learner_params.get_path('model.loss'),
optimizer=OptimizerClass(**dict(learner_params.get_path('model.optimizer.parameters', {}))),
metrics=learner_params.get_path('model.metrics')
)
return model