def prepare(self): """Prepare dataset for the usage. Returns ------- self """ if not self.meta_container.exists(): scene_label = 'home' evaluation_chunks = ListDictContainer( filename=os.path.join(self.local_path, 'chime_home', 'evaluation_chunks_refined.csv')).load( fields=['id', 'filename', 'set_id']) audio_files = {} for item in dcase_cross_val_data: audio_filename = os.path.join( 'chime_home', 'chunks', item['filename'] + self.sample_mode + '.wav') annotation_filename = os.path.join('chime_home', 'chunks', item['filename'] + '.csv') if audio_filename not in audio_files: audio_files[audio_filename] = { 'audio': audio_filename, 'meta': annotation_filename } meta_data = MetaDataContainer() for audio_filename, data in iteritems(audio_files): current_meta_data = DictContainer(filename=os.path.join( self.local_path, data['meta'])).load() tags = [] for i, tag in enumerate(current_meta_data['majorityvote']): if tag != 'S' and tag != 'U': tags.append(self.tagcode_to_taglabel(tag)) name = os.path.split(audio_filename)[1] segment_name = name[0:name.find('_chunk')] chunk_name = name[name.find('_chunk') + 1:].split('.')[0] item = MetaDataItem({ 'filename': audio_filename, 'scene_label': scene_label, 'tags': ';'.join(tags) + ';', 'identifier': segment_name }) self.process_meta_item(item=item, absolute_path=False) meta_data.append(item) # Save meta meta_data.save(filename=self.meta_file) # Load meta and cross validation self.load() all_folds_found = True train_filename = self.evaluation_setup_filename(setup_part='train') test_filename = self.evaluation_setup_filename(setup_part='test') eval_filename = self.evaluation_setup_filename(setup_part='evaluate') if not os.path.isfile(train_filename): all_folds_found = False if not os.path.isfile(test_filename): all_folds_found = False if not os.path.isfile(eval_filename): all_folds_found = False if not all_folds_found: Path().makedirs(path=self.evaluation_setup_path) # Train train_filename = self.evaluation_setup_filename(setup_part='train') train_meta = MetaDataContainer(filename=train_filename) for filename in self.train_files(): train_meta.append(self.file_meta(filename)[0]) train_meta.save() # Test test_filename = self.evaluation_setup_filename(setup_part='test') test_meta = MetaDataContainer(filename=test_filename) for filename in self.test_files(): test_meta.append( MetaDataItem( {'filename': self.absolute_to_relative_path(filename)})) test_meta.save() # Evaluate eval_filename = self.evaluation_setup_filename( setup_part='evaluate') eval_meta = MetaDataContainer(filename=eval_filename) for filename in self.test_files(): eval_meta.append(self.file_meta(filename)[0]) eval_meta.save() # Load meta and cross validation self.load() return self
def prepare(self): """Prepare dataset for the usage. Returns ------- self """ if not self.meta_container.exists(): meta_data = MetaDataContainer() for filename in self.audio_files: raw_path, raw_filename = os.path.split(filename) relative_path = self.absolute_to_relative_path(raw_path) meta_data.append( MetaDataItem({ 'filename': os.path.join(relative_path, raw_filename), 'scene_label': os.path.splitext(os.path.split(filename)[1])[0][:-2], })) meta_data.save(filename=self.meta_file) self.load_meta() all_folds_found = True for fold in self.folds(): train_filename = self.evaluation_setup_filename(setup_part='train', fold=fold) test_filename = self.evaluation_setup_filename(setup_part='test', fold=fold) if not os.path.isfile(train_filename): all_folds_found = False if not os.path.isfile(test_filename): all_folds_found = False if not all_folds_found: Path().makedirs(path=self.evaluation_setup_path) classes = [] files = [] for item in self.meta: classes.append(item.scene_label) files.append(item.filename) files = numpy.array(files) from sklearn.model_selection import StratifiedShuffleSplit sss = StratifiedShuffleSplit(n_splits=self.crossvalidation_folds, test_size=0.3, random_state=0) fold = 1 for train_index, test_index in sss.split(X=numpy.zeros( len(classes)), y=classes): train_files = files[train_index] test_files = files[test_index] train_filename = self.evaluation_setup_filename( setup_part='train', fold=fold) test_filename = self.evaluation_setup_filename( setup_part='test', fold=fold) eval_filename = self.evaluation_setup_filename( setup_part='evaluate', fold=fold) # Create meta containers and save them # Train train_meta = MetaDataContainer(filename=train_filename) for filename in train_files: train_meta += self.meta_container.filter(filename=filename) train_meta.save() # Test test_meta = MetaDataContainer(filename=test_filename) for filename in test_files: test_meta.append( MetaDataItem({ 'filename': self.absolute_to_relative_path(filename) })) test_meta.save() # Evaluate eval_meta = MetaDataContainer(filename=eval_filename) for filename in test_files: eval_meta += self.meta_container.filter(filename=filename) eval_meta.save() fold += 1 # Load meta and cross validation self.load() return self
def prepare(self): """Prepare dataset for the usage. Returns ------- self """ if not self.meta_container.exists(): scene_label = 'home' dcase_cross_val_data = ListDictContainer(filename=os.path.join( self.local_path, 'chime_home', 'development_chunks_refined_crossval_dcase2016.csv')).load( fields=['id', 'filename', 'set_id']) audio_files = {} for item in dcase_cross_val_data: audio_filename = os.path.join( 'chime_home', 'chunks', item['filename'] + self.sample_mode + '.wav') annotation_filename = os.path.join('chime_home', 'chunks', item['filename'] + '.csv') if audio_filename not in audio_files: audio_files[audio_filename] = { 'audio': audio_filename, 'meta': annotation_filename } meta_data = MetaDataContainer() for audio_filename, data in iteritems(audio_files): current_meta_data = DictContainer(filename=os.path.join( self.local_path, data['meta'])).load() tags = [] for i, tag in enumerate(current_meta_data['majorityvote']): if tag != 'S' and tag != 'U': tags.append(self.tagcode_to_taglabel(tag)) name = os.path.split(audio_filename)[1] segment_name = name[0:name.find('_chunk')] chunk_name = name[name.find('_chunk') + 1:].split('.')[0] item = MetaDataItem({ 'filename': audio_filename, 'scene_label': scene_label, 'tags': ';'.join(tags) + ';', 'identifier': segment_name }) self.process_meta_item(item=item, absolute_path=False) meta_data.append(item) # Save meta meta_data.save(filename=self.meta_file) # Load meta and cross validation self.load() all_folds_found = True for fold in range(1, self.crossvalidation_folds + 1): train_filename = self.evaluation_setup_filename(setup_part='train', fold=fold) test_filename = self.evaluation_setup_filename(setup_part='test', fold=fold) eval_filename = self.evaluation_setup_filename( setup_part='evaluate', fold=fold) if not os.path.isfile(train_filename): all_folds_found = False if not os.path.isfile(test_filename): all_folds_found = False if not os.path.isfile(eval_filename): all_folds_found = False if not all_folds_found: Path().makedirs(path=self.evaluation_setup_path) dcase_crossval = { 1: [], 2: [], 3: [], 4: [], 5: [], } dcase_cross_val_data = ListDictContainer(filename=os.path.join( self.local_path, 'chime_home', 'development_chunks_refined_crossval_dcase2016.csv')).load( fields=['id', 'filename', 'set_id']) for item in dcase_cross_val_data: dcase_crossval[int(item['set_id']) + 1].append( self.relative_to_absolute_path( os.path.join( 'chime_home', 'chunks', item['filename'] + self.sample_mode + '.wav'))) for fold in range(1, self.crossvalidation_folds + 1): # Collect training and testing files train_files = [] for f in range(1, self.crossvalidation_folds + 1): if f is not fold: train_files += dcase_crossval[f] test_files = dcase_crossval[fold] # Create meta containers and save them # Train train_filename = self.evaluation_setup_filename( setup_part='train', fold=fold) train_meta = MetaDataContainer(filename=train_filename) for filename in train_files: item = self.file_meta(filename)[0] self.process_meta_item(item=item, absolute_path=False) train_meta.append(item) train_meta.save() # Test test_filename = self.evaluation_setup_filename( setup_part='test', fold=fold) test_meta = MetaDataContainer(filename=test_filename) for filename in test_files: item = MetaDataItem( {'filename': self.absolute_to_relative_path(filename)}) test_meta.append(item) test_meta.save() # Evaluate eval_filename = self.evaluation_setup_filename( setup_part='evaluate', fold=fold) eval_meta = MetaDataContainer(filename=eval_filename) for filename in test_files: item = self.file_meta(filename)[0] self.process_meta_item(item=item, absolute_path=False) eval_meta.append(item) eval_meta.save() # Load meta and cross validation self.load() return self
def extract_packages(self): """Extract the dataset packages Raises ------ IOError Local package was not found. Returns ------- self """ # Make sure evaluation_setup directory exists Path().makedirs( path=os.path.join(self.local_path, self.evaluation_setup_folder)) log = FancyLogger() item_access_log_filename = os.path.join(self.local_path, 'item_access_error.log.csv') if 'audio' in self.included_content_types or self.included_content_types == [ 'all' ]: # mean process audio log.title("Download_data") log.info( "Once database is downloaded, do not forget to check your missing_files" ) non_existing_videos = pandas.DataFrame( columns=["filename", "error"]) log.line("check files exist or download data") # Collect file ids for package in self.package_list: if package.get('content_type') == "meta": base_filepath = os.path.splitext( package.get('filename').split('/')[-1])[0] if 'train' in package.get('filename'): result_audio_directory = os.path.join( self.local_path, 'dataset/audio/train', base_filepath) else: result_audio_directory = os.path.join( self.local_path, 'dataset/audio/test') missing_files = download(package.get('filename'), result_audio_directory, n_jobs=3) if not missing_files.empty: non_existing_videos = non_existing_videos.append( missing_files, ignore_index=True) # Save list of non-accessible videos ListDictContainer(non_existing_videos.to_dict(orient="records"), filename=item_access_log_filename).save( fields=['filename', 'error']) # Evaluation setup filenames train_filename_fold1 = self.evaluation_setup_filename( setup_part='train', fold=1, file_extension='csv') test_filename_fold1 = self.evaluation_setup_filename( setup_part='test', fold=1, file_extension='csv') train_filename_fold2 = self.evaluation_setup_filename( setup_part='train', fold=2, file_extension='csv') test_filename_fold2 = self.evaluation_setup_filename( setup_part='test', fold=2, file_extension='csv') evaluate_filename = self.evaluation_setup_filename( setup_part='evaluate', fold=2, file_extension='csv') # Check that evaluation setup exists evaluation_setup_exists = True if not os.path.isfile(train_filename_fold1) or not os.path.isfile(test_filename_fold1) \ or not os.path.isfile(train_filename_fold2) or not os.path.isfile(test_filename_fold2) \ or not os.path.isfile(evaluate_filename) or not self.meta_container.exists(): evaluation_setup_exists = False if not evaluation_setup_exists: # Evaluation setup was not found, generate one item_access_log_filename = os.path.join( self.local_path, 'item_access_error.log.csv') non_existing_videos = ListDictContainer().load( filename=item_access_log_filename, delimiter=',').get_field_unique('filename') train_meta_weak_fold1 = MetaDataContainer() audio_path = 'dataset/audio/train/weak' for item in MetaDataContainer().load(os.path.join( self.local_path, 'dataset/metadata/train/' 'weak.csv'), fields=["filename", "tags"], csv_header=True): if item.filename not in non_existing_videos: if not item.filename.endswith( self.default_audio_extension): item.filename = os.path.join( audio_path, os.path.splitext(item.filename)[0] + '.' + self.default_audio_extension) else: item.filename = Path(path=item.filename).modify( path_base=audio_path) # Only collect items which exists if audio present if 'audio' in self.included_content_types or 'all' in self.included_content_types: if os.path.isfile( os.path.join(self.local_path, item.filename)): train_meta_weak_fold1.append(item) else: train_meta_weak_fold1.append(item) train_meta_weak_fold1.save(filename=train_filename_fold1, csv_header=True, file_format="CSV") test_meta_unlabel_fold1 = MetaDataContainer() audio_path = 'dataset/audio/train/unlabel_in_domain' for item in MetaDataContainer().load(os.path.join( self.local_path, 'dataset/metadata/train/' 'unlabel_in_domain.csv'), csv_header=True): if item.filename not in non_existing_videos: # If not the right extension, change it if not item.filename.endswith( self.default_audio_extension): item.filename = os.path.join( audio_path, os.path.splitext(item.filename)[0] + '.' + self.default_audio_extension) else: item.filename = Path(path=item.filename).modify( path_base=audio_path) # Only collect items which exists if audio present if 'audio' in self.included_content_types or 'all' in self.included_content_types: if os.path.isfile( os.path.join(self.local_path, item.filename)): test_meta_unlabel_fold1.append(item) else: test_meta_unlabel_fold1.append(item) test_meta_unlabel_fold1.save(filename=test_filename_fold1, csv_header=True, file_format="CSV") # Fold 2 train is all the data used in fold 1 train_meta_weak_fold2 = MetaDataContainer() train_meta_weak_fold2 += MetaDataContainer().load( train_filename_fold1, csv_header=True, file_format="CSV") for item in MetaDataContainer().load(test_filename_fold1, csv_header=True, file_format="CSV"): item.tags = [] train_meta_weak_fold2.append(item) train_meta_weak_fold2.save(filename=train_filename_fold2, csv_header=True) # Evaluate meta is the groundtruth file with test annotations test.csv evaluate_meta = MetaDataContainer() audio_path = 'dataset/audio/test' for item in MetaDataContainer().load(os.path.join( self.local_path, 'dataset/metadata/test/test.csv'), csv_header=True): if item.filename not in non_existing_videos: if not item.filename.endswith( self.default_audio_extension): item.filename = os.path.join( audio_path, os.path.splitext(item.filename)[0] + '.' + self.default_audio_extension) else: item.filename = Path(path=item.filename).modify( path_base=audio_path) # Only collect items which exists if 'audio' in self.included_content_types or 'all' in self.included_content_types: if os.path.isfile( os.path.join(self.local_path, item.filename)): evaluate_meta.append(item) else: evaluate_meta.append(item) evaluate_meta.save(filename=evaluate_filename, csv_header=True, file_format="CSV") # Test meta is filenames of evaluation, labels will be predicted test_meta_strong_fold2 = MetaDataContainer() for filename in evaluate_meta.unique_files: test_meta_strong_fold2.append( MetaDataItem({'filename': filename})) test_meta_strong_fold2.save(filename=test_filename_fold2, csv_header=True, file_format="CSV") # meta_data is the default meta container containing all files of the dataset meta_data = MetaDataContainer() meta_data += MetaDataContainer().load(train_filename_fold1, csv_header=True, file_format="CSV") meta_data += MetaDataContainer().load(test_filename_fold1, csv_header=True, file_format="CSV") meta_data += MetaDataContainer().load(test_filename_fold2, csv_header=True, file_format="CSV") # Save meta meta_data.save(filename=self.meta_file) log.foot() return self
def prepare(self): """Prepare dataset for the usage. Returns ------- self """ # Make sure evaluation_setup directory exists Path().makedirs( path=os.path.join(self.local_path, self.evaluation_setup_folder)) reference_data_file = os.path.join( self.local_path, 'groundtruth_strong_label_evaluation_set.csv') if not self.meta_container.exists() and os.path.exists( reference_data_file): # Reference data is present and but meta data is empty meta_data = MetaDataContainer() ref_data = MetaDataContainer().load(filename=reference_data_file) for item in ref_data: # Modify audio file path item.filename = os.path.join( 'Y' + os.path.splitext(item.filename)[0] + '.' + self.default_audio_extension) # Set scene label item.scene_label = 'youtube' # Only collect items which exists if os.path.isfile(os.path.join(self.local_path, item.filename)): meta_data.append(item) # Save meta data meta_data.save(filename=self.meta_container.filename) # Load meta and cross validation self.load() test_filename = self.evaluation_setup_filename(setup_part='test', scene_label='youtube', file_extension='txt') evaluate_filename = self.evaluation_setup_filename( setup_part='evaluate', scene_label='youtube', file_extension='txt') # Check that evaluation setup exists evaluation_setup_exists = True if not os.path.isfile(test_filename) or not os.path.isfile( evaluate_filename): evaluation_setup_exists = False if not evaluation_setup_exists: if os.path.exists(reference_data_file): ref_data = MetaDataContainer().load( filename=reference_data_file) evaluate_meta = MetaDataContainer() for item in ref_data: # Modify audio file path if not item.filename.endswith( self.default_audio_extension): item.filename = os.path.join( 'audio', 'Y' + os.path.splitext(item.filename)[0] + '.' + self.default_audio_extension) # Set scene label item.scene_label = 'youtube' self.process_meta_item(item=item, absolute_path=False) evaluate_meta.append(item) evaluate_meta.save(filename=self.evaluation_setup_filename( setup_part='evaluate', scene_label='youtube', file_extension='txt')) audio_files = Path().file_list(path=self.local_path, extensions=self.audio_extensions) test_meta = MetaDataContainer() for audio_file in audio_files: item = MetaDataItem({ 'filename': os.path.split(audio_file)[1], 'scene_label': 'youtube' }) self.process_meta_item(item=item, absolute_path=False) test_meta.append(item) test_meta.save(filename=self.evaluation_setup_filename( setup_part='test', scene_label='youtube', file_extension='txt')) # Load meta and cross validation self.load() return self
def prepare(self): """Prepare dataset for the usage. Returns ------- self """ if is_jupyter(): from tqdm import tqdm_notebook as tqdm else: from tqdm import tqdm # Make sure audio directory exists Path().makedirs(path=os.path.join(self.local_path, 'audio')) # Make sure evaluation_setup directory exists Path().makedirs( path=os.path.join(self.local_path, self.evaluation_setup_folder)) if 'audio' in self.included_content_types: # Collect file ids files = [] files += ListDictContainer(filename=os.path.join( self.local_path, 'testing_set.csv')).load( fields=['query_id', 'segment_start', 'segment_end']) files += ListDictContainer(filename=os.path.join( self.local_path, 'training_set.csv')).load( fields=['query_id', 'segment_start', 'segment_end']) file_progress = tqdm(files, desc="{0: <25s}".format('Files'), file=sys.stdout, leave=False, disable=self.disable_progress_bar, ascii=self.use_ascii_progress_bar) non_existing_videos = {} # Load list of already identified non-accessible videos item_access_log_filename = os.path.join( self.local_path, 'item_access_error.log.csv') if os.path.isfile(item_access_log_filename): for item in ListDictContainer( filename=item_access_log_filename).load( fields=['query_id', 'error']): non_existing_videos[item['query_id']] = item # Check that audio files exists for file_data in file_progress: audio_filename = os.path.join( self.local_path, 'audio', 'Y{query_id}_{segment_start}_{segment_end}.{extension}'. format(query_id=file_data['query_id'], segment_start=file_data['segment_start'], segment_end=file_data['segment_end'], extension=self.default_audio_extension)) # Download segment if it does not exists if not os.path.isfile(audio_filename) and file_data[ 'query_id'] not in non_existing_videos: try: AudioContainer().load_from_youtube( query_id=file_data['query_id'], start=file_data['segment_start'], stop=file_data['segment_end']).save( filename=audio_filename) except IOError as e: non_existing_videos[file_data['query_id']] = { 'error': str(e.message).replace('\n', ' '), 'query_id': file_data['query_id'] } # Save list of non-accessible videos ListDictContainer(list(non_existing_videos.values()), filename=item_access_log_filename).save( fields=['query_id', 'error']) # Evaluation setup filenames train_filename = self.evaluation_setup_filename(setup_part='train', fold=1, scene_label='youtube', file_extension='txt') test_filename = self.evaluation_setup_filename(setup_part='test', fold=1, scene_label='youtube', file_extension='txt') evaluate_filename = self.evaluation_setup_filename( setup_part='evaluate', fold=1, scene_label='youtube', file_extension='txt') # Check that evaluation setup exists evaluation_setup_exists = True if not os.path.isfile(train_filename) or not os.path.isfile( test_filename) or not os.path.isfile(evaluate_filename): evaluation_setup_exists = False if not evaluation_setup_exists: # Evaluation setup was not found, generate one fold = 1 train_meta = MetaDataContainer() for item in MetaDataContainer().load( os.path.join(self.local_path, 'groundtruth_weak_label_training_set.csv')): if not item.filename.endswith(self.default_audio_extension): item.filename = os.path.join( 'audio', 'Y' + os.path.splitext(item.filename)[0] + '.' + self.default_audio_extension) # Set scene label item.scene_label = 'youtube' # Translate event onset and offset, weak labels item.offset -= item.onset item.onset -= item.onset # Only collect items which exists if audio present if 'audio' in self.included_content_types: if os.path.isfile( os.path.join(self.local_path, item.filename)): train_meta.append(item) else: train_meta.append(item) train_meta.save( filename=self.evaluation_setup_filename(setup_part='train', fold=fold, scene_label='youtube', file_extension='txt')) evaluate_meta = MetaDataContainer() for item in MetaDataContainer().load( os.path.join(self.local_path, 'groundtruth_strong_label_testing_set.csv')): if not item.filename.endswith(self.default_audio_extension): item.filename = os.path.join( 'audio', 'Y' + os.path.splitext(item.filename)[0] + '.' + self.default_audio_extension) # Set scene label item.scene_label = 'youtube' # Only collect items which exists if 'audio' in self.included_content_types: if os.path.isfile( os.path.join(self.local_path, item.filename)): evaluate_meta.append(item) else: evaluate_meta.append(item) evaluate_meta.save( filename=self.evaluation_setup_filename(setup_part='evaluate', fold=fold, scene_label='youtube', file_extension='txt')) test_meta = MetaDataContainer() for item in evaluate_meta: test_meta.append(MetaDataItem({'filename': item.filename})) test_meta.save( filename=self.evaluation_setup_filename(setup_part='test', fold=fold, scene_label='youtube', file_extension='txt')) # Load meta and cross validation self.load() if not self.meta_container.exists(): fold = 1 meta_data = MetaDataContainer() meta_data += MetaDataContainer().load( self.evaluation_setup_filename(setup_part='train', fold=fold, scene_label='youtube', file_extension='txt')) meta_data += MetaDataContainer().load( self.evaluation_setup_filename(setup_part='evaluate', fold=fold, scene_label='youtube', file_extension='txt')) # Save meta meta_data.save(filename=self.meta_file) # Load meta and cross validation self.load() return self
def evaluate_double_threshold( model_path: list, features: str = "features/logmel_64/test.ark", result_filename='dev_double.txt', test_labels: str = "metadata/test/test.csv", threshold=[0.75, 0.2], window=1, hop_size=0.02): from dcase_util.data import ProbabilityEncoder, ManyHotEncoder from dcase_util.containers import MetaDataContainer from thresholding import activity_detection from collections import defaultdict # Put into single list element if model_path is a single string, otherwise evaluate as fusion model_paths = model_path if type(model_path) == list else [model_path] fname_to_probabilities = defaultdict(list) for path in model_paths: model_dirname = os.path.dirname(path) meta_container_resultfile = os.path.join( model_dirname, "label_outputs_double_threshold.txt") metacontainer = MetaDataContainer(filename=meta_container_resultfile) cur_fname_to_probabilities, many_hot_encoder = _forward_model( path, features) for k, v in cur_fname_to_probabilities.items(): fname_to_probabilities[k].append(v) windows = {k: window for k in many_hot_encoder.label_list} if isinstance(window, str): windows = torch.load(window) # Average all the outputs for k, probs in fname_to_probabilities.items(): lengths = tuple(len(prob) for prob in probs) max_length = max(lengths) if len(set(lengths)) != 1: factors = (max_length / np.array(lengths)).astype(int) idxs = np.where(factors != 1)[0] for idx in idxs: probs[idx] = probs[idx].repeat( factors[idx], axis=0) left_over_pads = max_length - (factors[idx] * lengths[idx]) # In case of one array having uneven amount of frames ... pad if left_over_pads != 0: probs[idx] = np.pad( probs[idx], ((0, left_over_pads), (0, 0)), mode='reflect') # Average predictions of the models ( or just return single instance ) fname_to_probabilities[k] = np.mean(probs, axis=0) for k, probabilities in fname_to_probabilities.items(): for i, label in enumerate(many_hot_encoder.label_list): window_size = windows[label] estimated_events = activity_detection( probabilities[:, i], threshold[0], threshold[1], window_size) for [onset, offset] in estimated_events: metacontainer.append({'event_label': label, 'onset': onset * hop_size, 'offset': offset * hop_size, 'filename': os.path.basename(k) }) metacontainer.save() estimated_event_list = MetaDataContainer().load( filename=meta_container_resultfile) reference_event_list = MetaDataContainer().load(filename=test_labels) event_based_metric = event_based_evaluation( reference_event_list, estimated_event_list) onset_scores = precision_recall_fscore_on_offset( reference_event_list, estimated_event_list, offset=False) offset_scores = precision_recall_fscore_on_offset( reference_event_list, estimated_event_list, onset=False) onset_offset_scores = precision_recall_fscore_on_offset( reference_event_list, estimated_event_list) # Utt wise Accuracy precision_labels = precision_recall_fscore_on_offset( reference_event_list, estimated_event_list, onset=False, offset=False, label=True) print(event_based_metric.__str__()) print("{:>10}-Precision: {:.1%} Recall {:.1%} F-Score {:.1%}".format("UttLabel", *precision_labels)) print("{:>10}-Precision: {:.1%} Recall {:.1%} F-Score {:.1%}".format("Onset", *onset_scores)) print("{:>10}-Precision: {:.1%} Recall {:.1%} F-Score {:.1%}".format("Offset", *offset_scores)) print("{:>10}-Precision: {:.1%} Recall {:.1%} F-Score {:.1%}".format("On-Offset", *onset_offset_scores)) result_filename = os.path.join(model_dirname, result_filename) with open(result_filename, 'w') as wp: wp.write(event_based_metric.__str__()) wp.write('\n') wp.write("{:>10}: Precision: {:.1%} Recall {:.1%} F-Score {:.1%}\n".format( "UttLabel", *precision_labels)) wp.write( "{:>10}: Precision: {:.1%} Recall {:.1%} F-Score {:.1%}\n".format("Onset", *onset_scores)) wp.write( "{:>10}: Precision: {:.1%} Recall {:.1%} F-Score {:.1%}\n".format("Offset", *offset_scores)) wp.write("{:>10}: Precision: {:.1%} Recall {:.1%} F-Score {:.1%}\n".format( "On-Offset", *onset_offset_scores))
def evaluate_threshold( model_path: str, features: str = "features/logmel_64/test.ark", result_filename='dev.txt', test_labels: str = "metadata/test/test.csv", threshold=0.5, window=1, hop_size=0.02): from dcase_util.data import ProbabilityEncoder, DecisionEncoder, ManyHotEncoder from dcase_util.containers import MetaDataContainer from scipy.signal import medfilt modeldump = torch.load( model_path, map_location=lambda storage, loc: storage) model = modeldump['model'] config_parameters = modeldump['config'] scaler = modeldump['scaler'] many_hot_encoder = modeldump['encoder'] model_dirname = os.path.dirname(model_path) meta_container_resultfile = os.path.join( model_dirname, "pred_nowindow.txt") metacontainer = MetaDataContainer(filename=meta_container_resultfile) kaldi_string = parsecopyfeats( features, **config_parameters['feature_args']) model = model.to(device).eval() probability_encoder = ProbabilityEncoder() decision_encoder = DecisionEncoder( label_list=many_hot_encoder.label_list ) binarization_type = 'global_threshold' if isinstance( threshold, float) else 'class_threshold' # If class thresholds are given, then use those if isinstance(threshold, str): threshold = torch.load(threshold) windows = {k: window for k in many_hot_encoder.label_list} if isinstance(window, str): windows = torch.load(window) with torch.no_grad(): for k, feat in kaldi_io.read_mat_ark(kaldi_string): # Add batch dim feat = torch.from_numpy( scaler.transform(feat)).to(device).unsqueeze(0) feat = model(feat) probabilities = torch.sigmoid(feat).cpu().numpy().squeeze(0) frame_decisions = probability_encoder.binarization( probabilities=probabilities, binarization_type=binarization_type, threshold=threshold, time_axis=0, ) for i, label in enumerate(many_hot_encoder.label_list): label_frame_decisions = medfilt( frame_decisions[:, i], kernel_size=windows[label]) # Found only zeros, no activity, go on if (label_frame_decisions == 0).all(): continue estimated_events = decision_encoder.find_contiguous_regions( activity_array=label_frame_decisions ) for [onset, offset] in estimated_events: metacontainer.append({'event_label': label, 'onset': onset * hop_size, 'offset': offset * hop_size, 'filename': os.path.basename(k) }) metacontainer.save() estimated_event_list = MetaDataContainer().load( filename=meta_container_resultfile) reference_event_list = MetaDataContainer().load(filename=test_labels) event_based_metric = event_based_evaluation( reference_event_list, estimated_event_list) onset_scores = precision_recall_fscore_on_offset( reference_event_list, estimated_event_list, offset=False) offset_scores = precision_recall_fscore_on_offset( reference_event_list, estimated_event_list, onset=False) onset_offset_scores = precision_recall_fscore_on_offset( reference_event_list, estimated_event_list) # Utt wise Accuracy precision_labels = precision_recall_fscore_on_offset( reference_event_list, estimated_event_list, onset=False, offset=False, label=True) print(event_based_metric.__str__()) print("{:>10}-Precision: {:.1%} Recall {:.1%} F-Score {:.1%}".format("UttLabel", *precision_labels)) print("{:>10}-Precision: {:.1%} Recall {:.1%} F-Score {:.1%}".format("Onset", *onset_scores)) print("{:>10}-Precision: {:.1%} Recall {:.1%} F-Score {:.1%}".format("Offset", *offset_scores)) print("{:>10}-Precision: {:.1%} Recall {:.1%} F-Score {:.1%}".format("On-Offset", *onset_offset_scores)) result_filename = os.path.join(model_dirname, result_filename) with open(result_filename, 'w') as wp: wp.write(event_based_metric.__str__()) wp.write('\n') wp.write("{:>10}: Precision: {:.1%} Recall {:.1%} F-Score {:.1%}\n".format( "UttLabel", *precision_labels)) wp.write( "{:>10}: Precision: {:.1%} Recall {:.1%} F-Score {:.1%}\n".format("Onset", *onset_scores)) wp.write( "{:>10}: Precision: {:.1%} Recall {:.1%} F-Score {:.1%}\n".format("Offset", *offset_scores)) wp.write("{:>10}: Precision: {:.1%} Recall {:.1%} F-Score {:.1%}\n".format( "On-Offset", *onset_offset_scores))