def test__should_filter_by_duration_with_randomness(self): # speaker_id, clipped_utterance_file_name, clipped_utterance_duration, audio_id, snr utterances = [ (1, "file_1.wav", 4, "2010123", 13), (2, "file_2.wav", 2, "2010124", 11), (3, "file_3.wav", 4, "2010125", 18), (4, "file_4.wav", 4, "2010126", 19), (4, "file_4.wav", 2, "2010126", 24), (4, "file_4.wav", 2, "2010126", 24), (5, "file_4.wav", 2, "2010126", 24), (5, "file_4.wav", 2, "2010126", 24), (6, "file_4.wav", 2, "2010126", 24), (6, "file_4.wav", 2, "2010126", 24), (6, "file_4.wav", 2, "2010126", 24), ] data_filter = DataFilter() filtered1 = data_filter.by_duration(utterances, 7 / 3600, "true", 1) filtered2 = data_filter.by_duration(utterances, 7 / 3600, "true", 1) expected_utterances = [ (1, "file_1.wav", 4, "2010123", 13), (2, "file_2.wav", 2, "2010124", 11), ] print(str(filtered1)) print(str(filtered2)) self.assertNotEqual(expected_utterances, filtered1) self.assertNotEqual(filtered1, filtered2)
def test__should_apply_filters_with_by_snr_then_by_speaker(self): utterances = [ (1, "file_10.wav", 4, "1", 13), (1, "file_11.wav", 1, "2", 13), (1, "file_12.wav", 2, "3", 13), (1, "file_13.wav", 4, "4", 13), (2, "file_2.wav", 4, "5", 11), (2, "file_2.wav", 4, "5", 18), (3, "file_3.wav", 2, "6", 18), (3, "file_4.wav", 4, "7", 18), (4, "file_5.wav", 4, "8", 19), (4, "file_6.wav", 4, "9", 24), (4, "file_7.wav", 4, "10", 24), (5, "file_50.wav", 5, "10", 25), (5, "file_51.wav", 4, "10", 26), (5, "file_52.wav", 5, "10", 27), (6, "file_53.wav", 5, "10", 25), (6, "file_54.wav", 4, "10", 26), (6, "file_55.wav", 5, "10", 27), ] filters = { "by_source": "swayamprabha_chapter_30", "by_snr": { "gte": 13, "lte": 26 }, "by_speaker": { "lte_per_speaker_duration": 8 / 60, "gte_per_speaker_duration": 5 / 60, "with_threshold": 0, } # 'then_by_duration': 35 } expected_utterances = [ (1, "file_10.wav", 4, "1", 13), (1, "file_11.wav", 1, "2", 13), (1, "file_12.wav", 2, "3", 13), (3, "file_3.wav", 2, "6", 18), (3, "file_4.wav", 4, "7", 18), (4, "file_5.wav", 4, "8", 19), (4, "file_6.wav", 4, "9", 24), (5, "file_50.wav", 5, "10", 25), (6, "file_53.wav", 5, "10", 25), ] data_filter = DataFilter() filtered = data_filter.apply_filters(filters, utterances) self.assertEqual(type(expected_utterances), type(filtered)) # check they are the same type self.assertEqual(len(expected_utterances), len(filtered)) # check they are the same length self.assertEqual(expected_utterances, filtered)
def test__should_apply_filters_with_by_snr_then_by_duration_randomness( self): utterances = [ (1, "file_10.wav", 4, "1", 13), (1, "file_11.wav", 1, "2", 13), (1, "file_12.wav", 2, "3", 13), (1, "file_13.wav", 4, "4", 13), (2, "file_2.wav", 4, "5", 11), (2, "file_2.wav", 4, "5", 18), (3, "file_3.wav", 2, "6", 18), (3, "file_4.wav", 4, "7", 18), (4, "file_5.wav", 4, "8", 19), (4, "file_6.wav", 4, "9", 24), (4, "file_7.wav", 4, "10", 24), (5, "file_50.wav", 5, "10", 25), (5, "file_51.wav", 4, "10", 26), (5, "file_52.wav", 5, "10", 27), (6, "file_53.wav", 5, "10", 25), (6, "file_54.wav", 4, "10", 26), (6, "file_55.wav", 5, "10", 27), ] filters = { "by_source": "swayamprabha_chapter_30", "by_snr": { "gte": 13, "lte": 26 }, "by_duration": 21, "with_randomness": "true", } expected_utterances = [ (1, "file_10.wav", 4, "1", 13), (1, "file_11.wav", 1, "2", 13), (1, "file_12.wav", 2, "3", 13), (1, "file_13.wav", 4, "4", 13), (2, "file_2.wav", 4, "5", 18), (3, "file_3.wav", 2, "6", 18), (3, "file_4.wav", 4, "7", 18), ] data_filter = DataFilter() filtered1 = data_filter.apply_filters(filters, utterances) filtered2 = data_filter.apply_filters(filters, utterances) self.assertEqual(type(expected_utterances), type(filtered1)) # check they are the same type self.assertNotEqual(expected_utterances, filtered1) self.assertNotEqual(expected_utterances, filtered1) self.assertNotEqual(expected_utterances, filtered2) self.assertNotEqual(filtered1, filtered2)
def __init__(self, postgres_client, gcs_instance, **kwargs): self.postgres_client = postgres_client self.gcs_instance = gcs_instance self.data_tagger_config = None self.data_filter = DataFilter() Logger.info("Total available cpu count:" + str(multiprocessing.cpu_count())) self.data_mover = MediaFilesMover( GCPFileSystem(self.gcs_instance), multiprocessing.cpu_count() / ESTIMATED_CPU_SHARE, ) self.catalogue_dao = CatalogueDao(self.postgres_client) super().__init__(**kwargs)
def test__should_apply_filters_only_source(self): utterances = [ (1, "file_10.wav", 4, "1", 13), (1, "file_11.wav", 1, "2", 13), (1, "file_12.wav", 2, "3", 13), (1, "file_13.wav", 4, "4", 13), (2, "file_2.wav", 4, "5", 11), (3, "file_3.wav", 2, "6", 18), (3, "file_4.wav", 4, "7", 18), (4, "file_5.wav", 4, "8", 19), (4, "file_6.wav", 4, "9", 24), (4, "file_7.wav", 4, "10", 24), (5, "file_50.wav", 5, "10", 25), (5, "file_51.wav", 4, "10", 26), (5, "file_52.wav", 5, "10", 27), (6, "file_53.wav", 5, "10", 25), (6, "file_54.wav", 4, "10", 26), (6, "file_55.wav", 5, "10", 27), ] filters = {} expected_utterances = [ (1, "file_10.wav", 4, "1", 13), (1, "file_11.wav", 1, "2", 13), (1, "file_12.wav", 2, "3", 13), (1, "file_13.wav", 4, "4", 13), (2, "file_2.wav", 4, "5", 11), (3, "file_3.wav", 2, "6", 18), (3, "file_4.wav", 4, "7", 18), (4, "file_5.wav", 4, "8", 19), (4, "file_6.wav", 4, "9", 24), (4, "file_7.wav", 4, "10", 24), (5, "file_50.wav", 5, "10", 25), (5, "file_51.wav", 4, "10", 26), (5, "file_52.wav", 5, "10", 27), (6, "file_53.wav", 5, "10", 25), (6, "file_54.wav", 4, "10", 26), (6, "file_55.wav", 5, "10", 27), ] data_filter = DataFilter() filtered = data_filter.apply_filters(filters, utterances) self.assertEqual(type(expected_utterances), type(filtered)) # check they are the same type self.assertEqual(len(expected_utterances), len(filtered)) # check they are the same length self.assertEqual(expected_utterances, filtered)
def test__should_filter_by_speaker_duration(self): # speaker_id, clipped_utterance_file_name, clipped_utterance_duration, audio_id, snr utterances = [ (1, "file_10.wav", 40, "1", 13), (1, "file_11.wav", 10, "2", 13), (1, "file_12.wav", 20, "3", 13), (1, "file_13.wav", 4, "4", 13), (2, "file_2.wav", 40, "5", 11), (3, "file_3.wav", 20, "6", 18), (3, "file_4.wav", 40, "7", 18), (4, "file_5.wav", 40, "8", 19), (4, "file_6.wav", 40, "9", 24), (4, "file_7.wav", 40, "10", 24), (5, "file_50.wav", 50, "10", 25), (5, "file_51.wav", 40, "10", 26), (5, "file_52.wav", 5, "10", 27), (6, "file_53.wav", 15, "10", 25), (6, "file_54.wav", 25, "10", 26), (6, "file_55.wav", 40, "10", 27), ] data_filter = DataFilter() filtered = list( data_filter.by_per_speaker_duration( utterances, { "lte_per_speaker_duration": 1, "gte_per_speaker_duration": 0, "with_threshold": 2 / 60, }, )) expected_utterances = [ (1, "file_10.wav", 40, "1", 13), (1, "file_11.wav", 10, "2", 13), (2, "file_2.wav", 40, "5", 11), (3, "file_3.wav", 20, "6", 18), (3, "file_4.wav", 40, "7", 18), (4, "file_5.wav", 40, "8", 19), (5, "file_50.wav", 50, "10", 25), (6, "file_53.wav", 15, "10", 25), (6, "file_54.wav", 25, "10", 26), ] self.assertEqual(type(expected_utterances), type(filtered)) # check they are the same type self.assertEqual(len(expected_utterances), len(filtered)) # check they are the same length self.assertEqual(expected_utterances, filtered)
def test__should_filter_by_duration(self): # speaker_id, clipped_utterance_file_name, clipped_utterance_duration, audio_id, snr utterances = [ (1, "file_1.wav", 4, "2010123", 13), (2, "file_2.wav", 2, "2010124", 11), (3, "file_3.wav", 4, "2010125", 18), (4, "file_4.wav", 4, "2010126", 19), (4, "file_4.wav", 2, "2010126", 24), ] data_filter = DataFilter() filtered = data_filter.by_duration(utterances, 7 / 3600) expected_utterances = [ (1, "file_1.wav", 4, "2010123", 13), (2, "file_2.wav", 2, "2010124", 11), ] print(str(expected_utterances)) self.assertEqual(expected_utterances, filtered)
def test__should_apply_filters_only_source_and_utterance_duration_and_then_by_snr( self, ): utterances = [ (1, "file_10.wav", 4, "1", 13), (1, "file_11.wav", 1, "2", 13), (1, "file_12.wav", 2, "3", 13), (1, "file_13.wav", 4, "4", 13), (2, "file_2.wav", 4, "5", 11), (3, "file_3.wav", 2, "6", 18), (3, "file_4.wav", 4, "7", 18), (4, "file_5.wav", 4, "8", 19), (4, "file_6.wav", 4, "9", 24), (4, "file_7.wav", 4, "10", 24), (5, "file_50.wav", 5, "10", 25), (5, "file_51.wav", 4, "10", 26), (5, "file_52.wav", 5, "10", 27), (6, "file_53.wav", 5, "10", 25), (6, "file_54.wav", 4, "10", 26), (6, "file_55.wav", 5, "10", 27), ] filters = { "by_utterance_duration": { "gte": 2, "lte": 4 }, "by_snr": { "gte": 13, "lte": 19 }, } expected_utterances = [ (1, "file_10.wav", 4, "1", 13), (1, "file_12.wav", 2, "3", 13), (1, "file_13.wav", 4, "4", 13), (3, "file_3.wav", 2, "6", 18), (3, "file_4.wav", 4, "7", 18), (4, "file_5.wav", 4, "8", 19), ] data_filter = DataFilter() filtered = data_filter.apply_filters(filters, utterances) self.assertEqual(type(expected_utterances), type(filtered)) # check they are the same type self.assertEqual(expected_utterances, filtered)
def test__should_filter_by_snr(self): # speaker_id, clipped_utterance_file_name, clipped_utterance_duration, audio_id, snr utterances = [ (1, "file_1.wav", 10, 202009112003117071, 13), (2, "file_2.wav", 11, 202009112003117071, 11), (3, "file_3.wav", 12, 202009112003117071, 45), (4, "file_4.wav", 13, 202009112003117071, 19), (4, "file_4.wav", 6, 202009112003117071, 21), (4, "file_4.wav", 13, 202009112003117071, 100), ] data_filter = DataFilter() filtered = list(data_filter.by_snr(utterances, {"gte": 15, "lte": 50})) expected_utterances = [ (3, "file_3.wav", 12, 202009112003117071, 45), (4, "file_4.wav", 13, 202009112003117071, 19), (4, "file_4.wav", 6, 202009112003117071, 21), ] print(type(filtered[0][3])) self.assertEqual(expected_utterances, filtered)
def test__should_return_empty_list_if_zero_utterances_pased(self): # speaker_id, clipped_utterance_file_name, clipped_utterance_duration, audio_id, snr utterances = [] data_filter = DataFilter() filters = { "by_source": "swayamprabha_chapter_30", "by_snr": { "gte": 13, "lte": 26 }, "by_speaker": { "lte_per_speaker_duration": 8 / 60, "gte_per_speaker_duration": 5 / 60, "with_threshold": 0, }, "by_duration": 21 / 3600, } filtered = list(data_filter.apply_filters(filters, utterances)) expected_utterances = [] self.assertEqual(expected_utterances, filtered)
def test__should_filter_by_utterance_duration(self): # speaker_id, clipped_utterance_file_name, clipped_utterance_duration, audio_id, snr utterances = [ (1, "file_1.wav", 1, "2010123", 13), (2, "file_2.wav", 2, "2010124", 11), (3, "file_3.wav", 4, "2010125", 18), (4, "file_4.wav", 4, "2010126", 19), (4, "file_4.wav", 2, "2010126", 24), (5, "file_4.wav", 5, "2010126", 24), ] data_filter = DataFilter() utterance_filter = {"lte": 4, "gte": 2} filtered = list( data_filter.by_utterance_duration(utterances, utterance_filter)) expected_utterances = [ (2, "file_2.wav", 2, "2010124", 11), (3, "file_3.wav", 4, "2010125", 18), (4, "file_4.wav", 4, "2010126", 19), (4, "file_4.wav", 2, "2010126", 24), ] self.assertEqual(expected_utterances, filtered)
class DataMarker(BaseProcessor): """ 1. Load Configuration 2. Filter data baased on criteria 2. Tag/Mark data in the DB 3. Move marked data """ @staticmethod def get_instance(data_processor_instance, gcs_instance, **kwargs): return DataMarker(data_processor_instance, gcs_instance, **kwargs) def __init__(self, postgres_client, gcs_instance, **kwargs): self.postgres_client = postgres_client self.gcs_instance = gcs_instance self.data_tagger_config = None self.data_filter = DataFilter() Logger.info("Total available cpu count:" + str(multiprocessing.cpu_count())) self.data_mover = MediaFilesMover( GCPFileSystem(self.gcs_instance), multiprocessing.cpu_count() / ESTIMATED_CPU_SHARE, ) self.catalogue_dao = CatalogueDao(self.postgres_client) super().__init__(**kwargs) def process(self, **kwargs): """ Main function for running all processing that takes places in the data marker """ Logger.info("*************Starting data marker****************") self.data_tagger_config = self.postgres_client.config_dict.get( CONFIG_NAME) source, filter_criteria = self.get_config(**kwargs) Logger.info("Fetching utterances for source:" + source) utterances = self.catalogue_dao.get_utterances_by_source( source, "Clean") filtered_utterances = self.data_filter.apply_filters( filter_criteria, utterances) Logger.info("updating utterances that need to be staged, count=" + str(len(filtered_utterances))) if len(filtered_utterances) > 0: rows_updated = ( self.catalogue_dao.update_utterances_staged_for_transcription( filtered_utterances, source)) Logger.info("Rows updated:" + str(rows_updated)) else: Logger.info("No utterances found for filter criteria") source_dir = filter_criteria.get("landing_source_dir", source) landing_path_with_source = ( f"{self.data_tagger_config.get(LANDING_BASE_PATH)}/{source_dir}") source_path_with_source = ( f"{self.data_tagger_config.get(SOURCE_BASE_PATH)}/{source}") files = self.to_files(filtered_utterances, source_path_with_source) Logger.info("Staging utterances to dir:" + landing_path_with_source) self.data_mover.move_media_files(files, landing_path_with_source) Logger.info("************* Data marker completed ****************") def to_files(self, utterances, source_path_with_source): list( map(lambda u: f"{source_path_with_source}/{u[3]}/clean/{u[1]}", utterances)) return list( map(lambda u: f"{source_path_with_source}/{u[3]}/clean/{u[1]}", utterances)) def get_config(self, **kwargs): filter_criteria = kwargs.get(FILTER_CRITERIA, {}) source = kwargs.get("source") if source is None: raise Exception("filter by source is mandatory") return source, filter_criteria.get(FILTER_CRITERIA)