def __init__( self, meters_list=None, find_feet=None, meters_filter=None, with_mir=True ): if not meters_list: meters_list = _load_yaml(meters_filename) if with_mir: mir_meters = _load_yaml(mir_meters_filename) meters_list = meters_list + mir_meters self._scans_with_feet = _gen_possible_feet(meters_list) find_feet = self.find_feet if meters_filter: meters_list = meters_filter(meters_list) Scanner.__init__( self, GraphTransliterator.from_yaml_file(transcription_filename), GraphTransliterator.from_yaml_file(long_parser_filename), GraphTransliterator.from_yaml_file(short_parser_filename), _load_yaml(constraints_filename), meters_list, find_feet=find_feet, post_scan_filter=filter_scans, )
def load_transliterator(source, **kwargs): """Loads transliterator (format, parameter).""" format, parameter = source if format == "bundled": mod = __import__("graphtransliterator.transliterators") transliterators_mod = mod.transliterators transliterator_class = getattr(transliterators_mod, parameter) return transliterator_class(**kwargs) elif format == "json": return GraphTransliterator.loads(parameter, **kwargs) elif format == "json_file": with open(parameter, "r") as f: return GraphTransliterator.loads(f.read(), **kwargs) elif format == "yaml_file": return GraphTransliterator.from_yaml_file(parameter, **kwargs)
norm_string = re.sub(r"ṭ", "ṭ", norm_string) translated = norm_string.translate( norm_string.maketrans(normalization_dict)) norm_string = re.sub(r"-", " ", translated) norm_string = re.sub(r'( ){2,}', " ", norm_string) return norm_string df = pd.read_csv(input_file, sep='\t') df_augmented = df.copy() df_augmented["normalized"] = df["sentence"] = df.apply( lambda row: normalize(row['sentence'], orthography), axis=1) if orthography in [ 'tifinagh_ahaggar', 'tifinagh_ahaggar_lig', 'tifinagh_ircam', 'arabic' ]: gt = GraphTransliterator.from_yaml_file(paths[orthography]) df_augmented['transliteration'] = df["sentence"] = df_augmented.apply( lambda row: gt.transliterate(row['normalized']), axis=1) df.to_csv(output_folder + '/' + os.path.basename(input_file), sep='\t', index=False, header=True) input_base = os.path.splitext(os.path.basename(input_file))[0] with open(output_folder + '/' + input_base + "_compare.txt", "w+") as f: for (idx, row) in df_augmented.iterrows(): f.write(row.sentence + "\n") f.write("\t" + row.normalized + "\n") if orthography in [ 'tifinagh_ahaggar', 'tifinagh_ahaggar_lig', 'tifinagh_ircam', 'arabic'
def test_GraphTransliterator(tmpdir): """Test GraphTransliterator.""" yaml_str = r""" tokens: a: [token, class1] b: [token, class2] u: [token] ' ': [wb] rules: a: A b: B <wb> u: \N{DEVANAGARI LETTER U} onmatch_rules: - <class1> + <class2>: "," - <class1> + <token>: \N{DEVANAGARI SIGN VIRAMA} whitespace: default: ' ' token_class: 'wb' consolidate: true metadata: author: Author """ input_dict = yaml.safe_load(yaml_str) assert "a" in GraphTransliterator.from_easyreading_dict(input_dict).tokens.keys() gt = GraphTransliterator.from_easyreading_dict(input_dict) assert gt.onmatch_rules[0].production == "," assert gt.tokens assert gt.rules assert gt.whitespace assert gt.whitespace.default assert gt.whitespace.token_class assert gt.whitespace.consolidate assert gt.metadata["author"] == "Author" assert type(gt.graph) == DirectedGraph yaml_file = tmpdir.join("yaml_test.yaml") yaml_filename = str(yaml_file) yaml_file.write(yaml_str) assert yaml_file.read() == yaml_str assert GraphTransliterator.from_yaml_file(yaml_filename) assert len(set(GraphTransliterator.from_easyreading_dict(input_dict).tokens)) == 4 assert GraphTransliterator.from_yaml(yaml_str).transliterate("ab") == "A,B" assert ( GraphTransliterator.from_yaml_file(yaml_filename).transliterate("ab") == "A,B" ) assert ( GraphTransliterator.from_easyreading_dict( { "tokens": {"a": ["class_a"], "b": ["class_b"], " ": ["wb"]}, "onmatch_rules": [{"<class_a> + <class_b>": ","}], "whitespace": { "default": " ", "token_class": "wb", "consolidate": True, }, "rules": {"a": "A", "b": "B"}, } ).transliterate("ab") == "A,B" )
for i in range(len(final) - a_id): #print(final[a_id+i]) final2.append(final[a_id + i]) return final2 ## ConfusionMatrix confusion_matrix = ConfusionDictionary() possibilities = set(x for x in confusion_matrix.getPreds()) for x in confusion_matrix.getGolds(): possibilities.add(x) import json from graphtransliterator import GraphTransliterator gt = GraphTransliterator.from_yaml_file( "/Users/mosaix/orthographic-ASR/transliterate/transliterators/latin_prealignment.yml" ) tf = GraphTransliterator.from_yaml_file( "/Users/mosaix/orthographic-ASR/transliterate/transliterators/tifinagh_to_latin.yml" ) no_lm_store = {} gold_aligned = [] pred_aligned = [] with open('transliterate/output/latin_norm/no_lm/inferences.json') as f: data = json.load(f) with open('transliterate/output/latin_norm/no_lm/alignments.txt', "w+") as l: for i in data: try: wavfile = i['wav_filename'].split('/')[-1] compare_tuple = (gt.transliterate(i['src']),