def predict(): if request.method == 'POST': print(request.form.get('NewYork')) try: NewYork = request.form['NewYork'] California = request.form['California'] Florida = request.form['Florida'] NewYorkstrip = NewYork.strip() NewYorkLower = NewYorkstrip.lower() f = open("data/Alldata.csv", "r") s = f.read() dic = ast.literal_eval(s) my_list = [] words = NewYorkLower.split() for c in words: trn = Transliterator(source=California.strip(), target=Florida.strip(), build_lookup=True) eng = trn.transform(c.lower()) my_list.append(dic.get(c,eng)) a = my_list listToStr = ' '.join(map(str, a)) except ValueError: return "Please check if the values are entered correctly" return render_template('home.html', prediction = listToStr)
def transliterate(): word = request.args.get('word', default="congress", type=str) trn = Transliterator(source='eng', target='hin', build_lookup=True, decode='beamsearch') best_transliterated_list = trn.transform(word, k_best=5) return {"transliteration": best_transliterated_list}
def read_kumaretal_2019_agg_downloads(path, mode, romanize=False): st_time = time() global FIELDS, MAX_CHAR_LEN from indictrans import Transliterator trn_hin2eng = Transliterator(source='hin', target='eng') n_trimmed, n_romanized = 0, 0 Example = namedtuple(f"{mode}_example", FIELDS, defaults=(None, ) * len(FIELDS)) examples = [] lines = read_csv_file(path, has_header=False) for i, line in enumerate(lines): uid, txt, label = line[0], line[1], line[2] if not txt: continue if romanize: new_txt = trn_hin2eng.transform(txt) if txt != new_txt: n_romanized += 1 txt = new_txt new_txt = clean_generic(txt) if new_txt.strip() == "": new_txt = txt if len(new_txt) > MAX_CHAR_LEN: n_trimmed += 1 newtokens, currsum = [], 0 for tkn in new_txt.split(): # 1 for space if currsum + len(tkn) + 1 <= MAX_CHAR_LEN: newtokens.append(tkn) currsum += len(tkn) + 1 else: break new_txt = " ".join(newtokens) example = Example(dataset="kumaretal_2019_agg", task="classification", split_type=mode, uid=uid, text=txt, label=label, text_pp=new_txt) examples.append(example) progress_bar(len(examples), len(lines), ["time"], [time() - st_time]) if romanize: print( f"len of {mode} data: {len(examples)} and # of trimmed instances: {n_trimmed} " f"and # of romanized instances: {n_romanized}") else: print( f"len of {mode} data: {len(examples)} and # of trimmed instances: {n_trimmed}" ) return examples
def test_ind2ru(self): """Test Indic-to-[Roman, Urdu] ML models""" for lang_pair in self.src2trg: src = lang_pair[0] trg = lang_pair[1] trans = Transliterator(source=src, target=trg) with io.open('%s/%s_%s.testpairs' % (self.test_dir, src, trg), encoding='utf-8') as fp: for line in fp: word, expected = line.split() self.assertEqual(trans.transform(word), expected)
def test_ru2ind(self): """Test [Roman, Urdu]-to-Indic ML models""" for lang_pair in self.trg2src: src = lang_pair[0] trg = lang_pair[1] trans = Transliterator(source=src, target=trg) with io.open('%s/%s_%s.testpairs' % (self.test_dir, trg, src), encoding='utf-8') as fp: for line in fp: expected, word = line.split() self.assertEqual(trans.transform(word), expected)
def test_ru2ind(self): """Test [Roman, Urdu]-to-Indic ML models""" for lang_pair in self.trg2src: src = lang_pair[0] trg = lang_pair[1] trans = Transliterator(source=src, target=trg) with io.open('%s/%s_%s.testpairs' % (self.test_dir, trg, src), encoding='utf-8') as fp: for line in fp: expected, word = line.split() self.assertEqual(trans.transform(word), expected)
def test_ind2ru(self): """Test Indic-to-[Roman, Urdu] ML models""" for lang_pair in self.src2trg: src = lang_pair[0] trg = lang_pair[1] trans = Transliterator(source=src, target=trg) with io.open('%s/%s_%s.testpairs' % (self.test_dir, src, trg), encoding='utf-8') as fp: for line in fp: word, expected = line.split() self.assertEqual(trans.transform(word), expected)
def __init__(self, text, source, target): self.text = text self.source = source self.target = target self.isEngSource = (self.source == 'eng') if self.isEngSource: self.usdictionary = enchant.Dict("en_US") self.gbdictionary = enchant.Dict("en_GB") self.validated = self.source in self.codes and self.target in self.codes if self.validated: self.engine = Transliterator(source=self.source, target=self.target)
def get_converters(): from indictrans import Transliterator converters = { 'hi2ur': Transliterator(source='hin', target='urd', rb=False), #, build_lookup=True), 'ur2hi': Transliterator(source='urd', target='hin', rb=False), #, build_lookup=True), 'hi2en': Transliterator(source='hin', target='eng', rb=False), #, build_lookup=True), 'ur2en': Transliterator(source='urd', target='eng', rb=False), #, build_lookup=True), } return converters
def test_kbest(self): """Make sure `k-best` works without failure""" k_best = range(2, 15) r2i = Transliterator(source='eng', target='hin', decode='beamsearch') i2r = Transliterator(source='hin', target='eng', decode='beamsearch') for k in k_best: hin = r2i.transform('indictrans', k_best=k) eng = i2r.transform(hin[0], k_best=k) self.assertTrue(len(hin) == k) self.assertTrue(len(eng) == k)
def get_converters(): from google_trans_new import google_translator from indictrans import Transliterator converters = { 'g_translator': google_translator(url_suffix="com.pk"), 'hi2ur': Transliterator(source='hin', target='urd', rb=False), #, build_lookup=True), 'ur2hi': Transliterator(source='urd', target='hin', rb=False), #, build_lookup=True), 'hi2en': Transliterator(source='hin', target='eng', rb=False), #, build_lookup=True), 'ur2en': Transliterator(source='urd', target='eng', rb=False), #, build_lookup=True), } return converters
class IndictransTransliterator: def __init__(self): self.trn = Transliterator(source='hin', target='eng', decode='beamsearch', build_lookup=True) self.trans_dict = {} def transliterate(self, original): transliterations = self.get_all(original) return random.choice(transliterations) def get_all(self, original): if original in self.trans_dict: return self.trans_dict[original] else: transliterations = self.trn.transform(original, k_best=5) self.trans_dict[original] = transliterations return transliterations @staticmethod def _is_deva(unicode_tok): """Returns True if |unicode_tok| contains a Devanagari character""" for c in unicode_tok: if int('0900', 16) <= ord(c) <= int('097f', 16): return True return False
def read_iitp_product_reviews_hi_sa_downloads(path, mode): st_time = time() global FIELDS, MAX_CHAR_LEN from indictrans import Transliterator trn_hin2eng = Transliterator(source='hin', target='eng') n_trimmed = 0 Example = namedtuple(f"{mode}_example", FIELDS, defaults=(None, ) * len(FIELDS)) examples = [] lines = [line.strip() for line in open(path, "r")] for i, line in enumerate(lines): line = line.strip() if not line: continue vals = line.split(",") label = vals[0] txt = ",".join(vals[1:]) txt = trn_hin2eng.transform(txt) new_txt = "".join([char for char in txt]) if len(new_txt) > MAX_CHAR_LEN: n_trimmed += 1 newtokens, currsum = [], 0 for tkn in new_txt.split(): # 1 for space if currsum + len(tkn) + 1 <= MAX_CHAR_LEN: newtokens.append(tkn) currsum += len(tkn) + 1 else: break new_txt = " ".join(newtokens) example = Example(dataset="iitp_product_reviews_hi_sa", task="classification", split_type=mode, uid=len(examples), text=txt, label=label, text_pp=new_txt) examples.append(example) progress_bar(len(examples), len(lines), ["time"], [time() - st_time]) print( f"len of {mode} data: {len(examples)} and # of trimmed instances: {n_trimmed}" ) return examples
def perform_action(query): query = str(query).lower() display_map = {} kan_trans = Transliterator(source="kan", target="eng", build_lookup=True) hin_trans = Transliterator(source="hin", target="eng", build_lookup=True) query = transliterate(query, hin_trans, kan_trans) if get_lang_cs(query, lit) == "en_hi": intent = load_and_query_classifier(query) if get_lang_cs(query, lit) == "en_ka": intent = kannada_load_and_query_classifier(query) #print(intent) #print ("INTENT identified:"+" "+intent) ner_type = ["HOTEL", "RESTAURANT", "TRAVEL_BOOKING", "REMINDER"] # flag = 1 for kind in ner_type: if kind == intent: return ner_module.response(query, intent) if get_lang_cs(query, lit) == "en_hi": #print "This is a query of type:en_hi" word_map = preprocess_pipeline(query) elif get_lang_cs(query, lit) == "en_ka": #print "This is a query of type:en_ka" word_map = keyword_extract(query) if intent in ["SYMPTOMS", "TREATMENT", "PREVENTION"]: if query.find("corona") >= 0 or query.find("covid") >= 0 or query.find( "crna") >= 0: if intent == "SYMPTOMS": return covid19.symptoms elif intent == "TREATMENT": return covid19.treatment elif intent == "PREVENTION": return covid19.prevention #print(word_map) display_map["code_switch_type"] = get_lang_cs(query, lit) display_map["transliterated"] = query display_map["intent"] = intent return "Response:" + "\n" + str(display_map) + "\n" + google_search( query, word_map, intent)
def read_hinglishpedia_downloads(path1, path2, mode, standardizing_tags={}): st_time = time() global FIELDS, MAX_CHAR_LEN n_trimmed = 0 FIELDS += [ fieldname for fieldname in [ "tgt", ] if fieldname not in FIELDS ] Example = namedtuple(f"{mode}_example", FIELDS, defaults=(None, ) * len(FIELDS)) examples = [] from indictrans import Transliterator trn_hin2eng = Transliterator(source='hin', target='eng') txt_lines = [line.strip() for line in open(path1, "r")] tag_lines = [line.strip() for line in open(path2, "r")] for i, (txt, tags) in tqdm(enumerate(zip(txt_lines, tag_lines))): if not txt: continue txt = trn_hin2eng.transform(txt) example = Example(dataset="hinglishpedia", task="classification", split_type=mode, uid=len(examples), text=txt, langids=" ".join([ standardizing_tags[lid] if lid in standardizing_tags else "other" for lid in tags.split() ]), text_pp=txt) examples.append(example) # progress_bar(len(examples), len(txt_lines), ["time"], [time() - st_time]) print( f"len of {mode} data: {len(examples)} and # of trimmed instances: {n_trimmed}" ) return examples
def get_transliteration(vocab, headers): trans = {} if headers is None: trn = Transliterator(source='eng', target='hin', build_lookup=True) trans = {item: trn.transform(item) for item in vocab} else: base_url = 'https://api.cognitive.microsofttranslator.com' path = '/transliterate?api-version=3.0&language=hi&fromScript=Latn&toScript=Deva' count = 0 body = [] constructed_url = base_url + path query = '' while (count <= 6500): for i in range(count, (count + 500), 50): for j in range(i, i + 50): query += vocab[j] + ' ' body.append({'text': query.strip()}) query = '' response = requests.post(constructed_url, headers=headers, json=body) result = response.json() for j, i in enumerate(result): trans.update({body[j]['text']: i['text']}) body = [] count += 500 for i in range(count, len(vocab), 50): for j in range(i, i + 50): if j < len(vocab): query += vocab[j] + ' ' body.append({'text': query.strip()}) query = '' response = requests.post(constructed_url, headers=headers, json=body) result = response.json() for j, i in enumerate(result): trans.update({body[j]['text']: i['text']}) return trans
def __init__(self, lang='ta'): self.lang = lang self.normalizer = BaseNormalizer(lang) # This language map was created using Google's googletrans module. Create the file alltextlang.txt by calling # detect_lang_and_store in feature_utils.py self.lmap = self.load_language_maps( os.path.join(os.path.dirname(sys.path[0]), '../resources/data/alltextslang.txt')) self.soundexer = Soundex() self.ta_trans = Transliterator(source='eng', target='tam', build_lookup=True) self.ml_trans = Transliterator(source='eng', target='mal', build_lookup=True) self.sym_spell = SymSpell(max_dictionary_edit_distance=2, prefix_length=7) self.sym_spell.load_dictionary( '../../src/extern/data/etymdict.csv.vocab.tsv.gz', term_index=0, count_index=1, separator="\t") super().__init__()
def test_kbest(self): """Make sure `k-best` works without failure""" k_best = range(2, 15) r2i = Transliterator(source='eng', target='hin', decode='beamsearch') i2r = Transliterator(source='hin', target='eng', decode='beamsearch') for k in k_best: hin = r2i.transform('indictrans', k_best=k) eng = i2r.transform(hin[0], k_best=k) self.assertTrue(len(hin) == k) self.assertTrue(len(eng) == k)
def test_rtrans(self): """Test Indic-to-Indic ML and Rule-Based models.""" with io.open('%s/indic-test' % self.test_dir, encoding='utf-8') as fp: # first line contains language codes lang_codes = fp.readline().split() lang2word = dict( zip(lang_codes, [[] for i in range(len(lang_codes))])) for line in fp: line = line.split() for i, word in enumerate(line): lang2word[lang_codes[i]].append(word) for src in lang_codes: for trg in lang_codes: if src == trg: continue s2t_ml = Transliterator(source=src, target=trg, rb=False) s2t_rb = Transliterator(source=src, target=trg, rb=True) for word in lang2word[src]: s2t_ml.transform(word) s2t_rb.transform(word)
def start_lsh(): create = True sqlite_file = "/home/hkesavam/new_lid/code/gen_data/db_lid" conn = sqlite3.connect(sqlite_file) cur = conn.cursor() data = cur.execute("select distinct(soundex) from dev_table where count > 5") data = list(data) #soundex_dict = pickle.load(open("/home/hkesavam/new_lid/code/gen_data/data_dev/dev_to_soundex.pkl", "r")) #data = soundex_dict.keys() #data = ["e16512", "e16532", "hello", "hell"] lsh = MinHashLSH(threshold=0.5, num_perm=32)# Create MinHash objects minhashes = {} tot_wr_count = 0 if create: for c, i in enumerate(data): minhash = MinHash(num_perm=32) for d in i[0]: try: d = d.encode("utf-8") minhash.update(d) except: print "Enterin continue" continue lsh.insert(c, minhash) minhashes[c] = minhash print len(data) print("Dumping") #cPickle.dump(new_data, open("data", "wb"), -1) cPickle.dump(lsh, open("/home/hkesavam/new_lid/code/gen_data/lsh_model", "wb"), -1) print "Finished dumping" if not(create): data = cPickle.load( open("/home/hkesavam/new_lid/code/gen_data/data", "rb")) lsh = cPickle.load( open("/home/hkesavam/new_lid/code/gen_data/lsh_model", "rb")) hin_soundex_inst = Soundex() hin_trans = Transliterator(source='eng', target='hin', build_lookup=True) return cur, data, lsh, hin_soundex_inst, hin_trans
def test_rtrans(self): """Test Indic-to-Indic ML and Rule-Based models.""" with io.open('%s/indic-test' % self.test_dir, encoding='utf-8') as fp: # first line contains language codes lang_codes = fp.readline().split() lang2word = dict(zip(lang_codes, [[] for i in range(len(lang_codes))])) for line in fp: line = line.split() for i, word in enumerate(line): lang2word[lang_codes[i]].append(word) for src in lang_codes: for trg in lang_codes: if src == trg: continue s2t_ml = Transliterator(source=src, target=trg, rb=False) s2t_rb = Transliterator(source=src, target=trg, rb=True) for word in lang2word[src]: s2t_ml.transform(word) s2t_rb.transform(word)
#!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import unicode_literals import csv from indictrans import Transliterator trn = Transliterator(source='hin', target='eng', build_lookup=True) # print eng with open('dataSet.csv') as csvfile: reader = csv.reader(csvfile) for row in reader: hin = row[1].decode('utf-8').replace('\n', " ") eng = trn.transform(hin) print row[0], eng.encode('unicode-escape'), row[2]
from indictrans import Transliterator vocab = [] with open('all_roman.txt', 'r') as infile: con = infile.readlines() vocab = [x.strip('\n') for x in con] trn = Transliterator(source='eng', target='hin') with open('transliterations.txt', 'w+') as outfile: for word in vocab: deva = trn.transform(word) outfile.write(word + "\t" + deva + "\n")
class FeatureExtractor(BaseEstimator, TransformerMixin): """Extract review text, emojis and emoji sentiment. Takes a sequence of strings and produces a dict of values. Keys are `review`, `emojis`, and `emoji-sentiment`. """ def __init__(self, lang='ta'): self.lang = lang self.normalizer = BaseNormalizer(lang) # This language map was created using Google's googletrans module. Create the file alltextlang.txt by calling # detect_lang_and_store in feature_utils.py self.lmap = self.load_language_maps( os.path.join(os.path.dirname(sys.path[0]), '../resources/data/alltextslang.txt')) self.soundexer = Soundex() self.ta_trans = Transliterator(source='eng', target='tam', build_lookup=True) self.ml_trans = Transliterator(source='eng', target='mal', build_lookup=True) self.sym_spell = SymSpell(max_dictionary_edit_distance=2, prefix_length=7) self.sym_spell.load_dictionary( '../../src/extern/data/etymdict.csv.vocab.tsv.gz', term_index=0, count_index=1, separator="\t") super().__init__() def load_language_maps(self, mapfile): lmap = {} with open(mapfile, 'r') as mapf: for line in mapf: text, lang, conf = line.rstrip().split('\t') lmap[text] = (lang, float(conf)) return lmap def get_language_tag(self, text): return self.lmap.get(text, ('unknown', 0.0)) def fit(self, x, y=None): return self def transform(self, reviews): features = np.recarray( shape=(len(reviews), ), dtype=[ ('review', object), ('emojis', object), ('emoji_sentiment', object), ('lang_tag', object), ('len_range', object), ('soundexes', object), ], ) for i, review in enumerate(reviews): features['review'][i] = self.normalizer.normalize(text=review) emojis, sentiment = get_emojis_from_text(review) features['emojis'][i] = ' '.join(emojis) features['emoji_sentiment'][i] = sentiment lang, conf = self.get_language_tag(review.strip()) if lang == self.lang or lang == (self.lang + 'en'): # google agrees with some confidence agreement = 1 elif conf < 0.5: # google says not-tamil, but weakly agreement = 0.5 else: # google clearly says not-tamil agreement = 0 features['lang_tag'][i] = {'lang': lang, 'agreement': agreement} features['len_range'][i] = get_doc_len_range(review) if self.lang == 'ta': review_trans = self.ta_trans.transform(review) for word in review_trans.split(): suggestions = self.sym_spell.lookup(word, Verbosity.CLOSEST, max_edit_distance=2, include_unknown=True) if len(suggestions) > 0 and suggestions[0].distance < 3: print(word, suggestions[0].term) # no match with dictionary, we need a more comprehensive dictionary plus phonetic similarity elif self.lang == 'ml': review_trans = self.ml_trans.transform(review) else: review_trans = review # TODO: introduce spell correct here for added normalisation # print(lang, review_trans) features['soundexes'][i] = ' '.join([ self.soundexer.soundex(word) for word in review_trans.split() ]) return features
from sklearn.metrics import classification_report from libindic.soundex import Soundex from lib.feature_utils import load_docs, get_emojis_from_text, get_doc_len_range sys.path.append( os.path.join(os.path.dirname(sys.path[0]), 'extern', 'indic_nlp_library')) from indicnlp.normalize.indic_normalize import BaseNormalizer try: from indictrans import Transliterator except ImportError: print( 'Please install indic-trans from git: https://github.com/libindic/indic-trans' ) ta_trans = Transliterator(source='eng', target='tam', build_lookup=True) ml_trans = Transliterator(source='eng', target='mal', build_lookup=True) # The maximum number of words to be used. (most frequent) MAX_NB_WORDS = 50000 # Max number of words in each review. MAX_SEQUENCE_LENGTH = 150 # This is fixed. EMBEDDING_DIM = 100 tokenizer = Tokenizer(num_words=MAX_NB_WORDS, filters='!"#$%&()*+,-./:;<=>?@[\]^_`{|}~', lower=True) soundexer = Soundex() def load_language_maps(mapfile): lmap = {}
# -*- coding: utf-8 -*- # @Author: claravania # @Date: 2018-10-15 11:03:20 # @Last Modified by: claravania # @Last Modified time: 2018-11-08 17:22:37 import codecs import os from indictrans import Transliterator ud_dir = '../data/ud-treebanks-v2.2' hi_tb = 'UD_Urdu-UDTB' trn = Transliterator(source='urd', target='eng') filenames = os.listdir(os.path.join(ud_dir, hi_tb)) for f in filenames: if f.endswith('.conllu') or f.endswith('.conllu.sample'): fname = os.path.join(ud_dir, hi_tb, f) print 'Reading ' + fname ftrn = fname + '.en' fout = codecs.open(ftrn, 'w', encoding='utf-8') count = 0 with codecs.open(fname, encoding='utf-8') as fin: for line in fin: count += 1
class Vida: """ Only focused on English to 15 Indic Language Transliteration Hindi (hin) Bengali (ben) Gujarati (guj) Punjabi (pun) Malayalam (mal) Kannada (kan) Tamil (tam) Telugu (tel) Oriya (ori) Marathi (mar) Assamese (ass) Konkani (kon) Bodo (bod) Nepali (nep) Urdu (urd) English (eng) """ languages = { "hin": "Hindi", "ben": "Bengali", "guj": "Gujarati", "pun": "Punjabi", "mal": "Malayalam", "kan": "Kannada", "tam": "Tamil", "tel": "Telugu", "ori": "Oriya", "mar": "Marathi", "ass": "Assamese", "kon": "Konkani", "bod": "Bodo", "nep": "Nepali", "urd": "Urdu", "eng": "English" } codes = languages.keys() @staticmethod def is_ascii(s): return all(ord(c) < 128 for c in s) def __init__(self, text, source, target): self.text = text self.source = source self.target = target self.isEngSource = (self.source == 'eng') if self.isEngSource: self.usdictionary = enchant.Dict("en_US") self.gbdictionary = enchant.Dict("en_GB") self.validated = self.source in self.codes and self.target in self.codes if self.validated: self.engine = Transliterator(source=self.source, target=self.target) def run(self): status = False message = "Couldn't transliterate the text." content = {} output = [] if not self.validated: message = "Please provide languages and their code." output = self.text else: text = self.text.split() try: for index in xrange(len(text)): word = text[index] if not self.isEngSource: word = word.decode('utf-8') output.insert( index, self.engine.transform(word).encode('utf-8')) else: if not Vida.is_ascii(word): word = word.decode('utf-8') if not self.usdictionary.check( word) and not self.gbdictionary.check(word): output.insert( index, self.engine.transform(word).encode('utf-8')) else: output.insert(index, word) status = True message = "Succesfully transliterated the code." except UnicodeDecodeError, e: Repo.exception(e) message = "Couldn't decode the language properly." except IndexError, e: Repo.exception(e) message = "Couldn't properly frame the sentence." output = ' '.join(output)
import io import re import sys import math import string import random import pickle from argparse import ArgumentParser from collections import Counter, defaultdict import dynet as dy import numpy as np from gensim.models.word2vec import KeyedVectors from indictrans import Transliterator trn = Transliterator(source='eng', target='hin', build_lookup=True) def is_lang_dist(dist_string): return ":" in dist_string def get_lang_dist(dist_string): dist = dict() pairs = dist_string.split(',') for p in pairs: lang, prob = p.split(':') dist[lang] = prob return dist
# If transliterated word is not a legitimate hindi word, add the closest hindi match to the predictions list if hindi_match: suggested_words.append([hindi_match[0], 1]) return suggested_words if __name__ == '__main__': # sample test script to test this program logic # initialize english and hindi dictionary object eng_dict = enchant.Dict('en_US') hin_dict = enchant.Dict('hi_IN') # initialize objects for language classifier and indictrans class trn = Transliterator(source='eng', target='hin') classifier = LanguageIdentifier.from_modelstring(model, norm_probs=True) # take sample sentence from the user sentence = list(input().split()) # transliterate every word in the sentence for word in sentence: print(transliterate_in_hindi(trn, word, eng_dict, hin_dict, classifier, 0))
def main(): from indictrans import Transliterator trn = Transliterator(source='eng', target='tel', build_lookup=True) file = open("DATASET/datasetforfinal.txt", "r") file_data = file.readlines() print(len(file_data))
#!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import unicode_literals from indictrans import Transliterator from polyglot_tokenizer import Tokenizer flag = True s = 'hin' t = 'eng' forward_transl_full = Transliterator(source=s, target=t, build_lookup=True) forward_transl_token = Transliterator(source=s, target=t, decode='beamsearch') back_transl_token = Transliterator(source=t, target=s, build_lookup=True) tk = Tokenizer(lang=s[:2]) tk_back = Tokenizer(lang=t[:2]) l = u"रज्ज के रुलाया" #\nरज्ज के हंसाया\n\nमैंने दिल खो' के इश्क़ कमाया\n" l = l.lower().strip() lines = l.split("\n") print(lines) output = [] if flag == True: for l in lines: json = {}
def code_transliterate(self): trn = Transliterator(source='hin', target='eng', build_lookup=True) eng = trn.transform(self) return eng
def __init__(self): self.trn = Transliterator(source='hin', target='eng', decode='beamsearch', build_lookup=True) self.trans_dict = {}