def _compute_has_valid_language(self):
""" Detect if text is writed in the language corresponding to the
language_id """
self.has_valid_language = False
if self.translated_text is not None and \
self.translation_language_id is not None:
s = self.translated_text.strip(' \t\n\r.')
if s:
# find the language name of text argument
detectlanguage.configuration.api_key = config.get(
'detect_language_api_key')
languageName = ""
langs = detectlanguage.languages()
try:
codeLang = detectlanguage.simple_detect(
self.translated_text)
except IndexError:
# Language could not be detected
return
for lang in langs:
if lang.get("code") == codeLang:
languageName = lang.get("name").lower()
break
supporter_langs = map(
lambda lang: lang.lower(),
self.supporter_languages_ids.mapped('name'))
self.has_valid_language = languageName in supporter_langs
def traduzir(request, titulo, descricao):
detectlanguage.configuration.api_key = "f2d7fc33a7f3cb0b045d83b6a4c36ad9"
de = detectlanguage.simple_detect(titulo)
client = MicrosoftTranslatorClient('sabialiedufes', 'HyJ6rFaIeEPd9q7ZjrWaixrKIUuAXuxIAroGL6YpRl8=')
t = client.TranslateText(titulo.encode('utf-8'), de, 'pt')
d = client.TranslateText(descricao.encode('utf-8'), de, 'pt')
return simplejson.dumps({'titulo' : t, 'descricao':d})
def detect_lang(self, text):
"""
Use detectlanguage API to find the language of the given text
:param text: text to detect
:return: res.lang compassion record if the language is found, or False
"""
detectlanguage.configuration.api_key = config.get("detect_language_api_key")
language_name = False
langs = detectlanguage.languages()
try:
code_lang = detectlanguage.simple_detect(text)
except (IndexError, detectlanguage.DetectLanguageError):
# Language could not be detected
return self.env["res.lang.compassion"]
for lang in langs:
if lang.get("code") == code_lang:
language_name = lang.get("name")
break
if not language_name:
return self.env["res.lang.compassion"]
return (
self.env["res.lang.compassion"]
.with_context({"lang": "en_US"})
.search([("name", "=ilike", language_name)], limit=1)
)
def _compute_has_valid_language(self):
""" Detect if text is written in the language corresponding to the
language_id """
self.has_valid_language = False
if self.translated_text is not None and \
self.translation_language_id is not None:
s = self.translated_text.strip(' \t\n\r.')
if s:
# find the language name of text argument
detectlanguage.configuration.api_key = config.get(
'detect_language_api_key')
languageName = ""
langs = detectlanguage.languages()
try:
codeLang = detectlanguage.simple_detect(
self.translated_text)
except IndexError:
# Language could not be detected
return
for lang in langs:
if lang.get("code") == codeLang:
languageName = lang.get("name").lower()
break
supporter_langs = map(
lambda lang: lang.lower(),
self.supporter_languages_ids.mapped('name'))
self.has_valid_language = languageName in supporter_langs
def servidor(environ, start_response):
start_response('200 OK', [('Content-Type', 'application/json')])
if environ["REQUEST_METHOD"] == "GET":
return json.dumps([False, translations])
elif environ["REQUEST_METHOD"] == "POST":
try:
request_body_size = int(environ.get('CONTENT_LENGTH', 0))
except (ValueError):
request_body_size = 0
request_body = environ['wsgi.input'].read(request_body_size)
data = json.loads(request_body)
text = data['text']
# Using 1rst API
language2 = detectlanguage.simple_detect(text)
# Using 2nd API
#language2 = detect(text)
print "Language Detected is - "+language2
try:
return ["{\"language\": \"" + str(language2)+"\"}"]
except:
return ["error "]
else:
return json.dumps([False, { 'msg': "Rest api error)" }])
def traduzir(request, texto, de, para):
detectlanguage.configuration.api_key = "f2d7fc33a7f3cb0b045d83b6a4c36ad9"
msg = detectlanguage.simple_detect(texto)
client = MicrosoftTranslatorClient(
'sabialiedufes', 'HyJ6rFaIeEPd9q7ZjrWaixrKIUuAXuxIAroGL6YpRl8=')
t = client.TranslateText(texto.encode('utf-8'), msg, para)
return simplejson.dumps({'texto': t})
async def speak(self, *text):
text = ' '.join([str(x) for x in text])
lang = detectlanguage.simple_detect(text).upper()
if lang not in self.supported:
await self.bot.say("Sorry, I do not speak {}.".format(
self.languages[lang.lower()]))
return
else:
print(f"text={text}, lang={lang}")
try:
# Translate English to French and everything else to English
desired_lang = ('EN', 'FR')[lang == 'EN']
text = pydeepl.translate(str(text), desired_lang, lang)
except Exception as e: # Internal error
await self.bot.say('Error: {}'.format(e))
raise
else:
pre = (
'Translation from DeepL',
'Traduction de DeepL')[lang == 'EN'] # Basic localization
await self.bot.say('{}```\n{}\n```'.format(pre, text))
def get(self, request):
StringToFindLocale = request.GET.get('message')
if not StringToFindLocale:
return HttpResponseBadRequest(
'Message Not Exist - Message to find locale does not exist')
detectResult = detectlanguage.simple_detect(StringToFindLocale)
return JsonResponse({"locale": detectResult})
def traduzir(request, titulo, descricao):
detectlanguage.configuration.api_key = "f2d7fc33a7f3cb0b045d83b6a4c36ad9"
de = detectlanguage.simple_detect(titulo)
client = MicrosoftTranslatorClient(
'sabialiedufes', 'HyJ6rFaIeEPd9q7ZjrWaixrKIUuAXuxIAroGL6YpRl8=')
t = client.TranslateText(titulo.encode('utf-8'), de, 'pt')
d = client.TranslateText(descricao.encode('utf-8'), de, 'pt')
return simplejson.dumps({'titulo': t, 'descricao': d})
def POST(self, text):
print text
# Using 1rst API
language2 = detectlanguage.simple_detect(text)
# Using 2nd API
#language2 = detect(text)
print "Language Detected is - " + language2
return {'language': language2}
def detect_motion_language(motion: Motion) -> bool:
language = detectlanguage.simple_detect(motion.motion + ' ' +
motion.infoslide)
if not language:
return False
motion.language = Language.objects.get_or_create(name=language)[0]
motion.save()
return True
def clean(self):
cleaned_data = super(TranslationForm, self).clean()
locale = self.cleaned_data['locale']
value = self.cleaned_data['value']
LocaleOfValue = detectlanguage.simple_detect(value)
# check {value}'s language is equal to {locale}
if LocaleOfValue != locale:
self.message = 'Language Does Not Match - [ "value" : ' + value + ' ]\'s language is not equal to "' + locale + '"'
raise forms.ValidationError(self.message)
return cleaned_data
def ld_identify(text):
detectlanguage.configuration.api_key = ld_keyvalue
code = ""
try:
code = detectlanguage.simple_detect(text)
except:
return ""
lanlist = detectlanguage.languages()
result = ""
for landict in lanlist:
if landict['code'] == code:
result = landict['name']
break
return result
def storeFileDetails():
data = {
"Name": request.form['name'].encode('utf-8'),
"Content": request.form['content'].encode('utf-8'),
"Sentences": request.form['sentences'].encode('utf-8'),
"toLanguage": request.form['tolanguage'].encode('utf-8'),
"fileLanguage": request.form['fromlanguage'].encode('utf-8')
}
file_language = detectlanguage.simple_detect(data["Content"])
file_name = data["Name"].split("\\")
file_sentences = data["Sentences"].split(",")
#data["content"] = request.form['content'].encode('utf-8');
#data["sentences"] = request.form['sentences'].encode('utf-8');
#data["language"] = request.form['language'].encode('utf-8');
#separated_data = data.split('!@#$%,')
#dataNew = {separated_data[0].split(":")}
translate = mongo.db.translation
existing_file = translate.find_one({
"fname": file_name[-1],
"user": session["username"],
"transtolang": data["toLanguage"],
"filelang": file_language
})
translate.insert({
'fname': file_name[-1],
'fcontent': data["Content"],
'sentences': file_sentences,
'transtolang': data["toLanguage"],
'filelang': file_language,
'tProgress': "In Progress",
'user': session["username"]
})
if existing_file is None:
return jsonify({'result': "File uploaded.", 'sentences': "Thank you!"})
return jsonify({
'result':
"You have already uploaded " + file_name[-1],
'sentences':
"to be translated into " + data["toLanguage"] +
". Please change the file name or language of translation. Thank you!"
})
def get_all_quotations(author, may_require_translation):
#author,may_require_translation = choice(authors)
global auto_suggest
try:
quotes = wikiquote.quotes(author)
except Exception as e:
print(f"Exception retrieving from wikiquote: {e}")
return
quote_list = list()
for quote in quotes:
quote = quote.replace(chr(173), "") # appears to be extended ascii 173 in Churchil quotes (? others)
if may_require_translation:
lang_code = detectlanguage.simple_detect(quote)
if lang_code != "en":
language = lang_map.get(lang_code, "No language code match")
#translation = translate_client.translate(quote, "en")
response = translate_client.translate_text(
parent=parent,
contents=[quote],
mime_type='text/plain',
source_language_code=lang_code,
target_language_code='en')
translation = response.translations[0]
translation = f"{translation}".replace("translated_text: ", "").replace('"', '')
else:
language = ""
translation = ""
else:
language = ""
translation = ""
s = f"Translated from {language}\n" if language else ""
z = f"\n\n{s}" if translation else ""
quote_list.append(f"{translation}{z}{quote}")
# return a list of quotes
return quote_list
def get_quotation(author, may_require_translation):
#author,may_require_translation = choice(authors)
global auto_suggest
try:
quote = choice(wikiquote.quotes(author))
except Exception as e:
print(f"Exception retrieving from wikiquote: {e}")
quote = f"Couldn't retrieve the quotation from {author}. Received exception: {html.escape(repr(e))}"
quote = quote.replace(chr(173), "") # appears to be extended ascii 173 in Churchil quotes (? others)
if may_require_translation:
lang_code = detectlanguage.simple_detect(quote)
if lang_code != "en":
language = lang_map.get(lang_code, "No language code match")
#translation = translate_client.translate(quote, "en")
response = translate_client.translate_text(
parent=parent,
contents=[quote],
mime_type='text/plain',
source_language_code=lang_code,
target_language_code='en')
translation = response.translations[0]
translation = f"{translation}".replace("translated_text: ", "").replace('"', '')
else:
language = ""
translation = ""
else:
language = ""
translation = ""
s = f"Translated from {language}\n" if language else ""
z = f"\n\n{s}" if translation else ""
lines = textwrap.wrap(f"{translation}{z}{quote}", max_chars_line, initial_indent=indent, subsequent_indent=indent)
line_count = len(lines)
lines = "\n".join(lines)
return lines, line_count
def create_translation():
if not request.json or not 'text' in request.json:
abort(400)
print request.json
text = request.json['text']
print text
# Using 1rst API
language2 = detectlanguage.simple_detect(text)
# Using 2nd API
#language2 = detect(text)
print "Language Detected is - " + language2
language = {
'language': request.json.get('language', language2),
}
translation = {
'id': translations[-1]['id'] + 1,
'text': request.json['text'],
'language': request.json.get('language', language2),
}
translations.append(translation)
return jsonify(language), 201
def detect_lang(self, text):
"""
Use detectlanguage API to find the language of the given text
:param text: text to detect
:return: res.lang compassion record if the language is found, or False
"""
detectlanguage.configuration.api_key = config.get(
'detect_language_api_key')
language_name = False
langs = detectlanguage.languages()
try:
code_lang = detectlanguage.simple_detect(text)
except (IndexError, detectlanguage.DetectLanguageError):
# Language could not be detected
return False
for lang in langs:
if lang.get("code") == code_lang:
language_name = lang.get("name")
break
if not language_name:
return False
return self.env['res.lang.compassion'].search(
[('name', '=ilike', language_name)], limit=1)
word_freqs[w] = 1
#populate word_scores with average of all scores from tweets (value 1-10)
for w in word_totals:
try:
word = str(w)
word_scores[word] = float(word_totals[word]) / float(word_freqs[word])
except:
pass
#get 10 sample tweets from twitter
tweet_list = []
tweet_count = 20
for tweet in iterator:
try:
text = json.loads(json.dumps((tweet)))[u'text']
if detectlanguage.simple_detect(text) == "en":
tweet_count -= 1
tweet_list.append(text)
except:
pass
if tweet_count <= 0:
break
for t in tweet_list:
print t
print calculate_score(t, word_scores)
print ""
import detectlanguage
detectlanguage.configuration.api_key = "7e8ac16dc6ab196f2449de5fd7d7f70b"
# Enable secure mode (SSL) if you are passing sensitive data
# detectlanguage.configuration.secure = True
print(detectlanguage.simple_detect("Buenos dias señor"))
print(detectlanguage.detect("سلام گلم"))
print(detectlanguage.detect("Oui"))
detectlanguage.user_status()
import detectlanguage
detectlanguage.configuration.api_key = "609c401c57d4180ddbf29d57ad9b273c"
print(detectlanguage.detect("Buenos dias señor"))
print(detectlanguage.simple_detect("तक को बनाया स्टार"))
print(detectlanguage.user_status())
def test_simple_detect(self):
result = detectlanguage.simple_detect("Hello world")
eq_('en', result)
doc[champ] = donnes
else:
print("file ignored ", fil, champ)
elif bre['label'] in OT:
file = [truc for truc in lstUnk if
bre['label'] == truc.split('-')[1].replace('.txt', "") and "Consistent" not in truc][0]
with open(file, 'r', encoding='utf8') as fichier:
contenu = fichier.read()
if len(contenu) > 0:
phrase = contenu.split('.')[0]
if len(phrase) > 0 and len(phrase.split()) > 5: # more than 5 words
try:
if detectlanguage.user_status()['requests'] < detectlanguage.user_status() ['daily_requests_limit'] -1:
lang = detectlanguage.simple_detect(phrase)
else:
time.sleep(86400) # wait ONE day!
lang = detectlanguage.simple_detect(phrase)
except:
time.sleep(2)
try:
if detectlanguage.user_status()['requests'] < detectlanguage.user_status()[
'daily_requests_limit'] - 1:
lang = detectlanguage.simple_detect(phrase)
else:
time.sleep(86400) # wait ONE day!
lang = detectlanguage.simple_detect(phrase)
except:
lang = ''
else:
def _language_detect(message):
return detectlanguage.simple_detect(message)
def detect_language(text):
"""detects the language a text string is in and returns the language as a string"""
output = unicodedata.normalize('NFD', text).encode('ascii', 'ignore')
lang_code = detectlanguage.simple_detect(output)
return langs[lang_code]
def test_simple_detect(self):
result = detectlanguage.simple_detect("Hello world")
eq_('en', result)
def gui():
test = e1.get()
testy = word_tokenize(test)
sent = []
lan = []
hin = []
translator = Translator()
gs = goslate.Goslate()
with open(
'/home/charul/recent_projects/code_switching/normalisation_dataset.csv',
'r') as file:
reader = csv.reader(file, delimiter=' ')
for row in reader:
sent.append(row[0]) # stores the first column of all the rows
lan.append(row[1]) # stores the 2nd column of all the rows
hin.append(row[2]) # storesthe 3rd column of all the rows
result = ''
con_hin = ''
for words in testy:
maxi = 0.90
match = ''
for i in range(0, 15966):
if (difflib.SequenceMatcher(None, sent[i], words).ratio() >
maxi): # so that we get the word having highest ratio
maxi = difflib.SequenceMatcher(None, sent[i], words).ratio()
match = hin[i]
if (
match == ''
): # if the typed word is not present in dataset then the typed word is sent as output
#match = words
if (detectlanguage.simple_detect(words) != 'en'):
match = words
else:
x = translator.translate(words, dest='hi')
con_hin = x.text
match = con_hin
result = result + match + " "
print(result)
# code for translating the language
y = gs.translate(result, 'en')
#x=translator.translate(result, dest='en')
#y= x.text # for extracting only converted text and omitting the other output parameters
print(y)
text1.insert(INSERT,
y + "\n") # for displaying the result in the text box in GUI
print()
#Code for speech recognition
speak_('Speech recog')
#POS tagging
output_file = open('hindi.input.txt', 'w')
output_file.write(result)
output_file.close()
os.chdir('/home/charul/recent_projects/code_switching/Hindi_POS')
os.system('make tag')
def detect(content): return detectlanguage.simple_detect(content)
def get_quotation():
global authors_left
author, may_require_translation = choice(authors_left)
try:
quote = choice(wikiquote.quotes(author))
except Exception as e:
print(f"Exception retrieving from wikiquote: {e}")
quote = f"Couldn't retrieve the quotation from {author}. Received exception: {html.escape(repr(e))}"
quote = quote.replace(
chr(173),
"") # appears to be extended ascii 173 in Churchil quotes (? others):w
if may_require_translation:
lang_code = detectlanguage.simple_detect(quote)
if lang_code != "en":
language = lang_map.get(lang_code, "No language code match")
#translation = translate_client.translate(quote, "en")
response = translate_client.translate_text(
parent=parent,
contents=[quote],
mime_type='text/plain', # mime types: text/plain, text/html
#source_language_code=lang_code+'-'+language,
source_language_code=lang_code,
#target_language_code='en-US')
target_language_code='en')
translation = response.translations[0]
translation = f"{translation}".replace("translated_text: ",
"").replace('"', '')
print(translation)
else:
language = ""
translation = ""
else:
language = ""
translation = ""
s = f"Translated from {language}<br/>" if language else ""
z = " \n \n" if translation else ""
#lines = textwrap.wrap(repr(translation) quote + "\n" + repr(translation), max_chars_line)
lines = textwrap.wrap(f"{translation}{z}{quote}", max_chars_line)
text = "<br/>".join(lines)
data = {"text": f"{s}{text}<br/>", "footer": author}
try:
publish_lyrics(payload=json.dumps(data))
except Exception as e:
print(e)
try:
bio = wikipedia.summary(author)
except Exception as e:
print(f"Couldn't retrieve {author} bio from wikipedia: {e}")
text = f"Couldn't retrieve {author} bio from wikipedia: {html.escape(repr(e))}"
else:
index = bio.find(".", 400)
if index != -1:
bio = bio[:index + 1]
lines = textwrap.wrap(bio, max_chars_line)
text = "<br/>".join(lines)
data = {"header": f"{author}<br/>", "text": text}
try:
publish_bio(payload=json.dumps(data))
except Exception as e:
print(e)
print(wikipedia.summary(author))
try:
page = wikipedia.page(author)
images = page.images
except Exception as e:
print(f"Could not retrieve page/images for {author}")
print(f"Exception retrieving from wikipedia: {e}")
data = {"uri": "searching"}
else:
while 1:
uri = choice(images)
if uri[-4:].lower() in [".jpg", ".png"]:
break
else:
images.remove(uri)
data = {"header": author, "uri": uri, "type": "image"}
print(data)
try:
publish_images(payload=json.dumps(data))
except Exception as e:
print(e)
authors_left.remove((author, may_require_translation))
if not authors_left:
authors_left = authors[:]
print("Starting author cycle again")
else:
print(f"Number of authors left is {len(authors_left)}")
