def __init__(self, internal_name, controller):
super(TerminologyModel, self).__init__(controller)
self.internal_name = internal_name
self.client = HTTPClient()
self.client.set_virtaal_useragent()
self.load_config()
if not os.path.isdir(self.TERMDIR):
os.mkdir(self.TERMDIR)
self.main_controller = controller.main_controller
self.term_controller = controller
self.matcher = None
self.init_matcher()
lang_controller = self.main_controller.lang_controller
self.source_lang = lang_controller.source_lang.code
self.target_lang = lang_controller.target_lang.code
self._connect_ids.append((
lang_controller.connect('source-lang-changed', self._on_lang_changed, 'source'),
lang_controller
))
self._connect_ids.append((
lang_controller.connect('target-lang-changed', self._on_lang_changed, 'target'),
lang_controller
))
self.update_terms()
def __init__(self, url, max_candidates=3, min_similarity=75, max_length=1000):
gobject.GObject.__init__(self)
HTTPClient.__init__(self)
self.max_candidates = max_candidates
self.min_similarity = min_similarity
self.comparer = LevenshteinComparer(max_length)
self.last_suggestions = None
self.url = url
self.source_lang = None
self.target_lang = None
def __init__(self, internal_name, controller):
self.internal_name = internal_name
super(TMModel, self).__init__(controller)
self.load_config()
if not self.config['api_key']:
self._disable_all(
"An API key is needed to use the Google Translate plugin")
return
self.client = HTTPClient()
self._languages = set()
langreq = RESTRequest(self.url_getlanguages % self.config, '')
self.client.add(langreq)
langreq.connect('http-success',
lambda langreq, response: self.got_languages(response))
def _download_checker(self, language):
"""A Windows-only way to obtain new dictionaries."""
if 'APPDATA' not in os.environ:
# We won't have an idea of where to save it, so let's give up now
return
if language in self.clients:
# We already tried earlier, or started the process
return
if not self.languages:
if self._lang_list not in self.clients:
# We don't yet have a list of available languages
url = self._base_URL + self._lang_list #index page listing all the dictionaries
callback = lambda *args: self._process_index(language=language, *args)
client = HTTPClient()
client.set_virtaal_useragent()
client.get(url, callback)
self.clients[self._lang_list] = client
# self._process_index will call this again, so we can exit
return
# Let's see if a dictionary is available for this language:
for l in self.languages:
if l == language or l.startswith(language+'_'):
self.clients[language] = None
logging.debug("Will use %s to spell check %s", l, language)
language = l
break
else:
# No dictionary available
# Indicate that we never need to try this language:
logging.debug("Found no suitable language for spell checking")
self.clients[language] = None
return
# Now download the actual files after we have determined that it is
# available
self.clients[language] = HTTPClient()
self.clients[language].set_virtaal_useragent()
callback = lambda *args: self._process_tarball(language=language, *args)
url = self._dict_URL % language
if logging.root.level != logging.DEBUG:
self.clients[language].get(url, callback)
else:
def error_log(request, result):
logging.debug('Could not get %s: status %d' % (url, request.status))
self.clients[language].get(url, callback, error_callback=error_log)
def __init__(self, internal_name, controller):
self.internal_name = internal_name
self.language_pairs = []
self.load_config()
self.client = HTTPClient()
self.url_getpairs = "%(url)s/listPairs?appId=%(appid)s" % {"url": self.url, "appid": self.config["appid"]}
self.url_translate = "%(url)s/translate" % {"url": self.url}
self.appid = self.config['appid']
langreq = RESTRequest(self.url_getpairs, '')
self.client.add(langreq)
langreq.connect(
'http-success',
lambda langreq, response: self.got_language_pairs(response)
)
super(TMModel, self).__init__(controller)
def __init__(self, max_candidates=3, min_similarity=75, max_length=1000):
gobject.GObject.__init__(self)
HTTPClient.__init__(self)
self.max_candidates = max_candidates
self.min_similarity = min_similarity
self.comparer = LevenshteinComparer(max_length)
self.last_suggestions = [] # used by the open-tran terminology backend
self._languages = set()
self.source_lang = None
self.target_lang = None
#detect supported language
self.url_getlanguages = 'http://open-tran.eu/json/supported'
self.url_translate = 'http://%s.%s.open-tran.eu/json/suggest'
langreq = RESTRequest(self.url_getlanguages, id='')
self.add(langreq)
langreq.connect('http-success',
lambda langreq, response: self.got_languages(response))
def _build_client(self, url, clients_id, callback, error_callback=None):
from virtaal.support.httpclient import HTTPClient
client = HTTPClient()
client.set_virtaal_useragent()
self.clients[clients_id] = client
if logging.root.level != logging.DEBUG:
client.get(url, callback)
else:
def error_log(request, result):
logging.debug('Could not get %s: status %d' %
(url, request.status))
client.get(url, callback, error_callback=error_log)
def __init__(self, internal_name, controller):
self.internal_name = internal_name
self.languages = []
self.load_config()
self.client = HTTPClient()
self.url_getlanguages = "%(url)s/GetLanguages?appId=%(appid)s" % {
"url": self.config['url'],
"appid": self.config["appid"]
}
self.url_translate = "%(url)s/Translate" % {"url": self.config['url']}
self.appid = self.config['appid']
langreq = RESTRequest(self.url_getlanguages,
'',
method='GET',
data=urllib.urlencode(''),
headers=None)
self.client.add(langreq)
langreq.connect('http-success',
lambda langreq, response: self.got_languages(response))
super(TMModel, self).__init__(controller)
def _build_client(self, url, clients_id, callback, error_callback=None):
from virtaal.support.httpclient import HTTPClient
client = HTTPClient()
client.set_virtaal_useragent()
self.clients[clients_id] = client
if logging.root.level != logging.DEBUG:
client.get(url, callback)
else:
def error_log(request, result):
logging.debug('Could not get %s: status %d' % (url, request.status))
client.get(url, callback, error_callback=error_log)
def __init__(self, max_candidates=3, min_similarity=75, max_length=1000):
gobject.GObject.__init__(self)
HTTPClient.__init__(self)
self.max_candidates = max_candidates
self.min_similarity = min_similarity
self.comparer = LevenshteinComparer(max_length)
self.last_suggestions = None # used by the open-tran terminology backend
self._languages = set()
self.source_lang = None
self.target_lang = None
#detect supported language
self.url_getlanguages = 'http://open-tran.eu/json/supported'
self.url_translate = 'http://%s.%s.open-tran.eu/json/suggest'
langreq = RESTRequest(self.url_getlanguages, id='', method='GET', data=urllib.urlencode(''))
self.add(langreq)
langreq.connect(
'http-success',
lambda langreq, response: self.got_languages(response)
)
def __init__(self, internal_name, controller):
self.internal_name = internal_name
self.language_pairs = []
self.load_config()
self.client = HTTPClient()
self.url_getpairs = "%(url)s/listPairs?appId=%(appid)s" % {"url": self.url, "appid": self.config["appid"]}
self.url_translate = "%(url)s/translate" % {"url": self.url}
self.appid = self.config["appid"]
langreq = RESTRequest(self.url_getpairs, "", method="GET", data=urllib.urlencode(""), headers=None)
self.client.add(langreq)
langreq.connect("http-success", lambda langreq, response: self.got_language_pairs(response))
super(TMModel, self).__init__(controller)
def __init__(self, internal_name, controller):
self.internal_name = internal_name
super(TMModel, self).__init__(controller)
self.load_config()
if not self.config['api_key']:
self._disable_all("An API key is needed to use the Google Translate plugin")
return
self.client = HTTPClient()
self._languages = set()
langreq = RESTRequest(self.url_getlanguages % self.config, '')
self.client.add(langreq)
langreq.connect(
'http-success',
lambda langreq, response: self.got_languages(response)
)
def __init__(self, internal_name, controller):
self.internal_name = internal_name
self.languages = []
self.load_config()
self.client = HTTPClient()
self.url_getlanguages = "%(url)s/GetLanguages?appId=%(appid)s" % {"url": self.config['url'], "appid": self.config["appid"]}
self.url_translate = "%(url)s/Translate" % {"url": self.config['url']}
self.appid = self.config['appid']
langreq = RESTRequest(self.url_getlanguages, '', method='GET', data=urllib.urlencode(''), headers=None)
self.client.add(langreq)
langreq.connect(
'http-success',
lambda langreq, response: self.got_languages(response)
)
super(TMModel, self).__init__(controller)
class TMModel(BaseTMModel):
"""This is a Google Translate translation memory model.
The plugin uses the U{Google AJAX Languages API<http://code.google.com/apis/ajaxlanguage/>}
to query Google's machine translation services. The implementation makes use of the
U{RESTful<http://code.google.com/apis/ajaxlanguage/documentation/#fonje>} interface for
Non-JavaScript environments.
"""
__gtype_name__ = 'GoogleTranslateTMModel'
#l10n: The name of Google Translate in your language (translated in most languages). See http://translate.google.com/
display_name = _('Google Translate')
description = _(
"Unreviewed machine translations from Google's translation service")
default_config = {'api_key': ''}
translate_url = "https://www.googleapis.com/language/translate/v2?key=%(key)s&q=%(message)s&source=%(from)s&target=%(to)s"
languages_url = "https://www.googleapis.com/language/translate/v2/languages?key=%(key)s"
# INITIALIZERS #
def __init__(self, internal_name, controller):
self.internal_name = internal_name
super(TMModel, self).__init__(controller)
self.load_config()
if not self.config['api_key']:
self._disable_all(
"An API key is needed to use the Google Translate plugin")
return
self.client = HTTPClient()
self._languages = set()
langreq = RESTRequest(self.url_getlanguages % self.config, '')
self.client.add(langreq)
langreq.connect('http-success',
lambda langreq, response: self.got_languages(response))
# METHODS #
def query(self, tmcontroller, unit):
query_str = unit.source
# Google's Terms of Service says the whole URL must be less than "2K"
# characters.
query_str = query_str[:2000 - len(self.translate_url)]
source_lang = code_translation.get(self.source_lang,
self.source_lang).replace('_', '-')
target_lang = code_translation.get(self.target_lang,
self.target_lang).replace('_', '-')
if source_lang not in self._languages or target_lang not in self._languages:
logging.debug('language pair not supported: %s => %s' %
(source_lang, target_lang))
return
if self.cache.has_key(query_str):
self.emit('match-found', query_str, self.cache[query_str])
else:
real_url = self.translate_url % {
'key': self.config['api_key'],
'message': urllib.quote_plus(query_str.encode('utf-8')),
'from': source_lang,
'to': target_lang,
}
req = RESTRequest(real_url, '')
self.client.add(req)
# Google's Terms of Service says we need a proper HTTP referrer
req.curl.setopt(pycurl.REFERER, virtaal_referrer)
req.connect(
'http-success', lambda req, response: self.got_translation(
response, query_str))
req.connect(
'http-client-error',
lambda req, response: self.got_error(response, query_str))
req.connect(
'http-server-error',
lambda req, response: self.got_error(response, query_str))
def got_translation(self, val, query_str):
"""Handle the response from the web service now that it came in."""
# In December 2011 version 1 of the API was deprecated, and we had to
# release code to handle the eminent disappearance of the API. Although
# version 2 is now supported, the code is a bit more careful (as most
# code probably should be) and in case of error we make the list of
# supported languages empty so that no unnecesary network activity is
# performed if we can't communicate with the available API any more.
try:
data = json.loads(val)
# We try to access the members to validate that the dictionary is
# formed in the way we expect.
data['data']
data['data']['translations']
text = data['data']['translations'][0]['translatedText']
except Exception, e:
self._disable_all("Error with json response: %s" % e)
return
target_unescaped = unescape_html_entities(text)
if not isinstance(target_unescaped, unicode):
target_unescaped = unicode(target_unescaped, 'utf-8')
match = {
'source': query_str,
'target': target_unescaped,
#l10n: Try to keep this as short as possible. Feel free to transliterate.
'tmsource': _('Google')
}
self.cache[query_str] = [match]
self.emit('match-found', query_str, [match])
def __init__(self, url):
gobject.GObject.__init__(self)
HTTPClient.__init__(self)
self.url = url + '/RPC2'
self.multilang = False
class TMModel(BaseTMModel):
"""This is a Google Translate translation memory model.
The plugin uses the U{Google AJAX Languages API<http://code.google.com/apis/ajaxlanguage/>}
to query Google's machine translation services. The implementation makes use of the
U{RESTful<http://code.google.com/apis/ajaxlanguage/documentation/#fonje>} interface for
Non-JavaScript environments.
"""
__gtype_name__ = "GoogleTranslateTMModel"
# l10n: The name of Google Translate in your language (translated in most languages). See http://translate.google.com/
display_name = _("Google Translate")
description = _("Unreviewed machine translations from Google's translation service")
translate_url = (
"http://ajax.googleapis.com/ajax/services/language/translate?v=1.0&q=%(message)s&langpair=%(from)s%%7C%(to)s"
)
# INITIALIZERS #
def __init__(self, internal_name, controller):
self.internal_name = internal_name
super(TMModel, self).__init__(controller)
self.client = HTTPClient()
# METHODS #
def query(self, tmcontroller, unit):
query_str = unit.source
# Google's Terms of Service says no more than 5000 characters
query_str = query_str[:5000]
source_lang = code_translation.get(self.source_lang, self.source_lang).replace("_", "-")
target_lang = code_translation.get(self.target_lang, self.target_lang).replace("_", "-")
if source_lang not in _languages or target_lang not in _languages:
logging.debug("language pair not supported: %s => %s" % (source_lang, target_lang))
return
if self.cache.has_key(query_str):
self.emit("match-found", query_str, self.cache[query_str])
else:
real_url = self.translate_url % {
"message": urllib.quote_plus(query_str.encode("utf-8")),
"from": source_lang,
"to": target_lang,
}
req = RESTRequest(real_url, "", method="GET", data=urllib.urlencode(""), headers=None)
self.client.add(req)
# Google's Terms of Service says we need a proper HTTP referrer
req.curl.setopt(pycurl.REFERER, virtaal_referrer)
req.connect("http-success", lambda req, response: self.got_translation(response, query_str))
def got_translation(self, val, query_str):
"""Handle the response from the web service now that it came in."""
data = json.loads(val)
if data["responseStatus"] != 200:
logging.debug("Failed to translate '%s':\n%s" % (query_str, data["responseDetails"]))
return
target_unescaped = unescape_html_entities(data["responseData"]["translatedText"])
if not isinstance(target_unescaped, unicode):
target_unescaped = unicode(target_unescaped, "utf-8")
match = {
"source": query_str,
"target": target_unescaped,
# l10n: Try to keep this as short as possible. Feel free to transliterate.
"tmsource": _("Google"),
}
self.cache[query_str] = [match]
self.emit("match-found", query_str, [match])
def __init__(self, internal_name, controller):
self.internal_name = internal_name
super(TMModel, self).__init__(controller)
self.client = HTTPClient()
class TMModel(BaseTMModel):
"""This is a Google Translate translation memory model.
The plugin uses the U{Google AJAX Languages API<http://code.google.com/apis/ajaxlanguage/>}
to query Google's machine translation services. The implementation makes use of the
U{RESTful<http://code.google.com/apis/ajaxlanguage/documentation/#fonje>} interface for
Non-JavaScript environments.
"""
__gtype_name__ = 'GoogleTranslateTMModel'
#l10n: The name of Google Translate in your language (translated in most languages). See http://translate.google.com/
display_name = _('Google Translate')
description = _("Unreviewed machine translations from Google's translation service")
translate_url = "http://ajax.googleapis.com/ajax/services/language/translate?v=1.0&q=%(message)s&langpair=%(from)s%%7C%(to)s"
# INITIALIZERS #
def __init__(self, internal_name, controller):
self.internal_name = internal_name
super(TMModel, self).__init__(controller)
self.client = HTTPClient()
# METHODS #
def query(self, tmcontroller, unit):
query_str = unit.source
# Google's Terms of Service says no more than 5000 characters
query_str = query_str[:5000]
source_lang = code_translation.get(self.source_lang, self.source_lang).replace('_', '-')
target_lang = code_translation.get(self.target_lang, self.target_lang).replace('_', '-')
if source_lang not in _languages or target_lang not in _languages:
logging.debug('language pair not supported: %s => %s' % (source_lang, target_lang))
return
if self.cache.has_key(query_str):
self.emit('match-found', query_str, self.cache[query_str])
else:
real_url = self.translate_url % {
'message': urllib.quote_plus(query_str.encode('utf-8')),
'from': source_lang,
'to': target_lang,
}
req = RESTRequest(real_url, '')
self.client.add(req)
# Google's Terms of Service says we need a proper HTTP referrer
req.curl.setopt(pycurl.REFERER, virtaal_referrer)
req.connect(
'http-success',
lambda req, response: self.got_translation(response, query_str)
)
req.connect(
'http-client-error',
lambda req, response: self.got_error(response, query_str)
)
req.connect(
'http-server-error',
lambda req, response: self.got_error(response, query_str)
)
def got_translation(self, val, query_str):
"""Handle the response from the web service now that it came in."""
# Since we expect Google to shut down the service in December 2011, we
# try to be extra careful with error handling, and actually expect
# problems. If we encounter a problem, we make the list of languages
# empty so that no other requests would be attempted.
try:
data = json.loads(val)
# We try to access the members to validate that the dictionary is
# formed in the way we expect.
data['responseStatus']
data['responseData']['translatedText']
except Exception, e:
self._disable_all("Error with json response: %s" % e)
return
if data['responseStatus'] != 200:
logging.debug("Failed to translate '%s':\n%s" % (query_str, data['responseDetails']))
self._disable_all("responseStatus not 200")
return
target_unescaped = unescape_html_entities(data['responseData']['translatedText'])
if not isinstance(target_unescaped, unicode):
target_unescaped = unicode(target_unescaped, 'utf-8')
match = {
'source': query_str,
'target': target_unescaped,
#l10n: Try to keep this as short as possible. Feel free to transliterate.
'tmsource': _('Google')
}
self.cache[query_str] = [match]
self.emit('match-found', query_str, [match])
class TMModel(BaseTMModel):
"""This is the translation memory model."""
__gtype_name__ = "ApertiumTMModel"
display_name = _("Apertium")
description = _("Unreviewed machine translations from Apertium")
url = "http://api.apertium.org/json"
default_config = {"appid": ""}
# INITIALISERS #
def __init__(self, internal_name, controller):
self.internal_name = internal_name
self.language_pairs = []
self.load_config()
self.client = HTTPClient()
self.url_getpairs = "%(url)s/listPairs?appId=%(appid)s" % {"url": self.url, "appid": self.config["appid"]}
self.url_translate = "%(url)s/translate" % {"url": self.url}
self.appid = self.config["appid"]
langreq = RESTRequest(self.url_getpairs, "", method="GET", data=urllib.urlencode(""), headers=None)
self.client.add(langreq)
langreq.connect("http-success", lambda langreq, response: self.got_language_pairs(response))
super(TMModel, self).__init__(controller)
# METHODS #
def query(self, tmcontroller, unit):
"""Send the query to the web service. The response is handled by means
of a call-back because it happens asynchronously."""
pair = (self.source_lang, self.target_lang)
if pair not in self.language_pairs:
return
query_str = unit.source
if self.cache.has_key(query_str):
self.emit("match-found", query_str, self.cache[query_str])
else:
values = {
"appId": self.appid,
"q": query_str,
"langpair": "%s|%s" % (self.source_lang, self.target_lang),
"markUnknown": "no",
"format": "html",
}
req = RESTRequest(
self.url_translate + "?" + urllib.urlencode(values),
"",
method="GET",
data=urllib.urlencode(""),
headers=None,
)
self.client.add(req)
req.connect("http-success", lambda req, response: self.got_translation(response, query_str))
def got_language_pairs(self, val):
"""Handle the response from the web service to set up language pairs."""
data = json.loads(val)
if data["responseStatus"] != 200:
logging.debug("Failed to get languages:\n%s", (data["responseDetails"]))
return
self.language_pairs = [(pair["sourceLanguage"], pair["targetLanguage"]) for pair in data["responseData"]]
def got_translation(self, val, query_str):
"""Handle the response from the web service now that it came in."""
data = json.loads(val)
if data["responseStatus"] != 200:
logging.debug("Failed to translate '%s':\n%s", (query_str, data["responseDetails"]))
return
target = data["responseData"]["translatedText"]
target = unescape_html_entities(target)
if target.endswith("\n") and not query_str.endswith("\n"):
target = target[:-1] # chop of \n
if not isinstance(target, unicode):
target = unicode(target, "utf-8")
match = {
"source": query_str,
"target": target,
# l10n: Try to keep this as short as possible. Feel free to transliterate in CJK languages for optimal vertical display.
"tmsource": _("Apertium"),
}
self.cache[query_str] = [match]
self.emit("match-found", query_str, [match])
def __init__(self, url):
gobject.GObject.__init__(self)
HTTPClient.__init__(self)
self.url = url + '/RPC2'
self.multilang = False
class TerminologyModel(BaseTerminologyModel):
"""A terminology back-end to access the Translate.org.za-managed terminology."""
__gtype_name__ = 'AutoTermTerminology'
display_name = _('Localization Terminology')
description = _('Selected localization terminology')
_l10n_URL = 'http://terminology.locamotion.org/l10n/%(srclang)s/%(tgtlang)s'
TERMDIR = os.path.join(pan_app.get_config_dir(), 'autoterm')
# INITIALIZERS #
def __init__(self, internal_name, controller):
super(TerminologyModel, self).__init__(controller)
self.internal_name = internal_name
self.client = HTTPClient()
self.client.set_virtaal_useragent()
self.load_config()
if not os.path.isdir(self.TERMDIR):
os.mkdir(self.TERMDIR)
self.main_controller = controller.main_controller
self.term_controller = controller
self.matcher = None
self.init_matcher()
lang_controller = self.main_controller.lang_controller
self.source_lang = lang_controller.source_lang.code
self.target_lang = lang_controller.target_lang.code
self._connect_ids.append((
lang_controller.connect('source-lang-changed', self._on_lang_changed, 'source'),
lang_controller
))
self._connect_ids.append((
lang_controller.connect('target-lang-changed', self._on_lang_changed, 'target'),
lang_controller
))
self.update_terms()
def init_matcher(self, filename=''):
"""
Initialize the matcher to be used by the C{TerminologyPlaceable} parser.
"""
if self.matcher in TerminologyPlaceable.matchers:
TerminologyPlaceable.matchers.remove(self.matcher)
if os.path.isfile(filename):
logging.debug('Loading terminology from %s' % (filename))
self.store = factory.getobject(filename)
else:
logging.debug('Creating empty terminology store')
self.store = TranslationStore()
self.store.makeindex()
self.matcher = terminologymatcher(self.store)
TerminologyPlaceable.matchers.append(self.matcher)
# ACCESSORS #
def _get_curr_term_filename(self, srclang=None, tgtlang=None, ext=None):
if srclang is None:
srclang = self.source_lang
if tgtlang is None:
tgtlang = self.target_lang
if not ext:
ext = 'po'
base = '%s__%s' % (srclang, tgtlang)
for filename in os.listdir(self.TERMDIR):
if filename.startswith(base):
return filename
return base + os.extsep + ext
curr_term_filename = property(_get_curr_term_filename)
# METHODS #
def update_terms(self, srclang=None, tgtlang=None):
"""Update the terminology file for the given language or all if none specified."""
if srclang is None:
srclang = self.source_lang
if tgtlang is None:
tgtlang = self.target_lang
if srclang is None and tgtlang is None:
# Update all files
return
if srclang is None or tgtlang is None:
raise ValueError('Both srclang and tgtlang must be specified')
if not self.is_update_needed(srclang, tgtlang):
logging.debug('Skipping update for (%s, %s) language pair' % (srclang, tgtlang))
localfile = self._get_curr_term_filename(srclang, tgtlang)
localfile = os.path.join(self.TERMDIR, localfile)
self.init_matcher(localfile)
return
self._update_term_file(srclang, tgtlang)
def is_update_needed(self, srclang, tgtlang):
localfile = self._get_curr_term_filename(srclang, tgtlang)
localfile = os.path.join(self.TERMDIR, localfile)
if not os.path.isfile(localfile):
return True
stats = os.stat(localfile)
from datetime import datetime
return (time.mktime(datetime.now().timetuple()) - stats.st_mtime) > THREE_DAYS
def _check_for_update(self, srclang, tgtlang):
localfile = self._get_curr_term_filename(srclang, tgtlang)
localfile = os.path.join(self.TERMDIR, localfile)
etag = None
if os.path.isfile(localfile) and localfile in self.config:
etag = self.config[os.path.abspath(localfile)]
url = self._l10n_URL % {'srclang': srclang, 'tgtlang': tgtlang}
if not os.path.isfile(localfile):
localfile = None
callback = lambda *args: self._process_header(localfile=localfile, *args)
if logging.root.level != logging.DEBUG:
self.client.get(url, callback, etag)
else:
def error_log(request, result):
logging.debug('Could not get %s: status %d' % (url, request.status))
self.client.get(url, callback, etag, error_callback=error_log)
def _get_ext_from_url(self, url):
from urlparse import urlparse
parsed = urlparse(url)
#dir, filename = os.path.split(parsed.path)
#rewritten for compatibility with Python 2.4:
dir, filename = os.path.split(parsed[2])
if not filename or '.' not in filename:
return None
ext = filename.split('.')[-1]
if not ext:
ext = None
return ext
def _get_ext_from_store_guess(self, content):
from StringIO import StringIO
from translate.storage.factory import _guessextention
s = StringIO(content)
try:
return _guessextention(s)
except ValueError:
pass
return None
def _process_header(self, request, result, localfile=None):
if request.status == 304:
logging.debug('ETag matches for file %s :)' % (localfile))
elif request.status == 200:
if not localfile:
ext = self._get_ext_from_url(request.get_effective_url())
if ext is None:
ext = self._get_ext_from_store_guess(result)
if ext is None:
logging.debug('Unable to determine extension for store. Defaulting to "po".')
ext = 'po'
localfile = self._get_curr_term_filename(ext=ext)
localfile = os.path.join(self.TERMDIR, localfile)
logging.debug('Saving to %s' % (localfile))
open(localfile, 'w').write(result)
# Find ETag header and save the value
headers = request.result_headers.getvalue().splitlines()
etag = ''
etagline = [l for l in headers if l.lower().startswith('etag:')]
if etagline:
etag = etagline[0][7:-1]
self.config[os.path.abspath(localfile)] = etag
else:
logging.debug('Unhandled status code: %d' % (request.status))
localfile = ''
if os.path.isfile(localfile):
# Update mtime
os.utime(localfile, None)
self.init_matcher(localfile)
def _update_term_file(self, srclang, tgtlang):
"""Update the terminology file for the given languages."""
self.init_matcher() # Make sure that the matcher is empty until we have an update
filename = self._get_curr_term_filename(srclang, tgtlang)
localfile = os.path.join(self.TERMDIR, filename)
self._check_for_update(srclang, tgtlang)
# SIGNAL HANDLERS #
def _on_lang_changed(self, lang_controller, lang, which):
setattr(self, '%s_lang' % (which), lang)
self.update_terms(self.source_lang, self.target_lang)
def __init__(self, internal_name, controller):
self.internal_name = internal_name
super(TMModel, self).__init__(controller)
self.client = HTTPClient()
class TMModel(BaseTMModel):
"""This is the translation memory model."""
__gtype_name__ = 'ApertiumTMModel'
display_name = _('Apertium')
description = _('Unreviewed machine translations from Apertium')
url = "http://api.apertium.org/json"
default_config = {
"appid" : "",
}
# INITIALISERS #
def __init__(self, internal_name, controller):
self.internal_name = internal_name
self.language_pairs = []
self.load_config()
self.client = HTTPClient()
self.url_getpairs = "%(url)s/listPairs?appId=%(appid)s" % {"url": self.url, "appid": self.config["appid"]}
self.url_translate = "%(url)s/translate" % {"url": self.url}
self.appid = self.config['appid']
langreq = RESTRequest(self.url_getpairs, '')
self.client.add(langreq)
langreq.connect(
'http-success',
lambda langreq, response: self.got_language_pairs(response)
)
super(TMModel, self).__init__(controller)
# METHODS #
def query(self, tmcontroller, unit):
"""Send the query to the web service. The response is handled by means
of a call-back because it happens asynchronously."""
pair = (self.source_lang, self.target_lang)
if pair not in self.language_pairs:
return
query_str = unit.source
if self.cache.has_key(query_str):
self.emit('match-found', query_str, self.cache[query_str])
else:
values = {
'appId': self.appid,
'q': query_str,
'langpair': "%s|%s" % (self.source_lang, self.target_lang),
'markUnknown': "no",
'format': 'html',
}
req = RESTRequest(self.url_translate + "?" + urllib.urlencode(values), '')
self.client.add(req)
req.connect(
'http-success',
lambda req, response: self.got_translation(response, query_str)
)
def got_language_pairs(self, val):
"""Handle the response from the web service to set up language pairs."""
data = json.loads(val)
if data['responseStatus'] != 200:
import logging
logging.debug("Failed to get languages:\n%s", (data['responseDetails']))
return
self.language_pairs = [(pair['sourceLanguage'], pair['targetLanguage']) for pair in data['responseData']]
def got_translation(self, val, query_str):
"""Handle the response from the web service now that it came in."""
data = json.loads(val)
if data['responseStatus'] != 200:
import logging
logging.debug("Failed to translate '%s':\n%s", (query_str, data['responseDetails']))
return
target = data['responseData']['translatedText']
target = unescape_html_entities(target)
if target.endswith("\n") and not query_str.endswith("\n"):
target = target[:-1]# chop of \n
if not isinstance(target, unicode):
target = unicode(target, 'utf-8')
match = {
'source': query_str,
'target': target,
#l10n: Try to keep this as short as possible. Feel free to transliterate in CJK languages for optimal vertical display.
'tmsource': _('Apertium'),
}
self.cache[query_str] = [match]
self.emit('match-found', query_str, [match])
class TMModel(BaseTMModel):
"""This is the translation memory model."""
__gtype_name__ = 'MicrosoftTranslatorTMModel'
display_name = _('Microsoft Translator')
description = _('Unreviewed machine translations from Microsoft Translator')
default_config = {
"url" : "http://api.microsofttranslator.com/V1/Http.svc",
"appid" : "7286B45B8C4816BDF75DC007C1952DDC11C646C1",
}
# INITIALISERS #
def __init__(self, internal_name, controller):
self.internal_name = internal_name
self.languages = []
self.load_config()
self.client = HTTPClient()
self.url_getlanguages = "%(url)s/GetLanguages?appId=%(appid)s" % {"url": self.config['url'], "appid": self.config["appid"]}
self.url_translate = "%(url)s/Translate" % {"url": self.config['url']}
self.appid = self.config['appid']
langreq = RESTRequest(self.url_getlanguages, '', method='GET', data=urllib.urlencode(''), headers=None)
self.client.add(langreq)
langreq.connect(
'http-success',
lambda langreq, response: self.got_languages(response)
)
super(TMModel, self).__init__(controller)
# METHODS #
def query(self, tmcontroller, unit):
"""Send the query to the web service. The response is handled by means
of a call-back because it happens asynchronously."""
if self.source_lang not in self.languages or self.target_lang not in self.languages:
return
query_str = unit.source
if self.cache.has_key(query_str):
self.emit('match-found', query_str, self.cache[query_str])
else:
values = {
'appId': self.appid,
'text': query_str,
'from': self.source_lang,
'to': self.target_lang
}
req = RESTRequest(self.url_translate + "?" + urllib.urlencode(values), '', method='GET', \
data=urllib.urlencode(''), headers=None)
self.client.add(req)
req.connect(
'http-success',
lambda req, response: self.got_translation(response, query_str)
)
def got_languages(self, val):
"""Handle the response from the web service to set up language pairs."""
val = strip_bom(val.decode('utf-8')).strip()
self.languages = [lang for lang in val.split('\r\n')]
def got_translation(self, val, query_str):
"""Handle the response from the web service now that it came in."""
if not isinstance(val, unicode):
val = unicode(val, 'utf-8')
val = strip_bom(val)
match = {
'source': query_str,
'target': val,
#l10n: Try to keep this as short as possible. Feel free to transliterate in CJK languages for optimal vertical display.
'tmsource': _('Microsoft'),
}
self.cache[query_str] = [match]
self.emit('match-found', query_str, [match])
class TMModel(BaseTMModel):
"""This is the translation memory model."""
__gtype_name__ = 'MicrosoftTranslatorTMModel'
display_name = _('Microsoft Translator')
description = _(
'Unreviewed machine translations from Microsoft Translator')
default_config = {
"url": "http://api.microsofttranslator.com/V1/Http.svc",
"appid": "7286B45B8C4816BDF75DC007C1952DDC11C646C1",
}
# INITIALISERS #
def __init__(self, internal_name, controller):
self.internal_name = internal_name
self.languages = []
self.load_config()
self.client = HTTPClient()
self.url_getlanguages = "%(url)s/GetLanguages?appId=%(appid)s" % {
"url": self.config['url'],
"appid": self.config["appid"]
}
self.url_translate = "%(url)s/Translate" % {"url": self.config['url']}
self.appid = self.config['appid']
langreq = RESTRequest(self.url_getlanguages,
'',
method='GET',
data=urllib.urlencode(''),
headers=None)
self.client.add(langreq)
langreq.connect('http-success',
lambda langreq, response: self.got_languages(response))
super(TMModel, self).__init__(controller)
# METHODS #
def query(self, tmcontroller, unit):
"""Send the query to the web service. The response is handled by means
of a call-back because it happens asynchronously."""
if self.source_lang not in self.languages or self.target_lang not in self.languages:
return
query_str = unit.source
if self.cache.has_key(query_str):
self.emit('match-found', query_str, self.cache[query_str])
else:
values = {
'appId': self.appid,
'text': query_str,
'from': self.source_lang,
'to': self.target_lang
}
req = RESTRequest(self.url_translate + "?" + urllib.urlencode(values), '', method='GET', \
data=urllib.urlencode(''), headers=None)
self.client.add(req)
req.connect(
'http-success', lambda req, response: self.got_translation(
response, query_str))
def got_languages(self, val):
"""Handle the response from the web service to set up language pairs."""
val = strip_bom(val.decode('utf-8')).strip()
self.languages = [lang for lang in val.split('\r\n')]
def got_translation(self, val, query_str):
"""Handle the response from the web service now that it came in."""
if not isinstance(val, unicode):
val = unicode(val, 'utf-8')
val = strip_bom(val)
match = {
'source': query_str,
'target': val,
#l10n: Try to keep this as short as possible. Feel free to transliterate in CJK languages for optimal vertical display.
'tmsource': _('Microsoft'),
}
self.cache[query_str] = [match]
self.emit('match-found', query_str, [match])
class TMModel(BaseTMModel):
"""This is a Google Translate translation memory model.
The plugin uses the U{Google AJAX Languages API<http://code.google.com/apis/ajaxlanguage/>}
to query Google's machine translation services. The implementation makes use of the
U{RESTful<http://code.google.com/apis/ajaxlanguage/documentation/#fonje>} interface for
Non-JavaScript environments.
"""
__gtype_name__ = 'GoogleTranslateTMModel'
#l10n: The name of Google Translate in your language (translated in most languages). See http://translate.google.com/
display_name = _('Google Translate')
description = _("Unreviewed machine translations from Google's translation service")
default_config = {'api_key': ''}
translate_url = "https://www.googleapis.com/language/translate/v2?key=%(key)s&q=%(message)s&source=%(from)s&target=%(to)s"
languages_url = "https://www.googleapis.com/language/translate/v2/languages?key=%(key)s"
# INITIALIZERS #
def __init__(self, internal_name, controller):
self.internal_name = internal_name
super(TMModel, self).__init__(controller)
self.load_config()
if not self.config['api_key']:
self._disable_all("An API key is needed to use the Google Translate plugin")
return
self.client = HTTPClient()
self._languages = set()
langreq = RESTRequest(self.url_getlanguages % self.config, '')
self.client.add(langreq)
langreq.connect(
'http-success',
lambda langreq, response: self.got_languages(response)
)
# METHODS #
def query(self, tmcontroller, unit):
query_str = unit.source
# Google's Terms of Service says the whole URL must be less than "2K"
# characters.
query_str = query_str[:2000 - len(self.translate_url)]
source_lang = code_translation.get(self.source_lang, self.source_lang).replace('_', '-')
target_lang = code_translation.get(self.target_lang, self.target_lang).replace('_', '-')
if source_lang not in self._languages or target_lang not in self._languages:
logging.debug('language pair not supported: %s => %s' % (source_lang, target_lang))
return
if self.cache.has_key(query_str):
self.emit('match-found', query_str, self.cache[query_str])
else:
real_url = self.translate_url % {
'key': self.config['api_key'],
'message': urllib.quote_plus(query_str.encode('utf-8')),
'from': source_lang,
'to': target_lang,
}
req = RESTRequest(real_url, '')
self.client.add(req)
# Google's Terms of Service says we need a proper HTTP referrer
req.curl.setopt(pycurl.REFERER, virtaal_referrer)
req.connect(
'http-success',
lambda req, response: self.got_translation(response, query_str)
)
req.connect(
'http-client-error',
lambda req, response: self.got_error(response, query_str)
)
req.connect(
'http-server-error',
lambda req, response: self.got_error(response, query_str)
)
def got_translation(self, val, query_str):
"""Handle the response from the web service now that it came in."""
# In December 2011 version 1 of the API was deprecated, and we had to
# release code to handle the eminent disappearance of the API. Although
# version 2 is now supported, the code is a bit more careful (as most
# code probably should be) and in case of error we make the list of
# supported languages empty so that no unnecesary network activity is
# performed if we can't communicate with the available API any more.
try:
data = json.loads(val)
# We try to access the members to validate that the dictionary is
# formed in the way we expect.
data['data']
data['data']['translations']
text = data['data']['translations'][0]['translatedText']
except Exception, e:
self._disable_all("Error with json response: %s" % e)
return
target_unescaped = unescape_html_entities(text)
if not isinstance(target_unescaped, unicode):
target_unescaped = unicode(target_unescaped, 'utf-8')
match = {
'source': query_str,
'target': target_unescaped,
#l10n: Try to keep this as short as possible. Feel free to transliterate.
'tmsource': _('Google')
}
self.cache[query_str] = [match]
self.emit('match-found', query_str, [match])
def __init__(self, base_url):
HTTPClient.__init__(self)
self.base_url = base_url
class TMModel(BaseTMModel):
"""This is a Google Translate translation memory model.
The plugin uses the U{Google AJAX Languages API<http://code.google.com/apis/ajaxlanguage/>}
to query Google's machine translation services. The implementation makes use of the
U{RESTful<http://code.google.com/apis/ajaxlanguage/documentation/#fonje>} interface for
Non-JavaScript environments.
"""
__gtype_name__ = 'GoogleTranslateTMModel'
#l10n: The name of Google Translate in your language (translated in most languages). See http://translate.google.com/
display_name = _('Google Translate')
description = _(
"Unreviewed machine translations from Google's translation service")
translate_url = "http://ajax.googleapis.com/ajax/services/language/translate?v=1.0&q=%(message)s&langpair=%(from)s%%7C%(to)s"
# INITIALIZERS #
def __init__(self, internal_name, controller):
self.internal_name = internal_name
super(TMModel, self).__init__(controller)
self.client = HTTPClient()
# METHODS #
def query(self, tmcontroller, unit):
query_str = unit.source
# Google's Terms of Service says no more than 5000 characters
query_str = query_str[:5000]
source_lang = code_translation.get(self.source_lang,
self.source_lang).replace('_', '-')
target_lang = code_translation.get(self.target_lang,
self.target_lang).replace('_', '-')
if source_lang not in _languages or target_lang not in _languages:
logging.debug('language pair not supported: %s => %s' %
(source_lang, target_lang))
return
if self.cache.has_key(query_str):
self.emit('match-found', query_str, self.cache[query_str])
else:
real_url = self.translate_url % {
'message': urllib.quote_plus(query_str.encode('utf-8')),
'from': source_lang,
'to': target_lang,
}
req = RESTRequest(real_url,
'',
method='GET',
data=urllib.urlencode(''),
headers=None)
self.client.add(req)
# Google's Terms of Service says we need a proper HTTP referrer
req.curl.setopt(pycurl.REFERER, virtaal_referrer)
req.connect(
'http-success', lambda req, response: self.got_translation(
response, query_str))
def got_translation(self, val, query_str):
"""Handle the response from the web service now that it came in."""
data = json.loads(val)
if data['responseStatus'] != 200:
logging.debug("Failed to translate '%s':\n%s" %
(query_str, data['responseDetails']))
return
target_unescaped = unescape_html_entities(
data['responseData']['translatedText'])
if not isinstance(target_unescaped, unicode):
target_unescaped = unicode(target_unescaped, 'utf-8')
match = {
'source': query_str,
'target': target_unescaped,
#l10n: Try to keep this as short as possible. Feel free to transliterate.
'tmsource': _('Google')
}
self.cache[query_str] = [match]
self.emit('match-found', query_str, [match])
def __init__(self, base_url):
HTTPClient.__init__(self)
self.base_url = base_url
