def json_req(self, url, http_post=False, skip_auth=False, raw=False,
**kwargs):
'''
Performs JSON request.
'''
# Encode params
if len(kwargs) > 0:
params = urlencode(
{key: val.encode('utf-8') for key, val in kwargs.items()}
)
else:
params = ''
# Store for exception handling
self.request_url = url
self.request_params = params
# Append parameters
if len(params) > 0 and not http_post:
url = '?'.join((url, params))
# Create request object with custom headers
request = Request(url)
request.timeout = 0.5
request.add_header('User-Agent', USER_AGENT)
# Optional authentication
if not skip_auth:
self.authenticate(request)
# Fire request
if http_post:
handle = urlopen(request, params.encode('utf-8'))
else:
handle = urlopen(request)
# Read and possibly convert response
text = handle.read()
# Needed for Microsoft
if text[:3] == b'\xef\xbb\xbf':
text = text.decode('UTF-8-sig')
else:
text = text.decode('utf-8')
# Replace literal \t
text = text.strip().replace(
'\t', '\\t'
).replace(
'\r', '\\r'
)
# Needed for Google
while ',,' in text or '[,' in text:
text = text.replace(',,', ',null,').replace('[,', '[')
if raw:
return text
# Parse JSON
response = json.loads(text)
# Return data
return response
def download_translations(self, source, language, text, unit, user):
"""Download list of possible translations from a service."""
# should the machine translation service be used?
# (rather than only the term database)
enable_mt = False
if isinstance(settings.MT_SAP_USE_MT, bool):
enable_mt = settings.MT_SAP_USE_MT
# build the json body
request_data_as_bytes = json.dumps(
{
'targetLanguages': [language],
'sourceLanguage': source,
'enableMT': enable_mt,
'enableTranslationQualityEstimation': enable_mt,
'units': [{'value': text}]
},
ensure_ascii=False
).encode('utf-8')
# create the request
translation_url = settings.MT_SAP_BASE_URL + 'translate'
request = Request(
translation_url if six.PY3 else translation_url.encode("utf-8")
)
request.timeout = 0.5
request.add_header('User-Agent', USER_AGENT.encode('utf-8'))
request.add_header('Referer', get_site_url().encode('utf-8'))
request.add_header('Content-Type', 'application/json; charset=utf-8')
request.add_header('Content-Length', len(request_data_as_bytes))
request.add_header('Accept', 'application/json; charset=utf-8')
self.authenticate(request)
# Read and possibly convert response
content = urlopen(
request, request_data_as_bytes
).read().decode('utf-8')
# Replace literal \t
content = content.strip().replace(
'\t', '\\t'
).replace(
'\r', '\\r'
)
response = json.loads(content)
translations = []
# prepare the translations for weblate
for item in response['units']:
for translation in item['translations']:
translations.append((
translation['value'],
translation.get('qualityIndex', 100),
self.name,
text
))
return translations
def download_translations(self, source, language, text, unit, user):
"""Download list of possible translations from a service."""
# should the machine translation service be used?
# (rather than only the term database)
enable_mt = False
if isinstance(settings.MT_SAP_USE_MT, bool):
enable_mt = settings.MT_SAP_USE_MT
# build the json body
request_data_as_bytes = json.dumps(
{
'targetLanguages': [language],
'enableMT': enable_mt,
'enableTranslationQualityEstimation': enable_mt,
'units': [{'value': text}]
},
ensure_ascii=False
).encode('utf-8')
# create the request
translation_url = settings.MT_SAP_BASE_URL + 'translate'
request = Request(translation_url)
request.timeout = 0.5
request.add_header('User-Agent', USER_AGENT.encode('utf-8'))
request.add_header('Referer', get_site_url().encode('utf-8'))
request.add_header('Content-Type', 'application/json; charset=utf-8')
request.add_header('Content-Length', len(request_data_as_bytes))
request.add_header('Accept', 'application/json; charset=utf-8')
self.authenticate(request)
# Read and possibly convert response
content = urlopen(
request, request_data_as_bytes
).read().decode('utf-8')
# Replace literal \t
content = content.strip().replace(
'\t', '\\t'
).replace(
'\r', '\\r'
)
response = json.loads(content)
translations = []
# prepare the translations for weblate
for unit in response['units']:
for translation in unit['translations']:
translations.append((
translation['value'],
translation.get('qualityIndex', 100),
self.name,
text
))
return translations
def soap_req(self, action, **kwargs):
template = get_template('machine/microsoft_terminology_{}.xml'.format(
action.lower()))
payload = template.render(kwargs)
request = Request(self.MS_TM_API_URL, payload.encode('utf-8'))
request.timeout = 0.5
request.add_header('SOAPAction',
'"{}"'.format(self.MS_TM_SOAP_HEADER + action))
request.add_header('Content-Type', 'text/xml; charset=utf-8')
return urlopen(request)
def download_avatar_image(user, size):
"""Download avatar image from remote server."""
url = avatar_for_email(user.email, size)
request = Request(url)
request.timeout = 0.5
request.add_header('User-Agent', USER_AGENT)
# Fire request
handle = urlopen(request)
# Read and possibly convert response
return handle.read()
def download_avatar_image(user, size):
"""Download avatar image from remote server."""
url = avatar_for_email(user.email, size)
request = Request(url)
request.timeout = 0.5
request.add_header('User-Agent', USER_AGENT)
# Fire request
handle = urlopen(request)
# Read and possibly convert response
return bytes(handle.read())
def soap_req(self, action, **kwargs):
template = get_template(
'machine/microsoft_terminology_{}.xml'.format(action.lower())
)
payload = template.render(kwargs)
request = Request(self.MS_TM_API_URL, payload.encode('utf-8'))
request.timeout = 0.5
request.add_header(
'SOAPAction', '"{}"'.format(self.MS_TM_SOAP_HEADER + action)
)
request.add_header('Content-Type', 'text/xml; charset=utf-8')
return urlopen(request)
def soap_req(self, url, http_post=False, skip_auth=False, **kwargs):
soap_action = kwargs.get('soap_action', '')
url = self.MS_TM_API_URL
action = self.MS_TM_SOAP_HEADER + soap_action
headers = {
'SOAPAction': ('"{action}"').format(action=action),
'Content-Type': 'text/xml; charset=utf-8'
}
if soap_action == 'GetLanguages':
payload = {
'xmlns': self.MS_TM_SOAP_XMLNS,
'soap_action': soap_action
}
template = get_template(
'trans/machine/microsoft_terminology_get_langs.jinja')
elif soap_action == 'GetTranslations':
source = kwargs.get('source', '')
language = kwargs.get('language', '')
text = kwargs.get('text', '')
max_result = 5
if soap_action and source and language and text:
payload = {
'action': action,
'url': url,
'xmlns': self.MS_TM_SOAP_XMLNS,
'uuid': uuid4(),
'text': text,
'from_lang': source,
'to_lang': language,
'max_result': max_result
}
template = get_template(
'trans/machine/microsoft_terminology_translate.jinja')
else:
raise MachineTranslationError(
'Wrong SOAP request: "{soap_action}."').format(
soap_action=soap_action)
try:
payload = template.render(payload)
request = Request(url)
request.timeout = 0.5
for header, value in headers.iteritems():
request.add_header(header, value)
request.add_data(payload)
handle = urlopen(request)
except Exception as error:
raise MachineTranslationError('{err}'.format(err=error))
return handle
def get_github_email(access_token):
"""Get real email from GitHub"""
request = Request('https://api.github.com/user/emails')
request.timeout = 1.0
request.add_header('User-Agent', USER_AGENT)
request.add_header('Authorization', 'token {0}'.format(access_token))
handle = urlopen(request)
data = json.loads(handle.read().decode('utf-8'))
email = None
for entry in data:
# Skip not verified ones
if not entry['verified']:
continue
email = entry['email']
if entry['primary']:
break
return email
def get_github_email(access_token):
"""Get real email from GitHub"""
request = Request('https://api.github.com/user/emails')
request.timeout = 1.0
request.add_header('User-Agent', USER_AGENT)
request.add_header(
'Authorization',
'token {0}'.format(access_token)
)
handle = urlopen(request)
data = json.loads(handle.read().decode('utf-8'))
email = None
for entry in data:
# Skip not verified ones
if not entry['verified']:
continue
email = entry['email']
if entry['primary']:
break
return email
def _get_dummy_request(self):
r = Request('http://example.com')
r.timeout = 10
return r
def _get_dummy_request(self):
r = Request('http://example.com')
r.timeout = 10
return r
