def json_req(self, url, http_post=False, skip_auth=False, raw=False, **kwargs): ''' Performs JSON request. ''' # Encode params if len(kwargs) > 0: params = urlencode( {key: val.encode('utf-8') for key, val in kwargs.items()} ) else: params = '' # Store for exception handling self.request_url = url self.request_params = params # Append parameters if len(params) > 0 and not http_post: url = '?'.join((url, params)) # Create request object with custom headers request = Request(url) request.timeout = 0.5 request.add_header('User-Agent', USER_AGENT) # Optional authentication if not skip_auth: self.authenticate(request) # Fire request if http_post: handle = urlopen(request, params.encode('utf-8')) else: handle = urlopen(request) # Read and possibly convert response text = handle.read() # Needed for Microsoft if text[:3] == b'\xef\xbb\xbf': text = text.decode('UTF-8-sig') else: text = text.decode('utf-8') # Replace literal \t text = text.strip().replace( '\t', '\\t' ).replace( '\r', '\\r' ) # Needed for Google while ',,' in text or '[,' in text: text = text.replace(',,', ',null,').replace('[,', '[') if raw: return text # Parse JSON response = json.loads(text) # Return data return response
def download_translations(self, source, language, text, unit, user): """Download list of possible translations from a service.""" # should the machine translation service be used? # (rather than only the term database) enable_mt = False if isinstance(settings.MT_SAP_USE_MT, bool): enable_mt = settings.MT_SAP_USE_MT # build the json body request_data_as_bytes = json.dumps( { 'targetLanguages': [language], 'sourceLanguage': source, 'enableMT': enable_mt, 'enableTranslationQualityEstimation': enable_mt, 'units': [{'value': text}] }, ensure_ascii=False ).encode('utf-8') # create the request translation_url = settings.MT_SAP_BASE_URL + 'translate' request = Request( translation_url if six.PY3 else translation_url.encode("utf-8") ) request.timeout = 0.5 request.add_header('User-Agent', USER_AGENT.encode('utf-8')) request.add_header('Referer', get_site_url().encode('utf-8')) request.add_header('Content-Type', 'application/json; charset=utf-8') request.add_header('Content-Length', len(request_data_as_bytes)) request.add_header('Accept', 'application/json; charset=utf-8') self.authenticate(request) # Read and possibly convert response content = urlopen( request, request_data_as_bytes ).read().decode('utf-8') # Replace literal \t content = content.strip().replace( '\t', '\\t' ).replace( '\r', '\\r' ) response = json.loads(content) translations = [] # prepare the translations for weblate for item in response['units']: for translation in item['translations']: translations.append(( translation['value'], translation.get('qualityIndex', 100), self.name, text )) return translations
def download_translations(self, source, language, text, unit, user): """Download list of possible translations from a service.""" # should the machine translation service be used? # (rather than only the term database) enable_mt = False if isinstance(settings.MT_SAP_USE_MT, bool): enable_mt = settings.MT_SAP_USE_MT # build the json body request_data_as_bytes = json.dumps( { 'targetLanguages': [language], 'enableMT': enable_mt, 'enableTranslationQualityEstimation': enable_mt, 'units': [{'value': text}] }, ensure_ascii=False ).encode('utf-8') # create the request translation_url = settings.MT_SAP_BASE_URL + 'translate' request = Request(translation_url) request.timeout = 0.5 request.add_header('User-Agent', USER_AGENT.encode('utf-8')) request.add_header('Referer', get_site_url().encode('utf-8')) request.add_header('Content-Type', 'application/json; charset=utf-8') request.add_header('Content-Length', len(request_data_as_bytes)) request.add_header('Accept', 'application/json; charset=utf-8') self.authenticate(request) # Read and possibly convert response content = urlopen( request, request_data_as_bytes ).read().decode('utf-8') # Replace literal \t content = content.strip().replace( '\t', '\\t' ).replace( '\r', '\\r' ) response = json.loads(content) translations = [] # prepare the translations for weblate for unit in response['units']: for translation in unit['translations']: translations.append(( translation['value'], translation.get('qualityIndex', 100), self.name, text )) return translations
def soap_req(self, action, **kwargs): template = get_template('machine/microsoft_terminology_{}.xml'.format( action.lower())) payload = template.render(kwargs) request = Request(self.MS_TM_API_URL, payload.encode('utf-8')) request.timeout = 0.5 request.add_header('SOAPAction', '"{}"'.format(self.MS_TM_SOAP_HEADER + action)) request.add_header('Content-Type', 'text/xml; charset=utf-8') return urlopen(request)
def download_avatar_image(user, size): """Download avatar image from remote server.""" url = avatar_for_email(user.email, size) request = Request(url) request.timeout = 0.5 request.add_header('User-Agent', USER_AGENT) # Fire request handle = urlopen(request) # Read and possibly convert response return handle.read()
def download_avatar_image(user, size): """Download avatar image from remote server.""" url = avatar_for_email(user.email, size) request = Request(url) request.timeout = 0.5 request.add_header('User-Agent', USER_AGENT) # Fire request handle = urlopen(request) # Read and possibly convert response return bytes(handle.read())
def soap_req(self, action, **kwargs): template = get_template( 'machine/microsoft_terminology_{}.xml'.format(action.lower()) ) payload = template.render(kwargs) request = Request(self.MS_TM_API_URL, payload.encode('utf-8')) request.timeout = 0.5 request.add_header( 'SOAPAction', '"{}"'.format(self.MS_TM_SOAP_HEADER + action) ) request.add_header('Content-Type', 'text/xml; charset=utf-8') return urlopen(request)
def soap_req(self, url, http_post=False, skip_auth=False, **kwargs): soap_action = kwargs.get('soap_action', '') url = self.MS_TM_API_URL action = self.MS_TM_SOAP_HEADER + soap_action headers = { 'SOAPAction': ('"{action}"').format(action=action), 'Content-Type': 'text/xml; charset=utf-8' } if soap_action == 'GetLanguages': payload = { 'xmlns': self.MS_TM_SOAP_XMLNS, 'soap_action': soap_action } template = get_template( 'trans/machine/microsoft_terminology_get_langs.jinja') elif soap_action == 'GetTranslations': source = kwargs.get('source', '') language = kwargs.get('language', '') text = kwargs.get('text', '') max_result = 5 if soap_action and source and language and text: payload = { 'action': action, 'url': url, 'xmlns': self.MS_TM_SOAP_XMLNS, 'uuid': uuid4(), 'text': text, 'from_lang': source, 'to_lang': language, 'max_result': max_result } template = get_template( 'trans/machine/microsoft_terminology_translate.jinja') else: raise MachineTranslationError( 'Wrong SOAP request: "{soap_action}."').format( soap_action=soap_action) try: payload = template.render(payload) request = Request(url) request.timeout = 0.5 for header, value in headers.iteritems(): request.add_header(header, value) request.add_data(payload) handle = urlopen(request) except Exception as error: raise MachineTranslationError('{err}'.format(err=error)) return handle
def get_github_email(access_token): """Get real email from GitHub""" request = Request('https://api.github.com/user/emails') request.timeout = 1.0 request.add_header('User-Agent', USER_AGENT) request.add_header('Authorization', 'token {0}'.format(access_token)) handle = urlopen(request) data = json.loads(handle.read().decode('utf-8')) email = None for entry in data: # Skip not verified ones if not entry['verified']: continue email = entry['email'] if entry['primary']: break return email
def get_github_email(access_token): """Get real email from GitHub""" request = Request('https://api.github.com/user/emails') request.timeout = 1.0 request.add_header('User-Agent', USER_AGENT) request.add_header( 'Authorization', 'token {0}'.format(access_token) ) handle = urlopen(request) data = json.loads(handle.read().decode('utf-8')) email = None for entry in data: # Skip not verified ones if not entry['verified']: continue email = entry['email'] if entry['primary']: break return email
def _get_dummy_request(self): r = Request('http://example.com') r.timeout = 10 return r