def __init__(self, queue, config): super(TestrunThread, self).__init__() self.queue = queue self.proxy_factory = ProxyFactory(config=dict( db_uri=config['db_uri']), logger=logger )
def __request__(): url = request.form['url'] proxy_factory = ProxyFactory( config=dict(db_uri=os.environ.get('DB_URI')), logger=logger ) logger.info('proxy_factory={}'.format(proxy_factory)) keys = filter(lambda x: x != 'url', request.form.keys()) params = {key: request.form[key] for key in keys} #keys = filter(lambda x: x not in ['url', 'q'], request.form.keys()) #params = {key: request.form[key] for key in keys} #data = dict(q=request.form['q']) headers = { 'Referer': 'http://translate.google.com/', #'User-Agent': "Mozilla/5.0 (X11; Linux x86_64; rv:5.0) Gecko/20100101 Firefox/5.0)", 'User-Agent': request.headers['User-Agent'], #'Content-Length': str(sys.getsizeof(text)) } try: #req = requests.post(url, params=params, data=data, headers=request.headers, timeout=5) #req = requests.get(url, params=payload, headers=headers) req = proxy_factory.make_request(url, headers=headers, params=params, req_type=requests.post, timeout=2, pool_size=10) if req == None: logger.error('Request failed (req=None)') return 'Request failed', 500 return req.text, req.status_code except Exception as e: logger.exception(e) return str(e), 500
class TestrunThread(Thread): def __init__(self, queue, config): super(TestrunThread, self).__init__() self.queue = queue self.proxy_factory = ProxyFactory(config=dict( db_uri=config['db_uri']), logger=logger ) def run(self): while True: logger.info('Qsize = {} (approx.)'.format(self.queue.qsize())) url, proxy = self.queue.get() logger.info('Test run: Fetching %s via %s' % (url, proxy)) self.proxy_factory.make_request(url, proxy=proxy, timeout=3) self.queue.task_done()
def __translate__(text, source, target, client='x', user_agent=DEFAULT_USER_AGENT): """ text: text to be translated source: source language target: target language """ if source == target: return text from hallucination import ProxyFactory proxy_factory = ProxyFactory( db_engine=db.engine, logger=logger ) if not re.match(r'Mozilla/\d+\.\d+ \(.*', user_agent): user_agent = 'Mozilla/5.0 (%s)' % user_agent headers = { 'Referer': 'http://translate.google.com', 'User-Agent': user_agent, 'Content-Length': str(sys.getsizeof(text)) } payload = { 'client': client, 'sl': source, 'tl': target, 'text': text, } url = 'http://translate.google.com/translate_a/t' req = None try: req = proxy_factory.make_request(url, headers=headers, params=payload, req_type=requests.post, timeout=2, pool_size=10) except Exception as e: logger.exception(e) if req == None: # if request via proxy fails req = requests.post(url, headers=headers, data=payload) if req.status_code != 200: raise HTTPException( ('Google Translate returned HTTP {}'.format(req.status_code)), req.status_code) if client == 'x': data = json.loads(req.text) try: #if target == 'ja': # sentences = data['sentences'] sentences = data['sentences'] except: sentences = data['results'][0]['sentences'] result = ' '.join(map(lambda x: x['trans'], sentences)) # Remove unneccessary white spaces return '\n'.join(map(lambda x: x.strip(), result.split('\n'))) elif client == 't': return parse_javascript(req.text) else: raise Exception("Unsupported client '{}'".format(client))