def __init__(self, queue, config):
super(TestrunThread, self).__init__()
self.queue = queue
self.proxy_factory = ProxyFactory(config=dict(
db_uri=config['db_uri']),
logger=logger
)
def __request__():
url = request.form['url']
proxy_factory = ProxyFactory(
config=dict(db_uri=os.environ.get('DB_URI')),
logger=logger
)
logger.info('proxy_factory={}'.format(proxy_factory))
keys = filter(lambda x: x != 'url', request.form.keys())
params = {key: request.form[key] for key in keys}
#keys = filter(lambda x: x not in ['url', 'q'], request.form.keys())
#params = {key: request.form[key] for key in keys}
#data = dict(q=request.form['q'])
headers = {
'Referer': 'http://translate.google.com/',
#'User-Agent': "Mozilla/5.0 (X11; Linux x86_64; rv:5.0) Gecko/20100101 Firefox/5.0)",
'User-Agent': request.headers['User-Agent'],
#'Content-Length': str(sys.getsizeof(text))
}
try:
#req = requests.post(url, params=params, data=data, headers=request.headers, timeout=5)
#req = requests.get(url, params=payload, headers=headers)
req = proxy_factory.make_request(url, headers=headers, params=params,
req_type=requests.post, timeout=2, pool_size=10)
if req == None:
logger.error('Request failed (req=None)')
return 'Request failed', 500
return req.text, req.status_code
except Exception as e:
logger.exception(e)
return str(e), 500
class TestrunThread(Thread):
def __init__(self, queue, config):
super(TestrunThread, self).__init__()
self.queue = queue
self.proxy_factory = ProxyFactory(config=dict(
db_uri=config['db_uri']),
logger=logger
)
def run(self):
while True:
logger.info('Qsize = {} (approx.)'.format(self.queue.qsize()))
url, proxy = self.queue.get()
logger.info('Test run: Fetching %s via %s' % (url, proxy))
self.proxy_factory.make_request(url, proxy=proxy, timeout=3)
self.queue.task_done()
def __translate__(text, source, target, client='x', user_agent=DEFAULT_USER_AGENT):
"""
text: text to be translated
source: source language
target: target language
"""
if source == target:
return text
from hallucination import ProxyFactory
proxy_factory = ProxyFactory(
db_engine=db.engine,
logger=logger
)
if not re.match(r'Mozilla/\d+\.\d+ \(.*', user_agent):
user_agent = 'Mozilla/5.0 (%s)' % user_agent
headers = {
'Referer': 'http://translate.google.com',
'User-Agent': user_agent,
'Content-Length': str(sys.getsizeof(text))
}
payload = {
'client': client,
'sl': source,
'tl': target,
'text': text,
}
url = 'http://translate.google.com/translate_a/t'
req = None
try:
req = proxy_factory.make_request(url, headers=headers, params=payload,
req_type=requests.post, timeout=2, pool_size=10)
except Exception as e:
logger.exception(e)
if req == None:
# if request via proxy fails
req = requests.post(url, headers=headers, data=payload)
if req.status_code != 200:
raise HTTPException(
('Google Translate returned HTTP {}'.format(req.status_code)),
req.status_code)
if client == 'x':
data = json.loads(req.text)
try:
#if target == 'ja':
# sentences = data['sentences']
sentences = data['sentences']
except:
sentences = data['results'][0]['sentences']
result = ' '.join(map(lambda x: x['trans'], sentences))
# Remove unneccessary white spaces
return '\n'.join(map(lambda x: x.strip(), result.split('\n')))
elif client == 't':
return parse_javascript(req.text)
else:
raise Exception("Unsupported client '{}'".format(client))
