def setup_application(args):
if args.stat_period_max_age:
BaseHandler.stat_period_max_age = timedelta(0,
args.stat_period_max_age,
0)
setup_handler(args.pairs_path, args.nonpairs_path, args.lang_names,
args.missing_freqs, args.timeout, args.max_pipes_per_pair,
args.min_pipes_per_pair, args.max_users_per_pipe,
args.max_idle_secs, args.restart_pipe_after,
args.max_doc_pipes, args.verbosity, args.scalemt_logs,
args.unknown_memory_limit, args.api_keys)
handlers = [
(r'/', RootHandler),
(r'/list', ListHandler),
(r'/listPairs', ListHandler),
(r'/stats', StatsHandler),
(r'/pairprefs', PairPrefsHandler),
(r'/translate', TranslateHandler),
(r'/translateChain', TranslateChainHandler),
(r'/translateDoc', TranslateDocHandler),
(r'/translatePage', TranslateWebpageHandler),
(r'/translateRaw', TranslateRawHandler),
(r'/analy[sz]e', AnalyzeHandler),
(r'/generate', GenerateHandler),
(r'/listLanguageNames', ListLanguageNamesHandler),
(r'/perWord', PerWordHandler),
(r'/calcCoverage', CoverageHandler),
(r'/identifyLang', IdentifyLangHandler),
(r'/getLocale', GetLocaleHandler),
(r'/pipedebug', PipeDebugHandler),
] # type: List[Tuple[str, Type[tornado.web.RequestHandler]]]
if importlib_util.find_spec('streamparser'):
handlers.append((r'/speller', SpellerHandler))
if all([args.wiki_username, args.wiki_password
]) and importlib_util.find_spec('requests'):
import requests
logging.info('Logging into Apertium Wiki with username %s',
args.wiki_username)
SuggestionHandler.SUGGEST_URL = 'User:'******'edit', 'info|revisions')
handlers.append((r'/suggest', SuggestionHandler))
# TODO: fix mypy. Application expects List but List is invariant and we use subclasses
return tornado.web.Application(handlers) # type:ignore
def setup_application(args):
if args.stat_period_max_age:
BaseHandler.STAT_PERIOD_MAX_AGE = timedelta(0, args.stat_period_max_age, 0)
setup_handler(args.port, args.pairs_path, args.nonpairs_path, args.lang_names, args.missing_freqs, args.timeout,
args.max_pipes_per_pair, args.min_pipes_per_pair, args.max_users_per_pipe, args.max_idle_secs,
args.restart_pipe_after, args.max_doc_pipes, args.verbosity, args.scalemt_logs,
args.unknown_memory_limit, args.api_keys)
handlers = [
(r'/', RootHandler),
(r'/list', ListHandler),
(r'/listPairs', ListHandler),
(r'/stats', StatsHandler),
(r'/translate', TranslateHandler),
(r'/translateChain', TranslateChainHandler),
(r'/translateDoc', TranslateDocHandler),
(r'/translatePage', TranslateWebpageHandler),
(r'/translateRaw', TranslateRawHandler),
(r'/analy[sz]e', AnalyzeHandler),
(r'/generate', GenerateHandler),
(r'/listLanguageNames', ListLanguageNamesHandler),
(r'/perWord', PerWordHandler),
(r'/calcCoverage', CoverageHandler),
(r'/identifyLang', IdentifyLangHandler),
(r'/getLocale', GetLocaleHandler),
(r'/pipedebug', PipeDebugHandler),
]
if importlib.util.find_spec('streamparser'):
handlers.append((r'/speller', SpellerHandler))
if all([args.wiki_username, args.wiki_password]) and importlib.util.find_spec('requests'):
import requests
logging.info('Logging into Apertium Wiki with username %s', args.wiki_username)
SuggestionHandler.SUGGEST_URL = 'User:'******'edit', 'info|revisions')
handlers.append((r'/suggest', SuggestionHandler))
return tornado.web.Application(handlers)
def post(self):
context = self.get_argument('context', None)
word = self.get_argument('word', None)
new_word = self.get_argument('newWord', None)
langpair = self.get_argument('langpair', None)
recap = self.get_argument('g-recaptcha-response', None)
if not new_word:
self.send_error(400, explanation='A suggestion is required')
return
if not recap:
self.send_error(400, explanation='The ReCAPTCHA is required')
return
if not all([context, word, langpair, new_word, recap]):
self.send_error(400, explanation='All arguments were not provided')
return
logging.info(
'Suggestion (%s): Context is %s \n Word: %s ; New Word: %s ',
langpair, context, word, new_word)
logging.info('Now verifying ReCAPTCHA.')
if not self.recaptcha_secret:
logging.error('No ReCAPTCHA secret provided!')
self.send_error(
400,
explanation='Server not configured correctly for suggestions')
return
if recap == BYPASS_TOKEN:
logging.info('Adding data to wiki with bypass token')
else:
# for nginx or when behind a proxy
x_real_ip = self.request.headers.get('X-Real-IP')
user_ip = x_real_ip or self.request.remote_ip
payload = {
'secret': self.recaptcha_secret,
'response': recap,
'remoteip': user_ip,
}
recap_request = self.wiki_session.post(RECAPTCHA_VERIFICATION_URL,
data=payload)
if recap_request.json()['success']:
logging.info('ReCAPTCHA verified, adding data to wiki')
else:
logging.info('ReCAPTCHA verification failed, stopping')
self.send_error(400,
explanation='ReCAPTCHA verification failed')
return
data = {
'context': context,
'langpair': langpair,
'word': word,
'newWord': new_word,
}
result = add_suggestion(self.wiki_session, self.SUGGEST_URL,
self.wiki_edit_token, data)
if result:
self.send_response({
'responseData': {
'status': 'Success',
},
'responseDetails': None,
'responseStatus': 200,
})
else:
logging.info('Page update failed, trying to get new edit token')
self.wiki_edit_token = wiki_get_token(
SuggestionHandler.wiki_session, 'edit', 'info|revisions')
logging.info('Obtained new edit token. Trying page update again.')
result = add_suggestion(self.wiki_session, self.SUGGEST_URL,
self.wiki_edit_token, data)
if result:
self.send_response({
'responseData': {
'status': 'Success',
},
'responseDetails': None,
'responseStatus': 200,
})
else:
self.send_error(400, explanation='Page update failed')
def post(self):
context = self.get_argument('context', None)
word = self.get_argument('word', None)
new_word = self.get_argument('newWord', None)
langpair = self.get_argument('langpair', None)
recap = self.get_argument('g-recaptcha-response', None)
if not new_word:
self.send_error(400, explanation='A suggestion is required')
return
if not recap:
self.send_error(400, explanation='The ReCAPTCHA is required')
return
if not all([context, word, langpair, new_word, recap]):
self.send_error(400, explanation='All arguments were not provided')
return
logging.info('Suggestion (%s): Context is %s \n Word: %s ; New Word: %s ', langpair, context, word, new_word)
logging.info('Now verifying ReCAPTCHA.')
if not self.recaptcha_secret:
logging.error('No ReCAPTCHA secret provided!')
self.send_error(400, explanation='Server not configured correctly for suggestions')
return
if recap == BYPASS_TOKEN:
logging.info('Adding data to wiki with bypass token')
else:
# for nginx or when behind a proxy
x_real_ip = self.request.headers.get('X-Real-IP')
user_ip = x_real_ip or self.request.remote_ip
payload = {
'secret': self.recaptcha_secret,
'response': recap,
'remoteip': user_ip,
}
recap_request = self.wiki_session.post(RECAPTCHA_VERIFICATION_URL, data=payload)
if recap_request.json()['success']:
logging.info('ReCAPTCHA verified, adding data to wiki')
else:
logging.info('ReCAPTCHA verification failed, stopping')
self.send_error(400, explanation='ReCAPTCHA verification failed')
return
data = {
'context': context, 'langpair': langpair,
'word': word, 'newWord': new_word,
}
result = add_suggestion(self.wiki_session,
self.SUGGEST_URL, self.wiki_edit_token,
data)
if result:
self.send_response({
'responseData': {
'status': 'Success',
},
'responseDetails': None,
'responseStatus': 200,
})
else:
logging.info('Page update failed, trying to get new edit token')
self.wiki_edit_token = wiki_get_token(
SuggestionHandler.wiki_session, 'edit', 'info|revisions')
logging.info('Obtained new edit token. Trying page update again.')
result = add_suggestion(self.wiki_session,
self.SUGGEST_URL, self.wiki_edit_token,
data)
if result:
self.send_response({
'responseData': {
'status': 'Success',
},
'responseDetails': None,
'responseStatus': 200,
})
else:
self.send_error(400, explanation='Page update failed')
def setup_application(args):
if args.daemon:
# regular content logs are output stderr
# python messages are mostly output to stdout
# hence swapping the filenames?
sys.stderr = open(os.path.join(args.log_path, 'apertium-apy.log'), 'a+')
sys.stdout = open(os.path.join(args.log_path, 'apertium-apy.err'), 'a+')
if args.scalemt_logs:
logger = logging.getLogger('scale-mt')
logger.propagate = False
smtlog = os.path.join(args.log_path, 'ScaleMTRequests.log')
logging_handler = logging_handlers.TimedRotatingFileHandler(smtlog, 'midnight', 0)
# internal attribute, should not use
logging_handler.suffix = '%Y-%m-%d' # type: ignore
logger.addHandler(logging_handler)
# if scalemt_logs is enabled, disable tornado.access logs
if args.daemon:
logging.getLogger('tornado.access').propagate = False
if args.stat_period_max_age:
BaseHandler.STAT_PERIOD_MAX_AGE = timedelta(0, args.stat_period_max_age, 0)
setup_handler(args.port, args.pairs_path, args.nonpairs_path, args.lang_names, args.missing_freqs, args.timeout,
args.max_pipes_per_pair, args.min_pipes_per_pair, args.max_users_per_pipe, args.max_idle_secs,
args.restart_pipe_after, args.max_doc_pipes, args.verbosity, args.scalemt_logs, args.unknown_memory_limit)
handlers = [
(r'/', RootHandler),
(r'/list', ListHandler),
(r'/listPairs', ListHandler),
(r'/stats', StatsHandler),
(r'/translate', TranslateHandler),
(r'/translateChain', TranslateChainHandler),
(r'/translateDoc', TranslateDocHandler),
(r'/translatePage', TranslateWebpageHandler),
(r'/translateRaw', TranslateRawHandler),
(r'/analy[sz]e', AnalyzeHandler),
(r'/generate', GenerateHandler),
(r'/listLanguageNames', ListLanguageNamesHandler),
(r'/perWord', PerWordHandler),
(r'/calcCoverage', CoverageHandler),
(r'/identifyLang', IdentifyLangHandler),
(r'/getLocale', GetLocaleHandler),
(r'/pipedebug', PipeDebugHandler),
]
if importlib.util.find_spec('streamparser'):
handlers.append((r'/speller', SpellerHandler))
if all([args.wiki_username, args.wiki_password]) and importlib.util.find_spec('requests'):
import requests
logging.info('Logging into Apertium Wiki with username %s', args.wiki_username)
SuggestionHandler.SUGGEST_URL = 'User:'******'edit', 'info|revisions')
handlers.append((r'/suggest', SuggestionHandler))
return tornado.web.Application(handlers)