def __init__(
self,
source_language='en',
target_language='mg',
database_file='default'):
self.output_database = DictionaryDatabaseManager(
database_file=database_file)
self.fast_tree = {}
self.target_language = target_language
self.source_language = source_language
def additional_data(self):
from api.databasemanager import DictionaryDatabaseManager
dbm = DictionaryDatabaseManager()
if self.id:
sql = f"select word_id, type, information from additional_word_information where word_id = {self.id}"
rq = dbm.session.execute(sql)
ret = {}
for w_id, adt, info in rq.fetchall():
if adt in ret:
ret[adt].append(info)
else:
ret[adt] = [info]
return ret
else:
return None
def do_import(self, workers=100):
input_database = DictionaryDatabaseManager(
database_file=self.export_path)
with input_database.engine.connect() as connection:
query = connection.execute("""
select
word.id,
word.word,
word.language,
word.part_of_speech,
definitions.definition,
definitions.definition_language
from
dictionary,
word,
definitions
where
dictionary.definition = definitions.id
and word.id = dictionary.word
and definition_language = 'mg'
""")
print('-- build tree --')
for w in query.fetchall():
word, language, part_of_speech, definition = w[1], w[2], w[
3], w[4]
key = (word, language, part_of_speech)
if key in self.fast_tree:
self.fast_tree[key].append(definition)
else:
self.fast_tree[key] = [definition]
print('-- using tree --')
for word, language, part_of_speech in self.fast_tree:
entry = Entry(entry=word,
language=language,
part_of_speech=part_of_speech,
definitions=self.fast_tree[(word, language,
part_of_speech)])
try:
self.worker(entry)
except Exception:
continue
def __init__(self, database_file='default'):
self.output_database = DictionaryDatabaseManager(
database_file=database_file)
self.fast_tree = set()
from datetime import datetime, timedelta
from random import randint
import pywikibot
import requests
from lxml import etree
from sqlalchemy import func
from api.databasemanager import DictionaryDatabaseManager, LanguageDatabaseManager
from api.decorator import time_this
from api.dictionary.model import Language
from api.dictionary.model import Word
from conf.entryprocessor.languagecodes import LANGUAGE_CODES
log = logging.getLogger(__file__)
dictionary_db_manager = DictionaryDatabaseManager()
language_db_manager = LanguageDatabaseManager()
language_session = language_db_manager.session
word_session = dictionary_db_manager.session
WORKING_WIKI = pywikibot.Site("mg", "wiktionary")
try:
username = "******" % pywikibot.config.usernames['wiktionary']['mg']
except KeyError:
username = "******"
TABLE_PATTERN = """
{| class=\"wikitable sortable\"
! Kaodim-piteny
! Anarana anglisy
type=int,
default=100)
args = parser.parse_args()
WORD_STORAGE = args.STORAGE
HOST = args.HOST
PORT = args.PORT
LOG = args.LOG_FILE
try:
LOG_LEVEL = log._nameToLevel[args.LOG_LEVEL.upper()]
except KeyError:
LOG_LEVEL = 10
log.basicConfig(filename=LOG, level=log.DEBUG)
dictionary_db_manager = DictionaryDatabaseManager(database_file=WORD_STORAGE,
db_header='')
routes = web.RouteTableDef()
app = web.Application(middlewares=[
json_error_handler,
auto_committer,
])
app['database'] = dictionary_db_manager
app['session_instance'] = dictionary_db_manager.session
app['autocommit'] = args.autocommit
app['commit_every'] = args.commit_every
app['commit_count'] = 0
app.router.add_route('GET', '/languages/list', get_language_list)
app.router.add_route('GET', '/languages/list/download', download_dictionary)
app.router.add_route('GET', '/definition/{definition_id}',
def __init__(self, database_file='default'):
self.input_database = DictionaryDatabaseManager(
database_file=database_file)