def get(self, source_id, source_lang, target_lang):
src_index = TMUtils.lang2es_index(source_lang)
tgt_index = TMUtils.lang2es_index(target_lang)
m_index = TMUtils.es_index2mapdb(src_index, tgt_index)
m_results = self.mongo_db[m_index].find({'source_id': source_id})
if not m_results or not m_results.count():
return None
return m_results[0]['target_id']
def _get_index(self, source_lang, target_lang, create_missing=False):
m_index = TMUtils.es_index2mapdb(TMUtils.lang2es_index(source_lang),
TMUtils.lang2es_index(target_lang))
if self.es.indices.exists(index=m_index): return m_index, False
# Try reverse index
r_index = TMUtils.es_index2mapdb(TMUtils.lang2es_index(target_lang),
TMUtils.lang2es_index(source_lang))
# Found reverse index - use it
if self.es.indices.exists(r_index): return r_index, True
if not create_missing: return None, None
# Neither direct, nor reverse index exist - create a direct one
try:
self.es.indices.create(m_index)
except:
pass
self.refresh_lang_graph()
return m_index, False
def add_segment(self, segment):
# Add MongoDB document
m_index = TMUtils.es_index2mapdb(
TMUtils.lang2es_index(segment.source_lang),
TMUtils.lang2es_index(segment.target_lang))
# TODO: do not update if creation date is older than existing one
m_result = self.mongo_db[m_index].update_one(
{'source_id': segment.source_id},
{'$set': self._segment2doc(segment)},
upsert=True) # insert if doesn't exist
return m_result
def add_segments(self, segments):
if not segments:
return
m_index = TMUtils.es_index2mapdb(
TMUtils.lang2es_index(segments[0].source_lang),
TMUtils.lang2es_index(segments[0].target_lang))
try:
db = self.server[m_index]
except:
db = self.server.create(m_index)
return db.update([self._segment2doc(s) for s in segments])
def get(self, source_id, source_lang, target_lang):
tname = TMUtils.es_index2mapdb(TMUtils.lang2es_index(source_lang),
TMUtils.lang2es_index(target_lang))
if not tname in self.tables:
raise Exception("Language pair : {} - {} doesn't exist".format(
source_lang, target_lang))
# TODO: implement bidirectional query
t = self.tables[tname]
res = self.conn.execute(
t.select(t.target_id).where(t.source_id == source_id))
if res:
return res.fetchone()[0]
return None
def add_segments(self, segments):
bulk = None
for segment in segments:
if not bulk:
m_index = TMUtils.es_index2mapdb(
TMUtils.lang2es_index(segment.source_lang),
TMUtils.lang2es_index(segment.target_lang))
bulk = self.mongo_db[m_index].initialize_unordered_bulk_op()
bulk.find({'source_id': segment.source_id}) \
.update_one({'$set': self._segment2doc(segment) })
try:
result = bulk.execute()
except BulkWriteError as bwe:
result = bwe.details
logging.error(bwe.details)
return result
def _segment2table(self, segment, suffix=None):
tname = TMUtils.es_index2mapdb(
TMUtils.lang2es_index(segment.source_lang),
TMUtils.lang2es_index(segment.target_lang))
if suffix: tname += suffix
if not tname in self.tables:
md = MetaData()
self.tables[tname] = Table(tname,
md,
Column('id', Integer, primary_key=True),
Column('source_id', GUID, index=True),
Column('target_id', GUID, index=True),
Column('creation_date', TIMESTAMP),
Column('change_date', TIMESTAMP),
mysql_engine='InnoDB',
mysql_charset='utf8')
md.bind = self.conn
self.tables[tname].create(checkfirst=True)
return self.tables[tname]
def get(self, source_id, source_lang, target_lang):
m_index = TMUtils.es_index2mapdb(TMUtils.lang2es_index(source_lang),
TMUtils.lang2es_index(target_lang))
doc = self.server[m_index].get(source_id.hex)
if doc: return uuid.UUID(doc['target_id'])
return None
def add_segment(self, segment):
m_index = TMUtils.es_index2mapdb(
TMUtils.lang2es_index(segment.source_lang),
TMUtils.lang2es_index(segment.target_lang))
return self.server[m_index].update([self._segment2doc(segment)])