def get(self, source_id, source_lang, target_lang): src_index = TMUtils.lang2es_index(source_lang) tgt_index = TMUtils.lang2es_index(target_lang) m_index = TMUtils.es_index2mapdb(src_index, tgt_index) m_results = self.mongo_db[m_index].find({'source_id': source_id}) if not m_results or not m_results.count(): return None return m_results[0]['target_id']
def _get_index(self, source_lang, target_lang, create_missing=False): m_index = TMUtils.es_index2mapdb(TMUtils.lang2es_index(source_lang), TMUtils.lang2es_index(target_lang)) if self.es.indices.exists(index=m_index): return m_index, False # Try reverse index r_index = TMUtils.es_index2mapdb(TMUtils.lang2es_index(target_lang), TMUtils.lang2es_index(source_lang)) # Found reverse index - use it if self.es.indices.exists(r_index): return r_index, True if not create_missing: return None, None # Neither direct, nor reverse index exist - create a direct one try: self.es.indices.create(m_index) except: pass self.refresh_lang_graph() return m_index, False
def add_segment(self, segment): # Add MongoDB document m_index = TMUtils.es_index2mapdb( TMUtils.lang2es_index(segment.source_lang), TMUtils.lang2es_index(segment.target_lang)) # TODO: do not update if creation date is older than existing one m_result = self.mongo_db[m_index].update_one( {'source_id': segment.source_id}, {'$set': self._segment2doc(segment)}, upsert=True) # insert if doesn't exist return m_result
def add_segments(self, segments): if not segments: return m_index = TMUtils.es_index2mapdb( TMUtils.lang2es_index(segments[0].source_lang), TMUtils.lang2es_index(segments[0].target_lang)) try: db = self.server[m_index] except: db = self.server.create(m_index) return db.update([self._segment2doc(s) for s in segments])
def get(self, source_id, source_lang, target_lang): tname = TMUtils.es_index2mapdb(TMUtils.lang2es_index(source_lang), TMUtils.lang2es_index(target_lang)) if not tname in self.tables: raise Exception("Language pair : {} - {} doesn't exist".format( source_lang, target_lang)) # TODO: implement bidirectional query t = self.tables[tname] res = self.conn.execute( t.select(t.target_id).where(t.source_id == source_id)) if res: return res.fetchone()[0] return None
def add_segments(self, segments): bulk = None for segment in segments: if not bulk: m_index = TMUtils.es_index2mapdb( TMUtils.lang2es_index(segment.source_lang), TMUtils.lang2es_index(segment.target_lang)) bulk = self.mongo_db[m_index].initialize_unordered_bulk_op() bulk.find({'source_id': segment.source_id}) \ .update_one({'$set': self._segment2doc(segment) }) try: result = bulk.execute() except BulkWriteError as bwe: result = bwe.details logging.error(bwe.details) return result
def _segment2table(self, segment, suffix=None): tname = TMUtils.es_index2mapdb( TMUtils.lang2es_index(segment.source_lang), TMUtils.lang2es_index(segment.target_lang)) if suffix: tname += suffix if not tname in self.tables: md = MetaData() self.tables[tname] = Table(tname, md, Column('id', Integer, primary_key=True), Column('source_id', GUID, index=True), Column('target_id', GUID, index=True), Column('creation_date', TIMESTAMP), Column('change_date', TIMESTAMP), mysql_engine='InnoDB', mysql_charset='utf8') md.bind = self.conn self.tables[tname].create(checkfirst=True) return self.tables[tname]
def get(self, source_id, source_lang, target_lang): m_index = TMUtils.es_index2mapdb(TMUtils.lang2es_index(source_lang), TMUtils.lang2es_index(target_lang)) doc = self.server[m_index].get(source_id.hex) if doc: return uuid.UUID(doc['target_id']) return None
def add_segment(self, segment): m_index = TMUtils.es_index2mapdb( TMUtils.lang2es_index(segment.source_lang), TMUtils.lang2es_index(segment.target_lang)) return self.server[m_index].update([self._segment2doc(segment)])