def do_test(self, source, target): if source is not None: self.storage.create_index(source, 'source') if target is not None: self.storage.create_index(target, 'target-cs') sindex = weblate.trans.search.get_source_index() self.assertIsNotNone(sindex) tindex = weblate.trans.search.get_target_index('cs') self.assertIsNotNone(tindex) writer = sindex.writer() writer.update_document( pk=1, source="source", context="context", location="location", ) writer.commit() writer = tindex.writer() writer.update_document( pk=1, target="target", comment="comment" ) writer.commit() for item in ('source', 'context', 'location', 'target'): self.assertEqual( fulltext_search(item, ['cs'], {item: True}), set([1]) )
def search(self, translation, params): """ High level wrapper for searching. """ base = self.all() if params['type'] != 'all': base = self.filter_type(params['type'], translation, params['ignored']) if 'lang' in params and params['lang']: base = base.filter(translation__language__code=params['lang']) if not params['q']: return base if params['search'] in ('exact', 'substring'): queries = [] modifier = '' if params['search'] != 'exact': modifier = '__icontains' for param in SEARCH_FILTERS: if params[param]: queries.append(param) query = reduce( lambda q, value: q | Q( **{'%s%s' % (value, modifier): params['q']}), queries, Q()) return base.filter(query) else: lang = self.all()[0].translation.language.code return base.filter( checksum__in=fulltext_search(params['q'], lang, params))
def more_like_this(self, unit, top=5): """ Finds closely similar units. """ if appsettings.MT_WEBLATE_LIMIT >= 0: queue = multiprocessing.Queue() proc = multiprocessing.Process( target=more_like_queue, args=(unit.pk, unit.source, top, queue) ) proc.start() proc.join(appsettings.MT_WEBLATE_LIMIT) if proc.is_alive(): proc.terminate() if queue.empty(): raise Exception('Request timed out.') more_results = queue.get() else: more_results = more_like(unit.pk, unit.source, top) same_results = fulltext_search( unit.get_source_plurals()[0], unit.translation.language.code, {'source': True} ) return self.filter( pk__in=more_results - same_results, translation__language=unit.translation.language, translated=True ).exclude( pk=unit.id )
def more_like_this(self, unit, top=5): """Find closely similar units.""" if settings.MT_WEBLATE_LIMIT >= 0: queue = multiprocessing.Queue() proc = multiprocessing.Process( target=more_like_queue, args=(unit.pk, unit.source, top, queue) ) proc.start() proc.join(settings.MT_WEBLATE_LIMIT) if proc.is_alive(): proc.terminate() if queue.empty(): raise Exception( 'Request for more like {0} timed out.'.format(unit.pk) ) more_results = queue.get() else: more_results = more_like(unit.pk, unit.source, top) same_results = fulltext_search( unit.get_source_plurals()[0], [unit.translation.language.code], {'source': True} ) return self.filter( pk__in=more_results - same_results, translation__language=unit.translation.language, translated=True ).exclude( pk=unit.id )
def same_source(self, unit): """Find units with same source.""" pks = fulltext_search(unit.get_source_plurals()[0], unit.translation.language.code, {'source': True}) return self.filter(pk__in=pks, translation__language=unit.translation.language, translated=True).exclude(pk=unit.id)
def same_source(self, unit): """ Finds units with same source. """ checksums = fulltext_search(unit.get_source_plurals()[0], unit.translation.language.code, True, False, False) return self.filter( checksum__in=checksums, translation__language=unit.translation.language, translated=True ).exclude(pk=unit.id)
def search(self, params, project=None, subproject=None, language=None, translation=None): """High level wrapper for searching.""" if translation is not None: if subproject is None: subproject = translation.subproject if language is None: language = translation.language if subproject is not None: if project is None: project = subproject.project base = self.prefetch() if params['type'] != 'all': base = self.filter_type(params['type'], project, language, params['ignored']) if (params.get('date') or params.get('exclude_user') or params.get('only_user')): base = base.review(params.get('date'), params.get('exclude_user'), params.get('only_user'), project, subproject, language, translation) if 'lang' in params and params['lang']: base = base.filter(translation__language__code=params['lang']) if not params['q']: result = base elif params['search'] in ('exact', 'substring'): queries = [] modifier = '' if params['search'] != 'exact': modifier = '__icontains' for param in SEARCH_FILTERS: if param in params and params[param]: queries.append(param) query = functools.reduce( lambda q, value: q | Q(**{'{0}{1}'.format(value, modifier): params['q']}), queries, Q()) result = base.filter(query) else: langs = set( self.values_list('translation__language__code', flat=True)) result = base.filter( pk__in=fulltext_search(params['q'], langs, params)) return result
def fulltext(self, query, source=True, context=True, translation=True): """ Performs full text search on defined set of fields. Returns queryset unless checksums is set. """ lang = self.all()[0].translation.language.code ret = fulltext_search(query, lang, source, context, translation) return ret
def same_source(self, unit): """Find units with same source.""" pks = fulltext_search(unit.get_source_plurals()[0], [unit.translation.language.code], {'source': True}) return self.filter( pk__in=pks, translation__language=unit.translation.language, state__gte=STATE_TRANSLATED, ).exclude(pk=unit.id)
def same_source(self, unit): """ Finds units with same source. """ checksums = fulltext_search(unit.get_source_plurals()[0], unit.translation.language.code, True, False, False) return self.filter(checksum__in=checksums, translation__language=unit.translation.language, translated=True).exclude(pk=unit.id)
def more_like_this(self, unit, top=5): """ Finds closely similar units. """ more_results = more_like(unit.checksum, unit.source, top) same_results = fulltext_search(unit.get_source_plurals()[0], unit.translation.language.code, True, False, False) checksums = more_results - same_results return self.filter( checksum__in=checksums, translation__language=unit.translation.language, translated=True ).exclude(pk=unit.id)
def same_source(self, unit): """Find units with same source.""" pks = fulltext_search( unit.get_source_plurals()[0], unit.translation.language.code, {'source': True} ) return self.filter( pk__in=pks, translation__language=unit.translation.language, translated=True ).exclude( pk=unit.id )
def fulltext(self, query, source=True, context=True, translation=True, checksums=False): ''' Performs full text search on defined set of fields. Returns queryset unless checksums is set. ''' lang = self.all()[0].translation.language.code ret = fulltext_search(query, lang, source, context, translation) if checksums: return ret return self.filter(checksum__in=ret)
def more_like_this(self, unit, top=5): """ Finds closely similar units. """ more_results = more_like(unit.checksum, unit.source, top) same_results = fulltext_search(unit.get_source_plurals()[0], unit.translation.language.code, True, False, False) checksums = more_results - same_results return self.filter(checksum__in=checksums, translation__language=unit.translation.language, translated=True).exclude(pk=unit.id)
def search(self, translation, params): """ High level wrapper for searching. """ base = self.all() if params['type'] != 'all': base = self.filter_type( params['type'], translation, params['ignored'] ) if 'lang' in params and params['lang']: base = base.filter(translation__language__code=params['lang']) if not params['q']: return base if params['search'] in ('exact', 'substring'): queries = [] modifier = '' if params['search'] != 'exact': modifier = '__icontains' for param in SEARCH_FILTERS: if params[param]: queries.append(param) query = functools.reduce( lambda q, value: q | Q(**{'%s%s' % (value, modifier): params['q']}), queries, Q() ) return base.filter(query) else: lang = self.values_list( 'translation__language__code', flat=True )[0] return base.filter( pk__in=fulltext_search( params['q'], lang, params ) )
def search(self, translation, params): """High level wrapper for searching.""" base = self.prefetch() if params['type'] != 'all': base = self.filter_type( params['type'], translation, params['ignored'] ) if 'lang' in params and params['lang']: base = base.filter(translation__language__code=params['lang']) if not params['q']: result = base elif params['search'] in ('exact', 'substring'): queries = [] modifier = '' if params['search'] != 'exact': modifier = '__icontains' for param in SEARCH_FILTERS: if param in params and params[param]: queries.append(param) query = functools.reduce( lambda q, value: q | Q(**{'{0}{1}'.format(value, modifier): params['q']}), queries, Q() ) result = base.filter(query) else: langs = set(self.values_list( 'translation__language__code', flat=True )) result = base.filter( pk__in=fulltext_search( params['q'], langs, params ) ) return result
def search(self, translation, params): """ High level wrapper for searching. """ base = self.all() if params['type'] != 'all': base = self.filter_type( params['type'], translation, params['ignored'] ) if not params['q']: return base if params['search'] in ('exact', 'substring'): queries = [] modifier = '' if params['search'] != 'exact': modifier = '__icontains' for param in SEARCH_FILTERS: if params[param]: queries.append(param) query = reduce( lambda q, value: q | Q(**{'%s%s' % (value, modifier): params['q']}), queries, Q() ) return base.filter(query) else: lang = self.all()[0].translation.language.code return base.filter( checksum__in=fulltext_search( params['q'], lang, params ) )
def search(self, params, project=None, component=None, language=None, translation=None): """High level wrapper for searching.""" if translation is not None: component = translation.component language = translation.language if component is not None: project = component.project base = self.prefetch() if params['type'] != 'all': base = self.filter_type( params['type'], project, language, params['ignored'] ) if (params.get('date') or params.get('exclude_user') or params.get('only_user')): base = base.review( params.get('date'), params.get('exclude_user'), params.get('only_user'), project, component, language, translation ) if 'lang' in params and params['lang']: base = base.filter(translation__language__code__in=params['lang']) if not params['q']: result = base elif params['search'] in ('exact', 'substring'): queries = [] modifier = '' if params['search'] != 'exact': modifier = '__icontains' for param in SEARCH_FILTERS: if param in params and params[param]: queries.append(param) query = functools.reduce( lambda q, value: q | Q(**{'{0}{1}'.format(value, modifier): params['q']}), queries, Q() ) result = base.filter(query) else: langs = set(self.values_list( 'translation__language__code', flat=True )) result = base.filter( pk__in=fulltext_search( params['q'], langs, params ) ) return result