def do_test(self, source, target):
if source is not None:
self.storage.create_index(source, 'source')
if target is not None:
self.storage.create_index(target, 'target-cs')
sindex = weblate.trans.search.get_source_index()
self.assertIsNotNone(sindex)
tindex = weblate.trans.search.get_target_index('cs')
self.assertIsNotNone(tindex)
writer = sindex.writer()
writer.update_document(
pk=1,
source="source",
context="context",
location="location",
)
writer.commit()
writer = tindex.writer()
writer.update_document(
pk=1,
target="target",
comment="comment"
)
writer.commit()
for item in ('source', 'context', 'location', 'target'):
self.assertEqual(
fulltext_search(item, ['cs'], {item: True}),
set([1])
)
def search(self, translation, params):
"""
High level wrapper for searching.
"""
base = self.all()
if params['type'] != 'all':
base = self.filter_type(params['type'], translation,
params['ignored'])
if 'lang' in params and params['lang']:
base = base.filter(translation__language__code=params['lang'])
if not params['q']:
return base
if params['search'] in ('exact', 'substring'):
queries = []
modifier = ''
if params['search'] != 'exact':
modifier = '__icontains'
for param in SEARCH_FILTERS:
if params[param]:
queries.append(param)
query = reduce(
lambda q, value: q | Q(
**{'%s%s' % (value, modifier): params['q']}), queries, Q())
return base.filter(query)
else:
lang = self.all()[0].translation.language.code
return base.filter(
checksum__in=fulltext_search(params['q'], lang, params))
def more_like_this(self, unit, top=5):
"""
Finds closely similar units.
"""
if appsettings.MT_WEBLATE_LIMIT >= 0:
queue = multiprocessing.Queue()
proc = multiprocessing.Process(
target=more_like_queue,
args=(unit.pk, unit.source, top, queue)
)
proc.start()
proc.join(appsettings.MT_WEBLATE_LIMIT)
if proc.is_alive():
proc.terminate()
if queue.empty():
raise Exception('Request timed out.')
more_results = queue.get()
else:
more_results = more_like(unit.pk, unit.source, top)
same_results = fulltext_search(
unit.get_source_plurals()[0],
unit.translation.language.code,
{'source': True}
)
return self.filter(
pk__in=more_results - same_results,
translation__language=unit.translation.language,
translated=True
).exclude(
pk=unit.id
)
def do_test(self, source, target):
if source is not None:
self.storage.create_index(source, 'source')
if target is not None:
self.storage.create_index(target, 'target-cs')
sindex = weblate.trans.search.get_source_index()
self.assertIsNotNone(sindex)
tindex = weblate.trans.search.get_target_index('cs')
self.assertIsNotNone(tindex)
writer = sindex.writer()
writer.update_document(
pk=1,
source="source",
context="context",
location="location",
)
writer.commit()
writer = tindex.writer()
writer.update_document(
pk=1,
target="target",
comment="comment"
)
writer.commit()
for item in ('source', 'context', 'location', 'target'):
self.assertEqual(
fulltext_search(item, ['cs'], {item: True}),
set([1])
)
def more_like_this(self, unit, top=5):
"""Find closely similar units."""
if settings.MT_WEBLATE_LIMIT >= 0:
queue = multiprocessing.Queue()
proc = multiprocessing.Process(
target=more_like_queue,
args=(unit.pk, unit.source, top, queue)
)
proc.start()
proc.join(settings.MT_WEBLATE_LIMIT)
if proc.is_alive():
proc.terminate()
if queue.empty():
raise Exception(
'Request for more like {0} timed out.'.format(unit.pk)
)
more_results = queue.get()
else:
more_results = more_like(unit.pk, unit.source, top)
same_results = fulltext_search(
unit.get_source_plurals()[0],
[unit.translation.language.code],
{'source': True}
)
return self.filter(
pk__in=more_results - same_results,
translation__language=unit.translation.language,
translated=True
).exclude(
pk=unit.id
)
def same_source(self, unit):
"""Find units with same source."""
pks = fulltext_search(unit.get_source_plurals()[0],
unit.translation.language.code, {'source': True})
return self.filter(pk__in=pks,
translation__language=unit.translation.language,
translated=True).exclude(pk=unit.id)
def same_source(self, unit):
"""
Finds units with same source.
"""
checksums = fulltext_search(unit.get_source_plurals()[0], unit.translation.language.code, True, False, False)
return self.filter(
checksum__in=checksums, translation__language=unit.translation.language, translated=True
).exclude(pk=unit.id)
def search(self,
params,
project=None,
subproject=None,
language=None,
translation=None):
"""High level wrapper for searching."""
if translation is not None:
if subproject is None:
subproject = translation.subproject
if language is None:
language = translation.language
if subproject is not None:
if project is None:
project = subproject.project
base = self.prefetch()
if params['type'] != 'all':
base = self.filter_type(params['type'], project, language,
params['ignored'])
if (params.get('date') or params.get('exclude_user')
or params.get('only_user')):
base = base.review(params.get('date'), params.get('exclude_user'),
params.get('only_user'), project, subproject,
language, translation)
if 'lang' in params and params['lang']:
base = base.filter(translation__language__code=params['lang'])
if not params['q']:
result = base
elif params['search'] in ('exact', 'substring'):
queries = []
modifier = ''
if params['search'] != 'exact':
modifier = '__icontains'
for param in SEARCH_FILTERS:
if param in params and params[param]:
queries.append(param)
query = functools.reduce(
lambda q, value: q |
Q(**{'{0}{1}'.format(value, modifier): params['q']}), queries,
Q())
result = base.filter(query)
else:
langs = set(
self.values_list('translation__language__code', flat=True))
result = base.filter(
pk__in=fulltext_search(params['q'], langs, params))
return result
def fulltext(self, query, source=True, context=True, translation=True):
"""
Performs full text search on defined set of fields.
Returns queryset unless checksums is set.
"""
lang = self.all()[0].translation.language.code
ret = fulltext_search(query, lang, source, context, translation)
return ret
def fulltext(self, query, source=True, context=True, translation=True):
"""
Performs full text search on defined set of fields.
Returns queryset unless checksums is set.
"""
lang = self.all()[0].translation.language.code
ret = fulltext_search(query, lang, source, context, translation)
return ret
def same_source(self, unit):
"""Find units with same source."""
pks = fulltext_search(unit.get_source_plurals()[0],
[unit.translation.language.code],
{'source': True})
return self.filter(
pk__in=pks,
translation__language=unit.translation.language,
state__gte=STATE_TRANSLATED,
).exclude(pk=unit.id)
def same_source(self, unit):
"""
Finds units with same source.
"""
checksums = fulltext_search(unit.get_source_plurals()[0],
unit.translation.language.code, True,
False, False)
return self.filter(checksum__in=checksums,
translation__language=unit.translation.language,
translated=True).exclude(pk=unit.id)
def more_like_this(self, unit, top=5):
"""
Finds closely similar units.
"""
more_results = more_like(unit.checksum, unit.source, top)
same_results = fulltext_search(unit.get_source_plurals()[0], unit.translation.language.code, True, False, False)
checksums = more_results - same_results
return self.filter(
checksum__in=checksums, translation__language=unit.translation.language, translated=True
).exclude(pk=unit.id)
def same_source(self, unit):
"""Find units with same source."""
pks = fulltext_search(
unit.get_source_plurals()[0],
unit.translation.language.code,
{'source': True}
)
return self.filter(
pk__in=pks,
translation__language=unit.translation.language,
translated=True
).exclude(
pk=unit.id
)
def fulltext(self, query, source=True, context=True, translation=True,
checksums=False):
'''
Performs full text search on defined set of fields.
Returns queryset unless checksums is set.
'''
lang = self.all()[0].translation.language.code
ret = fulltext_search(query, lang, source, context, translation)
if checksums:
return ret
return self.filter(checksum__in=ret)
def fulltext(self, query, source=True, context=True, translation=True,
checksums=False):
'''
Performs full text search on defined set of fields.
Returns queryset unless checksums is set.
'''
lang = self.all()[0].translation.language.code
ret = fulltext_search(query, lang, source, context, translation)
if checksums:
return ret
return self.filter(checksum__in=ret)
def more_like_this(self, unit, top=5):
"""
Finds closely similar units.
"""
more_results = more_like(unit.checksum, unit.source, top)
same_results = fulltext_search(unit.get_source_plurals()[0],
unit.translation.language.code, True,
False, False)
checksums = more_results - same_results
return self.filter(checksum__in=checksums,
translation__language=unit.translation.language,
translated=True).exclude(pk=unit.id)
def search(self, translation, params):
"""
High level wrapper for searching.
"""
base = self.all()
if params['type'] != 'all':
base = self.filter_type(
params['type'],
translation,
params['ignored']
)
if 'lang' in params and params['lang']:
base = base.filter(translation__language__code=params['lang'])
if not params['q']:
return base
if params['search'] in ('exact', 'substring'):
queries = []
modifier = ''
if params['search'] != 'exact':
modifier = '__icontains'
for param in SEARCH_FILTERS:
if params[param]:
queries.append(param)
query = functools.reduce(
lambda q, value:
q | Q(**{'%s%s' % (value, modifier): params['q']}),
queries,
Q()
)
return base.filter(query)
else:
lang = self.values_list(
'translation__language__code', flat=True
)[0]
return base.filter(
pk__in=fulltext_search(
params['q'],
lang,
params
)
)
def search(self, translation, params):
"""High level wrapper for searching."""
base = self.prefetch()
if params['type'] != 'all':
base = self.filter_type(
params['type'],
translation,
params['ignored']
)
if 'lang' in params and params['lang']:
base = base.filter(translation__language__code=params['lang'])
if not params['q']:
result = base
elif params['search'] in ('exact', 'substring'):
queries = []
modifier = ''
if params['search'] != 'exact':
modifier = '__icontains'
for param in SEARCH_FILTERS:
if param in params and params[param]:
queries.append(param)
query = functools.reduce(
lambda q, value:
q | Q(**{'{0}{1}'.format(value, modifier): params['q']}),
queries,
Q()
)
result = base.filter(query)
else:
langs = set(self.values_list(
'translation__language__code', flat=True
))
result = base.filter(
pk__in=fulltext_search(
params['q'],
langs,
params
)
)
return result
def search(self, translation, params):
"""
High level wrapper for searching.
"""
base = self.all()
if params['type'] != 'all':
base = self.filter_type(
params['type'],
translation,
params['ignored']
)
if not params['q']:
return base
if params['search'] in ('exact', 'substring'):
queries = []
modifier = ''
if params['search'] != 'exact':
modifier = '__icontains'
for param in SEARCH_FILTERS:
if params[param]:
queries.append(param)
query = reduce(
lambda q, value:
q | Q(**{'%s%s' % (value, modifier): params['q']}),
queries,
Q()
)
return base.filter(query)
else:
lang = self.all()[0].translation.language.code
return base.filter(
checksum__in=fulltext_search(
params['q'],
lang,
params
)
)
def search(self, params, project=None, component=None,
language=None, translation=None):
"""High level wrapper for searching."""
if translation is not None:
component = translation.component
language = translation.language
if component is not None:
project = component.project
base = self.prefetch()
if params['type'] != 'all':
base = self.filter_type(
params['type'],
project,
language,
params['ignored']
)
if (params.get('date') or
params.get('exclude_user') or
params.get('only_user')):
base = base.review(
params.get('date'),
params.get('exclude_user'),
params.get('only_user'),
project, component, language, translation
)
if 'lang' in params and params['lang']:
base = base.filter(translation__language__code__in=params['lang'])
if not params['q']:
result = base
elif params['search'] in ('exact', 'substring'):
queries = []
modifier = ''
if params['search'] != 'exact':
modifier = '__icontains'
for param in SEARCH_FILTERS:
if param in params and params[param]:
queries.append(param)
query = functools.reduce(
lambda q, value:
q | Q(**{'{0}{1}'.format(value, modifier): params['q']}),
queries,
Q()
)
result = base.filter(query)
else:
langs = set(self.values_list(
'translation__language__code', flat=True
))
result = base.filter(
pk__in=fulltext_search(
params['q'],
langs,
params
)
)
return result
