def download_translations(self, source, language, text, unit, user):
"""Download list of possible translations from a service."""
memory = TranslationMemory()
return [
self.format_unit_match(*result)
for result in memory.lookup(source.code, language.code, text)
]
def test_delete_origin_command(self, params=None):
if params is None:
params = ['--origin', 'test']
add_document()
call_command('delete_memory', *params)
memory = TranslationMemory()
self.assertEqual(memory.doc_count(), 0)
def handle(self, *args, **options):
"""Translation memory import."""
langmap = None
if options['language_map']:
langmap = {
x: y for (x, y) in (
z.split(':', 1) for z in options['language_map'].split(',')
)
}
memory = TranslationMemory()
if options['file'].name.lower().endswith('.tmx'):
memory.import_tmx(options['file'], langmap)
elif options['file'].name.lower().endswith('.json'):
try:
data = json.load(options['file'])
except ValueError:
raise CommandError('Failed to parse JSON file!')
finally:
options['file'].close()
with memory.writer() as writer:
for entry in data:
writer.add_document(**entry)
else:
raise CommandError(
'Unsupported file, needs .json or .tmx extension'
)
def test_delete_origin_command(self, params=None):
if params is None:
params = ['--origin', 'test']
add_document()
call_command('delete_memory', *params)
memory = TranslationMemory()
self.assertEqual(memory.doc_count(), 0)
def get_context_data(self, **kwargs):
memory = TranslationMemory()
context = super(MemoryView, self).get_context_data(**kwargs)
context.update(self.objects)
entries = memory.list_documents(**self.objects)
context['num_entries'] = len(entries)
context['total_entries'] = memory.doc_count()
context['delete_url'] = self.get_url('memory-delete')
context['upload_url'] = self.get_url('memory-upload')
context['download_url'] = self.get_url('memory-download')
user = self.request.user
if check_perm(user, 'memory.delete', self.objects):
context['delete_form'] = DeleteForm()
if check_perm(user, 'memory.edit', self.objects):
context['upload_form'] = UploadForm()
if 'project' in self.objects:
context['import_form'] = ImportForm()
context['import_url'] = self.get_url('memory-import')
if 'use_file' in self.objects:
context['menu_items'] = MENU
context['menu_page'] = 'memory'
if 'use_file' in self.objects or (
'project' in self.objects and self.objects['project'].use_shared_tm
):
context['shared_entries'] = len(memory.list_documents(use_shared=True))
return context
def test_import_json_command(self):
call_command(
'import_memory',
get_test_file('memory.json')
)
memory = TranslationMemory()
self.assertEqual(memory.doc_count(), 1)
def memory_backup(indent=2):
if not os.path.exists(data_dir('backups')):
os.makedirs(data_dir('backups'))
filename = data_dir('backups', 'memory.json')
memory = TranslationMemory()
with open(filename, 'w') as handle:
memory.dump(handle, indent)
def rebuild(self):
memory = TranslationMemory()
data = list(memory.searcher.documents())
memory.empty()
with memory.writer() as writer:
for entry in data:
writer.add_document(**entry)
def form_valid(self, form):
if not check_perm(self.request.user, 'memory.delete', self.objects):
raise PermissionDenied()
memory = TranslationMemory()
memory.delete(**self.objects)
messages.success(self.request, _('Entries deleted.'))
return super(DeleteView, self).form_valid(form)
def rebuild(self):
memory = TranslationMemory()
data = list(memory.searcher.documents())
memory.empty()
with memory.writer() as writer:
for entry in data:
writer.add_document(**entry)
def handle(self, *args, **options):
if options['backup']:
memory_backup(options['indent'])
return
memory = TranslationMemory()
self.stdout.ending = None
memory.dump(self.stdout, indent=options['indent'])
self.stdout.write('\n')
def test_import_empty_json_command(self):
with self.assertRaises(CommandError):
call_command(
'import_memory',
get_test_file('memory-empty.json')
)
memory = TranslationMemory()
self.assertEqual(memory.doc_count(), 0)
def test_import_invalid_command(self):
with self.assertRaises(CommandError):
call_command(
'import_memory',
get_test_file('cs.po')
)
memory = TranslationMemory()
self.assertEqual(memory.doc_count(), 0)
def handle(self, *args, **options):
if options['backup']:
memory_backup(options['indent'])
return
memory = TranslationMemory()
self.stdout.ending = None
memory.dump(self.stdout, indent=options['indent'])
self.stdout.write('\n')
def handle(self, *args, **options):
"""Translation memory cleanup."""
memory = TranslationMemory()
if options['all']:
memory.empty()
elif options['origin'] or options['category']:
memory.delete(options['origin'], options['category'])
else:
raise CommandError('Please specify what you want to delete')
def form_valid(self, form):
if not check_perm(self.request.user, 'memory.delete', self.objects):
raise PermissionDenied()
memory = TranslationMemory()
memory.delete(**self.objects)
messages.success(
self.request, _('Entries deleted.')
)
return super(DeleteView, self).form_valid(form)
def handle(self, *args, **options):
memory = TranslationMemory()
memory.open_searcher()
self.stdout.ending = None
json.dump(
list(memory.searcher.documents()),
self.stdout,
indent=options['indent'],
)
self.stdout.write('\n')
def download_translations(self, source, language, text, unit, request):
"""Download list of possible translations from a service."""
memory = TranslationMemory()
results = memory.lookup(
source.code, language.code, text,
request.user,
unit.translation.component.project,
unit.translation.component.project.use_shared_tm,
)
return [self.format_unit_match(*result) for result in results]
def handle(self, *args, **options):
memory = TranslationMemory()
memory.open_searcher()
self.stdout.ending = None
json.dump(
list(memory.searcher.documents()),
self.stdout,
indent=options['indent'],
)
self.stdout.write('\n')
def form_valid(self, form):
if not check_perm(self.request.user, 'memory.edit', self.objects):
raise PermissionDenied()
try:
TranslationMemory.import_file(form.cleaned_data['file'],
**self.objects)
messages.success(
self.request,
_('File processed, the entries will appear shortly.'))
except MemoryImportError as error:
messages.error(self.request, force_text(error))
return super(UploadView, self).form_valid(form)
def download_translations(self, source, language, text, unit, request):
"""Download list of possible translations from a service."""
memory = TranslationMemory()
results = memory.lookup(
source.code,
language.code,
text,
request.user,
unit.translation.component.project,
unit.translation.component.project.use_shared_tm,
)
return [self.format_unit_match(*result) for result in results]
def get(self, request, *args, **kwargs):
memory = TranslationMemory()
fmt = request.GET.get('format', 'json')
data = [dict(x) for x in memory.list_documents(**self.objects)]
if fmt == 'tmx':
response = render(request,
'memory/dump.tmx', {'data': data},
content_type='application/x-tmx')
else:
fmt = 'json'
response = JsonResponse(data, safe=False)
response['Content-Disposition'] = CD_TEMPLATE.format(fmt)
return response
def form_valid(self, form):
if not check_perm(self.request.user, 'memory.edit', self.objects):
raise PermissionDenied()
try:
TranslationMemory.import_file(
self.request, form.cleaned_data['file'], **self.objects
)
messages.success(
self.request,
_('File processed, the entries will appear shortly.')
)
except MemoryImportError as error:
messages.error(self.request, force_text(error))
return super(UploadView, self).form_valid(form)
def handle(self, *args, **options):
"""Translation memory import."""
langmap = None
if options['language_map']:
langmap = {
x: y
for (x, y) in (z.split(':', 1)
for z in options['language_map'].split(','))
}
try:
TranslationMemory.import_file(None, options['file'], langmap)
except MemoryImportError as error:
raise CommandError('Import failed: {}'.format(error))
def handle(self, *args, **options):
"""Translation memory import."""
langmap = None
if options['language_map']:
langmap = {
x: y for (x, y) in (
z.split(':', 1) for z in options['language_map'].split(',')
)
}
try:
TranslationMemory.import_file(None, options['file'], langmap)
except MemoryImportError as error:
raise CommandError('Import failed: {}'.format(error))
def test_delete(self):
self.add_document()
memory = TranslationMemory()
self.assertEqual(memory.doc_count(), 1)
self.assertEqual(memory.delete('test'), 1)
self.assertEqual(memory.delete('missing'), 0)
memory = TranslationMemory()
self.assertEqual(memory.doc_count(), 0)
def get(self, request, *args, **kwargs):
memory = TranslationMemory()
fmt = request.GET.get('format', 'json')
data = [dict(x) for x in memory.list_documents(**self.objects)]
if fmt == 'tmx':
response = render(
request,
'memory/dump.tmx',
{'data': data},
content_type='application/x-tmx'
)
else:
fmt = 'json'
response = JsonResponse(data, safe=False)
response['Content-Disposition'] = CD_TEMPLATE.format(fmt)
return response
def test_import_map(self):
call_command(
'import_memory',
get_test_file('memory.tmx'),
language_map='en_US:en',
)
self.assertEqual(TranslationMemory().doc_count(), 2)
def optimize_index(self):
"""Optimize index structures"""
memory = TranslationMemory()
memory.index.optimize()
index = get_source_index()
index.optimize()
languages = Language.objects.have_translation()
for lang in languages:
index = get_target_index(lang.code)
index.optimize()
def update_memory_task(self, *args, **kwargs):
def fixup_strings(data):
result = {}
for key, value in data.items():
if isinstance(value, int):
result[key] = value
else:
result[key] = force_text(value)
return result
data = extract_batch_kwargs(*args, **kwargs)
memory = TranslationMemory()
try:
with memory.writer() as writer:
for item in data:
writer.add_document(**fixup_strings(item))
except LockError as exc:
raise self.retry(exc=exc)
def get_context_data(self, **kwargs):
memory = TranslationMemory()
context = super(MemoryView, self).get_context_data(**kwargs)
context.update(self.objects)
entries = memory.list_documents(**self.objects)
context['num_entries'] = len(entries)
context['total_entries'] = memory.doc_count()
context['delete_url'] = self.get_url('memory-delete')
context['upload_url'] = self.get_url('memory-upload')
context['download_url'] = self.get_url('memory-download')
user = self.request.user
if check_perm(user, 'memory.delete', self.objects):
context['delete_form'] = DeleteForm()
if check_perm(user, 'memory.edit', self.objects):
context['upload_form'] = UploadForm()
if 'project' in self.objects:
context['import_form'] = ImportForm()
context['import_url'] = self.get_url('memory-import')
return context
def update_memory_task(self, *args, **kwargs):
def fixup_strings(data):
result = {}
for key, value in data.items():
if isinstance(value, int):
result[key] = value
else:
result[key] = force_text(value)
return result
data = extract_batch_kwargs(*args, **kwargs)
memory = TranslationMemory()
try:
with memory.writer() as writer:
for item in data:
writer.add_document(**fixup_strings(item))
except LockError:
# Manually handle retries, it doesn't work
# with celery-batches
sleep(10)
for unit in data:
update_memory_task.delay(**unit)
def handle(self, *args, **options):
"""Translation memory cleanup."""
memory = TranslationMemory()
if options['all']:
memory.empty()
elif options['origin'] or options['category']:
memory.delete(options['origin'], options['category'])
else:
raise CommandError('Please specify what you want to delete')
def download_translations(self, source, language, text, unit, user):
"""Download list of possible translations from a service."""
memory = TranslationMemory.get_thread_instance()
memory.refresh()
results = memory.lookup(
source.code,
language.code,
text,
user,
unit.translation.component.project,
unit.translation.component.project.use_shared_tm,
)
for text, target, similarity, category, origin in results:
yield {
'text': target,
'quality': similarity,
'service': self.name,
'origin': get_category_name(category, origin),
'source': text,
}
def memory_backup(indent=2):
filename = os.path.join(data_dir('backups'), 'memory.json')
memory = TranslationMemory()
with open(filename, 'w') as handle:
memory.dump(handle, indent)
def memory_optimize():
memory = TranslationMemory()
memory.index.optimize()
def test_delete_all_command(self):
self.add_document()
call_command('delete_memory', '--all')
memory = TranslationMemory()
self.assertEqual(memory.doc_count(), 0)
def handle(self, *args, **options):
"""Translation memory cleanup."""
memory = TranslationMemory()
for item in memory.get_values(options['type']):
self.stdout.write(item)
def setUp(self):
TranslationMemory.cleanup()
def test_list(self):
memory = TranslationMemory()
self.assertEqual(list(memory.get_origins()), [])
self.add_document()
memory = TranslationMemory()
self.assertEqual(memory.get_origins(), ['test'])
def add_document(self):
memory = TranslationMemory()
with memory.writer() as writer:
writer.add_document(**TEST_DOCUMENT)
def handle(self, *args, **options):
"""Translation memory cleanup."""
memory = TranslationMemory()
if options['type'] == 'origin':
for origin in memory.get_origins():
self.stdout.write(origin)
def test_list(self):
memory = TranslationMemory()
self.assertEqual(memory.get_values('origin'), [])
add_document()
memory = TranslationMemory()
self.assertEqual(memory.get_values('origin'), ['test'])
def test_import_invalid_command(self):
with self.assertRaises(CommandError):
call_command('import_memory', get_test_file('cs.po'))
memory = TranslationMemory()
self.assertEqual(memory.doc_count(), 0)
def test_import_empty_json_command(self):
with self.assertRaises(CommandError):
call_command('import_memory', get_test_file('memory-empty.json'))
memory = TranslationMemory()
self.assertEqual(memory.doc_count(), 0)
def handle(self, *args, **options):
"""Translation memory cleanup."""
memory = TranslationMemory()
for item in memory.get_values(options['type']):
self.stdout.write(item)
def test_rebuild_command(self):
add_document()
call_command('optimize_memory', '--rebuild')
memory = TranslationMemory()
self.assertEqual(memory.doc_count(), 1)
