def parse_store(cls, storefile):
# try to load the given file via openpyxl
# catch at least the BadZipFile exception if an unsupported
# file has been given
try:
workbook = load_workbook(filename=storefile)
worksheet = workbook.active
except BadZipFile:
return None, None
output = StringIO()
writer = csv.writer(output, dialect="unix")
for row in worksheet.rows:
writer.writerow([cell.value for cell in row])
if isinstance(storefile, str):
name = os.path.basename(storefile) + ".csv"
else:
name = os.path.basename(storefile.name) + ".csv"
# return the new csv as bytes
content = output.getvalue().encode()
# Load the file as CSV
return super().parse_store(BytesIOMode(name, content))
def handle_replace(self, request, fileobj):
"""Replace file content with uploaded one."""
filecopy = fileobj.read()
fileobj.close()
fileobj = BytesIOMode(fileobj.name, filecopy)
with self.component.repository.lock:
if self.is_source:
self.component.commit_pending("replace file", request.user)
else:
self.commit_pending("replace file", request.user)
# This will throw an exception in case of error
store2 = self.load_store(fileobj)
store2.check_valid()
# Actually replace file content
self.store.save_atomic(
self.store.storefile, lambda handle: handle.write(filecopy)
)
# Commit to VCS
previous_revision = (self.component.repository.last_revision,)
if self.git_commit(
request.user, request.user.get_author_name(), store_hash=False
):
# Drop store cache
self.drop_store_cache()
self.handle_store_change(
request,
request.user,
previous_revision,
change=Change.ACTION_REPLACE_UPLOAD,
)
return (0, 0, self.unit_set.count(), len(list(store2.content_units)))
def handle_replace(self, request, fileobj):
"""Replace file content with uploaded one."""
filecopy = fileobj.read()
fileobj.close()
fileobj = BytesIOMode(fileobj.name, filecopy)
with self.component.repository.lock, transaction.atomic():
self.commit_pending('replace file', request.user)
# This will throw an exception in case of error
store2 = self.load_store(fileobj)
store2.check_valid()
# Actually replace file content
self.store.save_atomic(self.store.storefile,
lambda handle: handle.write(filecopy))
# Commit to VCS
if self.repo_needs_commit():
self.__git_commit(request.user.get_author_name(),
timezone.now())
# Drop store cache
self.drop_store_cache()
# Parse the file again
if self.is_template:
self.component.create_translations(request=request,
force=True)
else:
self.check_sync(force=True,
request=request,
change=Change.ACTION_UPLOAD)
self.invalidate_cache()
return (0, 0, self.unit_set.count(), len(list(store2.content_units)))
def test_changed(self):
self.edit_unit("Hello, world!\n", "Hi, World!\n", "en")
response = self.client.get(
reverse("download_translation", kwargs=self.kw_translation),
{"format": "csv"},
)
self.assertEqual(
response.content.decode(), EXPECTED_CSV.replace("Hello, world", "Hi, World")
)
handle = BytesIOMode(
"test.csv", UPLOAD_CSV.replace("Hello, world", "Hi, World").encode()
)
params = {
"file": handle,
"method": "translate",
"author_name": self.user.full_name,
"author_email": self.user.email,
}
response = self.client.post(
reverse("upload_translation", kwargs=self.kw_translation),
params,
follow=True,
)
self.assertContains(response, "(skipped: 0, not found: 0, updated: 1)")
def check_unit(self, nplurals=3, template=None, source_info=None, **kwargs):
if nplurals == 3:
formula = "n==0 ? 0 : n==1 ? 1 : 2"
else:
formula = "0"
lang = Language.objects.create(code="zz")
plural = Plural.objects.create(language=lang, number=nplurals, formula=formula)
project = Project(slug="test")
component = Component(
slug="comp",
project=project,
file_format="xliff",
template=template,
source_language=Language.objects.get(code="en"),
)
translation = Translation(language=lang, component=component, plural=plural)
# Fake file format to avoid need for actual files
translation.store = EmptyFormat(BytesIOMode("", b""))
unit = Unit(translation=translation, id_hash=-1, **kwargs)
if source_info:
for key, value in source_info.items():
setattr(unit, key, value)
# The dashes need special handling in XML based formats
unit.__dict__["unresolved_comments"] = [
Comment(comment="Weblate translator comment ---- ")
]
unit.__dict__["suggestions"] = [
Suggestion(target="Weblate translator suggestion")
]
else:
unit.__dict__["unresolved_comments"] = []
unit.source_unit = unit
exporter = self.get_exporter(lang, translation=translation)
exporter.add_unit(unit)
return self.check_export(exporter)
def check_unit(self, nplurals=3, template=None, **kwargs):
if nplurals == 3:
equation = 'n==0 ? 0 : n==1 ? 1 : 2'
else:
equation = '0'
lang = Language.objects.create(
code='zz',
)
plural = Plural.objects.create(
language=lang,
number=nplurals,
equation=equation
)
project = Project(
slug='test',
source_language=Language.objects.get(code='en'),
)
component = Component(
slug='comp',
project=project,
file_format='xliff',
template=template
)
translation = Translation(
language=lang,
component=component,
plural=plural,
)
# Fake file format to avoid need for actual files
translation.store = EmptyFormat(BytesIOMode('', b''))
unit = Unit(translation=translation, id_hash=-1, **kwargs)
exporter = self.get_exporter(lang, translation=translation)
exporter.add_unit(unit)
return self.check_export(exporter)
def parse_store(cls, storefile):
# try to load the given file via openpyxl
# catch at least the BadZipFile exception if an unsupported
# file has been given
try:
workbook = load_workbook(filename=storefile)
worksheet = workbook.active
except BadZipFile:
return None, None
if six.PY3:
output = six.StringIO()
else:
output = six.BytesIO()
writer = csv.writer(output)
for row in worksheet.rows:
writer.writerow([cls.encode(cell.value) for cell in row])
if isinstance(storefile, six.string_types):
name = os.path.basename(storefile) + ".csv"
else:
name = os.path.basename(storefile.name) + ".csv"
# return the new csv as bytes
content = output.getvalue()
if six.PY3:
content = content.encode("utf-8")
# Load the file as CSV
return super(XlsxFormat, cls).parse_store(BytesIOMode(name, content))
def try_load(
filename, content, original_format, template_store, as_template: bool = False
):
"""Try to load file by guessing type."""
# Start with original format and translate-toolkit based autodetection
formats = [original_format, AutodetectFormat]
detected_format = detect_filename(filename)
if detected_format is not None and detected_format != original_format:
# Insert detected filename into most probable location. In case the extension
# matches original, insert it after that as it is more likely that the upload
# is in the original format (for example if component is monolingual PO file,
# the uploaded PO file is more likely to be monolingual as well).
formats.insert(
1 if detected_format.extension() == original_format.extension() else 0,
detected_format,
)
# Provide fallback to bilingual class in case using monolingual
if (
original_format.bilingual_class
and original_format.bilingual_class != detected_format
):
formats.insert(1, original_format.bilingual_class)
failure = Exception("Bug!")
for file_format in formats:
if file_format.monolingual in (True, None) and (template_store or as_template):
try:
result = file_format.parse(
BytesIOMode(filename, content), template_store
)
result.check_valid()
# Skip if there is untranslated unit
# this can easily happen when importing bilingual
# storage which can be monolingual as well
if list(result.iterate_merge("")):
return result
except Exception as error:
failure = error
if file_format.monolingual in (False, None):
try:
result = file_format.parse(BytesIOMode(filename, content))
result.check_valid()
return result
except Exception as error:
failure = error
raise failure
def convertfile(storefile):
store = pofile()
# Fake input file with a blank filename
htmlparser = htmlfile(includeuntaggeddata=False,
inputfile=BytesIOMode("", storefile.read()))
for htmlunit in htmlparser.units:
thepo = store.addsourceunit(htmlunit.source)
thepo.addlocations(htmlunit.getlocations())
thepo.addnote(htmlunit.getnotes(), "developer")
store.removeduplicates("msgctxt")
return store
def check_unit(self,
nplurals=3,
template=None,
source_info=None,
**kwargs):
if nplurals == 3:
equation = 'n==0 ? 0 : n==1 ? 1 : 2'
else:
equation = '0'
lang = Language.objects.create(code='zz', )
plural = Plural.objects.create(language=lang,
number=nplurals,
equation=equation)
project = Project(
slug='test',
source_language=Language.objects.get(code='en'),
)
component = Component(slug='comp',
project=project,
file_format='xliff',
template=template)
translation = Translation(
language=lang,
component=component,
plural=plural,
)
# Fake file format to avoid need for actual files
translation.store = EmptyFormat(BytesIOMode('', b''))
unit = Unit(translation=translation, id_hash=-1, **kwargs)
if source_info:
for key, value in source_info.items():
setattr(unit, key, value)
unit.get_comments = fake_get_comments
unit.__dict__['suggestions'] = [
Suggestion(target='Weblate translator suggestion')
]
else:
unit.get_comments = empty_get_comments
exporter = self.get_exporter(lang, translation=translation)
exporter.add_unit(unit)
return self.check_export(exporter)
def convertfile(storefile, template_store):
store = pofile()
# Fake input file with a blank filename
htmlparser = htmlfile(inputfile=BytesIOMode("", storefile.read()))
for htmlunit in htmlparser.units:
locations = htmlunit.getlocations()
if template_store:
# Transalation
template = template_store.find_unit_mono("".join(locations))
if template is None:
# Skip locations not present in the source HTML file
continue
# Create unit with matching source
thepo = store.addsourceunit(template.source)
thepo.target = htmlunit.source
else:
# Source file
thepo = store.addsourceunit(htmlunit.source)
thepo.target = htmlunit.source
thepo.addlocations(htmlunit.getlocations())
thepo.addnote(htmlunit.getnotes(), "developer")
store.removeduplicates("msgctxt")
return store
def validate_new_unit_data( # noqa: C901
self,
context: str,
source: Union[str, List[str]],
target: Optional[Union[str, List[str]]] = None,
auto_context: bool = False,
extra_flags: Optional[str] = None,
):
extra = {}
if isinstance(source, str):
source = [source]
if isinstance(target, str):
target = [target]
if not self.component.has_template():
extra["source"] = join_plural(source)
if not auto_context and self.unit_set.filter(context=context, **extra).exists():
raise ValidationError(_("This string seems to already exist."))
# Avoid using source translations without a filename
if not self.filename:
try:
translation = self.component.translation_set.exclude(pk=self.pk)[0]
except IndexError:
raise ValidationError(
_("Failed adding string: %s") % _("No translation found.")
)
translation.validate_new_unit_data(
context,
source,
target,
auto_context=auto_context,
extra_flags=extra_flags,
)
return
# Always load a new copy of store
store = self.load_store()
old_units = len(store.all_units)
# Add new unit
store.new_unit(context, source, target, skip_build=True)
# Serialize the content
handle = BytesIOMode("", b"")
# Catch serialization error
try:
store.save_content(handle)
except Exception as error:
raise ValidationError(_("Failed adding string: %s") % error)
handle.seek(0)
# Parse new file (check that it is valid)
try:
newstore = self.load_store(handle)
except Exception as error:
raise ValidationError(_("Failed adding string: %s") % error)
# Verify there is a single unit added
if len(newstore.all_units) != old_units + 1:
raise ValidationError(
_("Failed adding string: %s") % _("Failed to parse new string")
)
# Find newly added unit (it can be on any position), but we assume
# the storage has consistent ordering
unit = None
for pos, current in enumerate(newstore.all_units):
if pos >= old_units or (
current.source != store.all_units[pos].source
and current.context != store.all_units[pos].context
):
unit = current
break
# Verify unit matches data
if unit is None:
raise ValidationError(
_("Failed adding string: %s") % _("Failed to parse new string")
)
created_source = split_plural(unit.source)
if unit.context != context and (
self.component.has_template()
or self.component.file_format_cls.set_context_bilingual
):
raise ValidationError(
{"context": _('Context would be created as "%s"') % unit.context}
)
if created_source != source:
raise ValidationError(
{"source": _("Source would be created as %s") % created_source}
)
def extract_document(content):
return bytes(
OpenDocumentFormat.convertfile(BytesIOMode("test.odt", content))
).decode()
def extract_document(content):
return bytes(IDMLFormat.convertfile(BytesIOMode("test.idml", content))).decode()