def __init__(self, tmdbfile, tmfiles, max_candidates=3, min_similarity=75,
max_length=1000, prefix="", source_lang=None, target_lang=None):
self.tmdb = tmdb.TMDB(tmdbfile, max_candidates, min_similarity, max_length)
#load files into db
if isinstance(tmfiles, list):
[self.tmdb.add_store(factory.getobject(tmfile), source_lang, target_lang) \
for tmfile in tmfiles]
elif tmfiles:
self.tmdb.add_store(factory.getobject(tmfiles), source_lang, target_lang)
#initialize url dispatcher
self.rest = selector.Selector(prefix=prefix)
self.rest.add("/{slang}/{tlang}/unit/{uid:any}",
GET=self.translate_unit,
POST=self.update_unit,
PUT=self.add_unit,
DELETE=self.forget_unit
)
self.rest.add("/{slang}/{tlang}/store/{sid:any}",
GET=self.get_store_stats,
PUT=self.upload_store,
POST=self.add_store,
DELETE=self.forget_store)
def _make_ts_diff(old_ts_path, new_ts_path):
old = factory.getobject(old_ts_path)
new = factory.getobject(new_ts_path)
# this is for added / removed
old_ids = set(old.getids())
new_ids = set(new.getids())
added = new_ids - old_ids
removed = old_ids - new_ids
old_trans_count = 0
for unit_id in old_ids:
if old.findid(unit_id).istranslated():
old_trans_count += 1
new_trans_count = 0
for unit_id in new_ids:
if new.findid(unit_id).istranslated():
new_trans_count += 1
return {
"old_trans_count": old_trans_count,
"new_trans_count": new_trans_count,
"old_units": len(old_ids),
"new_units": len(new_ids),
"added": list(added),
"removed": list(removed),
}
def main(dir, output, pstr):
import random
percentage = float(pstr)
from translate.storage import factory
unitslist = []
create_intermediates(dir)
print "Creating final TMX..."
for f in os.listdir(dir):
if f.endswith(".tmx") and not os.path.isdir(f):
try:
filepath = os.path.join(dir,f)
minicorpus = factory.getobject(filepath)
for u in minicorpus.units:
u.addnote("Origin: "+f)
unitslist.append(minicorpus.units)
except ValueError:
print "Could not convert to factory."
continue
sample_list = create_sample(unitslist, percentage)
sample = []
for l in sample_list:
sample.extend(l)
random.shuffle(sample)
if os.path.exists(output):
os.remove(output)
newcorpus = factory.getobject(output)
for u in sample:
newunit = newcorpus.UnitClass.buildfromunit(u)
newcorpus.addunit(newunit)
newcorpus.save()
def _load_files(self, tmfiles, source_lang, target_lang):
from translate.storage import factory
if isinstance(tmfiles, list):
[self.tmdb.add_store(factory.getobject(tmfile), source_lang, target_lang) \
for tmfile in tmfiles]
elif tmfiles:
self.tmdb.add_store(factory.getobject(tmfiles), source_lang, target_lang)
def _checkerrors(self, filename, fileid, configid, checker, store):
def geterrors():
self.cur.execute("""SELECT
name,
unitindex
FROM uniterrors WHERE fileid=? and configid=?
ORDER BY unitindex;""", (fileid, configid))
return self.cur.fetchone(), self.cur
first, cur = geterrors()
if first is not None:
return first, cur
# This could happen if we haven't done the checks before, or the
# file changed, or we are using a different configuration
if callable(store):
store = store()
else:
store = store or factory.getobject(filename)
if os.path.exists(suggestion_filename(filename)):
checker.setsuggestionstore(factory.getobject(suggestion_filename(filename), ignore=suggestion_extension()))
self._cachestorechecks(fileid, store, checker, configid)
return geterrors()
def mergestore(inputfile, outputfile, templatefile, mergeblanks="no", mergefuzzy="no",
mergecomments="yes"):
try:
mergecomments = str2bool(mergecomments)
except ValueError:
raise ValueError("invalid mergecomments value: %r" % mergecomments)
try:
mergeblanks = str2bool(mergeblanks)
except ValueError:
raise ValueError("invalid mergeblanks value: %r" % mergeblanks)
try:
mergefuzzy = str2bool(mergefuzzy)
except ValueError:
raise ValueError("invalid mergefuzzy value: %r" % mergefuzzy)
inputstore = factory.getobject(inputfile)
if templatefile is None:
# just merge nothing
templatestore = type(inputstore)()
else:
templatestore = factory.getobject(templatefile)
outputstore = mergestores(templatestore, inputstore, mergeblanks,
mergefuzzy, mergecomments)
if outputstore.isempty():
return 0
outputstore.serialize(outputfile)
return 1
def scanstores(self):
"""Populate the internal store data."""
for dirname, filename in self.file_iter():
# TODO: Here os.path.join(dirname, filename) doesn't work.....
if filename == '':continue
if dirname=='':
filepath=os.path.join(dirname, filename)
else:
filepath=dirname+'/'+filename
strfile = wStringIO.StringIO(self.archive.read(filepath))
# let skip the other filetypes
root, ext = os.path.splitext(filename)
if self.specifyext:
if ext[1:].lower() != self.specifyext.lower():
strfile.filename = os.path.join(dirname, filename)
self.storedata.append(strfile)
else:
strfile.filename = filename
store = factory.getobject(strfile)
self.storedata.append(store)
else:
if ext[1:] not in factory.classes_str:
strfile.filename = os.path.join(dirname, filename)
self.storedata.append(strfile)
else:
strfile.filename = filename
store = factory.getobject(strfile)
self.storedata.append(store)
def test_odf2xliff2_inline():
"""Test for issue #3239."""
reference_xlf = factory.getobject(REFERENCE_XLF_INLINE)
odf2xliff.main(args(SOURCE_ODF_INLINE, GENERATED_XLF_TOOLKIT_INLINE))
generated_xlf_toolkit = factory.getobject(GENERATED_XLF_TOOLKIT_INLINE)
print_diff(reference_xlf, generated_xlf_toolkit)
assert reference_xlf == generated_xlf_toolkit
def pretranslate_file(input_file, output_file, template_file, tm=None, min_similarity=75, fuzzymatching=True):
"""Pretranslate any factory supported file with old translations and translation memory."""
input_store = factory.getobject(input_file)
template_store = None
if template_file is not None:
template_store = factory.getobject(template_file)
output = pretranslate_store(input_store, template_store, tm, min_similarity, fuzzymatching)
output_file.write(str(output))
return 1
def convertpot(input_file, output_file, template_file, tm=None, min_similarity=75, fuzzymatching=True, classes=factory.classes, **kwargs):
"""Main conversion function"""
input_store = factory.getobject(input_file, classes=classes)
template_store = None
if template_file is not None:
template_store = factory.getobject(template_file, classes=classes)
output_store = convert_stores(input_store, template_store, tm, min_similarity, fuzzymatching, **kwargs)
output_file.write(str(output_store))
return 1
def memory(tmfiles, max_candidates=1, min_similarity=75, max_length=1000):
"""Returns the TM store to use. Only initialises on first call."""
global tmmatcher
# Only initialise first time
if tmmatcher is None:
if isinstance(tmfiles, list):
tmstore = [factory.getobject(tmfile) for tmfile in tmfiles]
else:
tmstore = factory.getobject(tmfiles)
tmmatcher = match.matcher(tmstore, max_candidates=max_candidates, min_similarity=min_similarity, max_length=max_length)
return tmmatcher
def overwrite_file(request, relative_root_dir, django_file, upload_path):
"""overwrite with uploaded file"""
upload_dir = os.path.dirname(absolute_real_path(upload_path))
# Ensure that there is a directory into which we can dump the
# uploaded file.
if not os.path.exists(upload_dir):
os.makedirs(upload_dir)
# Get the file extensions of the uploaded filename and the
# current translation project
_upload_base, upload_ext = os.path.splitext(django_file.name)
_local_base, local_ext = os.path.splitext(upload_path)
# If the extension of the uploaded file matches the extension
# used in this translation project, then we simply write the
# file to the disc.
if upload_ext == local_ext:
outfile = open(absolute_real_path(upload_path), "wb")
try:
outfile.write(django_file.read())
finally:
outfile.close()
try:
#FIXME: we need a way to delay reparsing
store = Store.objects.get(file=upload_path)
store.update(
update_structure=True, update_translation=True,
conservative=False)
except Store.DoesNotExist:
# newfile, delay parsing
pass
else:
newstore = factory.getobject(django_file, classes=factory_classes)
if not newstore.units:
return
# If the extension of the uploaded file does not match the
# extension of the current translation project, we create
# an empty file (with the right extension)...
empty_store = factory.getobject(
absolute_real_path(upload_path), classes=factory_classes)
# And save it...
empty_store.save()
request.translation_project.scan_files()
# Then we open this newly created file and merge the
# uploaded file into it.
store = Store.objects.get(file=upload_path)
#FIXME: maybe there is a faster way to do this?
store.update(
update_structure=True, update_translation=True,
conservative=False, store=newstore)
store.sync(
update_structure=True, update_translation=True,
conservative=False)
def test_odf2xliff():
reference_xlf = factory.getobject(REFERENCE_XLF)
odf2xliff.main(args(SOURCE_ODF, GENERATED_XLF_TOOLKIT))
generated_xlf_toolkit = factory.getobject(GENERATED_XLF_TOOLKIT)
print_diff(reference_xlf, generated_xlf_toolkit)
assert reference_xlf == generated_xlf_toolkit
odf2xliff.main(args(SOURCE_ODF, GENERATED_XLF_ITOOLS))
generated_xlf_itools = factory.getobject(GENERATED_XLF_ITOOLS)
print_diff(reference_xlf, generated_xlf_itools)
assert reference_xlf == generated_xlf_itools
def test_output(self):
for posource in posources:
print("PO source file")
print(posource)
PO_FILE, MO_MSGFMT, MO_POCOMPILE = self.get_mo_and_po()
out_file = open(PO_FILE, 'w')
out_file.write(posource)
out_file.close()
subprocess.call(['msgfmt', PO_FILE, '-o', MO_MSGFMT])
subprocess.call(['pocompile', '--errorlevel=traceback', PO_FILE, MO_POCOMPILE])
store = factory.getobject(StringIO(posource))
if store.isempty() and not os.path.exists(MO_POCOMPILE):
# pocompile doesn't create MO files for empty PO files, so we
# can skip the checks here.
continue
mo_msgfmt_f = open(MO_MSGFMT)
mo_pocompile_f = open(MO_POCOMPILE)
try:
mo_msgfmt = mo_msgfmt_f.read()
print("msgfmt output:")
print(repr(mo_msgfmt))
mo_pocompile = mo_pocompile_f.read()
print("pocompile output:")
print(repr(mo_pocompile))
assert mo_msgfmt == mo_pocompile
finally:
mo_msgfmt_f.close()
mo_pocompile_f.close()
def update_file(self, filename):
# Adapted from Document.__init__()
from translate.storage import factory, statsdb
newstore = factory.getobject(filename)
oldfilename = self._trans_store.filename
oldfileobj = self._trans_store.fileobj
#get a copy of old stats before we convert
from translate.filters import checks
oldstats = statsdb.StatsCache().filestats(oldfilename, checks.UnitChecker(), self._trans_store)
from translate.convert import pot2po
self._trans_store = pot2po.convert_stores(newstore, self._trans_store, fuzzymatching=False)
self._trans_store.fileobj = oldfileobj #Let's attempt to keep the old file and name if possible
#FIXME: ugly tempfile hack, can we please have a pure store implementation of statsdb
import tempfile
import os
tempfd, tempfilename = tempfile.mkstemp()
os.write(tempfd, str(self._trans_store))
self.update_stats(filename=tempfilename)
os.close(tempfd)
os.remove(tempfilename)
self.controller.compare_stats(oldstats, self.stats)
# store filename or else save is confused
self._trans_store.filename = oldfilename
self._correct_header(self._trans_store)
self.nplurals = self._compute_nplurals(self._trans_store)
def convertpot(input_file, output_file, template_file, tm=None, min_similarity=75, fuzzymatching=True, classes=None, classes_str=factory.classes_str, **kwargs):
"""Main conversion function"""
input_store = factory.getobject(input_file, classes=classes, classes_str=classes_str)
try:
temp_store = factory.getobject(input_file, classes_str=classes_str)
except:
# StringIO and other file like objects will be closed after parsing
temp_store = None
template_store = None
if template_file is not None:
template_store = factory.getobject(template_file, classes_str=classes_str)
output_store = convert_stores(input_store, template_store, temp_store, tm, min_similarity, fuzzymatching, **kwargs)
output_file.write(str(output_store))
return 1
def get_units(self, filenames):
"""Gets the units to import and its total count."""
units = []
all_filenames = set()
for filename in filenames:
if not os.path.exists(filename):
self.stdout.write("File %s doesn't exist. Skipping it." %
filename)
continue
if os.path.isdir(filename):
for dirpath, dirs, fnames in os.walk(filename):
if (os.path.basename(dirpath) in
["CVS", ".svn", "_darcs", ".git", ".hg", ".bzr"]):
continue
for f in fnames:
all_filenames.add(os.path.join(dirpath, f))
else:
all_filenames.add(filename)
for filename in all_filenames:
store = factory.getobject(filename)
if not store.gettargetlanguage() and not self.target_language:
raise CommandError("Unable to determine target language for "
"'%s'. Try again specifying a fallback "
"target language with --target-language" %
filename)
self.filename = filename
units.extend([unit for unit in store.units if unit.istranslated()])
return units, len(units)
def test_output(self):
for posource in posources:
print("PO source file")
print(posource)
PO_FILE, MO_MSGFMT, MO_POCOMPILE = self.get_mo_and_po()
posource = posource.encode('utf-8')
with open(PO_FILE, 'wb') as out_file:
out_file.write(posource)
subprocess.call(['msgfmt', PO_FILE, '-o', MO_MSGFMT])
subprocess.call(['pocompile', '--errorlevel=traceback', PO_FILE, MO_POCOMPILE])
store = factory.getobject(BytesIO(posource))
if store.isempty() and not os.path.exists(MO_POCOMPILE):
# pocompile doesn't create MO files for empty PO files, so we
# can skip the checks here.
continue
with open(MO_MSGFMT, 'rb') as mo_msgfmt_f:
mo_msgfmt = mo_msgfmt_f.read()
print("msgfmt output:")
print(repr(mo_msgfmt))
with open(MO_POCOMPILE, 'rb') as mo_pocompile_f:
mo_pocompile = mo_pocompile_f.read()
print("pocompile output:")
print(repr(mo_pocompile))
assert mo_msgfmt == mo_pocompile
def build_mo(self):
"""Compile .mo files from available .po files"""
import subprocess
import gettext
from translate.storage import factory
for lang in self._langs:
lang = lang.rstrip()
po_path = os.path.join('pootle', 'locale', lang)
mo_path = os.path.join('pootle', 'locale', lang, 'LC_MESSAGES')
if not os.path.exists(mo_path):
os.makedirs(mo_path)
for po, mo in (('pootle.po', 'django.mo'),
('pootle_js.po', 'djangojs.mo')):
po_filename = os.path.join(po_path, po)
mo_filename = os.path.join(mo_path, mo)
try:
store = factory.getobject(po_filename)
gettext.c2py(store.getheaderplural()[1])
except Exception, e:
print "WARNING: %s, has invalid plural header: %s" % (lang, e)
try:
if not os.path.exists(mo_path):
os.makedirs(mo_path)
print "COMPILING: %s language" % lang
subprocess.Popen(['msgfmt', '--strict',
'-o', mo_filename, po_filename])
except Exception, e:
print "SKIPPING: %s, running msgfmt failed: %s" % (lang, e)
def test_output(self):
for posource in posources:
print "PO source file"
print posource
PO_FILE, MO_MSGFMT, MO_POCOMPILE = self.get_mo_and_po()
out_file = open(PO_FILE, "w")
out_file.write(posource)
out_file.close()
subprocess.call(["msgfmt", PO_FILE, "-o", MO_MSGFMT])
subprocess.call(["pocompile", PO_FILE, MO_POCOMPILE])
store = factory.getobject(StringIO.StringIO(posource))
if store.isempty() and not os.path.exists(MO_POCOMPILE):
# pocompile doesn't create MO files for empty PO files, so we
# can skip the checks here.
continue
mo_msgfmt_f = open(MO_MSGFMT)
mo_pocompile_f = open(MO_POCOMPILE)
try:
mo_msgfmt = mo_msgfmt_f.read()
print "msgfmt output:"
print repr(mo_msgfmt)
mo_pocompile = mo_pocompile_f.read()
print "pocompile output:"
print repr(mo_pocompile)
assert mo_msgfmt == mo_pocompile
finally:
mo_msgfmt_f.close()
mo_pocompile_f.close()
def handlefile(self, filename):
logging.info("About to process %s:" % filename)
try:
store = factory.getobject(filename)
source_lang = self.source_lang or store.getsourcelanguage()
target_lang = self.target_lang or store.gettargetlanguage()
project_style = self.project_style or store.getprojectstyle()
min_similarity = self.min_similarity
max_candidates = self.max_candidates
if not source_lang or not target_lang:
logging.error("Missing source or target language. Can't use "
"%s" % filename)
return
except Exception:
logging.exception("Error while processing %s" % filename)
return
translate_unit = current_app.tmdb.translate_unit
try:
for unit in store.units:
if unit.istranslatable():
# We need an explicit unicode (not multistring), otherwise
# psycopg2 can't adapt it:
translate_unit(unicode(unit.source), source_lang,
target_lang, project_style, min_similarity,
max_candidates)
except Exception:
logging.exception("Error when translating unit")
raise
def setup_file_and_db(self, file_contents=fr_terminology_extract):
cache = statsdb.StatsCache(os.path.join(self.path, "stats.db"))
filename = os.path.join(self.path, "test.po")
with open(filename, "w") as fh:
fh.write(file_contents)
f = factory.getobject(filename)
return f, cache
def convert():
if not Editor.isValid() or Editor.currentFile()=='': return
xliffpathname=Editor.currentFile()
(path, filename)=os.path.split(xliffpathname)
if not filename.endswith('.xlf'): return
store = factory.getobject(xliffpathname)
odfpathname=store.getfilenames()[0]
if odfpathname.startswith('NoName'):
print 'translate-toolkit is too old'
odfpathname=os.path.splitext(xliffpathname)[0]+'.odt'
if not os.path.exists(odfpathname): return
translatedodfpathname=os.path.splitext(odfpathname)[0]+'-'+Project.targetLangCode()+'.odt'
print 'translatedodfpathname %s' % translatedodfpathname
print 'odfpathname %s' % odfpathname
xliffinput=XliffInput(xliffpathname,Editor.currentFileContents())
odf=open(odfpathname,'rb')
xliff2odf.convertxliff(xliffinput, translatedodfpathname, odf)
ourpath=([p for p in sys.path if os.path.exists(p+'/xliff2odf.py')]+[''])[0]
os.system('python "'+ourpath+'/xliff2odf-standalone.py" "%s" "%s" &'%(translatedodfpathname, Editor.currentEntryId()))
def unit_iter(self):
"""Iterator over all the units in all the files in this directory."""
for dirname, filename in self.file_iter():
store = factory.getobject(path.join(dirname, filename))
#TODO: don't regenerate all the storage objects
for unit in store.unit_iter():
yield unit
def recacheunit(self, filename, checker, unit):
"""Recalculate all information for a specific unit. This is necessary
for updating all statistics when a translation of a unit took place,
for example.
This method assumes that everything was up to date before (file totals,
checks, checker config, etc."""
fileid = self._getfileid(filename, check_mod_info=False)
configid = self._get_config_id(fileid, checker)
unitid = unit.getid()
# get the unit index
totals_without_unit = self.file_totals[fileid] - \
FileTotals.new_record(*self.get_unit_stats(fileid, unitid))
self.cur.execute("""SELECT unitindex FROM units WHERE
fileid=? AND unitid=?;""", (fileid, unitid))
unitindex = self.cur.fetchone()[0]
self.cur.execute("""DELETE FROM units WHERE
fileid=? AND unitid=?;""", (fileid, unitid))
state = [self._cacheunitstats([unit], fileid, unitindex, totals_without_unit)]
# remove the current errors
self.cur.execute("""DELETE FROM uniterrors WHERE
fileid=? AND unitindex=?;""", (fileid, unitindex))
if os.path.exists(suggestion_filename(filename)):
checker.setsuggestionstore(factory.getobject(suggestion_filename(filename), ignore=suggestion_extension()))
state.extend(self._cacheunitschecks([unit], fileid, configid, checker, unitindex))
return state
def build_mo(self):
"""Compile .mo files from available .po files"""
import subprocess
import gettext
from translate.storage import factory
print "Preparing localization files"
for po_filename in glob.glob(path.join('po', 'pootle', '*', 'pootle.po')):
lang = path.split(path.split(po_filename)[0])[1]
lang_dir = path.join('mo', lang, 'LC_MESSAGES')
mo_filename = path.join(lang_dir, 'django.mo')
try:
store = factory.getobject(po_filename)
gettext.c2py(store.getheaderplural()[1])
except Exception, e:
print "skipping %s, probably invalid header: %s" % (lang, e)
try:
if not path.exists(lang_dir):
os.makedirs(lang_dir)
print "compiling %s language" % lang
subprocess.Popen(['msgfmt', '-c', '--strict', '-o', mo_filename, po_filename])
except Exception, e:
print "skipping %s, running msgfmt failed: %s" % (lang, e)
def _update_store_cache(self):
"""Add translation store to dictionary cache, replace old cached
version if needed."""
if self.exists():
mod_info = self.getpomtime()
else:
mod_info = 0
if (not hasattr(self, "_store_tuple") or
self._store_tuple.mod_info != mod_info):
try:
self._store_tuple = self._store_cache[self.path]
if self._store_tuple.mod_info != mod_info:
# if file is modified act as if it doesn't exist in cache
raise KeyError
except KeyError:
logging.debug(u"Cache miss for %s", self.path)
from translate.storage import factory
from pootle_store.filetypes import factory_classes
store_obj = factory.getobject(self.path,
ignore=self.field.ignore,
classes=factory_classes)
self._store_tuple = StoreTuple(store_obj, mod_info,
self.realpath)
self._store_cache[self.path] = self._store_tuple
def convertoo(inputfile, outputfile, templatefile, sourcelanguage=None,
targetlanguage=None, timestamp=None, includefuzzy=False,
multifilestyle="single", skip_source=False, filteraction=None,
outputthreshold=None):
inputstore = factory.getobject(inputfile)
if not convert.should_output_store(inputstore, outputthreshold):
return False
inputstore.filename = getattr(inputfile, 'name', '')
if not targetlanguage:
raise ValueError("You must specify the target language")
if not sourcelanguage:
if targetlanguage.isdigit():
sourcelanguage = "01"
else:
sourcelanguage = "en-US"
languages = (sourcelanguage, targetlanguage)
if templatefile is None:
raise ValueError("must have template file for oo files")
else:
convertor = reoo(templatefile, languages=languages,
timestamp=timestamp, includefuzzy=includefuzzy,
long_keys=multifilestyle != "single",
filteraction=filteraction)
outputstore = convertor.convertstore(inputstore)
# TODO: check if we need to manually delete missing items
outputfile.write(outputstore.__str__(skip_source, targetlanguage))
return True
def buildmatches(inputfile, outputfile, matcher):
"""Builds a .po.tm file for use in Pootle"""
#Can't use the same name: it might open the existing file!
outputfile = factory.getobject(outputfile, ignore=".tm")
#TODO: Do something useful with current content if file exists
#inputfile.units.sort(match.sourcelen)
try:
for unit in inputfile.units:
#if len(unit.source) > 70:
# break
if not unit.source:
continue
candidates = matcher.matches(unit.source)
for candidate in candidates:
source = candidate.source
target = candidate.target
newunit = outputfile.addsourceunit(source)
newunit.target = target
newunit.addnote(candidate.getnotes())
newunit.addlocations(unit.getlocations())
except exceptions.KeyboardInterrupt:
# Let's write what we have so far
return outputfile
return outputfile
def test_update_fuzzy(self):
dbunit = self._update_translation(0, {'fuzzy': True})
storeunit = dbunit.getorig()
self.assertTrue(dbunit.isfuzzy())
self.assertEqual(dbunit.isfuzzy(), storeunit.isfuzzy())
pofile = factory.getobject(self.store.file.path)
self.assertEqual(dbunit.isfuzzy(), pofile.units[dbunit.index].isfuzzy())
dbunit = self._update_translation(0, {'fuzzy': False})
storeunit = dbunit.getorig()
self.assertFalse(dbunit.isfuzzy())
self.assertEqual(dbunit.isfuzzy(), storeunit.isfuzzy())
pofile = factory.getobject(self.store.file.path)
self.assertEqual(dbunit.isfuzzy(), pofile.units[dbunit.index].isfuzzy())
def unit_iter(self):
"""Iterator over all the units in all the files in this zip file."""
for dirname, filename in self.file_iter():
strfile = BytesIO(self.archive.read('/'.join((dirname, filename))))
strfile.filename = filename
store = factory.getobject(strfile)
# TODO: don't regenerate all the storage objects
for unit in store.unit_iter():
yield unit
def convertsub(inputfile, outputfile, templatefile, includefuzzy=False):
inputstore = factory.getobject(inputfile)
if templatefile is None:
raise ValueError("must have template file for subtitle files")
else:
convertor = resub(templatefile, inputstore)
outputstring = convertor.convertstore(includefuzzy)
outputfile.write(outputstring)
return 1
def convertpot(input_file,
output_file,
template_file,
tm=None,
min_similarity=75,
fuzzymatching=True,
classes=factory.classes,
**kwargs):
"""Main conversion function"""
input_store = factory.getobject(input_file, classes=classes)
template_store = None
if template_file is not None:
template_store = factory.getobject(template_file, classes=classes)
output_store = convert_stores(input_store, template_store, tm,
min_similarity, fuzzymatching, **kwargs)
output_file.write(str(output_store))
return 1
def pretranslate_file(input_file,
output_file,
template_file,
tm=None,
min_similarity=75,
fuzzymatching=True):
"""Pretranslate any factory supported file with old translations and
translation memory.
"""
input_store = factory.getobject(input_file)
template_store = None
if template_file is not None:
template_store = factory.getobject(template_file)
output = pretranslate_store(input_store, template_store, tm,
min_similarity, fuzzymatching)
output.serialize(output_file)
return 1
def build_mo(self):
"""Compile .mo files from available .po files"""
import subprocess
import gettext
from translate.storage import factory
error_occured = False
for lang in self._langs:
lang = lang.rstrip()
po_path = os.path.join('pootle', 'locale', lang)
mo_path = os.path.join('pootle', 'locale', lang, 'LC_MESSAGES')
if not os.path.exists(mo_path):
os.makedirs(mo_path)
for po, mo in (('pootle.po', 'django.mo'), ('pootle_js.po',
'djangojs.mo')):
po_filename = os.path.join(po_path, po)
mo_filename = os.path.join(mo_path, mo)
if not os.path.exists(po_filename):
log.warn("%s: missing file %s", lang, po_filename)
continue
if not os.path.exists(mo_path):
os.makedirs(mo_path)
log.info("compiling %s", lang)
if self.check:
command = [
'msgfmt', '-c', '--strict', '-o', mo_filename,
po_filename
]
else:
command = [
'msgfmt', '--strict', '-o', mo_filename, po_filename
]
try:
subprocess.check_call(command, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
error_occured = True
except Exception as e:
log.warn("%s: skipping, running msgfmt failed: %s", lang,
e)
try:
store = factory.getobject(po_filename)
gettext.c2py(store.getheaderplural()[1])
except Exception:
log.warn("%s: invalid plural header in %s", lang,
po_filename)
if error_occured:
sys.exit(1)
def test_update_target(af_tutorial_po):
"""Tests that target changes are properly sync'ed to disk."""
db_unit = _update_translation(af_tutorial_po, 0, {'target': u'samaka'})
store_unit = db_unit.getorig()
assert db_unit.target == u'samaka'
assert db_unit.target == store_unit.target
po_file = factory.getobject(af_tutorial_po.file.path)
assert db_unit.target == po_file.units[db_unit.index].target
def upload_store(self, environ, start_response, sid, slang, tlang):
"""add units from uploaded file to tmdb"""
start_response("200 OK", [('Content-type', 'text/plain')])
data = StringIO.StringIO(environ['wsgi.input'].read(
int(environ['CONTENT_LENGTH'])))
data.name = sid
store = factory.getobject(data)
count = self.tmdb.add_store(store, slang, tlang)
response = "added %d units from %s" % (count, sid)
return [response]
def test_bz2file(self):
"""Test that we can open a gzip file correctly."""
if not BZ2File:
return
filename = os.path.join(self.testdir, self.filename + '.bz2')
bz2file = BZ2File(filename, mode="wb")
bz2file.write(self.file_content)
bz2file.close()
store = factory.getobject(filename)
assert isinstance(store, self.expected_instance)
def convertpo(inputfile, outputfile, templatefile, format=None, rewritestyle=None, ignoreoption=None):
"""Reads in inputfile, changes it to have debug strings, writes to outputfile."""
# note that templatefile is not used, but it is required by the converter...
inputstore = factory.getobject(inputfile)
if inputstore.isempty():
return 0
convertor = podebug(format=format, rewritestyle=rewritestyle, ignoreoption=ignoreoption)
outputstore = convertor.convertstore(inputstore)
outputfile.write(str(outputstore))
return 1
def convert_symbian(input_file, output_file, template_file, pot=False, duplicatestyle="msgctxt"):
store = factory.getobject(input_file)
location_index = build_location_index(store)
header_index = {
'Author': store.parseheader()['Last-Translator']
}
output = write_symbian(template_file, header_index, location_index)
for line in output:
output_file.write(line)
return 1
def convertpy(inputfile, outputfile, templatefile=None, includefuzzy=False,
outputthreshold=None):
inputstore = factory.getobject(inputfile)
if not convert.should_output_store(inputstore, outputthreshold):
return False
convertor = po2pydict()
outputstring = convertor.convertstore(inputstore, includefuzzy)
outputfile.write(outputstring.read())
return 1
def converttxt(inputfile, outputfile, templatefile, wrap=None, includefuzzy=False, encoding='utf-8'):
"""reads in stdin using fromfileclass, converts using convertorclass, writes to stdout"""
inputstore = factory.getobject(inputfile)
convertor = po2txt(wrap=wrap)
if templatefile is None:
outputstring = convertor.convertstore(inputstore, includefuzzy)
else:
templatestring = templatefile.read().decode(encoding)
outputstring = convertor.mergestore(inputstore, templatestring, includefuzzy)
outputfile.write(outputstring.encode('utf-8'))
return 1
def test_update_comment(self):
dbunit = self._update_translation(0, {'translator_comment': u'7amada'})
storeunit = dbunit.getorig()
self.assertEqual(dbunit.getnotes(origin="translator"), u'7amada')
self.assertEqual(dbunit.getnotes(origin="translator"),
storeunit.getnotes(origin="translator"))
pofile = factory.getobject(self.store.file.path)
self.assertEqual(
dbunit.getnotes(origin="translator"),
pofile.units[dbunit.index].getnotes(origin="translator"))
def runfilter(inputfile, outputfile, templatefile, checkfilter=None):
"""Reads in inputfile, filters using checkfilter, writes to outputfile."""
fromfile = factory.getobject(inputfile)
tofile = checkfilter.filterfile(fromfile)
if tofile.isempty():
return 0
tofile.serialize(outputfile)
return 1
def upload_store(sid, slang, tlang):
"""add units from uploaded file to tmdb"""
import StringIO
data = StringIO.StringIO(request.data)
data.name = sid
from translate.storage import factory
store = factory.getobject(data)
project_style = request.args.get('style', None)
count = current_app.tmdb.add_store(store, slang, tlang, project_style)
response = "added %d units from %s" % (count, sid)
return response
def test_update_fuzzy(af_tutorial_po):
"""Tests fuzzy state changes are stored and sync'ed."""
db_unit = _update_translation(af_tutorial_po, 0,
{'target': u'samaka', 'fuzzy': True})
store_unit = db_unit.getorig()
assert db_unit.isfuzzy()
assert db_unit.isfuzzy() == store_unit.isfuzzy()
po_file = factory.getobject(af_tutorial_po.file.path)
assert db_unit.isfuzzy() == po_file.units[db_unit.index].isfuzzy()
db_unit = _update_translation(af_tutorial_po, 0, {'fuzzy': False})
store_unit = db_unit.getorig()
assert not db_unit.isfuzzy()
assert db_unit.isfuzzy() == store_unit.isfuzzy()
po_file = factory.getobject(af_tutorial_po.file.path)
assert db_unit.isfuzzy() == po_file.units[db_unit.index].isfuzzy()
def test_update_target(project0_nongnu, store0):
"""Tests that target changes are properly sync'ed to disk."""
db_unit = _update_translation(store0, 0, {'target': u'samaka'})
store0.sync()
store_unit = store0.file.store.findid(db_unit.getid())
assert db_unit.target == u'samaka'
assert db_unit.target == store_unit.target
po_file = factory.getobject(store0.file.path)
assert db_unit.target == po_file.findid(db_unit.unitid).target
def segmentfile(inputfile, outputfile, templatefile, sourcelanguage="en", targetlanguage=None, stripspaces=True, onlyaligned=False):
"""reads in inputfile, segments it then, writes to outputfile"""
# note that templatefile is not used, but it is required by the converter...
inputstore = factory.getobject(inputfile)
if inputstore.isempty():
return 0
sourcelang = lang_factory.getlanguage(sourcelanguage)
targetlang = lang_factory.getlanguage(targetlanguage)
convertor = segment(sourcelang, targetlang, stripspaces=stripspaces, onlyaligned=onlyaligned)
outputstore = convertor.convertstore(inputstore)
outputfile.write(str(outputstore))
return 1
def test_update_fuzzy(project0_nongnu, store0):
"""Tests fuzzy state changes are stored and sync'ed."""
db_unit = _update_translation(
store0, 0,
{'target': u'samaka', 'fuzzy': True})
store_unit = store0.file.store.findid(db_unit.getid())
assert db_unit.isfuzzy()
assert db_unit.isfuzzy() == store_unit.isfuzzy()
po_file = factory.getobject(store0.file.path)
assert db_unit.isfuzzy() == po_file.findid(db_unit.getid()).isfuzzy()
db_unit = _update_translation(store0, 0, {'fuzzy': False})
store_unit = store0.file.store.findid(db_unit.getid())
assert not db_unit.isfuzzy()
assert db_unit.isfuzzy() == store_unit.isfuzzy()
po_file = factory.getobject(store0.file.path)
assert db_unit.isfuzzy() == po_file.findid(db_unit.getid()).isfuzzy()
def test_update_plural_target_dict(self):
dbunit = self._update_translation(2, {'target': {0: u'samaka', 1: u'samak'}})
storeunit = dbunit.getorig()
self.assertEqual(dbunit.target.strings, [u'samaka', u'samak'])
self.assertEqual(dbunit.target.strings, storeunit.target.strings)
pofile = factory.getobject(self.store.file.path)
self.assertEqual(dbunit.target.strings, pofile.units[dbunit.index].target.strings)
self.assertEqual(dbunit.target, u'samaka')
self.assertEqual(dbunit.target, storeunit.target)
self.assertEqual(dbunit.target, pofile.units[dbunit.index].target)
def writematches(inputfile, outputfile, templatefile, tm=None, max_candidates=4, min_similarity=75, max_length=1000):
if templatefile:
raise Warning("Template ignored")
inputfile = factory.getobject(inputfile)
if tm is None:
raise ValueError("Must have TM storage specified with --tm")
int_max_candidates = int(max_candidates)
int_min_similarity = int(min_similarity)
matcher = memory(tm, max_candidates=int_max_candidates, min_similarity=int_min_similarity, max_length=max_length)
output = buildmatches(inputfile, outputfile, matcher)
outputfile.writelines(str(output))
return 1
def upload_store(self, environ, start_response, sid, slang, tlang):
"""add units from uploaded file to tmdb"""
from translate.storage import factory
start_response("200 OK", [("Content-type", "text/plain")])
data = BytesIO(environ["wsgi.input"].read(
int(environ["CONTENT_LENGTH"])))
data.name = sid
store = factory.getobject(data)
count = self.tmdb.add_store(store, slang, tlang)
response = "added %d units from %s" % (count, sid)
return [response]
def mtfile(inputfile, outputfile, templatefile, dictionary_file, new_words,
all_words, project):
if not dictionary_file:
print("ERROR: missing dictionary file")
return 0
inputstore = factory.getobject(inputfile)
if inputstore.isempty():
return 0
convertor = word_substitute(dictionary_file, all_words, new_words, project)
outputstore = convertor.convertstore(inputstore)
outputstore.serialize(outputfile)
return 1
def test_update_comment(af_tutorial_po):
"""Tests translator comments are stored and sync'ed."""
db_unit = _update_translation(af_tutorial_po, 0,
{'translator_comment': u'7amada'})
store_unit = db_unit.getorig()
assert db_unit.getnotes(origin='translator') == u'7amada'
assert db_unit.getnotes(origin='translator') == \
store_unit.getnotes(origin='translator')
po_file = factory.getobject(af_tutorial_po.file.path)
assert db_unit.getnotes(origin='translator') == \
po_file.units[db_unit.index].getnotes(origin='translator')
def test_update_comment(project0_nongnu, store0):
"""Tests translator comments are stored and sync'ed."""
db_unit = _update_translation(store0, 0, {'translator_comment': u'7amada'})
store0.sync()
store_unit = store0.file.store.findid(db_unit.getid())
assert db_unit.getnotes(origin='translator') == u'7amada'
assert (db_unit.getnotes(origin='translator') == store_unit.getnotes(
origin='translator'))
po_file = factory.getobject(store0.file.path)
assert (db_unit.getnotes(origin='translator') == po_file.findid(
db_unit.getid()).getnotes(origin='translator'))
def test_update_fuzzy(project0_disk, store0):
"""Tests fuzzy state changes are stored and sync'ed."""
db_unit = _update_translation(store0, 0, {
"target": u"samaka",
"fuzzy": True
})
store_unit = store0.file.store.findid(db_unit.getid())
assert db_unit.isfuzzy()
assert db_unit.isfuzzy() == store_unit.isfuzzy()
po_file = factory.getobject(store0.file.path)
assert db_unit.isfuzzy() == po_file.findid(db_unit.getid()).isfuzzy()
db_unit = _update_translation(store0, 0, {"fuzzy": False})
store_unit = store0.file.store.findid(db_unit.getid())
assert not db_unit.isfuzzy()
assert db_unit.isfuzzy() == store_unit.isfuzzy()
po_file = factory.getobject(store0.file.path)
assert db_unit.isfuzzy() == po_file.findid(db_unit.getid()).isfuzzy()
def test_update_comment(project0_disk, store0):
"""Tests translator comments are stored and sync'ed."""
db_unit = _update_translation(store0, 0, {"translator_comment": u"7amada"})
store0.sync()
store_unit = store0.file.store.findid(db_unit.getid())
assert db_unit.getnotes(origin="translator") == u"7amada"
assert db_unit.getnotes(origin="translator") == store_unit.getnotes(
origin="translator")
po_file = factory.getobject(store0.file.path)
assert db_unit.getnotes(origin="translator") == po_file.findid(
db_unit.getid()).getnotes(origin="translator")
def convertini(inputfile,
outputfile,
templatefile,
includefuzzy=False,
dialect="default"):
inputstore = factory.getobject(inputfile)
if templatefile is None:
raise ValueError("must have template file for ini files")
else:
convertor = reini(templatefile, inputstore, dialect)
outputstring = convertor.convertstore(includefuzzy)
outputfile.write(outputstring)
return 1
def convertpot(input_file, output_file, template_file, tm=None,
min_similarity=75, fuzzymatching=True, classes=None,
classes_str=factory.classes_str, **kwargs):
"""Main conversion function."""
input_store = factory.getobject(input_file, classes=classes,
classes_str=classes_str)
try:
temp_store = factory.getobject(input_file, classes_str=classes_str)
except:
# StringIO and other file like objects will be closed after parsing
temp_store = None
template_store = None
if template_file is not None:
template_store = factory.getobject(template_file,
classes_str=classes_str)
output_store = convert_stores(input_store, template_store, temp_store, tm,
min_similarity, fuzzymatching, **kwargs)
output_file.write(str(output_store))
return 1
def load_files(self):
if self.matcher in TerminologyPlaceable.matchers:
TerminologyPlaceable.matchers.remove(self.matcher)
self.stores = []
for filename in self.config['files']:
if not filename:
continue
if not os.path.isfile(filename):
logging.debug('Not a file: "%s"' % (filename))
self.stores.append(factory.getobject(filename))
self.matcher = terminologymatcher(self.stores)
TerminologyPlaceable.matchers.append(self.matcher)