def __init__(self, tmdbfile, tmfiles, max_candidates=3, min_similarity=75,
            max_length=1000, prefix="", source_lang=None, target_lang=None):

        self.tmdb = tmdb.TMDB(tmdbfile, max_candidates, min_similarity, max_length)

        #load files into db
        if isinstance(tmfiles, list):
            [self.tmdb.add_store(factory.getobject(tmfile), source_lang, target_lang) \
                    for tmfile in tmfiles]
        elif tmfiles:
            self.tmdb.add_store(factory.getobject(tmfiles), source_lang, target_lang)

        #initialize url dispatcher
        self.rest = selector.Selector(prefix=prefix)
        self.rest.add("/{slang}/{tlang}/unit/{uid:any}",
                      GET=self.translate_unit,
                      POST=self.update_unit,
                      PUT=self.add_unit,
                      DELETE=self.forget_unit
                      )

        self.rest.add("/{slang}/{tlang}/store/{sid:any}",
                      GET=self.get_store_stats,
                      PUT=self.upload_store,
                      POST=self.add_store,
                      DELETE=self.forget_store)
def _make_ts_diff(old_ts_path, new_ts_path):
    old = factory.getobject(old_ts_path)
    new = factory.getobject(new_ts_path)

    # this is for added / removed
    old_ids = set(old.getids())
    new_ids = set(new.getids())
    added = new_ids - old_ids
    removed = old_ids - new_ids

    old_trans_count = 0
    for unit_id in old_ids:
        if old.findid(unit_id).istranslated():
            old_trans_count += 1

    new_trans_count = 0
    for unit_id in new_ids:
        if new.findid(unit_id).istranslated():
            new_trans_count += 1

    return {
        "old_trans_count": old_trans_count,
        "new_trans_count": new_trans_count,
        "old_units": len(old_ids),
        "new_units": len(new_ids),
        "added": list(added),
        "removed": list(removed),
    }
def main(dir, output, pstr):
	
	import random
	percentage = float(pstr)
	from translate.storage import factory
	unitslist = []
	create_intermediates(dir)
	
	print "Creating final TMX..."
	for f in os.listdir(dir):
		if f.endswith(".tmx") and not os.path.isdir(f):
			try:
				filepath = os.path.join(dir,f)
				minicorpus = factory.getobject(filepath)
				for u in minicorpus.units:
					u.addnote("Origin: "+f)
				unitslist.append(minicorpus.units)
			except ValueError:
				print "Could not convert to factory."
				continue
	
	sample_list = create_sample(unitslist, percentage)
	sample = []
	for l in sample_list:
		sample.extend(l)
	random.shuffle(sample)
	
	if os.path.exists(output):
		os.remove(output)
	newcorpus = factory.getobject(output)
	for u in sample:
		newunit = newcorpus.UnitClass.buildfromunit(u)
		newcorpus.addunit(newunit)
	newcorpus.save()
Beispiel #4
0
 def _load_files(self, tmfiles, source_lang, target_lang):
     from translate.storage import factory
     if isinstance(tmfiles, list):
         [self.tmdb.add_store(factory.getobject(tmfile), source_lang, target_lang) \
                 for tmfile in tmfiles]
     elif tmfiles:
         self.tmdb.add_store(factory.getobject(tmfiles), source_lang, target_lang)
Beispiel #5
0
    def _checkerrors(self, filename, fileid, configid, checker, store):

        def geterrors():
            self.cur.execute("""SELECT
                name,
                unitindex
                FROM uniterrors WHERE fileid=? and configid=?
                ORDER BY unitindex;""", (fileid, configid))
            return self.cur.fetchone(), self.cur

        first, cur = geterrors()
        if first is not None:
            return first, cur

        # This could happen if we haven't done the checks before, or the
        # file changed, or we are using a different configuration
        if callable(store):
            store = store()
        else:
            store = store or factory.getobject(filename)

        if os.path.exists(suggestion_filename(filename)):
            checker.setsuggestionstore(factory.getobject(suggestion_filename(filename), ignore=suggestion_extension()))
        self._cachestorechecks(fileid, store, checker, configid)
        return geterrors()
Beispiel #6
0
def mergestore(inputfile, outputfile, templatefile, mergeblanks="no", mergefuzzy="no",
               mergecomments="yes"):
    try:
        mergecomments = str2bool(mergecomments)
    except ValueError:
        raise ValueError("invalid mergecomments value: %r" % mergecomments)
    try:
        mergeblanks = str2bool(mergeblanks)
    except ValueError:
        raise ValueError("invalid mergeblanks value: %r" % mergeblanks)
    try:
        mergefuzzy = str2bool(mergefuzzy)
    except ValueError:
        raise ValueError("invalid mergefuzzy value: %r" % mergefuzzy)
    inputstore = factory.getobject(inputfile)
    if templatefile is None:
        # just merge nothing
        templatestore = type(inputstore)()
    else:
        templatestore = factory.getobject(templatefile)
    outputstore = mergestores(templatestore, inputstore, mergeblanks,
                              mergefuzzy, mergecomments)
    if outputstore.isempty():
        return 0
    outputstore.serialize(outputfile)
    return 1
Beispiel #7
0
 def scanstores(self):
     """Populate the internal store data."""
     for dirname, filename in self.file_iter():
         # TODO: Here os.path.join(dirname, filename) doesn't work.....
         if filename == '':continue
         if dirname=='':
             filepath=os.path.join(dirname, filename)
         else:
             filepath=dirname+'/'+filename
         strfile = wStringIO.StringIO(self.archive.read(filepath))
         # let skip the other filetypes            
         root, ext = os.path.splitext(filename)
         if self.specifyext:
             if ext[1:].lower() != self.specifyext.lower(): 
                 strfile.filename = os.path.join(dirname, filename)
                 self.storedata.append(strfile)
             else:
                 strfile.filename = filename
                 store = factory.getobject(strfile)
                 self.storedata.append(store)
         else:
             if ext[1:] not in factory.classes_str: 
                 strfile.filename = os.path.join(dirname, filename)
                 self.storedata.append(strfile)
             else:
                 strfile.filename = filename
                 store = factory.getobject(strfile)
                 self.storedata.append(store)
def test_odf2xliff2_inline():
    """Test for issue #3239."""
    reference_xlf = factory.getobject(REFERENCE_XLF_INLINE)

    odf2xliff.main(args(SOURCE_ODF_INLINE, GENERATED_XLF_TOOLKIT_INLINE))
    generated_xlf_toolkit = factory.getobject(GENERATED_XLF_TOOLKIT_INLINE)
    print_diff(reference_xlf, generated_xlf_toolkit)
    assert reference_xlf == generated_xlf_toolkit
Beispiel #9
0
def pretranslate_file(input_file, output_file, template_file, tm=None, min_similarity=75, fuzzymatching=True):
    """Pretranslate any factory supported file with old translations and translation memory."""
    input_store = factory.getobject(input_file)
    template_store = None
    if template_file is not None:
        template_store = factory.getobject(template_file)

    output = pretranslate_store(input_store, template_store, tm, min_similarity, fuzzymatching)
    output_file.write(str(output))
    return 1
Beispiel #10
0
def convertpot(input_file, output_file, template_file, tm=None, min_similarity=75, fuzzymatching=True, classes=factory.classes, **kwargs):
    """Main conversion function"""

    input_store = factory.getobject(input_file, classes=classes)
    template_store = None
    if template_file is not None:
        template_store = factory.getobject(template_file, classes=classes)
    output_store = convert_stores(input_store, template_store, tm, min_similarity, fuzzymatching, **kwargs)
    output_file.write(str(output_store))
    return 1
Beispiel #11
0
def memory(tmfiles, max_candidates=1, min_similarity=75, max_length=1000):
    """Returns the TM store to use. Only initialises on first call."""
    global tmmatcher
    # Only initialise first time
    if tmmatcher is None:
        if isinstance(tmfiles, list):
            tmstore = [factory.getobject(tmfile) for tmfile in tmfiles]
        else:
            tmstore = factory.getobject(tmfiles)
        tmmatcher = match.matcher(tmstore, max_candidates=max_candidates, min_similarity=min_similarity, max_length=max_length)
    return tmmatcher
Beispiel #12
0
def overwrite_file(request, relative_root_dir, django_file, upload_path):
    """overwrite with uploaded file"""
    upload_dir = os.path.dirname(absolute_real_path(upload_path))
    # Ensure that there is a directory into which we can dump the
    # uploaded file.
    if not os.path.exists(upload_dir):
        os.makedirs(upload_dir)

    # Get the file extensions of the uploaded filename and the
    # current translation project
    _upload_base, upload_ext = os.path.splitext(django_file.name)
    _local_base, local_ext = os.path.splitext(upload_path)
    # If the extension of the uploaded file matches the extension
    # used in this translation project, then we simply write the
    # file to the disc.
    if upload_ext == local_ext:
        outfile = open(absolute_real_path(upload_path), "wb")
        try:
            outfile.write(django_file.read())
        finally:
            outfile.close()
            try:
                #FIXME: we need a way to delay reparsing
                store = Store.objects.get(file=upload_path)
                store.update(
                    update_structure=True, update_translation=True,
                    conservative=False)
            except Store.DoesNotExist:
                # newfile, delay parsing
                pass
    else:
        newstore = factory.getobject(django_file, classes=factory_classes)
        if not newstore.units:
            return

        # If the extension of the uploaded file does not match the
        # extension of the current translation project, we create
        # an empty file (with the right extension)...
        empty_store = factory.getobject(
            absolute_real_path(upload_path), classes=factory_classes)
        # And save it...
        empty_store.save()
        request.translation_project.scan_files()
        # Then we open this newly created file and merge the
        # uploaded file into it.
        store = Store.objects.get(file=upload_path)
        #FIXME: maybe there is a faster way to do this?
        store.update(
            update_structure=True, update_translation=True,
            conservative=False, store=newstore)
        store.sync(
            update_structure=True, update_translation=True,
            conservative=False)
Beispiel #13
0
def test_odf2xliff():
    reference_xlf = factory.getobject(REFERENCE_XLF)

    odf2xliff.main(args(SOURCE_ODF, GENERATED_XLF_TOOLKIT))
    generated_xlf_toolkit = factory.getobject(GENERATED_XLF_TOOLKIT)
    print_diff(reference_xlf, generated_xlf_toolkit)
    assert reference_xlf == generated_xlf_toolkit

    odf2xliff.main(args(SOURCE_ODF, GENERATED_XLF_ITOOLS))
    generated_xlf_itools = factory.getobject(GENERATED_XLF_ITOOLS)
    print_diff(reference_xlf, generated_xlf_itools)
    assert reference_xlf == generated_xlf_itools
Beispiel #14
0
    def test_output(self):
        for posource in posources:
            print("PO source file")
            print(posource)
            PO_FILE, MO_MSGFMT, MO_POCOMPILE = self.get_mo_and_po()

            out_file = open(PO_FILE, 'w')
            out_file.write(posource)
            out_file.close()

            subprocess.call(['msgfmt', PO_FILE, '-o', MO_MSGFMT])
            subprocess.call(['pocompile', '--errorlevel=traceback', PO_FILE, MO_POCOMPILE])

            store = factory.getobject(StringIO(posource))
            if store.isempty() and not os.path.exists(MO_POCOMPILE):
                # pocompile doesn't create MO files for empty PO files, so we
                # can skip the checks here.
                continue

            mo_msgfmt_f = open(MO_MSGFMT)
            mo_pocompile_f = open(MO_POCOMPILE)

            try:
                mo_msgfmt = mo_msgfmt_f.read()
                print("msgfmt output:")
                print(repr(mo_msgfmt))
                mo_pocompile = mo_pocompile_f.read()
                print("pocompile output:")
                print(repr(mo_pocompile))
                assert mo_msgfmt == mo_pocompile
            finally:
                mo_msgfmt_f.close()
                mo_pocompile_f.close()
Beispiel #15
0
    def update_file(self, filename):
        # Adapted from Document.__init__()
        from translate.storage import factory, statsdb
        newstore = factory.getobject(filename)
        oldfilename = self._trans_store.filename
        oldfileobj = self._trans_store.fileobj

        #get a copy of old stats before we convert
        from translate.filters import checks
        oldstats = statsdb.StatsCache().filestats(oldfilename, checks.UnitChecker(), self._trans_store)

        from translate.convert import pot2po
        self._trans_store = pot2po.convert_stores(newstore, self._trans_store, fuzzymatching=False)
        self._trans_store.fileobj = oldfileobj #Let's attempt to keep the old file and name if possible

        #FIXME: ugly tempfile hack, can we please have a pure store implementation of statsdb
        import tempfile
        import os
        tempfd, tempfilename = tempfile.mkstemp()
        os.write(tempfd, str(self._trans_store))
        self.update_stats(filename=tempfilename)
        os.close(tempfd)
        os.remove(tempfilename)

        self.controller.compare_stats(oldstats, self.stats)

        # store filename or else save is confused
        self._trans_store.filename = oldfilename
        self._correct_header(self._trans_store)
        self.nplurals = self._compute_nplurals(self._trans_store)
Beispiel #16
0
def convertpot(input_file, output_file, template_file, tm=None, min_similarity=75, fuzzymatching=True, classes=None, classes_str=factory.classes_str, **kwargs):
    """Main conversion function"""

    input_store = factory.getobject(input_file, classes=classes, classes_str=classes_str)
    try:
        temp_store = factory.getobject(input_file, classes_str=classes_str)
    except:
        # StringIO and other file like objects will be closed after parsing
        temp_store = None

    template_store = None
    if template_file is not None:
        template_store = factory.getobject(template_file, classes_str=classes_str)
    output_store = convert_stores(input_store, template_store, temp_store, tm, min_similarity, fuzzymatching, **kwargs)
    output_file.write(str(output_store))
    return 1
Beispiel #17
0
    def get_units(self, filenames):
        """Gets the units to import and its total count."""
        units = []
        all_filenames = set()

        for filename in filenames:
            if not os.path.exists(filename):
                self.stdout.write("File %s doesn't exist. Skipping it." %
                                  filename)
                continue

            if os.path.isdir(filename):
                for dirpath, dirs, fnames in os.walk(filename):
                    if (os.path.basename(dirpath) in
                        ["CVS", ".svn", "_darcs", ".git", ".hg", ".bzr"]):

                        continue

                    for f in fnames:
                        all_filenames.add(os.path.join(dirpath, f))
            else:
                all_filenames.add(filename)

        for filename in all_filenames:
            store = factory.getobject(filename)
            if not store.gettargetlanguage() and not self.target_language:
                raise CommandError("Unable to determine target language for "
                                   "'%s'. Try again specifying a fallback "
                                   "target language with --target-language" %
                                   filename)

            self.filename = filename
            units.extend([unit for unit in store.units if unit.istranslated()])

        return units, len(units)
Beispiel #18
0
    def test_output(self):
        for posource in posources:
            print("PO source file")
            print(posource)
            PO_FILE, MO_MSGFMT, MO_POCOMPILE = self.get_mo_and_po()
            posource = posource.encode('utf-8')

            with open(PO_FILE, 'wb') as out_file:
                out_file.write(posource)

            subprocess.call(['msgfmt', PO_FILE, '-o', MO_MSGFMT])
            subprocess.call(['pocompile', '--errorlevel=traceback', PO_FILE, MO_POCOMPILE])

            store = factory.getobject(BytesIO(posource))
            if store.isempty() and not os.path.exists(MO_POCOMPILE):
                # pocompile doesn't create MO files for empty PO files, so we
                # can skip the checks here.
                continue

            with open(MO_MSGFMT, 'rb') as mo_msgfmt_f:
                mo_msgfmt = mo_msgfmt_f.read()
            print("msgfmt output:")
            print(repr(mo_msgfmt))

            with open(MO_POCOMPILE, 'rb') as mo_pocompile_f:
                mo_pocompile = mo_pocompile_f.read()
            print("pocompile output:")
            print(repr(mo_pocompile))

            assert mo_msgfmt == mo_pocompile
Beispiel #19
0
    def build_mo(self):
        """Compile .mo files from available .po files"""
        import subprocess
        import gettext
        from translate.storage import factory

        for lang in self._langs:
            lang = lang.rstrip()

            po_path = os.path.join('pootle', 'locale', lang)
            mo_path = os.path.join('pootle', 'locale', lang, 'LC_MESSAGES')

            if not os.path.exists(mo_path):
                os.makedirs(mo_path)

            for po, mo in (('pootle.po', 'django.mo'),
                                     ('pootle_js.po', 'djangojs.mo')):
                po_filename = os.path.join(po_path, po)
                mo_filename = os.path.join(mo_path, mo)

                try:
                    store = factory.getobject(po_filename)
                    gettext.c2py(store.getheaderplural()[1])
                except Exception, e:
                    print "WARNING: %s, has invalid plural header: %s" % (lang, e)

                try:
                    if not os.path.exists(mo_path):
                        os.makedirs(mo_path)
                    print "COMPILING: %s language" % lang
                    subprocess.Popen(['msgfmt', '--strict',
                                      '-o', mo_filename, po_filename])
                except Exception, e:
                    print "SKIPPING: %s, running msgfmt failed: %s" % (lang, e)
Beispiel #20
0
    def test_output(self):
        for posource in posources:
            print "PO source file"
            print posource
            PO_FILE, MO_MSGFMT, MO_POCOMPILE = self.get_mo_and_po()

            out_file = open(PO_FILE, "w")
            out_file.write(posource)
            out_file.close()

            subprocess.call(["msgfmt", PO_FILE, "-o", MO_MSGFMT])
            subprocess.call(["pocompile", PO_FILE, MO_POCOMPILE])

            store = factory.getobject(StringIO.StringIO(posource))
            if store.isempty() and not os.path.exists(MO_POCOMPILE):
                # pocompile doesn't create MO files for empty PO files, so we
                # can skip the checks here.
                continue

            mo_msgfmt_f = open(MO_MSGFMT)
            mo_pocompile_f = open(MO_POCOMPILE)

            try:
                mo_msgfmt = mo_msgfmt_f.read()
                print "msgfmt output:"
                print repr(mo_msgfmt)
                mo_pocompile = mo_pocompile_f.read()
                print "pocompile output:"
                print repr(mo_pocompile)
                assert mo_msgfmt == mo_pocompile
            finally:
                mo_msgfmt_f.close()
                mo_pocompile_f.close()
Beispiel #21
0
    def handlefile(self, filename):
        logging.info("About to process %s:" % filename)
        try:
            store = factory.getobject(filename)
            source_lang = self.source_lang or store.getsourcelanguage()
            target_lang = self.target_lang or store.gettargetlanguage()
            project_style = self.project_style or store.getprojectstyle()
            min_similarity = self.min_similarity
            max_candidates = self.max_candidates

            if not source_lang or not target_lang:
                logging.error("Missing source or target language. Can't use "
                              "%s" % filename)
                return
        except Exception:
            logging.exception("Error while processing %s" % filename)
            return

        translate_unit = current_app.tmdb.translate_unit
        try:
            for unit in store.units:
                if unit.istranslatable():
                    # We need an explicit unicode (not multistring), otherwise
                    # psycopg2 can't adapt it:
                    translate_unit(unicode(unit.source), source_lang,
                                   target_lang, project_style, min_similarity,
                                   max_candidates)
        except Exception:
            logging.exception("Error when translating unit")
            raise
 def setup_file_and_db(self, file_contents=fr_terminology_extract):
     cache = statsdb.StatsCache(os.path.join(self.path, "stats.db"))
     filename = os.path.join(self.path, "test.po")
     with open(filename, "w") as fh:
         fh.write(file_contents)
     f = factory.getobject(filename)
     return f, cache
Beispiel #23
0
def convert():
    if not Editor.isValid() or Editor.currentFile()=='': return

    xliffpathname=Editor.currentFile()
    (path, filename)=os.path.split(xliffpathname)
    if not filename.endswith('.xlf'): return

    store = factory.getobject(xliffpathname)
    odfpathname=store.getfilenames()[0]
    
    if odfpathname.startswith('NoName'):
        print 'translate-toolkit is too old'
        odfpathname=os.path.splitext(xliffpathname)[0]+'.odt'
    if not os.path.exists(odfpathname): return


    translatedodfpathname=os.path.splitext(odfpathname)[0]+'-'+Project.targetLangCode()+'.odt'
    print 'translatedodfpathname %s' % translatedodfpathname
    print 'odfpathname %s' % odfpathname
    xliffinput=XliffInput(xliffpathname,Editor.currentFileContents())
    odf=open(odfpathname,'rb')

    xliff2odf.convertxliff(xliffinput, translatedodfpathname, odf)

    ourpath=([p for p in sys.path if os.path.exists(p+'/xliff2odf.py')]+[''])[0]
    os.system('python "'+ourpath+'/xliff2odf-standalone.py" "%s" "%s" &'%(translatedodfpathname, Editor.currentEntryId()))
Beispiel #24
0
 def unit_iter(self):
     """Iterator over all the units in all the files in this directory."""
     for dirname, filename in self.file_iter():
         store = factory.getobject(path.join(dirname, filename))
         #TODO: don't regenerate all the storage objects
         for unit in store.unit_iter():
             yield unit
    def recacheunit(self, filename, checker, unit):
        """Recalculate all information for a specific unit. This is necessary
        for updating all statistics when a translation of a unit took place,
        for example.

        This method assumes that everything was up to date before (file totals,
        checks, checker config, etc."""
        fileid = self._getfileid(filename, check_mod_info=False)
        configid = self._get_config_id(fileid, checker)
        unitid = unit.getid()
        # get the unit index
        totals_without_unit = self.file_totals[fileid] - \
                                   FileTotals.new_record(*self.get_unit_stats(fileid, unitid))
        self.cur.execute("""SELECT unitindex FROM units WHERE
            fileid=? AND unitid=?;""", (fileid, unitid))
        unitindex = self.cur.fetchone()[0]
        self.cur.execute("""DELETE FROM units WHERE
            fileid=? AND unitid=?;""", (fileid, unitid))
        state = [self._cacheunitstats([unit], fileid, unitindex, totals_without_unit)]
        # remove the current errors
        self.cur.execute("""DELETE FROM uniterrors WHERE
            fileid=? AND unitindex=?;""", (fileid, unitindex))
        if os.path.exists(suggestion_filename(filename)):
            checker.setsuggestionstore(factory.getobject(suggestion_filename(filename), ignore=suggestion_extension()))
        state.extend(self._cacheunitschecks([unit], fileid, configid, checker, unitindex))
        return state
Beispiel #26
0
    def build_mo(self):
        """Compile .mo files from available .po files"""
        import subprocess
        import gettext
        from translate.storage import factory

        print "Preparing localization files"
        for po_filename in glob.glob(path.join('po', 'pootle', '*', 'pootle.po')):
            lang = path.split(path.split(po_filename)[0])[1]
            lang_dir = path.join('mo', lang, 'LC_MESSAGES')
            mo_filename = path.join(lang_dir, 'django.mo')

            try:
                store = factory.getobject(po_filename)
                gettext.c2py(store.getheaderplural()[1])
            except Exception, e:
                print "skipping %s, probably invalid header: %s" % (lang, e)

            try:
                if not path.exists(lang_dir):
                    os.makedirs(lang_dir)
                print "compiling %s language" % lang
                subprocess.Popen(['msgfmt', '-c', '--strict', '-o', mo_filename, po_filename])
            except Exception, e:
                print "skipping %s, running msgfmt failed: %s" % (lang, e)
Beispiel #27
0
    def _update_store_cache(self):
        """Add translation store to dictionary cache, replace old cached
        version if needed."""
        if self.exists():
            mod_info = self.getpomtime()
        else:
            mod_info = 0
        if (not hasattr(self, "_store_tuple") or
            self._store_tuple.mod_info != mod_info):
            try:
                self._store_tuple = self._store_cache[self.path]
                if self._store_tuple.mod_info != mod_info:
                    # if file is modified act as if it doesn't exist in cache
                    raise KeyError
            except KeyError:
                logging.debug(u"Cache miss for %s", self.path)
                from translate.storage import factory
                from pootle_store.filetypes import factory_classes

                store_obj = factory.getobject(self.path,
                                              ignore=self.field.ignore,
                                              classes=factory_classes)
                self._store_tuple = StoreTuple(store_obj, mod_info,
                                               self.realpath)
                self._store_cache[self.path] = self._store_tuple
Beispiel #28
0
def convertoo(inputfile, outputfile, templatefile, sourcelanguage=None,
              targetlanguage=None, timestamp=None, includefuzzy=False,
              multifilestyle="single", skip_source=False, filteraction=None,
              outputthreshold=None):
    inputstore = factory.getobject(inputfile)

    if not convert.should_output_store(inputstore, outputthreshold):
        return False

    inputstore.filename = getattr(inputfile, 'name', '')
    if not targetlanguage:
        raise ValueError("You must specify the target language")
    if not sourcelanguage:
        if targetlanguage.isdigit():
            sourcelanguage = "01"
        else:
            sourcelanguage = "en-US"
    languages = (sourcelanguage, targetlanguage)
    if templatefile is None:
        raise ValueError("must have template file for oo files")
    else:
        convertor = reoo(templatefile, languages=languages,
                         timestamp=timestamp, includefuzzy=includefuzzy,
                         long_keys=multifilestyle != "single",
                         filteraction=filteraction)
    outputstore = convertor.convertstore(inputstore)
    # TODO: check if we need to manually delete missing items
    outputfile.write(outputstore.__str__(skip_source, targetlanguage))
    return True
Beispiel #29
0
def buildmatches(inputfile, outputfile, matcher):
    """Builds a .po.tm file for use in Pootle"""
    #Can't use the same name: it might open the existing file!
    outputfile = factory.getobject(outputfile, ignore=".tm")
    #TODO: Do something useful with current content if file exists

    #inputfile.units.sort(match.sourcelen)
    try:
        for unit in inputfile.units:
            #if len(unit.source) > 70:
            #    break
            if not unit.source:
                continue
            candidates = matcher.matches(unit.source)
            for candidate in candidates:
                source = candidate.source
                target = candidate.target
                newunit = outputfile.addsourceunit(source)
                newunit.target = target
                newunit.addnote(candidate.getnotes())
                newunit.addlocations(unit.getlocations())
    except exceptions.KeyboardInterrupt:
        # Let's write what we have so far
        return outputfile
    return outputfile
Beispiel #30
0
    def test_update_fuzzy(self):
        dbunit = self._update_translation(0, {'fuzzy': True})
        storeunit = dbunit.getorig()

        self.assertTrue(dbunit.isfuzzy())
        self.assertEqual(dbunit.isfuzzy(), storeunit.isfuzzy())
        pofile = factory.getobject(self.store.file.path)
        self.assertEqual(dbunit.isfuzzy(), pofile.units[dbunit.index].isfuzzy())

        dbunit = self._update_translation(0, {'fuzzy': False})
        storeunit = dbunit.getorig()

        self.assertFalse(dbunit.isfuzzy())
        self.assertEqual(dbunit.isfuzzy(), storeunit.isfuzzy())
        pofile = factory.getobject(self.store.file.path)
        self.assertEqual(dbunit.isfuzzy(), pofile.units[dbunit.index].isfuzzy())
Beispiel #31
0
 def unit_iter(self):
     """Iterator over all the units in all the files in this zip file."""
     for dirname, filename in self.file_iter():
         strfile = BytesIO(self.archive.read('/'.join((dirname, filename))))
         strfile.filename = filename
         store = factory.getobject(strfile)
         # TODO: don't regenerate all the storage objects
         for unit in store.unit_iter():
             yield unit
Beispiel #32
0
def convertsub(inputfile, outputfile, templatefile, includefuzzy=False):
    inputstore = factory.getobject(inputfile)
    if templatefile is None:
        raise ValueError("must have template file for subtitle files")
    else:
        convertor = resub(templatefile, inputstore)
    outputstring = convertor.convertstore(includefuzzy)
    outputfile.write(outputstring)
    return 1
Beispiel #33
0
def convertpot(input_file,
               output_file,
               template_file,
               tm=None,
               min_similarity=75,
               fuzzymatching=True,
               classes=factory.classes,
               **kwargs):
    """Main conversion function"""

    input_store = factory.getobject(input_file, classes=classes)
    template_store = None
    if template_file is not None:
        template_store = factory.getobject(template_file, classes=classes)
    output_store = convert_stores(input_store, template_store, tm,
                                  min_similarity, fuzzymatching, **kwargs)
    output_file.write(str(output_store))
    return 1
Beispiel #34
0
def pretranslate_file(input_file,
                      output_file,
                      template_file,
                      tm=None,
                      min_similarity=75,
                      fuzzymatching=True):
    """Pretranslate any factory supported file with old translations and
    translation memory.
    """
    input_store = factory.getobject(input_file)
    template_store = None
    if template_file is not None:
        template_store = factory.getobject(template_file)

    output = pretranslate_store(input_store, template_store, tm,
                                min_similarity, fuzzymatching)
    output.serialize(output_file)
    return 1
Beispiel #35
0
    def build_mo(self):
        """Compile .mo files from available .po files"""
        import subprocess
        import gettext
        from translate.storage import factory

        error_occured = False

        for lang in self._langs:
            lang = lang.rstrip()

            po_path = os.path.join('pootle', 'locale', lang)
            mo_path = os.path.join('pootle', 'locale', lang, 'LC_MESSAGES')

            if not os.path.exists(mo_path):
                os.makedirs(mo_path)

            for po, mo in (('pootle.po', 'django.mo'), ('pootle_js.po',
                                                        'djangojs.mo')):
                po_filename = os.path.join(po_path, po)
                mo_filename = os.path.join(mo_path, mo)

                if not os.path.exists(po_filename):
                    log.warn("%s: missing file %s", lang, po_filename)
                    continue

                if not os.path.exists(mo_path):
                    os.makedirs(mo_path)

                log.info("compiling %s", lang)
                if self.check:
                    command = [
                        'msgfmt', '-c', '--strict', '-o', mo_filename,
                        po_filename
                    ]
                else:
                    command = [
                        'msgfmt', '--strict', '-o', mo_filename, po_filename
                    ]
                try:
                    subprocess.check_call(command, stderr=subprocess.STDOUT)
                except subprocess.CalledProcessError as e:
                    error_occured = True
                except Exception as e:
                    log.warn("%s: skipping, running msgfmt failed: %s", lang,
                             e)

                try:
                    store = factory.getobject(po_filename)
                    gettext.c2py(store.getheaderplural()[1])
                except Exception:
                    log.warn("%s: invalid plural header in %s", lang,
                             po_filename)

        if error_occured:
            sys.exit(1)
Beispiel #36
0
def test_update_target(af_tutorial_po):
    """Tests that target changes are properly sync'ed to disk."""
    db_unit = _update_translation(af_tutorial_po, 0, {'target': u'samaka'})
    store_unit = db_unit.getorig()

    assert db_unit.target == u'samaka'
    assert db_unit.target == store_unit.target

    po_file = factory.getobject(af_tutorial_po.file.path)
    assert db_unit.target == po_file.units[db_unit.index].target
 def upload_store(self, environ, start_response, sid, slang, tlang):
     """add units from uploaded file to tmdb"""
     start_response("200 OK", [('Content-type', 'text/plain')])
     data = StringIO.StringIO(environ['wsgi.input'].read(
         int(environ['CONTENT_LENGTH'])))
     data.name = sid
     store = factory.getobject(data)
     count = self.tmdb.add_store(store, slang, tlang)
     response = "added %d units from %s" % (count, sid)
     return [response]
Beispiel #38
0
 def test_bz2file(self):
     """Test that we can open a gzip file correctly."""
     if not BZ2File:
         return
     filename = os.path.join(self.testdir, self.filename + '.bz2')
     bz2file = BZ2File(filename, mode="wb")
     bz2file.write(self.file_content)
     bz2file.close()
     store = factory.getobject(filename)
     assert isinstance(store, self.expected_instance)
Beispiel #39
0
def convertpo(inputfile, outputfile, templatefile, format=None, rewritestyle=None, ignoreoption=None):
    """Reads in inputfile, changes it to have debug strings, writes to outputfile."""
    # note that templatefile is not used, but it is required by the converter...
    inputstore = factory.getobject(inputfile)
    if inputstore.isempty():
        return 0
    convertor = podebug(format=format, rewritestyle=rewritestyle, ignoreoption=ignoreoption)
    outputstore = convertor.convertstore(inputstore)
    outputfile.write(str(outputstore))
    return 1
Beispiel #40
0
def convert_symbian(input_file, output_file, template_file, pot=False, duplicatestyle="msgctxt"):
    store = factory.getobject(input_file)
    location_index = build_location_index(store)
    header_index = {
        'Author': store.parseheader()['Last-Translator']
    }
    output = write_symbian(template_file, header_index, location_index)
    for line in output:
        output_file.write(line)
    return 1
Beispiel #41
0
def convertpy(inputfile, outputfile, templatefile=None, includefuzzy=False,
              outputthreshold=None):
    inputstore = factory.getobject(inputfile)

    if not convert.should_output_store(inputstore, outputthreshold):
        return False

    convertor = po2pydict()
    outputstring = convertor.convertstore(inputstore, includefuzzy)
    outputfile.write(outputstring.read())
    return 1
Beispiel #42
0
def converttxt(inputfile, outputfile, templatefile, wrap=None, includefuzzy=False, encoding='utf-8'):
    """reads in stdin using fromfileclass, converts using convertorclass, writes to stdout"""
    inputstore = factory.getobject(inputfile)
    convertor = po2txt(wrap=wrap)
    if templatefile is None:
        outputstring = convertor.convertstore(inputstore, includefuzzy)
    else:
        templatestring = templatefile.read().decode(encoding)
        outputstring = convertor.mergestore(inputstore, templatestring, includefuzzy)
    outputfile.write(outputstring.encode('utf-8'))
    return 1
Beispiel #43
0
    def test_update_comment(self):
        dbunit = self._update_translation(0, {'translator_comment': u'7amada'})
        storeunit = dbunit.getorig()

        self.assertEqual(dbunit.getnotes(origin="translator"), u'7amada')
        self.assertEqual(dbunit.getnotes(origin="translator"),
                         storeunit.getnotes(origin="translator"))
        pofile = factory.getobject(self.store.file.path)
        self.assertEqual(
            dbunit.getnotes(origin="translator"),
            pofile.units[dbunit.index].getnotes(origin="translator"))
Beispiel #44
0
def runfilter(inputfile, outputfile, templatefile, checkfilter=None):
    """Reads in inputfile, filters using checkfilter, writes to outputfile."""
    fromfile = factory.getobject(inputfile)
    tofile = checkfilter.filterfile(fromfile)

    if tofile.isempty():
        return 0

    tofile.serialize(outputfile)

    return 1
Beispiel #45
0
def upload_store(sid, slang, tlang):
    """add units from uploaded file to tmdb"""
    import StringIO
    data = StringIO.StringIO(request.data)
    data.name = sid
    from translate.storage import factory
    store = factory.getobject(data)
    project_style = request.args.get('style', None)
    count = current_app.tmdb.add_store(store, slang, tlang, project_style)
    response = "added %d units from %s" % (count, sid)
    return response
Beispiel #46
0
def test_update_fuzzy(af_tutorial_po):
    """Tests fuzzy state changes are stored and sync'ed."""
    db_unit = _update_translation(af_tutorial_po, 0,
                                 {'target': u'samaka', 'fuzzy': True})
    store_unit = db_unit.getorig()

    assert db_unit.isfuzzy()
    assert db_unit.isfuzzy() == store_unit.isfuzzy()

    po_file = factory.getobject(af_tutorial_po.file.path)
    assert db_unit.isfuzzy() == po_file.units[db_unit.index].isfuzzy()

    db_unit = _update_translation(af_tutorial_po, 0, {'fuzzy': False})
    store_unit = db_unit.getorig()

    assert not db_unit.isfuzzy()
    assert db_unit.isfuzzy() == store_unit.isfuzzy()

    po_file = factory.getobject(af_tutorial_po.file.path)
    assert db_unit.isfuzzy() == po_file.units[db_unit.index].isfuzzy()
Beispiel #47
0
def test_update_target(project0_nongnu, store0):
    """Tests that target changes are properly sync'ed to disk."""
    db_unit = _update_translation(store0, 0, {'target': u'samaka'})
    store0.sync()
    store_unit = store0.file.store.findid(db_unit.getid())

    assert db_unit.target == u'samaka'
    assert db_unit.target == store_unit.target

    po_file = factory.getobject(store0.file.path)
    assert db_unit.target == po_file.findid(db_unit.unitid).target
Beispiel #48
0
def segmentfile(inputfile, outputfile, templatefile, sourcelanguage="en", targetlanguage=None, stripspaces=True, onlyaligned=False):
    """reads in inputfile, segments it then, writes to outputfile"""
    # note that templatefile is not used, but it is required by the converter...
    inputstore = factory.getobject(inputfile)
    if inputstore.isempty():
        return 0
    sourcelang = lang_factory.getlanguage(sourcelanguage)
    targetlang = lang_factory.getlanguage(targetlanguage)
    convertor = segment(sourcelang, targetlang, stripspaces=stripspaces, onlyaligned=onlyaligned)
    outputstore = convertor.convertstore(inputstore)
    outputfile.write(str(outputstore))
    return 1
Beispiel #49
0
def test_update_fuzzy(project0_nongnu, store0):
    """Tests fuzzy state changes are stored and sync'ed."""
    db_unit = _update_translation(
        store0, 0,
        {'target': u'samaka', 'fuzzy': True})
    store_unit = store0.file.store.findid(db_unit.getid())

    assert db_unit.isfuzzy()
    assert db_unit.isfuzzy() == store_unit.isfuzzy()

    po_file = factory.getobject(store0.file.path)
    assert db_unit.isfuzzy() == po_file.findid(db_unit.getid()).isfuzzy()

    db_unit = _update_translation(store0, 0, {'fuzzy': False})
    store_unit = store0.file.store.findid(db_unit.getid())

    assert not db_unit.isfuzzy()
    assert db_unit.isfuzzy() == store_unit.isfuzzy()

    po_file = factory.getobject(store0.file.path)
    assert db_unit.isfuzzy() == po_file.findid(db_unit.getid()).isfuzzy()
    def test_update_plural_target_dict(self):
        dbunit = self._update_translation(2, {'target': {0: u'samaka', 1: u'samak'}})
        storeunit = dbunit.getorig()

        self.assertEqual(dbunit.target.strings, [u'samaka', u'samak'])
        self.assertEqual(dbunit.target.strings, storeunit.target.strings)
        pofile = factory.getobject(self.store.file.path)
        self.assertEqual(dbunit.target.strings, pofile.units[dbunit.index].target.strings)

        self.assertEqual(dbunit.target, u'samaka')
        self.assertEqual(dbunit.target, storeunit.target)
        self.assertEqual(dbunit.target, pofile.units[dbunit.index].target)
def writematches(inputfile, outputfile, templatefile, tm=None, max_candidates=4, min_similarity=75, max_length=1000):
    if templatefile:
        raise Warning("Template ignored")
    inputfile = factory.getobject(inputfile)
    if tm is None:
        raise ValueError("Must have TM storage specified with --tm")
    int_max_candidates = int(max_candidates)
    int_min_similarity = int(min_similarity)
    matcher = memory(tm, max_candidates=int_max_candidates, min_similarity=int_min_similarity, max_length=max_length)
    output = buildmatches(inputfile, outputfile, matcher)
    outputfile.writelines(str(output))
    return 1
Beispiel #52
0
    def upload_store(self, environ, start_response, sid, slang, tlang):
        """add units from uploaded file to tmdb"""
        from translate.storage import factory

        start_response("200 OK", [("Content-type", "text/plain")])
        data = BytesIO(environ["wsgi.input"].read(
            int(environ["CONTENT_LENGTH"])))
        data.name = sid
        store = factory.getobject(data)
        count = self.tmdb.add_store(store, slang, tlang)
        response = "added %d units from %s" % (count, sid)
        return [response]
Beispiel #53
0
def mtfile(inputfile, outputfile, templatefile, dictionary_file, new_words,
           all_words, project):
    if not dictionary_file:
        print("ERROR: missing dictionary file")
        return 0
    inputstore = factory.getobject(inputfile)
    if inputstore.isempty():
        return 0
    convertor = word_substitute(dictionary_file, all_words, new_words, project)
    outputstore = convertor.convertstore(inputstore)
    outputstore.serialize(outputfile)
    return 1
Beispiel #54
0
def test_update_comment(af_tutorial_po):
    """Tests translator comments are stored and sync'ed."""
    db_unit = _update_translation(af_tutorial_po, 0,
                                 {'translator_comment': u'7amada'})
    store_unit = db_unit.getorig()

    assert db_unit.getnotes(origin='translator') == u'7amada'
    assert db_unit.getnotes(origin='translator') == \
            store_unit.getnotes(origin='translator')

    po_file = factory.getobject(af_tutorial_po.file.path)
    assert db_unit.getnotes(origin='translator') == \
            po_file.units[db_unit.index].getnotes(origin='translator')
Beispiel #55
0
def test_update_comment(project0_nongnu, store0):
    """Tests translator comments are stored and sync'ed."""
    db_unit = _update_translation(store0, 0, {'translator_comment': u'7amada'})
    store0.sync()
    store_unit = store0.file.store.findid(db_unit.getid())

    assert db_unit.getnotes(origin='translator') == u'7amada'
    assert (db_unit.getnotes(origin='translator') == store_unit.getnotes(
        origin='translator'))

    po_file = factory.getobject(store0.file.path)
    assert (db_unit.getnotes(origin='translator') == po_file.findid(
        db_unit.getid()).getnotes(origin='translator'))
Beispiel #56
0
def test_update_fuzzy(project0_disk, store0):
    """Tests fuzzy state changes are stored and sync'ed."""
    db_unit = _update_translation(store0, 0, {
        "target": u"samaka",
        "fuzzy": True
    })
    store_unit = store0.file.store.findid(db_unit.getid())

    assert db_unit.isfuzzy()
    assert db_unit.isfuzzy() == store_unit.isfuzzy()

    po_file = factory.getobject(store0.file.path)
    assert db_unit.isfuzzy() == po_file.findid(db_unit.getid()).isfuzzy()

    db_unit = _update_translation(store0, 0, {"fuzzy": False})
    store_unit = store0.file.store.findid(db_unit.getid())

    assert not db_unit.isfuzzy()
    assert db_unit.isfuzzy() == store_unit.isfuzzy()

    po_file = factory.getobject(store0.file.path)
    assert db_unit.isfuzzy() == po_file.findid(db_unit.getid()).isfuzzy()
Beispiel #57
0
def test_update_comment(project0_disk, store0):
    """Tests translator comments are stored and sync'ed."""
    db_unit = _update_translation(store0, 0, {"translator_comment": u"7amada"})
    store0.sync()
    store_unit = store0.file.store.findid(db_unit.getid())

    assert db_unit.getnotes(origin="translator") == u"7amada"
    assert db_unit.getnotes(origin="translator") == store_unit.getnotes(
        origin="translator")

    po_file = factory.getobject(store0.file.path)
    assert db_unit.getnotes(origin="translator") == po_file.findid(
        db_unit.getid()).getnotes(origin="translator")
Beispiel #58
0
def convertini(inputfile,
               outputfile,
               templatefile,
               includefuzzy=False,
               dialect="default"):
    inputstore = factory.getobject(inputfile)
    if templatefile is None:
        raise ValueError("must have template file for ini files")
    else:
        convertor = reini(templatefile, inputstore, dialect)
    outputstring = convertor.convertstore(includefuzzy)
    outputfile.write(outputstring)
    return 1
Beispiel #59
0
def convertpot(input_file, output_file, template_file, tm=None,
        min_similarity=75, fuzzymatching=True, classes=None,
        classes_str=factory.classes_str, **kwargs):
    """Main conversion function."""
    input_store = factory.getobject(input_file, classes=classes,
                                    classes_str=classes_str)
    try:
        temp_store = factory.getobject(input_file, classes_str=classes_str)
    except:
        # StringIO and other file like objects will be closed after parsing
        temp_store = None

    template_store = None
    if template_file is not None:
        template_store = factory.getobject(template_file,
                                           classes_str=classes_str)

    output_store = convert_stores(input_store, template_store, temp_store, tm,
                                  min_similarity, fuzzymatching, **kwargs)
    output_file.write(str(output_store))

    return 1
Beispiel #60
0
    def load_files(self):
        if self.matcher in TerminologyPlaceable.matchers:
            TerminologyPlaceable.matchers.remove(self.matcher)

        self.stores = []
        for filename in self.config['files']:
            if not filename:
                continue
            if not os.path.isfile(filename):
                logging.debug('Not a file: "%s"' % (filename))
            self.stores.append(factory.getobject(filename))
        self.matcher = terminologymatcher(self.stores)
        TerminologyPlaceable.matchers.append(self.matcher)