def __init__(self, tmdbfile, tmfiles, max_candidates=3, min_similarity=75, max_length=1000, prefix="", source_lang=None, target_lang=None): self.tmdb = tmdb.TMDB(tmdbfile, max_candidates, min_similarity, max_length) #load files into db if isinstance(tmfiles, list): [self.tmdb.add_store(factory.getobject(tmfile), source_lang, target_lang) \ for tmfile in tmfiles] elif tmfiles: self.tmdb.add_store(factory.getobject(tmfiles), source_lang, target_lang) #initialize url dispatcher self.rest = selector.Selector(prefix=prefix) self.rest.add("/{slang}/{tlang}/unit/{uid:any}", GET=self.translate_unit, POST=self.update_unit, PUT=self.add_unit, DELETE=self.forget_unit ) self.rest.add("/{slang}/{tlang}/store/{sid:any}", GET=self.get_store_stats, PUT=self.upload_store, POST=self.add_store, DELETE=self.forget_store)
def _make_ts_diff(old_ts_path, new_ts_path): old = factory.getobject(old_ts_path) new = factory.getobject(new_ts_path) # this is for added / removed old_ids = set(old.getids()) new_ids = set(new.getids()) added = new_ids - old_ids removed = old_ids - new_ids old_trans_count = 0 for unit_id in old_ids: if old.findid(unit_id).istranslated(): old_trans_count += 1 new_trans_count = 0 for unit_id in new_ids: if new.findid(unit_id).istranslated(): new_trans_count += 1 return { "old_trans_count": old_trans_count, "new_trans_count": new_trans_count, "old_units": len(old_ids), "new_units": len(new_ids), "added": list(added), "removed": list(removed), }
def main(dir, output, pstr): import random percentage = float(pstr) from translate.storage import factory unitslist = [] create_intermediates(dir) print "Creating final TMX..." for f in os.listdir(dir): if f.endswith(".tmx") and not os.path.isdir(f): try: filepath = os.path.join(dir,f) minicorpus = factory.getobject(filepath) for u in minicorpus.units: u.addnote("Origin: "+f) unitslist.append(minicorpus.units) except ValueError: print "Could not convert to factory." continue sample_list = create_sample(unitslist, percentage) sample = [] for l in sample_list: sample.extend(l) random.shuffle(sample) if os.path.exists(output): os.remove(output) newcorpus = factory.getobject(output) for u in sample: newunit = newcorpus.UnitClass.buildfromunit(u) newcorpus.addunit(newunit) newcorpus.save()
def _load_files(self, tmfiles, source_lang, target_lang): from translate.storage import factory if isinstance(tmfiles, list): [self.tmdb.add_store(factory.getobject(tmfile), source_lang, target_lang) \ for tmfile in tmfiles] elif tmfiles: self.tmdb.add_store(factory.getobject(tmfiles), source_lang, target_lang)
def _checkerrors(self, filename, fileid, configid, checker, store): def geterrors(): self.cur.execute("""SELECT name, unitindex FROM uniterrors WHERE fileid=? and configid=? ORDER BY unitindex;""", (fileid, configid)) return self.cur.fetchone(), self.cur first, cur = geterrors() if first is not None: return first, cur # This could happen if we haven't done the checks before, or the # file changed, or we are using a different configuration if callable(store): store = store() else: store = store or factory.getobject(filename) if os.path.exists(suggestion_filename(filename)): checker.setsuggestionstore(factory.getobject(suggestion_filename(filename), ignore=suggestion_extension())) self._cachestorechecks(fileid, store, checker, configid) return geterrors()
def mergestore(inputfile, outputfile, templatefile, mergeblanks="no", mergefuzzy="no", mergecomments="yes"): try: mergecomments = str2bool(mergecomments) except ValueError: raise ValueError("invalid mergecomments value: %r" % mergecomments) try: mergeblanks = str2bool(mergeblanks) except ValueError: raise ValueError("invalid mergeblanks value: %r" % mergeblanks) try: mergefuzzy = str2bool(mergefuzzy) except ValueError: raise ValueError("invalid mergefuzzy value: %r" % mergefuzzy) inputstore = factory.getobject(inputfile) if templatefile is None: # just merge nothing templatestore = type(inputstore)() else: templatestore = factory.getobject(templatefile) outputstore = mergestores(templatestore, inputstore, mergeblanks, mergefuzzy, mergecomments) if outputstore.isempty(): return 0 outputstore.serialize(outputfile) return 1
def scanstores(self): """Populate the internal store data.""" for dirname, filename in self.file_iter(): # TODO: Here os.path.join(dirname, filename) doesn't work..... if filename == '':continue if dirname=='': filepath=os.path.join(dirname, filename) else: filepath=dirname+'/'+filename strfile = wStringIO.StringIO(self.archive.read(filepath)) # let skip the other filetypes root, ext = os.path.splitext(filename) if self.specifyext: if ext[1:].lower() != self.specifyext.lower(): strfile.filename = os.path.join(dirname, filename) self.storedata.append(strfile) else: strfile.filename = filename store = factory.getobject(strfile) self.storedata.append(store) else: if ext[1:] not in factory.classes_str: strfile.filename = os.path.join(dirname, filename) self.storedata.append(strfile) else: strfile.filename = filename store = factory.getobject(strfile) self.storedata.append(store)
def test_odf2xliff2_inline(): """Test for issue #3239.""" reference_xlf = factory.getobject(REFERENCE_XLF_INLINE) odf2xliff.main(args(SOURCE_ODF_INLINE, GENERATED_XLF_TOOLKIT_INLINE)) generated_xlf_toolkit = factory.getobject(GENERATED_XLF_TOOLKIT_INLINE) print_diff(reference_xlf, generated_xlf_toolkit) assert reference_xlf == generated_xlf_toolkit
def pretranslate_file(input_file, output_file, template_file, tm=None, min_similarity=75, fuzzymatching=True): """Pretranslate any factory supported file with old translations and translation memory.""" input_store = factory.getobject(input_file) template_store = None if template_file is not None: template_store = factory.getobject(template_file) output = pretranslate_store(input_store, template_store, tm, min_similarity, fuzzymatching) output_file.write(str(output)) return 1
def convertpot(input_file, output_file, template_file, tm=None, min_similarity=75, fuzzymatching=True, classes=factory.classes, **kwargs): """Main conversion function""" input_store = factory.getobject(input_file, classes=classes) template_store = None if template_file is not None: template_store = factory.getobject(template_file, classes=classes) output_store = convert_stores(input_store, template_store, tm, min_similarity, fuzzymatching, **kwargs) output_file.write(str(output_store)) return 1
def memory(tmfiles, max_candidates=1, min_similarity=75, max_length=1000): """Returns the TM store to use. Only initialises on first call.""" global tmmatcher # Only initialise first time if tmmatcher is None: if isinstance(tmfiles, list): tmstore = [factory.getobject(tmfile) for tmfile in tmfiles] else: tmstore = factory.getobject(tmfiles) tmmatcher = match.matcher(tmstore, max_candidates=max_candidates, min_similarity=min_similarity, max_length=max_length) return tmmatcher
def overwrite_file(request, relative_root_dir, django_file, upload_path): """overwrite with uploaded file""" upload_dir = os.path.dirname(absolute_real_path(upload_path)) # Ensure that there is a directory into which we can dump the # uploaded file. if not os.path.exists(upload_dir): os.makedirs(upload_dir) # Get the file extensions of the uploaded filename and the # current translation project _upload_base, upload_ext = os.path.splitext(django_file.name) _local_base, local_ext = os.path.splitext(upload_path) # If the extension of the uploaded file matches the extension # used in this translation project, then we simply write the # file to the disc. if upload_ext == local_ext: outfile = open(absolute_real_path(upload_path), "wb") try: outfile.write(django_file.read()) finally: outfile.close() try: #FIXME: we need a way to delay reparsing store = Store.objects.get(file=upload_path) store.update( update_structure=True, update_translation=True, conservative=False) except Store.DoesNotExist: # newfile, delay parsing pass else: newstore = factory.getobject(django_file, classes=factory_classes) if not newstore.units: return # If the extension of the uploaded file does not match the # extension of the current translation project, we create # an empty file (with the right extension)... empty_store = factory.getobject( absolute_real_path(upload_path), classes=factory_classes) # And save it... empty_store.save() request.translation_project.scan_files() # Then we open this newly created file and merge the # uploaded file into it. store = Store.objects.get(file=upload_path) #FIXME: maybe there is a faster way to do this? store.update( update_structure=True, update_translation=True, conservative=False, store=newstore) store.sync( update_structure=True, update_translation=True, conservative=False)
def test_odf2xliff(): reference_xlf = factory.getobject(REFERENCE_XLF) odf2xliff.main(args(SOURCE_ODF, GENERATED_XLF_TOOLKIT)) generated_xlf_toolkit = factory.getobject(GENERATED_XLF_TOOLKIT) print_diff(reference_xlf, generated_xlf_toolkit) assert reference_xlf == generated_xlf_toolkit odf2xliff.main(args(SOURCE_ODF, GENERATED_XLF_ITOOLS)) generated_xlf_itools = factory.getobject(GENERATED_XLF_ITOOLS) print_diff(reference_xlf, generated_xlf_itools) assert reference_xlf == generated_xlf_itools
def test_output(self): for posource in posources: print("PO source file") print(posource) PO_FILE, MO_MSGFMT, MO_POCOMPILE = self.get_mo_and_po() out_file = open(PO_FILE, 'w') out_file.write(posource) out_file.close() subprocess.call(['msgfmt', PO_FILE, '-o', MO_MSGFMT]) subprocess.call(['pocompile', '--errorlevel=traceback', PO_FILE, MO_POCOMPILE]) store = factory.getobject(StringIO(posource)) if store.isempty() and not os.path.exists(MO_POCOMPILE): # pocompile doesn't create MO files for empty PO files, so we # can skip the checks here. continue mo_msgfmt_f = open(MO_MSGFMT) mo_pocompile_f = open(MO_POCOMPILE) try: mo_msgfmt = mo_msgfmt_f.read() print("msgfmt output:") print(repr(mo_msgfmt)) mo_pocompile = mo_pocompile_f.read() print("pocompile output:") print(repr(mo_pocompile)) assert mo_msgfmt == mo_pocompile finally: mo_msgfmt_f.close() mo_pocompile_f.close()
def update_file(self, filename): # Adapted from Document.__init__() from translate.storage import factory, statsdb newstore = factory.getobject(filename) oldfilename = self._trans_store.filename oldfileobj = self._trans_store.fileobj #get a copy of old stats before we convert from translate.filters import checks oldstats = statsdb.StatsCache().filestats(oldfilename, checks.UnitChecker(), self._trans_store) from translate.convert import pot2po self._trans_store = pot2po.convert_stores(newstore, self._trans_store, fuzzymatching=False) self._trans_store.fileobj = oldfileobj #Let's attempt to keep the old file and name if possible #FIXME: ugly tempfile hack, can we please have a pure store implementation of statsdb import tempfile import os tempfd, tempfilename = tempfile.mkstemp() os.write(tempfd, str(self._trans_store)) self.update_stats(filename=tempfilename) os.close(tempfd) os.remove(tempfilename) self.controller.compare_stats(oldstats, self.stats) # store filename or else save is confused self._trans_store.filename = oldfilename self._correct_header(self._trans_store) self.nplurals = self._compute_nplurals(self._trans_store)
def convertpot(input_file, output_file, template_file, tm=None, min_similarity=75, fuzzymatching=True, classes=None, classes_str=factory.classes_str, **kwargs): """Main conversion function""" input_store = factory.getobject(input_file, classes=classes, classes_str=classes_str) try: temp_store = factory.getobject(input_file, classes_str=classes_str) except: # StringIO and other file like objects will be closed after parsing temp_store = None template_store = None if template_file is not None: template_store = factory.getobject(template_file, classes_str=classes_str) output_store = convert_stores(input_store, template_store, temp_store, tm, min_similarity, fuzzymatching, **kwargs) output_file.write(str(output_store)) return 1
def get_units(self, filenames): """Gets the units to import and its total count.""" units = [] all_filenames = set() for filename in filenames: if not os.path.exists(filename): self.stdout.write("File %s doesn't exist. Skipping it." % filename) continue if os.path.isdir(filename): for dirpath, dirs, fnames in os.walk(filename): if (os.path.basename(dirpath) in ["CVS", ".svn", "_darcs", ".git", ".hg", ".bzr"]): continue for f in fnames: all_filenames.add(os.path.join(dirpath, f)) else: all_filenames.add(filename) for filename in all_filenames: store = factory.getobject(filename) if not store.gettargetlanguage() and not self.target_language: raise CommandError("Unable to determine target language for " "'%s'. Try again specifying a fallback " "target language with --target-language" % filename) self.filename = filename units.extend([unit for unit in store.units if unit.istranslated()]) return units, len(units)
def test_output(self): for posource in posources: print("PO source file") print(posource) PO_FILE, MO_MSGFMT, MO_POCOMPILE = self.get_mo_and_po() posource = posource.encode('utf-8') with open(PO_FILE, 'wb') as out_file: out_file.write(posource) subprocess.call(['msgfmt', PO_FILE, '-o', MO_MSGFMT]) subprocess.call(['pocompile', '--errorlevel=traceback', PO_FILE, MO_POCOMPILE]) store = factory.getobject(BytesIO(posource)) if store.isempty() and not os.path.exists(MO_POCOMPILE): # pocompile doesn't create MO files for empty PO files, so we # can skip the checks here. continue with open(MO_MSGFMT, 'rb') as mo_msgfmt_f: mo_msgfmt = mo_msgfmt_f.read() print("msgfmt output:") print(repr(mo_msgfmt)) with open(MO_POCOMPILE, 'rb') as mo_pocompile_f: mo_pocompile = mo_pocompile_f.read() print("pocompile output:") print(repr(mo_pocompile)) assert mo_msgfmt == mo_pocompile
def build_mo(self): """Compile .mo files from available .po files""" import subprocess import gettext from translate.storage import factory for lang in self._langs: lang = lang.rstrip() po_path = os.path.join('pootle', 'locale', lang) mo_path = os.path.join('pootle', 'locale', lang, 'LC_MESSAGES') if not os.path.exists(mo_path): os.makedirs(mo_path) for po, mo in (('pootle.po', 'django.mo'), ('pootle_js.po', 'djangojs.mo')): po_filename = os.path.join(po_path, po) mo_filename = os.path.join(mo_path, mo) try: store = factory.getobject(po_filename) gettext.c2py(store.getheaderplural()[1]) except Exception, e: print "WARNING: %s, has invalid plural header: %s" % (lang, e) try: if not os.path.exists(mo_path): os.makedirs(mo_path) print "COMPILING: %s language" % lang subprocess.Popen(['msgfmt', '--strict', '-o', mo_filename, po_filename]) except Exception, e: print "SKIPPING: %s, running msgfmt failed: %s" % (lang, e)
def test_output(self): for posource in posources: print "PO source file" print posource PO_FILE, MO_MSGFMT, MO_POCOMPILE = self.get_mo_and_po() out_file = open(PO_FILE, "w") out_file.write(posource) out_file.close() subprocess.call(["msgfmt", PO_FILE, "-o", MO_MSGFMT]) subprocess.call(["pocompile", PO_FILE, MO_POCOMPILE]) store = factory.getobject(StringIO.StringIO(posource)) if store.isempty() and not os.path.exists(MO_POCOMPILE): # pocompile doesn't create MO files for empty PO files, so we # can skip the checks here. continue mo_msgfmt_f = open(MO_MSGFMT) mo_pocompile_f = open(MO_POCOMPILE) try: mo_msgfmt = mo_msgfmt_f.read() print "msgfmt output:" print repr(mo_msgfmt) mo_pocompile = mo_pocompile_f.read() print "pocompile output:" print repr(mo_pocompile) assert mo_msgfmt == mo_pocompile finally: mo_msgfmt_f.close() mo_pocompile_f.close()
def handlefile(self, filename): logging.info("About to process %s:" % filename) try: store = factory.getobject(filename) source_lang = self.source_lang or store.getsourcelanguage() target_lang = self.target_lang or store.gettargetlanguage() project_style = self.project_style or store.getprojectstyle() min_similarity = self.min_similarity max_candidates = self.max_candidates if not source_lang or not target_lang: logging.error("Missing source or target language. Can't use " "%s" % filename) return except Exception: logging.exception("Error while processing %s" % filename) return translate_unit = current_app.tmdb.translate_unit try: for unit in store.units: if unit.istranslatable(): # We need an explicit unicode (not multistring), otherwise # psycopg2 can't adapt it: translate_unit(unicode(unit.source), source_lang, target_lang, project_style, min_similarity, max_candidates) except Exception: logging.exception("Error when translating unit") raise
def setup_file_and_db(self, file_contents=fr_terminology_extract): cache = statsdb.StatsCache(os.path.join(self.path, "stats.db")) filename = os.path.join(self.path, "test.po") with open(filename, "w") as fh: fh.write(file_contents) f = factory.getobject(filename) return f, cache
def convert(): if not Editor.isValid() or Editor.currentFile()=='': return xliffpathname=Editor.currentFile() (path, filename)=os.path.split(xliffpathname) if not filename.endswith('.xlf'): return store = factory.getobject(xliffpathname) odfpathname=store.getfilenames()[0] if odfpathname.startswith('NoName'): print 'translate-toolkit is too old' odfpathname=os.path.splitext(xliffpathname)[0]+'.odt' if not os.path.exists(odfpathname): return translatedodfpathname=os.path.splitext(odfpathname)[0]+'-'+Project.targetLangCode()+'.odt' print 'translatedodfpathname %s' % translatedodfpathname print 'odfpathname %s' % odfpathname xliffinput=XliffInput(xliffpathname,Editor.currentFileContents()) odf=open(odfpathname,'rb') xliff2odf.convertxliff(xliffinput, translatedodfpathname, odf) ourpath=([p for p in sys.path if os.path.exists(p+'/xliff2odf.py')]+[''])[0] os.system('python "'+ourpath+'/xliff2odf-standalone.py" "%s" "%s" &'%(translatedodfpathname, Editor.currentEntryId()))
def unit_iter(self): """Iterator over all the units in all the files in this directory.""" for dirname, filename in self.file_iter(): store = factory.getobject(path.join(dirname, filename)) #TODO: don't regenerate all the storage objects for unit in store.unit_iter(): yield unit
def recacheunit(self, filename, checker, unit): """Recalculate all information for a specific unit. This is necessary for updating all statistics when a translation of a unit took place, for example. This method assumes that everything was up to date before (file totals, checks, checker config, etc.""" fileid = self._getfileid(filename, check_mod_info=False) configid = self._get_config_id(fileid, checker) unitid = unit.getid() # get the unit index totals_without_unit = self.file_totals[fileid] - \ FileTotals.new_record(*self.get_unit_stats(fileid, unitid)) self.cur.execute("""SELECT unitindex FROM units WHERE fileid=? AND unitid=?;""", (fileid, unitid)) unitindex = self.cur.fetchone()[0] self.cur.execute("""DELETE FROM units WHERE fileid=? AND unitid=?;""", (fileid, unitid)) state = [self._cacheunitstats([unit], fileid, unitindex, totals_without_unit)] # remove the current errors self.cur.execute("""DELETE FROM uniterrors WHERE fileid=? AND unitindex=?;""", (fileid, unitindex)) if os.path.exists(suggestion_filename(filename)): checker.setsuggestionstore(factory.getobject(suggestion_filename(filename), ignore=suggestion_extension())) state.extend(self._cacheunitschecks([unit], fileid, configid, checker, unitindex)) return state
def build_mo(self): """Compile .mo files from available .po files""" import subprocess import gettext from translate.storage import factory print "Preparing localization files" for po_filename in glob.glob(path.join('po', 'pootle', '*', 'pootle.po')): lang = path.split(path.split(po_filename)[0])[1] lang_dir = path.join('mo', lang, 'LC_MESSAGES') mo_filename = path.join(lang_dir, 'django.mo') try: store = factory.getobject(po_filename) gettext.c2py(store.getheaderplural()[1]) except Exception, e: print "skipping %s, probably invalid header: %s" % (lang, e) try: if not path.exists(lang_dir): os.makedirs(lang_dir) print "compiling %s language" % lang subprocess.Popen(['msgfmt', '-c', '--strict', '-o', mo_filename, po_filename]) except Exception, e: print "skipping %s, running msgfmt failed: %s" % (lang, e)
def _update_store_cache(self): """Add translation store to dictionary cache, replace old cached version if needed.""" if self.exists(): mod_info = self.getpomtime() else: mod_info = 0 if (not hasattr(self, "_store_tuple") or self._store_tuple.mod_info != mod_info): try: self._store_tuple = self._store_cache[self.path] if self._store_tuple.mod_info != mod_info: # if file is modified act as if it doesn't exist in cache raise KeyError except KeyError: logging.debug(u"Cache miss for %s", self.path) from translate.storage import factory from pootle_store.filetypes import factory_classes store_obj = factory.getobject(self.path, ignore=self.field.ignore, classes=factory_classes) self._store_tuple = StoreTuple(store_obj, mod_info, self.realpath) self._store_cache[self.path] = self._store_tuple
def convertoo(inputfile, outputfile, templatefile, sourcelanguage=None, targetlanguage=None, timestamp=None, includefuzzy=False, multifilestyle="single", skip_source=False, filteraction=None, outputthreshold=None): inputstore = factory.getobject(inputfile) if not convert.should_output_store(inputstore, outputthreshold): return False inputstore.filename = getattr(inputfile, 'name', '') if not targetlanguage: raise ValueError("You must specify the target language") if not sourcelanguage: if targetlanguage.isdigit(): sourcelanguage = "01" else: sourcelanguage = "en-US" languages = (sourcelanguage, targetlanguage) if templatefile is None: raise ValueError("must have template file for oo files") else: convertor = reoo(templatefile, languages=languages, timestamp=timestamp, includefuzzy=includefuzzy, long_keys=multifilestyle != "single", filteraction=filteraction) outputstore = convertor.convertstore(inputstore) # TODO: check if we need to manually delete missing items outputfile.write(outputstore.__str__(skip_source, targetlanguage)) return True
def buildmatches(inputfile, outputfile, matcher): """Builds a .po.tm file for use in Pootle""" #Can't use the same name: it might open the existing file! outputfile = factory.getobject(outputfile, ignore=".tm") #TODO: Do something useful with current content if file exists #inputfile.units.sort(match.sourcelen) try: for unit in inputfile.units: #if len(unit.source) > 70: # break if not unit.source: continue candidates = matcher.matches(unit.source) for candidate in candidates: source = candidate.source target = candidate.target newunit = outputfile.addsourceunit(source) newunit.target = target newunit.addnote(candidate.getnotes()) newunit.addlocations(unit.getlocations()) except exceptions.KeyboardInterrupt: # Let's write what we have so far return outputfile return outputfile
def test_update_fuzzy(self): dbunit = self._update_translation(0, {'fuzzy': True}) storeunit = dbunit.getorig() self.assertTrue(dbunit.isfuzzy()) self.assertEqual(dbunit.isfuzzy(), storeunit.isfuzzy()) pofile = factory.getobject(self.store.file.path) self.assertEqual(dbunit.isfuzzy(), pofile.units[dbunit.index].isfuzzy()) dbunit = self._update_translation(0, {'fuzzy': False}) storeunit = dbunit.getorig() self.assertFalse(dbunit.isfuzzy()) self.assertEqual(dbunit.isfuzzy(), storeunit.isfuzzy()) pofile = factory.getobject(self.store.file.path) self.assertEqual(dbunit.isfuzzy(), pofile.units[dbunit.index].isfuzzy())
def unit_iter(self): """Iterator over all the units in all the files in this zip file.""" for dirname, filename in self.file_iter(): strfile = BytesIO(self.archive.read('/'.join((dirname, filename)))) strfile.filename = filename store = factory.getobject(strfile) # TODO: don't regenerate all the storage objects for unit in store.unit_iter(): yield unit
def convertsub(inputfile, outputfile, templatefile, includefuzzy=False): inputstore = factory.getobject(inputfile) if templatefile is None: raise ValueError("must have template file for subtitle files") else: convertor = resub(templatefile, inputstore) outputstring = convertor.convertstore(includefuzzy) outputfile.write(outputstring) return 1
def pretranslate_file(input_file, output_file, template_file, tm=None, min_similarity=75, fuzzymatching=True): """Pretranslate any factory supported file with old translations and translation memory. """ input_store = factory.getobject(input_file) template_store = None if template_file is not None: template_store = factory.getobject(template_file) output = pretranslate_store(input_store, template_store, tm, min_similarity, fuzzymatching) output.serialize(output_file) return 1
def build_mo(self): """Compile .mo files from available .po files""" import subprocess import gettext from translate.storage import factory error_occured = False for lang in self._langs: lang = lang.rstrip() po_path = os.path.join('pootle', 'locale', lang) mo_path = os.path.join('pootle', 'locale', lang, 'LC_MESSAGES') if not os.path.exists(mo_path): os.makedirs(mo_path) for po, mo in (('pootle.po', 'django.mo'), ('pootle_js.po', 'djangojs.mo')): po_filename = os.path.join(po_path, po) mo_filename = os.path.join(mo_path, mo) if not os.path.exists(po_filename): log.warn("%s: missing file %s", lang, po_filename) continue if not os.path.exists(mo_path): os.makedirs(mo_path) log.info("compiling %s", lang) if self.check: command = [ 'msgfmt', '-c', '--strict', '-o', mo_filename, po_filename ] else: command = [ 'msgfmt', '--strict', '-o', mo_filename, po_filename ] try: subprocess.check_call(command, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: error_occured = True except Exception as e: log.warn("%s: skipping, running msgfmt failed: %s", lang, e) try: store = factory.getobject(po_filename) gettext.c2py(store.getheaderplural()[1]) except Exception: log.warn("%s: invalid plural header in %s", lang, po_filename) if error_occured: sys.exit(1)
def test_update_target(af_tutorial_po): """Tests that target changes are properly sync'ed to disk.""" db_unit = _update_translation(af_tutorial_po, 0, {'target': u'samaka'}) store_unit = db_unit.getorig() assert db_unit.target == u'samaka' assert db_unit.target == store_unit.target po_file = factory.getobject(af_tutorial_po.file.path) assert db_unit.target == po_file.units[db_unit.index].target
def upload_store(self, environ, start_response, sid, slang, tlang): """add units from uploaded file to tmdb""" start_response("200 OK", [('Content-type', 'text/plain')]) data = StringIO.StringIO(environ['wsgi.input'].read( int(environ['CONTENT_LENGTH']))) data.name = sid store = factory.getobject(data) count = self.tmdb.add_store(store, slang, tlang) response = "added %d units from %s" % (count, sid) return [response]
def test_bz2file(self): """Test that we can open a gzip file correctly.""" if not BZ2File: return filename = os.path.join(self.testdir, self.filename + '.bz2') bz2file = BZ2File(filename, mode="wb") bz2file.write(self.file_content) bz2file.close() store = factory.getobject(filename) assert isinstance(store, self.expected_instance)
def convertpo(inputfile, outputfile, templatefile, format=None, rewritestyle=None, ignoreoption=None): """Reads in inputfile, changes it to have debug strings, writes to outputfile.""" # note that templatefile is not used, but it is required by the converter... inputstore = factory.getobject(inputfile) if inputstore.isempty(): return 0 convertor = podebug(format=format, rewritestyle=rewritestyle, ignoreoption=ignoreoption) outputstore = convertor.convertstore(inputstore) outputfile.write(str(outputstore)) return 1
def convert_symbian(input_file, output_file, template_file, pot=False, duplicatestyle="msgctxt"): store = factory.getobject(input_file) location_index = build_location_index(store) header_index = { 'Author': store.parseheader()['Last-Translator'] } output = write_symbian(template_file, header_index, location_index) for line in output: output_file.write(line) return 1
def convertpy(inputfile, outputfile, templatefile=None, includefuzzy=False, outputthreshold=None): inputstore = factory.getobject(inputfile) if not convert.should_output_store(inputstore, outputthreshold): return False convertor = po2pydict() outputstring = convertor.convertstore(inputstore, includefuzzy) outputfile.write(outputstring.read()) return 1
def converttxt(inputfile, outputfile, templatefile, wrap=None, includefuzzy=False, encoding='utf-8'): """reads in stdin using fromfileclass, converts using convertorclass, writes to stdout""" inputstore = factory.getobject(inputfile) convertor = po2txt(wrap=wrap) if templatefile is None: outputstring = convertor.convertstore(inputstore, includefuzzy) else: templatestring = templatefile.read().decode(encoding) outputstring = convertor.mergestore(inputstore, templatestring, includefuzzy) outputfile.write(outputstring.encode('utf-8')) return 1
def test_update_comment(self): dbunit = self._update_translation(0, {'translator_comment': u'7amada'}) storeunit = dbunit.getorig() self.assertEqual(dbunit.getnotes(origin="translator"), u'7amada') self.assertEqual(dbunit.getnotes(origin="translator"), storeunit.getnotes(origin="translator")) pofile = factory.getobject(self.store.file.path) self.assertEqual( dbunit.getnotes(origin="translator"), pofile.units[dbunit.index].getnotes(origin="translator"))
def runfilter(inputfile, outputfile, templatefile, checkfilter=None): """Reads in inputfile, filters using checkfilter, writes to outputfile.""" fromfile = factory.getobject(inputfile) tofile = checkfilter.filterfile(fromfile) if tofile.isempty(): return 0 tofile.serialize(outputfile) return 1
def upload_store(sid, slang, tlang): """add units from uploaded file to tmdb""" import StringIO data = StringIO.StringIO(request.data) data.name = sid from translate.storage import factory store = factory.getobject(data) project_style = request.args.get('style', None) count = current_app.tmdb.add_store(store, slang, tlang, project_style) response = "added %d units from %s" % (count, sid) return response
def test_update_fuzzy(af_tutorial_po): """Tests fuzzy state changes are stored and sync'ed.""" db_unit = _update_translation(af_tutorial_po, 0, {'target': u'samaka', 'fuzzy': True}) store_unit = db_unit.getorig() assert db_unit.isfuzzy() assert db_unit.isfuzzy() == store_unit.isfuzzy() po_file = factory.getobject(af_tutorial_po.file.path) assert db_unit.isfuzzy() == po_file.units[db_unit.index].isfuzzy() db_unit = _update_translation(af_tutorial_po, 0, {'fuzzy': False}) store_unit = db_unit.getorig() assert not db_unit.isfuzzy() assert db_unit.isfuzzy() == store_unit.isfuzzy() po_file = factory.getobject(af_tutorial_po.file.path) assert db_unit.isfuzzy() == po_file.units[db_unit.index].isfuzzy()
def test_update_target(project0_nongnu, store0): """Tests that target changes are properly sync'ed to disk.""" db_unit = _update_translation(store0, 0, {'target': u'samaka'}) store0.sync() store_unit = store0.file.store.findid(db_unit.getid()) assert db_unit.target == u'samaka' assert db_unit.target == store_unit.target po_file = factory.getobject(store0.file.path) assert db_unit.target == po_file.findid(db_unit.unitid).target
def segmentfile(inputfile, outputfile, templatefile, sourcelanguage="en", targetlanguage=None, stripspaces=True, onlyaligned=False): """reads in inputfile, segments it then, writes to outputfile""" # note that templatefile is not used, but it is required by the converter... inputstore = factory.getobject(inputfile) if inputstore.isempty(): return 0 sourcelang = lang_factory.getlanguage(sourcelanguage) targetlang = lang_factory.getlanguage(targetlanguage) convertor = segment(sourcelang, targetlang, stripspaces=stripspaces, onlyaligned=onlyaligned) outputstore = convertor.convertstore(inputstore) outputfile.write(str(outputstore)) return 1
def test_update_fuzzy(project0_nongnu, store0): """Tests fuzzy state changes are stored and sync'ed.""" db_unit = _update_translation( store0, 0, {'target': u'samaka', 'fuzzy': True}) store_unit = store0.file.store.findid(db_unit.getid()) assert db_unit.isfuzzy() assert db_unit.isfuzzy() == store_unit.isfuzzy() po_file = factory.getobject(store0.file.path) assert db_unit.isfuzzy() == po_file.findid(db_unit.getid()).isfuzzy() db_unit = _update_translation(store0, 0, {'fuzzy': False}) store_unit = store0.file.store.findid(db_unit.getid()) assert not db_unit.isfuzzy() assert db_unit.isfuzzy() == store_unit.isfuzzy() po_file = factory.getobject(store0.file.path) assert db_unit.isfuzzy() == po_file.findid(db_unit.getid()).isfuzzy()
def test_update_plural_target_dict(self): dbunit = self._update_translation(2, {'target': {0: u'samaka', 1: u'samak'}}) storeunit = dbunit.getorig() self.assertEqual(dbunit.target.strings, [u'samaka', u'samak']) self.assertEqual(dbunit.target.strings, storeunit.target.strings) pofile = factory.getobject(self.store.file.path) self.assertEqual(dbunit.target.strings, pofile.units[dbunit.index].target.strings) self.assertEqual(dbunit.target, u'samaka') self.assertEqual(dbunit.target, storeunit.target) self.assertEqual(dbunit.target, pofile.units[dbunit.index].target)
def writematches(inputfile, outputfile, templatefile, tm=None, max_candidates=4, min_similarity=75, max_length=1000): if templatefile: raise Warning("Template ignored") inputfile = factory.getobject(inputfile) if tm is None: raise ValueError("Must have TM storage specified with --tm") int_max_candidates = int(max_candidates) int_min_similarity = int(min_similarity) matcher = memory(tm, max_candidates=int_max_candidates, min_similarity=int_min_similarity, max_length=max_length) output = buildmatches(inputfile, outputfile, matcher) outputfile.writelines(str(output)) return 1
def upload_store(self, environ, start_response, sid, slang, tlang): """add units from uploaded file to tmdb""" from translate.storage import factory start_response("200 OK", [("Content-type", "text/plain")]) data = BytesIO(environ["wsgi.input"].read( int(environ["CONTENT_LENGTH"]))) data.name = sid store = factory.getobject(data) count = self.tmdb.add_store(store, slang, tlang) response = "added %d units from %s" % (count, sid) return [response]
def mtfile(inputfile, outputfile, templatefile, dictionary_file, new_words, all_words, project): if not dictionary_file: print("ERROR: missing dictionary file") return 0 inputstore = factory.getobject(inputfile) if inputstore.isempty(): return 0 convertor = word_substitute(dictionary_file, all_words, new_words, project) outputstore = convertor.convertstore(inputstore) outputstore.serialize(outputfile) return 1
def test_update_comment(af_tutorial_po): """Tests translator comments are stored and sync'ed.""" db_unit = _update_translation(af_tutorial_po, 0, {'translator_comment': u'7amada'}) store_unit = db_unit.getorig() assert db_unit.getnotes(origin='translator') == u'7amada' assert db_unit.getnotes(origin='translator') == \ store_unit.getnotes(origin='translator') po_file = factory.getobject(af_tutorial_po.file.path) assert db_unit.getnotes(origin='translator') == \ po_file.units[db_unit.index].getnotes(origin='translator')
def test_update_comment(project0_nongnu, store0): """Tests translator comments are stored and sync'ed.""" db_unit = _update_translation(store0, 0, {'translator_comment': u'7amada'}) store0.sync() store_unit = store0.file.store.findid(db_unit.getid()) assert db_unit.getnotes(origin='translator') == u'7amada' assert (db_unit.getnotes(origin='translator') == store_unit.getnotes( origin='translator')) po_file = factory.getobject(store0.file.path) assert (db_unit.getnotes(origin='translator') == po_file.findid( db_unit.getid()).getnotes(origin='translator'))
def test_update_fuzzy(project0_disk, store0): """Tests fuzzy state changes are stored and sync'ed.""" db_unit = _update_translation(store0, 0, { "target": u"samaka", "fuzzy": True }) store_unit = store0.file.store.findid(db_unit.getid()) assert db_unit.isfuzzy() assert db_unit.isfuzzy() == store_unit.isfuzzy() po_file = factory.getobject(store0.file.path) assert db_unit.isfuzzy() == po_file.findid(db_unit.getid()).isfuzzy() db_unit = _update_translation(store0, 0, {"fuzzy": False}) store_unit = store0.file.store.findid(db_unit.getid()) assert not db_unit.isfuzzy() assert db_unit.isfuzzy() == store_unit.isfuzzy() po_file = factory.getobject(store0.file.path) assert db_unit.isfuzzy() == po_file.findid(db_unit.getid()).isfuzzy()
def test_update_comment(project0_disk, store0): """Tests translator comments are stored and sync'ed.""" db_unit = _update_translation(store0, 0, {"translator_comment": u"7amada"}) store0.sync() store_unit = store0.file.store.findid(db_unit.getid()) assert db_unit.getnotes(origin="translator") == u"7amada" assert db_unit.getnotes(origin="translator") == store_unit.getnotes( origin="translator") po_file = factory.getobject(store0.file.path) assert db_unit.getnotes(origin="translator") == po_file.findid( db_unit.getid()).getnotes(origin="translator")
def convertini(inputfile, outputfile, templatefile, includefuzzy=False, dialect="default"): inputstore = factory.getobject(inputfile) if templatefile is None: raise ValueError("must have template file for ini files") else: convertor = reini(templatefile, inputstore, dialect) outputstring = convertor.convertstore(includefuzzy) outputfile.write(outputstring) return 1
def convertpot(input_file, output_file, template_file, tm=None, min_similarity=75, fuzzymatching=True, classes=None, classes_str=factory.classes_str, **kwargs): """Main conversion function.""" input_store = factory.getobject(input_file, classes=classes, classes_str=classes_str) try: temp_store = factory.getobject(input_file, classes_str=classes_str) except: # StringIO and other file like objects will be closed after parsing temp_store = None template_store = None if template_file is not None: template_store = factory.getobject(template_file, classes_str=classes_str) output_store = convert_stores(input_store, template_store, temp_store, tm, min_similarity, fuzzymatching, **kwargs) output_file.write(str(output_store)) return 1
def load_files(self): if self.matcher in TerminologyPlaceable.matchers: TerminologyPlaceable.matchers.remove(self.matcher) self.stores = [] for filename in self.config['files']: if not filename: continue if not os.path.isfile(filename): logging.debug('Not a file: "%s"' % (filename)) self.stores.append(factory.getobject(filename)) self.matcher = terminologymatcher(self.stores) TerminologyPlaceable.matchers.append(self.matcher)