def build_mo(self): """Compile .mo files from available .po files""" import subprocess import gettext from translate.storage import factory for lang in self._langs: lang = lang.rstrip() po_path = os.path.join('pootle', 'locale', lang) mo_path = os.path.join('pootle', 'locale', lang, 'LC_MESSAGES') if not os.path.exists(mo_path): os.makedirs(mo_path) for po, mo in (('pootle.po', 'django.mo'), ('pootle_js.po', 'djangojs.mo')): po_filename = os.path.join(po_path, po) mo_filename = os.path.join(mo_path, mo) try: store = factory.getobject(po_filename) gettext.c2py(store.getheaderplural()[1]) except Exception, e: print "WARNING: %s, has invalid plural header: %s" % (lang, e) try: if not os.path.exists(mo_path): os.makedirs(mo_path) print "COMPILING: %s language" % lang subprocess.Popen(['msgfmt', '--strict', '-o', mo_filename, po_filename]) except Exception, e: print "SKIPPING: %s, running msgfmt failed: %s" % (lang, e)
def build_mo(self): """Compile .mo files from available .po files""" import subprocess import gettext from translate.storage import factory print "Preparing localization files" for po_filename in glob.glob(path.join('po', 'pootle', '*', 'pootle.po')): lang = path.split(path.split(po_filename)[0])[1] lang_dir = path.join('mo', lang, 'LC_MESSAGES') mo_filename = path.join(lang_dir, 'django.mo') try: store = factory.getobject(po_filename) gettext.c2py(store.getheaderplural()[1]) except Exception, e: print "skipping %s, probably invalid header: %s" % (lang, e) try: if not path.exists(lang_dir): os.makedirs(lang_dir) print "compiling %s language" % lang subprocess.Popen(['msgfmt', '-c', '--strict', '-o', mo_filename, po_filename]) except Exception, e: print "skipping %s, running msgfmt failed: %s" % (lang, e)
def validate_pluraleq(value): try: gettext.c2py(value if value else '0') except ValueError as error: raise ValidationError( _('Failed to evaluate plural equation: {}').format(error) )
def test_invalid_syntax(self): invalid_expressions = [ 'x>1', '(n>1', 'n>1)', '42**42**42', '0xa', '1.0', '1e2', 'n>0x1', '+n', '-n', 'n()', 'n(1)', '1+', 'nn', 'n n', ] for expr in invalid_expressions: with self.assertRaises(ValueError): gettext.c2py(expr)
def test_chained_comparison(self): # C doesn't chain comparison as Python so 2 == 2 == 2 gets different results f = gettext.c2py('n == n == n') self.assertEqual(''.join(str(f(x)) for x in range(3)), '010') f = gettext.c2py('1 < n == n') self.assertEqual(''.join(str(f(x)) for x in range(3)), '100') f = gettext.c2py('n == n < 2') self.assertEqual(''.join(str(f(x)) for x in range(3)), '010') f = gettext.c2py('0 < n < 2') self.assertEqual(''.join(str(f(x)) for x in range(3)), '111')
def build_mo(self): """Compile .mo files from available .po files""" import subprocess import gettext from translate.storage import factory error_occured = False for lang in self._langs: lang = lang.rstrip() po_path = os.path.join('pootle', 'locale', lang) mo_path = os.path.join('pootle', 'locale', lang, 'LC_MESSAGES') if not os.path.exists(mo_path): os.makedirs(mo_path) for po, mo in (('pootle.po', 'django.mo'), ('pootle_js.po', 'djangojs.mo')): po_filename = os.path.join(po_path, po) mo_filename = os.path.join(mo_path, mo) if not os.path.exists(po_filename): log.warn("%s: missing file %s", lang, po_filename) continue if not os.path.exists(mo_path): os.makedirs(mo_path) log.info("compiling %s", lang) if self.check: command = ['msgfmt', '-c', '--strict', '-o', mo_filename, po_filename] else: command = ['msgfmt', '--strict', '-o', mo_filename, po_filename] try: subprocess.check_call(command, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: error_occured = True except Exception as e: log.warn("%s: skipping, running msgfmt failed: %s", lang, e) try: store = factory.getobject(po_filename) gettext.c2py(store.getheaderplural()[1]) except Exception: log.warn("%s: invalid plural header in %s", lang, po_filename) if error_occured: sys.exit(1)
def parse_formula(plurals): matches = PLURAL_RE.match(plurals) if matches is None: raise ValueError('Failed to parse formula') number = int(matches.group(1)) formula = matches.group(2) if not formula: formula = '0' # Try to parse the formula gettext.c2py(formula) return number, formula
def test_security(self): raises = self.assertRaises # Test for a dangerous expression raises(ValueError, gettext.c2py, "os.chmod('/etc/passwd',0777)") # issue28563 raises(ValueError, gettext.c2py, '"(eval(foo) && ""') raises(ValueError, gettext.c2py, 'f"{os.system(\'sh\')}"') # Maximum recursion depth exceeded during compilation raises(ValueError, gettext.c2py, 'n+'*10000 + 'n') self.assertEqual(gettext.c2py('n+'*100 + 'n')(1), 101) # MemoryError during compilation raises(ValueError, gettext.c2py, '('*100 + 'n' + ')'*100) # Maximum recursion depth exceeded in C to Python translator raises(ValueError, gettext.c2py, '('*10000 + 'n' + ')'*10000) self.assertEqual(gettext.c2py('('*20 + 'n' + ')'*20)(1), 1)
def make_plural_function(expression): """Create a lambda function for a C-like plural expression.""" # Largest expressions we could find in practice were 113 characters # long. 500 is a reasonable value which is still 4 times more than # that, yet not incredibly long. if expression is None: raise BadPluralExpression("No plural expression given.") if len(expression) > 500: raise BadPluralExpression("Plural expression is too long.") # Guard against '**' usage: it's not useful in evaluating # plural forms, yet can be used to introduce a DoS. if expression.find('**') != -1: raise BadPluralExpression("Invalid operator: **.") # We allow digits, whitespace [ \t], parentheses, "n", and operators # as allowed by GNU gettext implementation as well. if not re.match('^[0-9 \t()n|&?:!=<>+%*/-]*$', expression): raise BadPluralExpression( "Plural expression contains disallowed characters.") try: function = gettext.c2py(expression) except (ValueError, SyntaxError) as e: raise BadPluralExpression(e.args[0]) return function
def test_division(self): f = gettext.c2py('2/n*3') self.assertEqual(f(1), 6) self.assertEqual(f(2), 3) self.assertEqual(f(3), 0) self.assertEqual(f(-1), -6) self.assertRaises(ZeroDivisionError, f, 0)
def test_fr(self): eq = self.assertEqual f = gettext.c2py("n>1") s = "".join([str(f(x)) for x in range(200)]) eq( s, "00111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", )
def test_pl(self): eq = self.assertEqual f = gettext.c2py("n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2") s = "".join([str(f(x)) for x in range(200)]) eq( s, "20111222222222222222221112222222111222222211122222221112222222111222222211122222221112222222111222222211122222222222222222111222222211122222221112222222111222222211122222221112222222111222222211122222", )
def test_sl(self): eq = self.assertEqual f = gettext.c2py("n%100==1 ? 0 : n%100==2 ? 1 : n%100==3 || n%100==4 ? 2 : 3") s = "".join([str(f(x)) for x in range(200)]) eq( s, "30122333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333012233333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333", )
def test_gd(self): eq = self.assertEqual f = gettext.c2py("n==1 ? 0 : n==2 ? 1 : 2") s = "".join([str(f(x)) for x in range(200)]) eq( s, "20122222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222", )
def test_lt(self): eq = self.assertEqual f = gettext.c2py("n%10==1 && n%100!=11 ? 0 : n%10>=2 && (n%100<10 || n%100>=20) ? 1 : 2") s = "".join([str(f(x)) for x in range(200)]) eq( s, "20111111112222222222201111111120111111112011111111201111111120111111112011111111201111111120111111112011111111222222222220111111112011111111201111111120111111112011111111201111111120111111112011111111", )
def parse_json(self, text): json = loads(text) self._catalog.update(json) for k, v in self._catalog[""].iteritems(): k = k.lower() if k == 'plural-forms': self.nplurals = int(self.re_nplurals.search(v).group(1)) self.plural = c2py(self.re_plural.search(v).group(1))
def test_hu(self): eq = self.assertEqual f = gettext.c2py("0") s = "".join([str(f(x)) for x in range(200)]) eq( s, "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", )
def gen(): tests = [] for plural_form in PLURAL_FORMS: expr = c2py(plural_form) tests.append({ 'pluralform': plural_form, 'fixture': [expr(n) for n in range(NUM + 1)] }) return json.dumps(tests)
def test_gd2(self): eq = self.assertEqual # Tests the combination of parentheses and "?:" f = gettext.c2py("n==1 ? 0 : (n==2 ? 1 : 2)") s = "".join([str(f(x)) for x in range(200)]) eq( s, "20122222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222", )
def test_plural_number(self): f = gettext.c2py('n != 1') self.assertEqual(f(1), 0) self.assertEqual(f(2), 1) self.assertEqual(f(1.0), 0) self.assertEqual(f(2.0), 1) self.assertEqual(f(1.1), 1) self.assertRaises(TypeError, f, '2') self.assertRaises(TypeError, f, b'2') self.assertRaises(TypeError, f, []) self.assertRaises(TypeError, f, object())
def test_equation(self): """Validate that all equations can be parsed by gettext""" # Verify we get an error on invalid syntax with self.assertRaises((SyntaxError, ValueError)): gettext.c2py('n==0 ? 1 2') for code, dummy, nplurals, pluraleq in self.all_data(): # Validate plurals can be parsed plural = gettext.c2py(pluraleq) # Get maximal plural calculated = max([plural(x) for x in range(200)]) + 1 # Check it matches ours self.assertEqual( calculated, nplurals, 'Invalid nplurals for {0}: {1} ({2}, {3})'.format( code, calculated, nplurals, pluraleq, ) )
def same_plural(self, plurals): """Compare whether given plurals formula matches""" matches = PLURAL_RE.match(plurals) if matches is None: return False if int(matches.group(1)) != self.nplurals: return False # Convert formulas to functions ours = gettext.c2py(self.pluralequation) theirs = gettext.c2py(matches.group(2)) # Compare equation results # It would be better to compare formulas, # but this was easier to implement and the performance # is still okay. for i in range(-10, 200): if ours(i) != theirs(i): return False return True
def test_plural_number(self): f = gettext.c2py('n != 1') self.assertEqual(f(1), 0) self.assertEqual(f(2), 1) with self.assertWarns(DeprecationWarning): self.assertEqual(f(1.0), 0) with self.assertWarns(DeprecationWarning): self.assertEqual(f(2.0), 1) with self.assertWarns(DeprecationWarning): self.assertEqual(f(1.1), 1) self.assertRaises(TypeError, f, '2') self.assertRaises(TypeError, f, b'2') self.assertRaises(TypeError, f, []) self.assertRaises(TypeError, f, object())
def build_mo(self): """Compile .mo files from available .po files""" import subprocess import gettext from translate.storage import factory for lang in self._langs: lang = lang.rstrip() po_path = os.path.join("pootle", "locale", lang) mo_path = os.path.join("pootle", "locale", lang, "LC_MESSAGES") if not os.path.exists(mo_path): os.makedirs(mo_path) for po, mo in (("pootle.po", "django.mo"), ("pootle_js.po", "djangojs.mo")): po_filename = os.path.join(po_path, po) mo_filename = os.path.join(mo_path, mo) if not os.path.exists(po_filename): log.warn("%s: missing file %s", lang, po_filename) continue if not os.path.exists(mo_path): os.makedirs(mo_path) log.info("compiling %s", lang) try: subprocess.call(["msgfmt", "--strict", "-o", mo_filename, po_filename], stderr=subprocess.STDOUT) except Exception as e: log.warn("%s: skipping, running msgfmt failed: %s", lang, e) try: store = factory.getobject(po_filename) gettext.c2py(store.getheaderplural()[1]) except Exception: log.warn("%s: invalid plural header in %s", lang, po_filename)
def get_plural_label(self, idx): """Return label for plural form.""" if len(self._plural_examples) == 0: func = gettext.c2py(self.pluralequation) for i in range(0, 1000): ret = func(i) if ret not in self._plural_examples: self._plural_examples[ret] = [] if len(self._plural_examples[ret]) >= 10: continue self._plural_examples[ret].append(str(i)) # Translators: Label for plurals with example counts return _('{name} (e.g. {examples})').format( name=self.get_plural_name(idx), examples=', '.join(self._plural_examples[idx]) )
def same_plural(self, number, equation): """Compare whether given plurals formula matches""" if number != self.number or not equation: return False # Convert formulas to functions ours = self.plural_function theirs = gettext.c2py(equation) # Compare equation results # It would be better to compare formulas, # but this was easier to implement and the performance # is still okay. for i in range(-10, 200): if ours(i) != theirs(i): return False return True
def validate_entry(lang, e): """Validates plural forms expression.""" m = re.match(r'nplurals=([0-9]+); plural=(.+);', e) if not m: raise ValueError("incorrect structure of plural form expression") c_expr = m.group(2) # verify the syntax is correct func = gettext.c2py(c_expr) # and that all indexes are used: count = int(m.group(1)) used = [0] * count for i in range(0,1001): fi = func(i) if fi >= count: raise ValueError("expression out of range (n=%d -> %d)" % (i, fi)) used[func(i)] = 1 if sum(used) != count: print(f"warning: {lang}: not all plural form values used (n=0..1001)")
def translate_message(self, singular, plural=None, n=1): for store in self.stores.iterator(): unit = store.findunit(singular) if unit is not None and unit.istranslated(): if unit.hasplural() and n != 1: pluralequation = self.language.pluralequation if pluralequation: pluralfn = gettext.c2py(pluralequation) target = unit.target.strings[pluralfn(n)] if target is not None: return target else: return unit.target # no translation found if n != 1 and plural is not None: return plural else: return singular
def test_equation(self): """Validates that all equations can be parsed by gettext""" # Verify we get an error on invalid syntax self.assertRaises( SyntaxError, gettext.c2py, 'n==0 ? 1 2' ) for language in Language.objects.all(): # Validate plurals can be parsed plural = gettext.c2py(language.pluralequation) # Get maximal plural nplurals = max([plural(x) for x in range(200)]) + 1 # Check it matches ours self.assertEqual( nplurals, language.nplurals, 'Invalid nplurals for {0}: {1} ({2}, {3})'.format( language.code, nplurals, language.nplurals, language.pluralequation ) )
def test_cs(self): eq = self.assertEqual f = gettext.c2py('(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2') s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "20111222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222")
def test_gd2(self): eq = self.assertEqual # Tests the combination of parentheses and "?:" f = gettext.c2py('n==1 ? 0 : (n==2 ? 1 : 2)') s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "20122222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222")
def test_fr(self): eq = self.assertEqual f = gettext.c2py('n>1') s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "00111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111")
def plural_function(self): return gettext.c2py(self.formula if self.formula else "0")
def test_decimal_number(self): self.assertEqual(gettext.c2py('0123')(1), 123)
def _parse (self, fp): """Slighly modified version of gettext.GNUTranslations._parse.""" unpack = struct.unpack filename = getattr (fp, "name", "") # Parse the .mo file header, which consists of 5 little endian 32 # bit words. self._catalog = catalog = {} self.plural = lambda n: int (n != 1) # germanic plural by default buf = fp.read () buflen = len (buf) # Are we big endian or little endian? magic = unpack ("<I", buf [:4]) [0] if magic == self.LE_MAGIC : version, msgcount, masteridx, transidx = unpack ("<4I", buf [4:20]) ii = "<II" elif magic == self.BE_MAGIC: version, msgcount, masteridx, transidx = unpack (">4I", buf [4:20]) ii = ">II" else: raise IOError (0, "Bad magic number", filename) # Now put all messages from the .mo file buffer into the catalog # dictionary. for i in range (0, msgcount) : mlen, moff = unpack (ii, buf [masteridx : masteridx + 8]) tlen, toff = unpack (ii, buf [transidx : transidx + 8]) mend = moff + mlen tend = toff + tlen if mend < buflen and tend < buflen: msg = buf [moff:mend] tmsg = buf [toff:tend] else: raise IOError (0, "File is corrupt", filename) # See if we're looking at GNU .mo conventions for metadata if not mlen : # Catalog description lastk = k = None for b_item in tmsg.split ('\n'.encode("ascii")) : item = b_item.decode().strip() if not item: continue if ":" in item : k, v = item.split (":", 1) k = k.strip ().lower () v = v.strip () self._info [k] = v lastk = k elif lastk : self._info [lastk] += "\n" + item if k == "content-type" : self._charset = v.split ("charset=") [1] elif k == "plural-forms" : v = v.split (";") plural = v [1].split ("plural=") [1] self.plural = gettext.c2py (plural) # Note: we unconditionally convert both msgids and msgstrs to # Unicode using the character encoding specified in the charset # parameter of the Content-Type header. The gettext documentation # strongly encourages msgids to be us-ascii, but some appliations # require alternative encodings (e.g. Zope's ZCML and ZPT). For # traditional gettext applications, the msgid conversion will # cause no problems since us-ascii should always be a subset of # the charset encoding. We may want to fall back to 8-bit msgids # if the Unicode conversion fails. charset = self._charset or 'ascii' sep = b"\x00" if sep in msg : # Plural forms msgid1, msgid2 = msg.split (sep) tmsg = tmsg.split (sep) msgid1 = str (msgid1, charset) msgid2 = str (msgid2, charset) tmsg = [str (x, charset) for x in tmsg] for i, msg in enumerate (tmsg) : catalog [(msgid1, i)] = msg ### In addtion to the two keys to the catalog as well to be ### able to have access to the singular and the last plural ### translation as well catalog [msgid1] = tmsg [ 0] catalog [msgid2] = tmsg [-1] else: msg = str (msg, charset) tmsg = str (tmsg, charset) catalog [msg] = tmsg # advance to next entry in the seek tables masteridx += 8 transidx += 8
def validate_plural_formula(value): try: gettext.c2py(value if value else "0") except ValueError as error: raise ValidationError( _("Could not evaluate plural formula: {}").format(error))
def validate_pluraleq(value): try: gettext.c2py(value if value else '0') except ValueError as error: raise ValidationError( _('Failed to evaluate plural equation: {}').format(error))
def test_lv(self): eq = self.assertEqual f = gettext.c2py('n%10==1 && n%100!=11 ? 0 : n != 0 ? 1 : 2') s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "20111111111111111111101111111110111111111011111111101111111110111111111011111111101111111110111111111011111111111111111110111111111011111111101111111110111111111011111111101111111110111111111011111111")
def test_ro(self): eq = self.assertEqual f = gettext.c2py('n==1 ? 0 : (n==0 || (n%100 > 0 && n%100 < 20)) ? 1 : 2') s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "10111111111111111111222222222222222222222222222222222222222222222222222222222222222222222222222222222111111111111111111122222222222222222222222222222222222222222222222222222222222222222222222222222222")
import gettext import vulntools def func(): vulntools.exit_vulnerable() vulntools.prepare_process() try: py = gettext.c2py("n()") except ValueError: vulntools.exit_fixed() else: py(func) vulntools.exit_error()
def plural_function(self): return gettext.c2py(self.equation if self.equation else '0')
def update_event(self, inp=-1): self.set_output_val(0, gettext.c2py(self.input(0)))
def __init__(self, fp, locale): if not isinstance(locale, Locale): locale = Locale.parse(locale) self.locale = locale super(POTranslations, self).__init__(fp) self.plural = c2py(get_plural(locale).plural_expr)
def test_lt(self): eq = self.assertEqual f = gettext.c2py('n%10==1 && n%100!=11 ? 0 : n%10>=2 && (n%100<10 || n%100>=20) ? 1 : 2') s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "20111111112222222222201111111120111111112011111111201111111120111111112011111111201111111120111111112011111111222222222220111111112011111111201111111120111111112011111111201111111120111111112011111111")
def test_nested_condition_operator(self): self.assertEqual(gettext.c2py('n?1?2:3:4')(0), 4) self.assertEqual(gettext.c2py('n?1?2:3:4')(1), 2) self.assertEqual(gettext.c2py('n?1:3?4:5')(0), 4) self.assertEqual(gettext.c2py('n?1:3?4:5')(1), 1)
def test_pl(self): eq = self.assertEqual f = gettext.c2py('n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2') s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "20111222222222222222221112222222111222222211122222221112222222111222222211122222221112222222111222222211122222222222222222111222222211122222221112222222111222222211122222221112222222111222222211122222")
def test_hu(self): eq = self.assertEqual f = gettext.c2py('0') s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
def test_sl(self): eq = self.assertEqual f = gettext.c2py('n%100==1 ? 0 : n%100==2 ? 1 : n%100==3 || n%100==4 ? 2 : 3') s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "30122333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333012233333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333")
def test_gd(self): eq = self.assertEqual f = gettext.c2py('n==1 ? 0 : n==2 ? 1 : 2') s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "20122222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222")
def plural_function(self): return gettext.c2py(self.equation)
def test_ru(self): eq = self.assertEqual f = gettext.c2py('n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2') s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "20111222222222222222201112222220111222222011122222201112222220111222222011122222201112222220111222222011122222222222222220111222222011122222201112222220111222222011122222201112222220111222222011122222")
def test_ar(self): eq = self.assertEqual f = gettext.c2py('n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 ? 4 : 5') s = ''.join([ str(f(x)) for x in range(200) ]) eq(s, "01233333333444444444444444444444444444444444444444444444444444444444444444444444444444444444444444445553333333344444444444444444444444444444444444444444444444444444444444444444444444444444444444444444")