def test_compile_story_with_amps(self): regular = OpenString('0', u"hello world", order=0) with_amp = OpenString('1', u"hello &world", order=1) with_amp_escaped = OpenString('2', u"hello <world", order=2) many_amps = OpenString('3', u"&ਟ&&", order=3) base_template = u""" <Story> <Content>{regular}</Content> <Content>{with_amp}</Content> <Content>{with_amp_escaped}</Content> <Content>{many_amps}</Content> </Story> """ template = base_template.format( regular=regular.template_replacement, with_amp=with_amp.template_replacement, with_amp_escaped=with_amp_escaped.template_replacement, many_amps=many_amps.template_replacement, ) expected_compiled_story = base_template.format( regular=u"hello world", with_amp=u"hello &world", with_amp_escaped=u"hello <world", many_amps=u"&ਟ&&", ) handler = self.HANDLER_CLASS() handler.stringset = [regular, with_amp, with_amp_escaped, many_amps] compiled_story = handler._compile_story(template) self.assertEqual(compiled_story, expected_compiled_story)
def test_error_in__repr__(self): open_string = OpenString('random_key', {6: 'this is wrong'}) self.assertEqual( open_string.__repr__(), b'Invalid string' )
def test_compile_story(self): simple_story_template = u""" <Story> <Content>9a1c7ee2c7ce38d4bbbaf29ab9f2ac1e_tr</Content> <Content>3afcdbfeb6ecfbdd0ba628696e3cc163_tr</Content> <Metadata></Metadata> <Content> <?ACE 7?></Content> </Story> """ simple_compiled_story = u""" <Story> <Content>Some string 1</Content> <Content>Some string 2</Content> <Metadata></Metadata> <Content> <?ACE 7?></Content> </Story> """ handler = self.HANDLER_CLASS() handler.stringset = [ OpenString(u"0", u"Some string 1", order=0), OpenString(u"1", u"Some string 2", order=1), ] compiled_story = handler._compile_story(simple_story_template) self.assertEqual(compiled_story, simple_compiled_story)
def test_compile_story_with_lts(self): regular = OpenString('0', u"hello world", order=0) with_lt = OpenString('1', u"hello <world", order=1) with_lt_escaped = OpenString('2', u"hello <world", order=2) with_mixed_lts = OpenString('3', u"hello <<world", order=3) base_template = u""" <Story> <Content>{regular}</Content> <Content>{with_lt}</Content> <Content>{with_lt_escaped}</Content> <Content>{with_mixed_lts}</Content> </Story> """ template = base_template.format( regular=regular.template_replacement, with_lt=with_lt.template_replacement, with_lt_escaped=with_lt_escaped.template_replacement, with_mixed_lts=with_mixed_lts.template_replacement, ) expected_compiled_story = base_template.format( regular=u"hello world", with_lt=u"hello <world", with_lt_escaped=u"hello <world", with_mixed_lts=u"hello <<world", ) handler = self.HANDLER_CLASS() handler.stringset = [regular, with_lt, with_lt_escaped, with_mixed_lts] compiled_story = handler._compile_story(template) self.assertEqual(compiled_story, expected_compiled_story)
def test_hash_is_calculated_from_components(self): random_key = generate_random_string() random_context = generate_random_string() random_string = "hello world" random_hash = hash((random_key, random_context, random_string)) open_string = OpenString(random_key, '') open_string.context = random_context open_string._strings[5] = random_string self.assertEqual(hash(open_string), random_hash)
def test_should_wrap_in_quotes_false_if_already_wrapped(self): openstring = OpenString('k', u' "Κάτι άλλο "') should_wrap, wrap_char = self.handler._should_wrap_in_quotes( openstring) self.assertFalse(should_wrap) self.assertIsNone(wrap_char) openstring = OpenString('k', u" 'Κάτι άλλο' ") should_wrap, wrap_char = self.handler._should_wrap_in_quotes( openstring) self.assertFalse(should_wrap) self.assertIsNone(wrap_char)
def test_should_wrap_in_quotes_if_starts_but_not_ends_with_quote(self): openstring = OpenString('k', u' " Κάτι άλλο ') should_wrap, wrap_char = self.handler._should_wrap_in_quotes( openstring) self.assertTrue(should_wrap) self.assertEqual(wrap_char, u"'") openstring = OpenString('k', u" ' Κάτι άλλο ") should_wrap, wrap_char = self.handler._should_wrap_in_quotes( openstring) self.assertTrue(should_wrap) self.assertEqual(wrap_char, u'"')
def test_template_replacement_returns_correct_suffix(self): open_string = OpenString('test', 'test') # Test when pluralized False open_string.pluralized = False template_replacement = open_string.template_replacement self.assertTrue(template_replacement.endswith('tr')) # Test when pluralized True open_string.pluralized = True template_replacement = open_string.template_replacement self.assertTrue(template_replacement.endswith('pl'))
def test_context_is_hashed_when_present(self): random_context = generate_random_string() random_key = generate_random_string() random_hash = md5(':'.join([random_key, random_context ]).encode('utf-8')).hexdigest() open_string = OpenString(random_key, 'test') open_string.context = random_context replacement = open_string.template_replacement hash_string = replacement.split('_')[0] self.assertEqual(hash_string, random_hash)
def test_skip_middle_of_list(self): string1 = self.random_string string2 = generate_random_string() string3 = generate_random_string() openstring1 = OpenString('..0..', string1, order=0) openstring3 = OpenString('..2..', string3, order=2) source = '["%s", "%s", "%s"]' % (string1, string2, string3) template, stringset = self.handler.parse(source) compiled = self.handler.compile(template, [openstring1, openstring3]) self.assertEqual(compiled, '["%s", "%s"]' % (string1, string3))
def test_unescaped(self): source = u""" { "a": { "string": "testtest", "context": "context", "developer_comment": "comments" }, "b": { "string": "testtest2" } } """ expected_compilation = u""" { "a": { "string": "testtest", "context": "other \\" context", "developer_comment": "other \\" comment" }, "b": { "string": "testtest2", "context": "other \\" context", "developer_comment": "other \\" comment" } } """ template, stringset = self.handler.parse(source) unescaped = [ OpenString( key=stringset[0].key, string_or_strings=stringset[0].string, context=u'other " context', developer_comment=u'other " comment', character_limit=stringset[0].character_limit, ), OpenString( key=stringset[1].key, string_or_strings=stringset[1].string, context=u'other " context', developer_comment=u'other " comment', character_limit=stringset[1].character_limit, ) ] compiled = self.handler.compile(template, unescaped) self.maxDiff = None self.assertEqual(compiled, expected_compilation)
def test_context_is_hashed_when_present(self): random_context = generate_random_string() random_key = generate_random_string() random_hash = md5( ':'.join([random_key, random_context]).encode('utf-8') ).hexdigest() open_string = OpenString(random_key, 'test') open_string.context = random_context replacement = open_string.template_replacement hash_string = replacement.split('_')[0] self.assertEqual(hash_string, random_hash)
def test_skip_middle_of_dict(self): string1 = self.random_string string2 = generate_random_string() string3 = generate_random_string() openstring1 = OpenString('a', string1, order=0) openstring3 = OpenString('c', string3, order=2) source = '{"a": "%s", "b": "%s", "c": "%s"}' % (string1, string2, string3) template, stringset = self.handler.parse(source) compiled = self.handler.compile(template, [openstring1, openstring3]) self.assertEqual(compiled, '{"a": "%s", "c": "%s"}' % (string1, string3))
def test_unicode(self): source = u""" { "a": { "string": "%s", "something_else": "\xa0" } } """ % self.random_string expected_compilation = u""" { "a": { "string": "\xa0", "something_else": "\xa0" } } """ template, stringset = self.handler.parse(source) with_updated_char_limit = [ OpenString( key=stringset[0].key, string_or_strings=u"\xa0", context=stringset[0].context, developer_comment=stringset[0].developer_comment, character_limit=stringset[0].character_limit, ) ] compiled = self.handler.compile(template, with_updated_char_limit) self.assertEqual(compiled, expected_compilation)
def test_template_with_values_defined_after_template(self): source = """ { "a": { "string": "%s" } } """ % self.random_string expected_compilation = """ { "a": { "string": "%s", "context": "the_context", "character_limit": 49, "developer_comment": "the_comment" } } """ % self.random_string template, stringset = self.handler.parse(source) updated_strings = [ OpenString( key=stringset[0].key, string_or_strings=stringset[0].strings, context="the_context", developer_comment="the_comment", character_limit=49, ) ] compiled = self.handler.compile(template, updated_strings) self.assertEqual(compiled, expected_compilation)
def _compile(self, payload): handler_name = payload['handler'] handler_class = self.handlers[handler_name] stringset_json = payload['stringset'] template = payload['template'] stringset = [] for string_json in stringset_json: key = string_json.pop('key') strings = { int(key): value for key, value in six.iteritems(string_json.pop('strings')) } del string_json['template_replacement'] stringset.append(OpenString(key, strings, **string_json)) handler = handler_class() try: compiled = handler.compile(template, stringset) except Exception: returned_payload = { 'action': None, 'compiled': "", 'compile_error': traceback.format_exc() } else: returned_payload = { 'action': None, 'compiled': compiled, 'compile_error': "" } return returned_payload
def test_rtl_missing_ppr(self): path = '{}/hello_world_no_ppr.docx'.format(self.TESTFILE_BASE) with open(path, 'rb') as f: content = f.read() handler = DocxHandler() template, stringset = handler.parse(content) openstring = stringset[0] # Compile with altered translation translation = u'<tx>Καλημέρα κόσμε </tx><tx href="https://el.transifex.com/">αυτός είναι ένας κρίκος</tx>' # noqa stringset = [OpenString(openstring.key, translation, order=1)] content = handler.compile(template, stringset, is_rtl=True) docx = DocxFile(content) soup = BeautifulSoup(docx.get_document(), 'xml') self.assertEqual(len(stringset), 1) self.assertEqual(len(soup.find_all("w:bidi")), 1) for pPr in soup.find_all("w:pPr"): self.assertEqual(len(pPr.findChildren("w:bidi")), 1) for bidi in pPr.findChildren("w:bidi"): self.assertEqual(bidi["w:val"], "1") self.assertTrue(len(soup.find_all("w:rtl")), 1) for rPr in soup.find_all("w:rPr"): self.assertEqual(len(rPr.findChildren("w:rtl")), 1) for rtl in rPr.findChildren("w:rtl"): self.assertEqual(rtl["w:val"], "1")
def test_lt(self): # Parse original file content = self.get_content('with_lt.docx') template, stringset = self.handler.parse(content) # Make sure extracted data is OK self.assertEqual(len(stringset), 1) openstring = stringset[0] self.assertEqual(openstring.order, 0) self.assertEqual(openstring.string, u'This is a < lessthan') self.assertEqual(openstring.string, openstring.key) # Compile with altered translation translation = U'THIS IS A < LESSTHAN' stringset = [OpenString(openstring.key, translation, order=0)] content = self.handler.compile(template, stringset) # Make sure compiled file has altered data docx = DocxFile(content) self.assertFalse("This is a" in docx.get_document()) self.assertFalse("lessthan" in docx.get_document()) self.assertTrue("THIS IS A" in docx.get_document()) self.assertTrue("LESSTHAN" in docx.get_document()) # Parse compiled file template, stringset = self.handler.parse(content) # Make sure compiled file has the correct translation self.assertEqual(len(stringset), 1) openstring = stringset[0] self.assertEqual(openstring.order, 0) self.assertEqual(openstring.string, translation) self.assertEqual(openstring.string, openstring.key)
def test_ampersand(self): content = self.get_content('with_ampersand.docx') template, stringset = self.handler.parse(content) # Make sure extracted data is OK self.assertEqual(len(stringset), 1) openstring = stringset[0] self.assertEqual(openstring.order, 0) self.assertEqual(openstring.string, u'This is an & ampersand') self.assertEqual(openstring.string, openstring.key) # Compile with altered translation translation = U'THIS IS AN & AMPERSAND' stringset = [OpenString(openstring.key, translation, order=0)] content = self.handler.compile(template, stringset) # Make sure compiled file has altered data docx = DocxFile(content) self.assertFalse("This is an" in docx.get_document()) self.assertFalse("ampersand" in docx.get_document()) self.assertTrue("THIS IS AN" in docx.get_document()) self.assertTrue("AMPERSAND" in docx.get_document()) # Parse compiled file template, stringset = self.handler.parse(content) # Make sure compiled file has the correct translation self.assertEqual(len(stringset), 1) openstring = stringset[0] self.assertEqual(openstring.order, 0) self.assertEqual(openstring.string, translation) self.assertEqual(openstring.string, openstring.key)
def test_plurals(self): random_key = generate_random_string() random_singular = generate_random_string() random_plural = generate_random_string() random_openstring = OpenString(random_key, {1: random_singular, 5: random_plural}, order=0) random_hash = random_openstring.template_replacement source = strip_leading_spaces(u""" <resources> <plurals name="{key}"> <item quantity="one">{singular}</item> <item quantity="other">{plural}</item> </plurals> </resources> """.format(key=random_key, singular=random_singular, plural=random_plural)) template, stringset = self.handler.parse(source) compiled = self.handler.compile(template, [random_openstring]) self.assertEqual( template, strip_leading_spaces(u''' <resources> <plurals name="{key}"> {hash_} </plurals> </resources> '''.format(key=random_key, hash_=random_hash)) ) self.assertEqual(len(stringset), 1) self.assertEqual(stringset[0].__dict__, random_openstring.__dict__) self.assertEqual(compiled, source)
def test_compile_for_target_language_clears_untranslatable_strings(self): string = OpenString(generate_random_string(), generate_random_string(), order=0) template = """ <resources> <string name="untranslatable" translatable="false">Untranslatable</string> <string name="{key}">{string}</string> </resources> """.format(key=string.key, string=string.template_replacement) # Try for source compiled = self.handler.compile(template, [string], is_source=True) self.assertEqual(compiled, """ <resources> <string name="untranslatable" translatable="false">Untranslatable</string> <string name="{key}">{string}</string> </resources> """.format(key=string.key, string=string.string)) # Try for translation compiled = self.handler.compile(template, [string], is_source=False) self.assertEqual(compiled, """ <resources> <string name="{key}">{string}</string> </resources> """.format(key=string.key, string=string.string))
def _handle_string_tag(self, tag, offset, comment): string = None if tag.inner.strip() != "": context = tag.attrs.get('product', "") string = OpenString(tag.attrs['name'], tag.inner, context=context, order=self._order, developer_comment=comment) self._order += 1 # ... <string name="foo">Hello .... # ^ self.transcriber.copy_until(offset + tag.inner_offset) # ... ing name="foo">Hello world</stri... # ^ if string is not None: self.transcriber.add(string.template_replacement) self.transcriber.skip(len(tag.inner)) else: self.transcriber.copy_until(offset + tag.inner_offset + len(tag.inner)) # ...ello World</string> # ^ self.transcriber.copy_until(offset + len(tag.content)) return string
def test_tools_locale(self): random_key = generate_random_string() random_string = generate_random_string() random_openstring = OpenString(random_key, random_string, order=0) random_hash = random_openstring.template_replacement source_python_template = u''' <resources tools:locale="{language_code}"> <string name="{key}">{string}</string> </resources> ''' source = source_python_template.format(language_code="en", key=random_key, string=random_string) template, stringset = self.handler.parse(source) compiled = self.handler.compile(template, [random_openstring], language_info={'code': "fr"}) self.assertEqual( template, source_python_template.format(language_code="en", key=random_key, string=random_hash) ) self.assertEqual(len(stringset), 1) self.assertEqual(stringset[0].__dict__, random_openstring.__dict__) self.assertEqual(compiled, source_python_template.format(language_code="fr", key=random_key, string=random_string))
def test_compile_ignores_removed_strings_for_dicts(self): # The original JsonHandler, when compiling back from a template, # removes any strings that are not passed as an argument in the # compile() function. StructuredJsonHandler on the other hand, simply # ignores those key-values and leave them as is in the template. This # test ensures that this is the case. # For more information, see _copy_until_and_remove_section() in both # handlers. string1 = self.random_string string2 = generate_random_string() openstring1 = self.random_openstring openstring2 = OpenString("b", string2, order=1) hash1 = self.random_hash hash2 = openstring2.template_replacement source = ('{"a": {"string":"%s"}, "b": {"string":"%s"}}' % (string1, string2)) template, stringset = self.handler.parse(source) compiled = self.handler.compile(template, [openstring1]) self.assertEqual( template, '{"a": {"string":"%s"}, "b": {"string":"%s"}}' % (hash1, hash2)) self.assertEqual(len(stringset), 2) self.assertEqual(stringset[0].__dict__, openstring1.__dict__) self.assertEqual(stringset[1].__dict__, openstring2.__dict__) self.assertEqual( compiled, '{"a": {"string":"%s"}, "b": {"string":"%s"}}' % (string1, hash2))
def test_string_array(self): random_name = generate_random_string() random_key = '{}[0]'.format(random_name) random_string = generate_random_string() random_openstring = OpenString(random_key, random_string, order=0) random_hash = random_openstring.template_replacement source_python_template = strip_leading_spaces(u''' <resources> <string-array name="{key}"> <item>{string}</item> </string-array> </resources> ''') source = source_python_template.format(key=random_name, string=random_string) template, stringset = self.handler.parse(source) compiled = self.handler.compile(template, [random_openstring]) self.assertEqual( template, source_python_template.format(key=random_name, string=random_hash)) self.assertEqual(len(stringset), 1) self.assertEqual(stringset[0].__dict__, random_openstring.__dict__) self.assertEqual(compiled, source)
def setUp(self): super(JsonTestCase, self).setUp() self.handler = JsonHandler() self.random_string = generate_random_string() self.random_openstring = OpenString("a", self.random_string, order=0) self.random_hash = self.random_openstring.template_replacement
def test_escape_control_characters(self): openstring = OpenString('k', u'者・最') # 2nd character is the backspace char self.handler._escape_invalid_chars(openstring) self.assertEqual(openstring.string, u'者\\u0008・最') openstring = OpenString('k', {5: u'者・最', 1: u'AA', 2: u'aa'}) self.handler._escape_invalid_chars(openstring) self.assertEqual( openstring.string, { 5: u'者\\u0008・最', 1: u'\\u0008AA', 2: u'\\u0008aa' }, )
def test_single_string_array_item_skipped(self): random_key = generate_random_string() random_string = generate_random_string() random_openstring = OpenString("{}[1]".format(random_key), random_string, order=0) random_hash = random_openstring.template_replacement source_template = u''' <resources> <string-array name="{key}"> <item /> <item>{string}</item> </string-array> </resources> ''' source = source_template.format(key=random_key, string=random_string) template, stringset = self.handler.parse(source) compiled = self.handler.compile(template, stringset) self.assertEqual(template, source_template.format(key=random_key, string=random_hash)) self.assertEqual(len(stringset), 1) self.assertEqual(stringset[0].__dict__, random_openstring.__dict__) self.assertEqual(compiled, source)
def test_python_values_are_ignored(self): source = '[true, "%s", 5e12]' % self.random_string random_openstring = OpenString('..1..', self.random_string, order=0) random_hash = random_openstring.template_replacement template, stringset = self.handler.parse(source) self.assertEqual(template, '[true, "%s", 5e12]' % random_hash) self.assertEqual(len(stringset), 1) self.assertEqual(stringset[0].__dict__, random_openstring.__dict__)
def test_space_control(self): content = self.get_content('special_cases_2.docx') template, stringset = self.handler.parse(content) self.assertEqual(len(stringset), 1) openstring = stringset[0] self.assertEqual(openstring.order, 0) self.assertEqual( openstring.string, u'one two <tx href="https://www.transifex.com/">three </tx><tx> four </tx>five' # noqa ) self.assertEqual(openstring.string, openstring.key) translation = u'ενα δύο <tx href="https://www.transifex.com/">τρία </tx><tx> τέσσερα </tx>πέντε' # noqa stringset = [OpenString(openstring.key, translation, order=1)] content = self.handler.compile(template, stringset) template, stringset = self.handler.parse(content) self.assertEqual(len(stringset), 1) openstring = stringset[0] self.assertEqual(openstring.order, 0) self.assertEqual( openstring.string, u'ενα δύο <tx href="https://www.transifex.com/">τρία </tx><tx> τέσσερα </tx>πέντε' ) self.assertEqual(openstring.string, openstring.key) translation = u'ενα δύο<tx href="https://www.transifex.com/">τρία</tx><tx>τέσσερα</tx>πέντε' # noqa stringset = [OpenString(openstring.key, translation, order=1)] content = self.handler.compile(template, stringset) template, stringset = self.handler.parse(content) self.assertEqual(len(stringset), 1) openstring = stringset[0] self.assertEqual(openstring.order, 0) self.assertEqual( openstring.string, u'ενα δύο<tx href="https://www.transifex.com/">τρία</tx><tx>τέσσερα</tx>πέντε' ) self.assertEqual(openstring.string, openstring.key)
def test_multiples_are_plural(self): test_strings = { i: generate_random_string() for i in six.moves.xrange(randint(5, 10)) } open_string = OpenString('test', test_strings) self.assertEqual(open_string._strings, test_strings) self.assertTrue(open_string.pluralized)
def test_plurals_are_returned(self): random_strings = { i: generate_random_string() for i in six.moves.xrange(randint(10, 20)) } open_string = OpenString('test', random_strings) self.assertTrue(open_string.pluralized) self.assertEqual(open_string.string, random_strings)
def parse(self, content, **kwargs): """ Parses the given YAML content to create stringset and template Steps are: 1. Load yaml content using our custom loader TxYamlLoader that in addition to the value for each key notes the `start` and `end` index of each node in the file and some metadata. 2. Flattens the output of the loader to be a list of the form: ``` [{ 'key': 'string_key1', 'value': 'string1', 'end': <end_index_of_node>, 'start': <start_index_value>, 'style': '|, >, ...' }, ... ] ``` 3. Iterates over the flattened list and for each entry creates an OpenString object, appends it to stringset and replace its value with the template_replacement in the template. 4. Returns the (template, stringset) tuple. """ template = [] stringset = [] # The first argument of the add_constructor method is the tag you want # to handle. If you provide None as an argument, all the unknown tags # will be handled by the constructor specified in the second argument. # We need this in order parse all unknown custom-tagged values as # strings. SafeConstructor.add_constructor(None, SafeConstructor.construct_yaml_str) yaml_data = self._load_yaml(content, loader=TxYamlLoader) yaml_data = self._get_yaml_data_to_parse(yaml_data) # Helper to store the processed data while parsing the file self._parsed_data = [] self._parse_yaml_data(yaml_data, '', '') self._parsed_data = sorted(self._parsed_data, key=lambda node: node.get('start')) end = 0 order = 0 for node in self._parsed_data: start = node.get('start') end_ = node.get('end') key = node.get('key') tag = node.get('tag') value = node.get('value') style = node.get('style') if not value: continue if isinstance(value, dict) and not all(six.itervalues(value)): continue string_object = OpenString( key, value, context=tag or '', flags=style, order=order, ) stringset.append(string_object) order += 1 template.append(u"{}{}".format(content[end:start], string_object.template_replacement)) comment = self._find_comment(content, end, start) string_object.developer_comment = comment end = end_ template.append(content[end:]) template = u''.join(template) return template, stringset