Beispiel #1
0
    def test_main(self):
        part1_data = {}
        # Hit the exception early
        try:
            open_urlresource(TESTDATAURL, encoding="utf-8")
        except IOError:
            self.skipTest("Could not retrieve " + TESTDATAURL)
        for line in open_urlresource(TESTDATAURL, encoding="utf-8"):
            if '#' in line:
                line = line.split('#')[0]
            line = line.strip()
            if not line:
                continue
            if line.startswith("@Part"):
                part = line.split()[0]
                continue
            if part == "@Part3":
                # XXX we don't support PRI #29 yet, so skip these tests for now
                continue
            try:
                c1, c2, c3, c4, c5 = [unistr(x) for x in line.split(';')[:-1]]
            except RangeError:
                # Skip unsupported characters;
                # try atleast adding c1 if we are in part1
                if part == "@Part1":
                    try:
                        c1 = unistr(line.split(';')[0])
                    except RangeError:
                        pass
                    else:
                        part1_data[c1] = 1
                continue

            # Perform tests
            self.assertTrue(c2 == NFC(c1) == NFC(c2) == NFC(c3), line)
            self.assertTrue(c4 == NFC(c4) == NFC(c5), line)
            self.assertTrue(c3 == NFD(c1) == NFD(c2) == NFD(c3), line)
            self.assertTrue(c5 == NFD(c4) == NFD(c5), line)
            self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == \
                            NFKC(c3) == NFKC(c4) == NFKC(c5),
                            line)
            self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == \
                            NFKD(c3) == NFKD(c4) == NFKD(c5),
                            line)

            # Record part 1 data
            if part == "@Part1":
                part1_data[c1] = 1

        # Perform tests for all other data
        for c in range(sys.maxunicode + 1):
            X = chr(c)
            if X in part1_data:
                continue
            self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c)
    def test_main(self):
        part1_data = {}
        # Hit the exception early
        try:
            open_urlresource(TESTDATAURL, encoding="utf-8")
        except IOError:
            self.skipTest("Could not retrieve " + TESTDATAURL)
        for line in open_urlresource(TESTDATAURL, encoding="utf-8"):
            if '#' in line:
                line = line.split('#')[0]
            line = line.strip()
            if not line:
                continue
            if line.startswith("@Part"):
                part = line.split()[0]
                continue
            if part == "@Part3":
                # XXX we don't support PRI #29 yet, so skip these tests for now
                continue
            try:
                c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]]
            except RangeError:
                # Skip unsupported characters;
                # try atleast adding c1 if we are in part1
                if part == "@Part1":
                    try:
                        c1 = unistr(line.split(';')[0])
                    except RangeError:
                        pass
                    else:
                        part1_data[c1] = 1
                continue

            # Perform tests
            self.assertTrue(c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3), line)
            self.assertTrue(c4 ==  NFC(c4) ==  NFC(c5), line)
            self.assertTrue(c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3), line)
            self.assertTrue(c5 ==  NFD(c4) ==  NFD(c5), line)
            self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == \
                            NFKC(c3) == NFKC(c4) == NFKC(c5),
                            line)
            self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == \
                            NFKD(c3) == NFKD(c4) == NFKD(c5),
                            line)

            # Record part 1 data
            if part == "@Part1":
                part1_data[c1] = 1

        # Perform tests for all other data
        for c in range(sys.maxunicode+1):
            X = chr(c)
            if X in part1_data:
                continue
            self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c)
Beispiel #3
0
    def test_named_sequences_full(self):
        # Check all the named sequences
        def check_version(testfile):
            hdr = testfile.readline()
            return unicodedata.unidata_version in hdr

        url = ("http://www.pythontest.net/unicode/%s/NamedSequences.txt" %
               unicodedata.unidata_version)
        try:
            testdata = support.open_urlresource(url,
                                                encoding="utf-8",
                                                check=check_version)
        except (OSError, HTTPException):
            self.skipTest("Could not retrieve " + url)
        self.addCleanup(testdata.close)
        for line in testdata:
            line = line.strip()
            if not line or line.startswith('#'):
                continue
            seqname, codepoints = line.split(';')
            codepoints = ''.join(chr(int(cp, 16)) for cp in codepoints.split())
            self.assertEqual(unicodedata.lookup(seqname), codepoints)
            with self.assertRaises(SyntaxError):
                self.checkletter(seqname, None)
            with self.assertRaises(KeyError):
                unicodedata.ucd_3_2_0.lookup(seqname)
Beispiel #4
0
def read_vectors(hash_name):
    with support.open_urlresource(URL.format(hash_name)) as f:
        for line in f:
            line = line.strip()
            if line.startswith('#') or not line:
                continue
            parts = line.split(',')
            parts[0] = bytes.fromhex(parts[0])
            yield parts
Beispiel #5
0
    def test_main(self):
        # Hit the exception early
        try:
            testdata = open_urlresource(TESTDATAURL, encoding="utf-8",
                                        check=check_version)
        except (OSError, HTTPException):
            self.fail(f"Could not retrieve {TESTDATAURL}")

        with testdata:
            self.run_normalization_tests(testdata)
    def test_main(self):
        # Hit the exception early
        try:
            testdata = open_urlresource(TESTDATAURL,
                                        encoding="utf-8",
                                        check=check_version)
        except (OSError, HTTPException):
            self.fail(f"Could not retrieve {TESTDATAURL}")

        with testdata:
            self.run_normalization_tests(testdata)
Beispiel #7
0
    def test_main(self):
        # Hit the exception early
        try:
            testdata = open_urlresource(TESTDATAURL, encoding="utf-8",
                                        check=check_version)
        except PermissionError:
            self.skipTest(f"Permission error when downloading {TESTDATAURL} "
                          f"into the test data directory")
        except (OSError, HTTPException):
            self.fail(f"Could not retrieve {TESTDATAURL}")

        with testdata:
            self.run_normalization_tests(testdata)
Beispiel #8
0
def read_vectors(hash_name):
    url = URL.format(hash_name)
    try:
        testdata = support.open_urlresource(url, encoding="utf-8")
    except (OSError, HTTPException):
        raise unittest.SkipTest("Could not retrieve {}".format(url))
    with testdata:
        for line in testdata:
            line = line.strip()
            if line.startswith('#') or not line:
                continue
            parts = line.split(',')
            parts[0] = bytes.fromhex(parts[0])
            yield parts
Beispiel #9
0
def read_vectors(hash_name):
    url = URL.format(hash_name)
    try:
        testdata = support.open_urlresource(url)
    except (OSError, HTTPException):
        raise unittest.SkipTest("Could not retrieve {}".format(url))
    with testdata:
        for line in testdata:
            line = line.strip()
            if line.startswith('#') or not line:
                continue
            parts = line.split(',')
            parts[0] = bytes.fromhex(parts[0])
            yield parts
Beispiel #10
0
    def test_main(self):
        # Hit the exception early
        try:
            testdata = open_urlresource(TESTDATAURL,
                                        encoding="utf-8",
                                        check=check_version)
        except PermissionError:
            self.skipTest(f"Permission error when downloading {TESTDATAURL} "
                          f"into the test data directory")
        except (OSError, HTTPException):
            self.fail(f"Could not retrieve {TESTDATAURL}")

        with testdata:
            self.run_normalization_tests(testdata)
    def test_main(self):
        part = None
        part1_data = {}
        # Hit the exception early
        try:
            testdata = open_urlresource(TESTDATAURL, encoding="utf-8", check=check_version)
        except (OSError, HTTPException):
            self.skipTest("Could not retrieve " + TESTDATAURL)
        self.addCleanup(testdata.close)
        for line in testdata:
            if "#" in line:
                line = line.split("#")[0]
            line = line.strip()
            if not line:
                continue
            if line.startswith("@Part"):
                part = line.split()[0]
                continue
            try:
                c1, c2, c3, c4, c5 = [unistr(x) for x in line.split(";")[:-1]]
            except RangeError:
                # Skip unsupported characters;
                # try at least adding c1 if we are in part1
                if part == "@Part1":
                    try:
                        c1 = unistr(line.split(";")[0])
                    except RangeError:
                        pass
                    else:
                        part1_data[c1] = 1
                continue

            # Perform tests
            self.assertTrue(c2 == NFC(c1) == NFC(c2) == NFC(c3), line)
            self.assertTrue(c4 == NFC(c4) == NFC(c5), line)
            self.assertTrue(c3 == NFD(c1) == NFD(c2) == NFD(c3), line)
            self.assertTrue(c5 == NFD(c4) == NFD(c5), line)
            self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5), line)
            self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5), line)

            # Record part 1 data
            if part == "@Part1":
                part1_data[c1] = 1

        # Perform tests for all other data
        for c in range(sys.maxunicode + 1):
            X = chr(c)
            if X in part1_data:
                continue
            self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c)
Beispiel #12
0
 def test_main(self):
     part = None
     part1_data = {}
     try:
         testdata = open_urlresource(TESTDATAURL,
                                     encoding='utf-8',
                                     check=check_version)
     except (OSError, HTTPException):
         self.skipTest('Could not retrieve ' + TESTDATAURL)
     self.addCleanup(testdata.close)
     for line in testdata:
         if '#' in line:
             line = line.split('#')[0]
         line = line.strip()
         if not line:
             continue
         if line.startswith('@Part'):
             part = line.split()[0]
             continue
         try:
             c1, c2, c3, c4, c5 = [unistr(x) for x in line.split(';')[:-1]]
         except RangeError:
             if part == '@Part1':
                 try:
                     c1 = unistr(line.split(';')[0])
                 except RangeError:
                     pass
                 else:
                     part1_data[c1] = 1
             continue
         self.assertTrue(c2 == NFC(c1) == NFC(c2) == NFC(c3), line)
         self.assertTrue(c4 == NFC(c4) == NFC(c5), line)
         self.assertTrue(c3 == NFD(c1) == NFD(c2) == NFD(c3), line)
         self.assertTrue(c5 == NFD(c4) == NFD(c5), line)
         self.assertTrue(
             c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5),
             line)
         self.assertTrue(
             c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5),
             line)
         if part == '@Part1':
             part1_data[c1] = 1
     for c in range(sys.maxunicode + 1):
         X = chr(c)
         if X in part1_data:
             continue
         self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c)
Beispiel #13
0
    def test_normalization(self):
        TESTDATAFILE = "NormalizationTest.txt"
        TESTDATAURL = f"http://www.pythontest.net/unicode/{unicodedata.unidata_version}/{TESTDATAFILE}"

        # Hit the exception early
        try:
            testdata = open_urlresource(TESTDATAURL,
                                        encoding="utf-8",
                                        check=self.check_version)
        except PermissionError:
            self.skipTest(f"Permission error when downloading {TESTDATAURL} "
                          f"into the test data directory")
        except (OSError, HTTPException):
            self.fail(f"Could not retrieve {TESTDATAURL}")

        with testdata:
            self.run_normalization_tests(testdata)
Beispiel #14
0
 def test_named_sequences_full(self):
     # Check all the named sequences
     url = ("http://www.unicode.org/Public/%s/ucd/NamedSequences.txt" %
            unicodedata.unidata_version)
     try:
         testdata = support.open_urlresource(url, encoding="utf-8",
                                             check=check_version)
     except (IOError, HTTPException):
         self.skipTest("Could not retrieve " + url)
     self.addCleanup(testdata.close)
     for line in testdata:
         line = line.strip()
         if not line or line.startswith('#'):
             continue
         seqname, codepoints = line.split(';')
         codepoints = ''.join(chr(int(cp, 16)) for cp in codepoints.split())
         self.assertEqual(unicodedata.lookup(seqname), codepoints)
         with self.assertRaises(SyntaxError):
             self.checkletter(seqname, None)
         with self.assertRaises(KeyError):
             unicodedata.ucd_3_2_0.lookup(seqname)
 def open_mapping_file(self):
     return support.open_urlresource(self.mapfileurl)
 def open_mapping_file(self):
     return support.open_urlresource(self.mapfileurl)
Beispiel #17
0
def test_main():
    # Hit the exception early
    open_urlresource(TESTDATAURL)
    run_unittest(NormalizationTest)
def test_main():
    # Hit the exception early
    open_urlresource(TESTDATAURL)
    run_unittest(NormalizationTest)
Beispiel #19
0
 def open_mapping_file(self):
     return support.open_urlresource(self.mapfileurl, encoding="utf-8")