def charmap(): global _charmap if _charmap is None: f = charmap_file() if not os.path.exists(f): _charmap = {} for i in range(0, sys.maxunicode + 1): cat = unicodedata.category(hunichr(i)) rs = _charmap.setdefault(cat, []) if rs and rs[-1][-1] == i - 1: rs[-1][-1] += 1 else: rs.append([i, i]) # We explicitly set the mtime to an arbitary value so as to get # a stable format for our charmap. data = sorted( (k, tuple((map(tuple, v)))) for k, v in _charmap.items()) # Write the Unicode table atomically fd, tmpfile = tempfile.mkstemp(dir=tmpdir()) os.close(fd) with GzipFile(tmpfile, 'wb', mtime=1) as o: o.write(pickle.dumps(data, pickle.HIGHEST_PROTOCOL)) os.rename(tmpfile, f) with GzipFile(f, 'rb') as i: _charmap = dict(pickle.loads(i.read())) assert _charmap is not None return _charmap
def charmap(): global _charmap if _charmap is None: f = charmap_file() if not os.path.exists(f): _charmap = {} for i in range(0, sys.maxunicode + 1): cat = unicodedata.category(hunichr(i)) rs = _charmap.setdefault(cat, []) if rs and rs[-1][-1] == i - 1: rs[-1][-1] += 1 else: rs.append([i, i]) # We explicitly set the mtime to an arbitary value so as to get # a stable format for our charmap. data = sorted( (k, tuple((map(tuple, v)))) for k, v in _charmap.items()) # Write the Unicode table atomically fd, tmpfile = tempfile.mkstemp(dir=tmpdir()) os.close(fd) with GzipFile(tmpfile, 'wb', mtime=1) as o: o.write(pickle.dumps(data, pickle.HIGHEST_PROTOCOL)) os.rename(tmpfile, f) with GzipFile(f, 'rb') as i: _charmap = dict(pickle.loads(i.read())) assert _charmap is not None return _charmap
def charmap(): global _charmap if _charmap is None: f = charmap_file() if not os.path.exists(f): tmp_charmap = {} for i in range(0, sys.maxunicode + 1): cat = unicodedata.category(hunichr(i)) rs = tmp_charmap.setdefault(cat, []) if rs and rs[-1][-1] == i - 1: rs[-1][-1] += 1 else: rs.append([i, i]) # We explicitly set the mtime to an arbitary value so as to get # a stable format for our charmap. data = sorted((k, tuple((map(tuple, v)))) for k, v in tmp_charmap.items()) # Write the Unicode table atomically fd, tmpfile = tempfile.mkstemp(dir=tmpdir()) os.close(fd) with GzipFile(tmpfile, "wb", mtime=1) as o: o.write(pickle.dumps(data, pickle.HIGHEST_PROTOCOL)) try: os.rename(tmpfile, f) except FileExistsError: # pragma: no cover # This exception is only raised on Windows, and coverage is # measured on Linux. pass with GzipFile(f, "rb") as i: _charmap = dict(pickle.loads(i.read())) assert _charmap is not None return _charmap
def charmap(): """Return a dict that maps a Unicode category, to a tuple of 2-tuples covering the codepoint intervals for characters in that category. >>> charmap()['Co'] ((57344, 63743), (983040, 1048573), (1048576, 1114109)) """ global _charmap # Best-effort caching in the face of missing files and/or unwritable # filesystems is fairly simple: check if loaded, else try loading, # else calculate and try writing the cache. if _charmap is None: f = charmap_file() try: with gzip.GzipFile(f, "rb") as i: tmp_charmap = dict(json.loads(i)) except Exception: tmp_charmap = {} for i in range(0, sys.maxunicode + 1): cat = unicodedata.category(hunichr(i)) rs = tmp_charmap.setdefault(cat, []) if rs and rs[-1][-1] == i - 1: rs[-1][-1] += 1 else: rs.append([i, i]) try: # Write the Unicode table atomically fd, tmpfile = tempfile.mkstemp(dir=tmpdir()) os.close(fd) # Explicitly set the mtime to get reproducible output with gzip.GzipFile(tmpfile, "wb", mtime=1) as o: result = json.dumps(sorted(tmp_charmap.items())) o.write(result.encode()) os.rename(tmpfile, f) except Exception: pass # convert between lists and tuples _charmap = { k: tuple(tuple(pair) for pair in pairs) for k, pairs in tmp_charmap.items() } # each value is a tuple of 2-tuples (that is, tuples of length 2) # and that both elements of that tuple are integers. for vs in _charmap.values(): ints = list(sum(vs, ())) assert all([isinstance(x, int) for x in ints]) assert ints == sorted(ints) assert all([len(tup) == 2 for tup in vs]) assert _charmap is not None return _charmap
def charmap(): """Return a dict that maps a Unicode category, to a tuple of 2-tuples covering the codepoint intervals for characters in that category. >>> charmap()['Co'] ((57344, 63743), (983040, 1048573), (1048576, 1114109)) """ global _charmap # Best-effort caching in the face of missing files and/or unwritable # filesystems is fairly simple: check if loaded, else try loading, # else calculate and try writing the cache. if _charmap is None: f = charmap_file() try: with gzip.GzipFile(f, "rb") as i: tmp_charmap = dict(json.loads(i)) except Exception: tmp_charmap = {} for i in range(0, sys.maxunicode + 1): cat = unicodedata.category(hunichr(i)) rs = tmp_charmap.setdefault(cat, []) if rs and rs[-1][-1] == i - 1: rs[-1][-1] += 1 else: rs.append([i, i]) try: # Write the Unicode table atomically fd, tmpfile = tempfile.mkstemp(dir=tmpdir()) os.close(fd) # Explicitly set the mtime to get reproducible output with gzip.GzipFile(tmpfile, "wb", mtime=1) as o: result = json.dumps(sorted(tmp_charmap.items())) o.write(result.encode()) os.rename(tmpfile, f) except Exception: pass # convert between lists and tuples _charmap = { k: tuple(tuple(pair) for pair in pairs) for k, pairs in tmp_charmap.items() } # each value is a tuple of 2-tuples (that is, tuples of length 2) # and that both elements of that tuple are integers. for vs in _charmap.values(): ints = list(sum(vs, ())) assert all([isinstance(x, int) for x in ints]) assert ints == sorted(ints) assert all([len(tup) == 2 for tup in vs]) assert _charmap is not None return _charmap
def charmap(): """Return a dict that maps a Unicode category, to a tuple of 2-tuples covering the codepoint intervals for characters in that category. >>> charmap()['Co'] ((57344, 63743), (983040, 1048573), (1048576, 1114109)) """ global _charmap # Best-effort caching in the face of missing files and/or unwritable # filesystems is fairly simple: check if loaded, else try loading, # else calculate and try writing the cache. if _charmap is None: f = charmap_file() try: with gzip.GzipFile(f, 'rb') as i: _charmap = dict(pickle.load(i)) except Exception: tmp_charmap = {} for i in range(0, sys.maxunicode + 1): cat = unicodedata.category(hunichr(i)) rs = tmp_charmap.setdefault(cat, []) if rs and rs[-1][-1] == i - 1: rs[-1][-1] += 1 else: rs.append([i, i]) _charmap = { k: tuple(tuple(pair) for pair in pairs) for k, pairs in tmp_charmap.items() } try: # Write the Unicode table atomically fd, tmpfile = tempfile.mkstemp(dir=tmpdir()) os.close(fd) # Explicitly set the mtime to get reproducible output with gzip.GzipFile(tmpfile, 'wb', mtime=1) as o: pickle.dump(sorted(_charmap.items()), o, pickle.HIGHEST_PROTOCOL) os.rename(tmpfile, f) except Exception: # pragma: no cover pass assert _charmap is not None return _charmap
def charmap(): """Return a dict that maps a Unicode category, to a tuple of 2-tuples covering the codepoint intervals for characters in that category. >>> charmap()['Co'] ((57344, 63743), (983040, 1048573), (1048576, 1114109)) """ global _charmap # Best-effort caching in the face of missing files and/or unwritable # filesystems is fairly simple: check if loaded, else try loading, # else calculate and try writing the cache. if _charmap is None: f = charmap_file() try: with gzip.GzipFile(f, 'rb') as i: _charmap = dict(pickle.load(i)) except Exception: tmp_charmap = {} for i in range(0, sys.maxunicode + 1): cat = unicodedata.category(hunichr(i)) rs = tmp_charmap.setdefault(cat, []) if rs and rs[-1][-1] == i - 1: rs[-1][-1] += 1 else: rs.append([i, i]) _charmap = {k: tuple((map(tuple, v))) for k, v in tmp_charmap.items()} try: # Write the Unicode table atomically fd, tmpfile = tempfile.mkstemp(dir=tmpdir()) os.close(fd) # Explicitly set the mtime to get reproducible output with gzip.GzipFile(tmpfile, 'wb', mtime=1) as o: pickle.dump(sorted(_charmap.items()), o, pickle.HIGHEST_PROTOCOL) os.rename(tmpfile, f) except Exception: # pragma: no cover pass assert _charmap is not None return _charmap
def charmap(): """Return a dict that maps a Unicode category, to a tuple of 2-tuples covering the codepoint intervals for characters in that category. >>> charmap()['Co'] ((57344, 63743), (983040, 1048573), (1048576, 1114109)) """ global _charmap if _charmap is None: f = charmap_file() if not os.path.exists(f): tmp_charmap = {} for i in range(0, sys.maxunicode + 1): cat = unicodedata.category(hunichr(i)) rs = tmp_charmap.setdefault(cat, []) if rs and rs[-1][-1] == i - 1: rs[-1][-1] += 1 else: rs.append([i, i]) # We explicitly set the mtime to an arbitrary value so as to get # a stable format for our charmap. data = sorted( (k, tuple((map(tuple, v)))) for k, v in tmp_charmap.items()) # Write the Unicode table atomically fd, tmpfile = tempfile.mkstemp(dir=tmpdir()) os.close(fd) with GzipFile(tmpfile, 'wb', mtime=1) as o: o.write(pickle.dumps(data, pickle.HIGHEST_PROTOCOL)) try: os.rename(tmpfile, f) except FileExistsError: # pragma: no cover # This exception is only raised on Windows, and coverage is # measured on Linux. pass with GzipFile(f, 'rb') as i: _charmap = dict(pickle.loads(i.read())) assert _charmap is not None return _charmap