def patch_hyphen(srcdir, dstdir, copy_unchanged=True):
"""Add hyphen-minus glyphs to fonts that need it.
This is to enable languages to be hyphenated properly,
since Minikin's itemizer currently shows tofus if an
automatically hyphenated word is displated in a font
that has neither HYPHEN nor HYPHEN-MINUS.
The list of font names comes from LANG_TO_SCRIPT in
tools/font/fontchain_lint.py.
(In practice only U+002D HYPHEN-MINUS is added, since Noto LGC fonts
don't have U+2010 HYPHEN.)
Bug: 21570828"""
# Names of fonts for which Android requires a hyphen.
# This list omits Japanese and Korean.
script_names = [
'Armenian', 'Ethiopic', 'Bengali', 'Gujarati', 'Devanagari',
'Kannada', 'Malayalam', 'Oriya', 'Gurmukhi', 'Tamil', 'Telugu']
HYPHENS = {0x002D, 0x2010}
for sn in script_names:
globexp = path.join(srcdir, 'Noto*%s-*.ttf' % sn)
fonts = glob.glob(globexp)
if not fonts:
raise ValueError('could not match ' + globexp)
fonts = [path.basename(f) for f in fonts]
for font_name in fonts:
lgc_font_name = font_name.replace(sn, '')
font_file = path.join(srcdir, font_name)
lgc_font_file = path.join(srcdir, lgc_font_name)
chars_to_add = (
(HYPHENS - coverage.character_set(font_file))
& coverage.character_set(lgc_font_file))
if chars_to_add:
print 'patch hyphens', font_name
merger.merge_chars_from_bank(
path.join(srcdir, font_name),
path.join(srcdir, lgc_font_name),
path.join(srcdir, font_name),
chars_to_add)
else:
if copy_unchanged:
shutil.copy2(
path.join(srcdir,font_name), path.join(dstdir, font_name))
print '%s already has hyphens, copying' % font_name
else:
print '%s already has hyphens' % font_name
def main(argv):
"""Fix all the fonts given in the command line.
If they are Lao fonts, make sure they have ZWSP and dotted circle. If they
are Khmer fonts, make sure they have ZWSP, joiners, and dotted circle."""
for font_name in argv[1:]:
if 'Khmer' in font_name:
script = 'Khmr'
elif 'Lao' in font_name:
script = 'Laoo'
needed_chars = set(opentype_data.SPECIAL_CHARACTERS_NEEDED[script])
lgc_font_name = (
os.path.basename(font_name).replace('Khmer', '').replace('Lao', ''))
lgc_font_name = os.path.join(_UNHINTED_FONTS_DIR, lgc_font_name)
font_charset = coverage.character_set(font_name)
missing_chars = needed_chars - font_charset
if missing_chars:
merge_chars_from_bank(
font_name,
lgc_font_name,
os.path.dirname(font_name)+'/new/'+os.path.basename(font_name),
missing_chars)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-f',
'--font',
help='font whose character map to restrict text to',
required=True)
parser.add_argument('-t',
'--text',
help='initial text, prepend @ to read from file')
args = parser.parse_args()
if args.text:
if args.text[0] == '@':
with codecs.open(args.text[1:], 'r', 'utf-8') as f:
text = f.read()
else:
text = args.text
else:
text = ''
if args.font:
charset = coverage.character_set(args.font)
name_map = _get_char_names(charset)
text = _build_text(name_map, text)
print('text: ' + text)
else:
charset = None
def test_lack_of_unassigned_chars(self):
"""Tests that unassigned characters are not in the fonts."""
for font in self.fonts:
charset = coverage.character_set(font)
self.assertNotIn(0x2072, charset)
self.assertNotIn(0x2073, charset)
self.assertNotIn(0x208F, charset)
def test_lack_of_arrows_and_combining_keycap(self):
"""Tests that arrows and combining keycap are not in the fonts."""
for font in self.fonts:
charset = coverage.character_set(font)
self.assertNotIn(0x20E3, charset) # COMBINING ENCLOSING KEYCAP
self.assertNotIn(0x2191, charset) # UPWARDS ARROW
self.assertNotIn(0x2193, charset) # DOWNWARDS ARROW
def test_lack_of_unassigned_chars(self):
"""Tests that unassigned characters are not in the fonts."""
for font in self.fonts:
charset = coverage.character_set(font)
self.assertNotIn(0x2072, charset)
self.assertNotIn(0x2073, charset)
self.assertNotIn(0x208F, charset)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'-f', '--font',
help='font whose character map to restrict text to',
required=True)
parser.add_argument(
'-t', '--text',
help='initial text, prepend @ to read from file')
args = parser.parse_args()
if args.text:
if args.text[0] == '@':
with codecs.open(args.text[1:], 'r', 'utf-8') as f:
text = f.read()
else:
text = args.text
else:
text = ''
if args.font:
charset = coverage.character_set(args.font)
name_map = _get_char_names(charset)
text = _build_text(name_map, text)
print 'text: ' + text
else:
charset = None
def test_inclusion_of_sound_recording_copyright(self):
"""Tests that sound recording copyright symbol is in the fonts."""
for font in self.fonts:
charset = coverage.character_set(font)
self.assertIn(
0x2117, charset, # SOUND RECORDING COPYRIGHT
'U+2117 not found in %s.' % font_data.font_name(font))
def test_inclusion(self):
"""Tests for characters which should be included."""
for font in self.fonts:
charset = coverage.character_set(font)
for char in self.include:
self.assertIn(char, charset)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"-f",
"--font",
help="font whose character map to restrict text to",
required=True,
)
parser.add_argument("-t",
"--text",
help="initial text, prepend @ to read from file")
args = parser.parse_args()
if args.text:
if args.text[0] == "@":
with codecs.open(args.text[1:], "r", "utf-8") as f:
text = f.read()
else:
text = args.text
else:
text = ""
if args.font:
charset = coverage.character_set(args.font)
name_map = _get_char_names(charset)
text = _build_text(name_map, text)
print("text: " + text)
else:
charset = None
def test_rendering(data,
font_file_name,
min_allowed,
max_allowed,
language=None):
"""Test the rendering of the input data in a given font.
The input data is first filtered for sequences supported in the font.
"""
font_characters = coverage.character_set(font_file_name)
# Hack to add ASCII digits, even if the font doesn't have them,
# to keep potential frequency info in the input intact
font_characters |= set(range(ord("0"), ord("9") + 1))
supported_chars_regex = _regular_expression_from_set(font_characters)
harfbuzz_input = []
for match in supported_chars_regex.finditer(data):
harfbuzz_input.append(match.group(0))
harfbuzz_input = "\n".join(harfbuzz_input)
return render.test_text_vertical_extents(harfbuzz_input, font_file_name,
min_allowed, max_allowed,
language)
def test_exclusion(self):
"""Tests that characters which should be excluded are absent."""
for font in self.fonts:
charset = coverage.character_set(font)
for char in self.exclude:
self.assertNotIn(char, charset)
def test_inclusion(self):
"""Tests for characters which should be included."""
for font in self.fonts:
charset = coverage.character_set(font)
for char in self.include:
self.assertIn(char, charset)
def setUp(self):
self.font_files, _ = self.loaded_fonts
charset = coverage.character_set(self.font_files[0])
self.marks_to_test = [
char for char in charset if unicode_data.combining(char) == 230
]
self.advance_cache = {}
def test_exclusion(self):
"""Tests that characters which should be excluded are absent."""
for font in self.fonts:
charset = coverage.character_set(font)
for char in self.exclude:
self.assertNotIn(char, charset)
def main(argv):
"""Fix all the fonts given in the command line.
If they are Lao fonts, make sure they have ZWSP and dotted circle. If they
are Khmer fonts, make sure they have ZWSP, joiners, and dotted circle."""
for font_name in argv[1:]:
if 'Khmer' in font_name:
script = 'Khmr'
elif 'Lao' in font_name:
script = 'Laoo'
needed_chars = set(opentype_data.SPECIAL_CHARACTERS_NEEDED[script])
lgc_font_name = (
os.path.basename(font_name).replace('Khmer', '').replace('Lao', ''))
lgc_font_name = os.path.join(_UNHINTED_FONTS_DIR, lgc_font_name)
font_charset = coverage.character_set(font_name)
missing_chars = needed_chars - font_charset
if missing_chars:
merge_chars_from_bank(
font_name,
lgc_font_name,
os.path.dirname(font_name)+'/new/'+os.path.basename(font_name),
missing_chars)
def test_inclusion_of_sound_recording_copyright(self):
"""Tests that sound recording copyright symbol is in the fonts."""
for font in self.fonts:
charset = coverage.character_set(font)
self.assertIn(
0x2117,
charset, # SOUND RECORDING COPYRIGHT
'U+2117 not found in %s.' % font_data.font_name(font))
def setUp(self):
self.font_files, _ = self.loaded_fonts
charset = coverage.character_set(self.font_files[0])
self.marks_to_test = [
char for char in charset
if unicode_data.category(char) in ['Lm', 'Sk']
]
self.advance_cache = {}
def test_non_inclusion_of_other_pua(self):
"""Tests that there are not other PUA characters except legacy ones."""
for font in self.fonts:
charset = coverage.character_set(font)
pua_chars = {
char for char in charset
if 0xE000 <= char <= 0xF8FF or 0xF0000 <= char <= 0x10FFFF}
self.assertTrue(pua_chars <= self.LEGACY_PUA)
def get_families(fonts):
"""Group fonts into families, separate into hinted and unhinted, select
representative."""
family_id_to_fonts = collections.defaultdict(set)
families = {}
for font in fonts:
family_id = noto_font_to_family_id(font)
family_id_to_fonts[family_id].add(font)
for family_id, fonts in family_id_to_fonts.items():
hinted_members = []
unhinted_members = []
rep_member = None
rep_backup = None # used in case all fonts are ttc fonts
for font in fonts:
if font.is_hinted:
hinted_members.append(font)
else:
unhinted_members.append(font)
if not rep_member:
if (font.weight == "Regular" and font.slope is None
and not (font.is_cjk and font.is_mono)
and not font.is_UI):
# We assume here that there's no difference between a hinted or
# unhinted rep_member in terms of what we use it for. The other
# filters are to ensure the fontTools font name is a good stand-in
# for the family name.
if font.fmt == "ttc" and not rep_backup:
rep_backup = font
else:
rep_member = font
rep_member = rep_member or rep_backup
if not rep_member:
raise ValueError("Family %s does not have a representative font." %
family_id)
name = get_font_family_name(rep_member.filepath)
if rep_member.fmt in {"ttf", "otf"}:
charset = coverage.character_set(rep_member.filepath)
else:
# was NotImplemented, but bool(NotImplemented) is True
charset = None
families[family_id] = NotoFamily(
name,
family_id,
rep_member,
charset,
hinted_members,
unhinted_members,
)
return families
def subset_font(source_file,
target_file,
include=None,
exclude=None,
options=None):
"""Subsets a font file.
Subsets a font file based on a specified character set. If only include is
specified, only characters from that set would be included in the output
font. If only exclude is specified, all characters except those in that
set will be included. If neither is specified, the character set will
remain the same, but inaccessible glyphs will be removed.
Args:
source_file: Input file name.
target_file: Output file name
include: The list of characters to include from the source font.
exclude: The list of characters to exclude from the source font.
options: A dictionary listing which options should be different from the
default.
Raises:
NotImplementedError: Both include and exclude were specified.
"""
opt = subset.Options()
opt.name_IDs = ["*"]
opt.name_legacy = True
opt.name_languages = ["*"]
opt.layout_features = ["*"]
opt.notdef_outline = True
opt.recalc_bounds = True
opt.recalc_timestamp = True
opt.canonical_order = True
opt.drop_tables = ["+TTFA"]
if options is not None:
for name, value in options.items():
setattr(opt, name, value)
if include is not None:
if exclude is not None:
raise NotImplementedError(
"Subset cannot include and exclude a set at the same time.")
target_charset = include
else:
if exclude is None:
exclude = []
source_charset = coverage.character_set(source_file)
target_charset = source_charset - set(exclude)
font = subset.load_font(source_file, opt)
subsetter = subset.Subsetter(options=opt)
subsetter.populate(unicodes=target_charset)
subsetter.subset(font)
subset.save_font(font, target_file, opt)
def test_sanity(self):
"""Test basic sanity of the method."""
font_file = tempfile.NamedTemporaryFile()
font = make_font('')
font.save(font_file.name)
charset = coverage.character_set(font_file.name)
self.assertTrue(ord(' ') in charset)
self.assertTrue(ord('A') in charset)
self.assertFalse(0x10B00 in charset)
def test_non_inclusion_of_other_pua(self):
"""Tests that there are not other PUA characters except legacy ones."""
for font in self.fonts:
charset = coverage.character_set(font)
pua_chars = {
char
for char in charset
if 0xE000 <= char <= 0xF8FF or 0xF0000 <= char <= 0x10FFFF
}
self.assertTrue(pua_chars <= self.LEGACY_PUA)
def get_cps_from_files(paths):
"""Return a tuple of the cps and the paths that we actually read."""
cps = set()
new_paths = set()
for f in paths:
ext = path.splitext(f)[1]
if ext not in ['.otf', '.ttf']:
continue
cps |= coverage.character_set(f)
new_paths.add(f)
return cps, sorted(new_paths)
def get_cps_from_files(paths):
"""Return a tuple of the cps and the paths that we actually read."""
cps = set()
new_paths = set()
for f in paths:
ext = path.splitext(f)[1]
if ext not in ['.otf', '.ttf']:
continue
cps |= coverage.character_set(f)
new_paths.add(f)
return cps, sorted(new_paths)
def get_families(fonts):
"""Group fonts into families, group by hinted and unhinted, select representative."""
families = {}
all_keys = set([font.key for font in fonts])
for key in all_keys:
members = {font for font in fonts
if font.key == key and font.variant != 'UI' and font.fmt in {'ttf', 'otf'}}
if not members:
mbrs = {font for font in fonts if font.key == key}
raise ValueError("no members for %s from %s" % (key, [f.filepath for f in mbrs]))
hinted_members = []
unhinted_members = []
rep_member = None
for font in members:
if font.hint_status == 'hinted':
hinted_members.append(font)
else:
unhinted_members.append(font)
if not rep_member:
if font.weight == 'Regular' and font.slope is None:
# We assume here that there's no difference between a hinted or unhinted
# rep_member in terms of what we use it for
rep_member = font
if not rep_member:
raise ValueError('Family %s does have a representative font.' % key)
if hinted_members and not len(hinted_members) in [1, 2, 4, 7]:
raise ValueError('Family %s has %d hinted_members (%s)' % (
key, len(hinted_members), [font.name for font in hinted_members]))
if unhinted_members and not len(unhinted_members) in [1, 2, 4, 7]:
raise ValueError('Family %s has %d unhinted_members (%s)' % (
key, len(unhinted_members), [font.name for font in unhinted_members]))
if hinted_members and unhinted_members and len(hinted_members) != len(unhinted_members):
raise ValueError('Family %s has %d hinted members but %d unhinted memberts' % (
key, len(hinted_members), len(unhinted_members)))
name = get_font_family_name(rep_member.filepath)
if rep_member.fmt in {'ttf', 'otf'}:
charset = coverage.character_set(rep_member.filepath)
else:
charset = NotImplemented
families[key] = Family(name, rep_member, charset, hinted_members, unhinted_members)
return families
def get_families(fonts):
"""Group fonts into families, separate into hinted and unhinted, select
representative."""
family_id_to_fonts = collections.defaultdict(set)
families = {}
for font in fonts:
family_id = noto_font_to_family_id(font)
family_id_to_fonts[family_id].add(font)
for family_id, fonts in family_id_to_fonts.iteritems():
hinted_members = []
unhinted_members = []
rep_member = None
rep_backup = None # used in case all fonts are ttc fonts
for font in fonts:
if font.is_hinted:
hinted_members.append(font)
else:
unhinted_members.append(font)
if not rep_member:
if font.weight == 'Regular' and font.slope is None and not (
font.is_cjk and font.is_mono) and not font.is_UI:
# We assume here that there's no difference between a hinted or
# unhinted rep_member in terms of what we use it for. The other
# filters are to ensure the fontTools font name is a good stand-in
# for the family name.
if font.fmt == 'ttc' and not rep_backup:
rep_backup = font
else:
rep_member = font
rep_member = rep_member or rep_backup
if not rep_member:
raise ValueError(
'Family %s does not have a representative font.' % family_id)
name = get_font_family_name(rep_member.filepath)
if rep_member.fmt in {'ttf', 'otf'}:
charset = coverage.character_set(rep_member.filepath)
else:
# was NotImplemented, but bool(NotImplemented) is True
charset = None
families[family_id] = NotoFamily(
name, family_id, rep_member, charset, hinted_members, unhinted_members)
return families
def main():
"""Checkes the coverage of all Roboto fonts."""
with open('res/char_requirements.tsv') as char_reqs_file:
char_reqs_data = char_reqs_file.read()
# The format of the data to be parsed is like the following:
# General Punctuation\t2000..206F\t111\t35\t54\t0\tEverything except 2028..202E, 2060..2064, and 2066..206F
# Currency Symbols\t20A0..20CF\t29\t5\t24\t1\t
required_set = set()
for line in char_reqs_data.split('\n'):
if line.startswith('#'): # Skip comment lines
continue
line = line.split('\t')
if not line[0]:
continue # Skip the first line and empty lines
block_range = line[1]
full_coverage_required = (line[5] == '1')
exceptions = line[6]
required_set.update(
_find_required_chars(block_range, full_coverage_required,
exceptions))
# Skip Unicode 8.0 characters
required_set = {
ch
for ch in required_set if float(unicode_data.age(ch)) <= 7.0
}
# Skip ASCII and C1 controls
required_set -= set(range(0, 0x20) + range(0x7F, 0xA0))
missing_char_found = False
for font in load_fonts():
font_coverage = coverage.character_set(font)
missing_chars = required_set - font_coverage
if missing_chars:
missing_char_found = True
font_name = font_data.font_name(font)
print 'Characters missing from %s:' % font_name
for char in sorted(missing_chars):
_print_char(char)
print
if missing_char_found:
sys.exit(1)
def main():
"""Checkes the coverage of all Roboto fonts."""
with open('res/char_requirements.tsv') as char_reqs_file:
char_reqs_data = char_reqs_file.read()
# The format of the data to be parsed is like the following:
# General Punctuation\t2000..206F\t111\t35\t54\t0\tEverything except 2028..202E, 2060..2064, and 2066..206F
# Currency Symbols\t20A0..20CF\t29\t5\t24\t1\t
required_set = set()
for line in char_reqs_data.split('\n'):
if line.startswith('#'): # Skip comment lines
continue
line = line.split('\t')
if not line[0]:
continue # Skip the first line and empty lines
block_range = line[1]
full_coverage_required = (line[5] == '1')
exceptions = line[6]
required_set.update(
_find_required_chars(block_range,
full_coverage_required,
exceptions))
# Skip Unicode 8.0 characters
required_set = {ch for ch in required_set
if float(unicode_data.age(ch)) <= 7.0}
# Skip ASCII and C1 controls
required_set -= set(range(0, 0x20) + range(0x7F, 0xA0))
missing_char_found = False
for font in load_fonts():
font_coverage = coverage.character_set(font)
missing_chars = required_set - font_coverage
if missing_chars:
missing_char_found = True
font_name = font_data.font_name(font)
print 'Characters missing from %s:' % font_name
for char in sorted(missing_chars):
_print_char(char)
print
if missing_char_found:
sys.exit(1)
def test_all_combinations(max_len,
font_file_name,
min_allowed,
max_allowed,
language=None):
"""Tests the rendering of all combinations up to certain length."""
font_characters = coverage.character_set(font_file_name)
font_characters -= set(range(0x00, 0x20)) # Remove ASCII controls
font_characters = [unichr(code) for code in font_characters]
font_characters = sorted(font_characters)
all_strings = []
for length in range(1, max_len + 1):
all_combinations = itertools.product(font_characters, repeat=length)
all_strings += ["".join(comb) for comb in all_combinations]
test_data = "\n".join(all_strings)
return test_rendering(test_data, font_file_name, min_allowed, max_allowed,
language)
def find_fonts():
font_name_regexp = re.compile(
'(NotoSans|NotoSerif|NotoNaskh|NotoKufi|Arimo|Cousine|Tinos)'
'(Mono)?'
'(.*?)'
'(UI|Eastern|Estrangela|Western)?'
'-'
'(|Black|Bold|DemiLight|Light|Medium|Regular|Thin)'
'(Italic)?'
'(-Windows)?'
'.[ot]t[cf]')
unicode_data.load_data()
for directory in [
path.join(FONT_DIR, 'hinted'),
path.join(FONT_DIR, 'unhinted'),
path.join(FONT_DIR, 'alpha'), CJK_DIR
]:
for filename in os.listdir(directory):
match = font_name_regexp.match(filename)
if match:
family, mono, script, variant, weight, style, platform = match.groups(
)
elif filename == 'NotoNastaliqUrduDraft.ttf':
family = 'NotoNastaliq'
script = 'Aran' # Arabic Nastaliq
weight = ''
style = variant = platform = None
else:
if not (filename == 'NotoSansCJK.ttc.zip'
or # All-comprehensive CJK
filename.endswith('.ttx') or filename.endswith('.git')
or filename.startswith('README.') or filename
in ['COPYING', 'LICENSE', 'NEWS', 'HISTORY']):
raise ValueError("unexpected filename in %s: '%s'." %
(directory, filename))
continue
if directory == CJK_DIR:
license_type = 'sil'
else:
license_type = 'apache'
if mono:
# we don't provide the Mono CJK on the website
continue
if script == "Historic":
# we don't provide this either
continue
if family in {'Arimo', 'Cousine', 'Tinos'}:
continue # Skip these three for the website
if family.startswith('Noto'):
family = family.replace('Noto', 'Noto ')
if weight == '':
weight = 'Regular'
assert platform is None
if script == '': # LGC
supported_scripts.update({'Latn', 'Grek', 'Cyrl'})
elif script == 'Aran':
supported_scripts.add(script)
elif script in {'JP', 'KR', 'SC', 'TC', 'CJK'}:
continue # Skip unified or old CJK fonts
else:
script = convert_to_four_letter(script)
supported_scripts.add(script)
file_path = path.join(directory, filename)
if filename.endswith('.ttf') or filename.endswith('.otf'):
charset = coverage.character_set(file_path)
else:
charset = NotImplemented
if directory == CJK_DIR:
hint_status = 'hinted'
elif directory.endswith('alpha'):
hint_status = 'unhinted'
else:
hint_status = path.basename(directory)
assert hint_status in ['hinted', 'unhinted']
key = family.replace(' ', '-')
if script:
key += '-' + script
if variant not in {None, 'UI'}:
key += '-' + variant
key = key.lower()
font = Font(file_path, hint_status, key, family, script, variant,
weight, style, platform, charset, license_type)
all_fonts.append(font)
def get_families(fonts):
"""Group fonts into families, separate into hinted and unhinted, select representative."""
family_id_to_fonts = collections.defaultdict(set)
families = {}
for font in fonts:
family_id = noto_font_to_family_id(font)
family_id_to_fonts[family_id].add(font)
for family_id, fonts in family_id_to_fonts.iteritems():
hinted_members = []
unhinted_members = []
rep_member = None
for font in fonts:
if font.is_hinted:
hinted_members.append(font)
else:
unhinted_members.append(font)
if not rep_member:
if font.weight == 'Regular' and font.slope is None and not font.is_mono:
# We assume here that there's no difference between a hinted or unhinted
# rep_member in terms of what we use it for. The other filters are to ensure
# the fontTools font name is a good stand-in for the family name.
rep_member = font
if not rep_member:
raise ValueError('Family %s does not have a representative font.' %
family_id)
if hinted_members and not len(hinted_members) in [
1, 2, 4, 7, 9
]: # 9 adds the two Mono variants
raise ValueError(
'Family %s has %d hinted_members (%s)' %
(family_id, len(hinted_members),
[path.basename(font.filepath) for font in hinted_members]))
if unhinted_members and not len(unhinted_members) in [1, 2, 4]:
raise ValueError(
'Family %s has %d unhinted_members (%s)' %
(family_id, len(unhinted_members),
[path.basename(font.filepath) for font in unhinted_members]))
if hinted_members and unhinted_members and len(hinted_members) != len(
unhinted_members):
# Let's not consider this an error for now. Just drop the members with the higher number
# of fonts, assuming it's a superset of the fonts in the smaller set, so that the fonts
# we provide and display are available to all users. This means website users will not be
# able to get these fonts via the website.
#
# We'll keep the representative font and not try to change it.
print 'Family %s has %d hinted members but %d unhinted memberts' % (
family_id, len(hinted_members), len(unhinted_members))
if len(hinted_members) < len(unhinted_members):
unhinted_members = []
else:
hinted_members = []
name = get_font_family_name(rep_member.filepath)
if rep_member.fmt in {'ttf', 'otf'}:
charset = coverage.character_set(rep_member.filepath)
else:
charset = NotImplemented
families[family_id] = NotoFamily(name, family_id, rep_member, charset,
hinted_members, unhinted_members)
return families
def get_families(fonts):
"""Group fonts into families, group by hinted and unhinted, select representative."""
families = {}
all_keys = set([font.key for font in fonts])
for key in all_keys:
members = {
font
for font in fonts if font.key == key and font.variant != 'UI'
and font.fmt in {'ttf', 'otf'}
}
if not members:
mbrs = {font for font in fonts if font.key == key}
raise ValueError("no members for %s from %s" %
(key, [f.filepath for f in mbrs]))
hinted_members = []
unhinted_members = []
rep_member = None
for font in members:
if font.hint_status == 'hinted':
hinted_members.append(font)
else:
unhinted_members.append(font)
if not rep_member:
if font.weight == 'Regular' and font.slope is None:
# We assume here that there's no difference between a hinted or unhinted
# rep_member in terms of what we use it for
rep_member = font
if not rep_member:
raise ValueError('Family %s does have a representative font.' %
key)
if hinted_members and not len(hinted_members) in [1, 2, 4, 7]:
raise ValueError('Family %s has %d hinted_members (%s)' %
(key, len(hinted_members),
[font.name for font in hinted_members]))
if unhinted_members and not len(unhinted_members) in [1, 2, 4, 7]:
raise ValueError('Family %s has %d unhinted_members (%s)' %
(key, len(unhinted_members),
[font.name for font in unhinted_members]))
if hinted_members and unhinted_members and len(hinted_members) != len(
unhinted_members):
raise ValueError(
'Family %s has %d hinted members but %d unhinted memberts' %
(key, len(hinted_members), len(unhinted_members)))
name = get_font_family_name(rep_member.filepath)
if rep_member.fmt in {'ttf', 'otf'}:
charset = coverage.character_set(rep_member.filepath)
else:
charset = NotImplemented
families[key] = Family(name, rep_member, charset, hinted_members,
unhinted_members)
return families
def patch_hyphen(srcdir, dstdir, copy_unchanged=True):
"""Add hyphen-minus glyphs to fonts that need it.
This is to enable languages to be hyphenated properly,
since Minikin's itemizer currently shows tofus if an
automatically hyphenated word is displated in a font
that has neither HYPHEN nor HYPHEN-MINUS.
The list of font names comes from LANG_TO_SCRIPT in
tools/font/fontchain_lint.py.
(In practice only U+002D HYPHEN-MINUS is added, since Noto LGC fonts
don't have U+2010 HYPHEN.)
Bug: 21570828"""
# Names of fonts for which Android requires a hyphen.
# This list omits Japanese and Korean.
script_names = [
"Armenian",
"Ethiopic",
"Bengali",
"Gujarati",
"Devanagari",
"Kannada",
"Malayalam",
"Oriya",
"Gurmukhi",
"Tamil",
"Telugu",
]
HYPHENS = {0x002D, 0x2010}
for sn in script_names:
globexp = path.join(srcdir, "Noto*%s-*.ttf" % sn)
fonts = glob.glob(globexp)
if not fonts:
continue
fonts = [path.basename(f) for f in fonts]
for font_name in fonts:
lgc_font_name = font_name.replace(sn, "")
font_file = path.join(srcdir, font_name)
lgc_font_file = path.join(srcdir, lgc_font_name)
chars_to_add = (HYPHENS - coverage.character_set(font_file)
) & coverage.character_set(lgc_font_file)
if chars_to_add:
print("patch hyphens", font_name)
merger.merge_chars_from_bank(
path.join(srcdir, font_name),
path.join(srcdir, lgc_font_name),
path.join(srcdir, font_name),
chars_to_add,
)
else:
if copy_unchanged:
shutil.copy2(path.join(srcdir, font_name),
path.join(dstdir, font_name))
print("%s already has hyphens, copying" % font_name)
else:
print("%s already has hyphens" % font_name)
def find_fonts():
font_name_regexp = re.compile(
'(NotoSans|NotoSerif|NotoNaskh|NotoKufi|Arimo|Cousine|Tinos)'
'(Mono)?'
'(.*?)'
'(UI|Eastern|Estrangela|Western)?'
'-'
'(|Black|Bold|DemiLight|Light|Medium|Regular|Thin)'
'(Italic)?'
'(-Windows)?'
'.[ot]t[cf]')
unicode_data.load_data()
for directory in [path.join(INDIVIDUAL_FONT_DIR, 'hinted'),
path.join(INDIVIDUAL_FONT_DIR, 'unhinted'),
path.join(FONT_DIR, 'alpha'),
CJK_DIR]:
for filename in os.listdir(directory):
match = font_name_regexp.match(filename)
if match:
family, mono, script, variant, weight, style, platform = match.groups()
elif filename == 'NotoNastaliqUrduDraft.ttf':
family = 'NotoNastaliq'
script = 'Aran' # Arabic Nastaliq
weight = ''
style = variant = platform = None
else:
if not (
filename == 'NotoSansCJK.ttc' or # All-comprehensive CJK
filename.endswith('.ttx') or
filename.startswith('README.') or
filename in ['COPYING', 'LICENSE', 'NEWS', 'HISTORY']):
raise ValueError("unexpected filename in %s: '%s'." %
(directory, filename))
continue
if directory == CJK_DIR:
license_type = 'sil'
else:
license_type = 'apache'
if mono:
# we don't provide the Mono CJK on the website
continue
if script == "Historic":
# we don't provide this either
continue
if family in {'Arimo', 'Cousine', 'Tinos'}:
continue # Skip these three for the website
if family.startswith('Noto'):
family = family.replace('Noto', 'Noto ')
if weight == '':
weight = 'Regular'
assert platform is None
if script == '': # LGC
supported_scripts.update({'Latn', 'Grek', 'Cyrl'})
elif script == 'Aran':
supported_scripts.add(script)
elif script in {'JP', 'KR', 'SC', 'TC', 'CJK'}:
continue # Skip unified or old CJK fonts
else:
script = convert_to_four_letter(script)
supported_scripts.add(script)
file_path = path.join(directory, filename)
if filename.endswith('.ttf') or filename.endswith('.otf'):
charset = coverage.character_set(file_path)
else:
charset = NotImplemented
if directory == CJK_DIR:
hint_status = 'hinted'
elif directory.endswith('alpha'):
hint_status = 'unhinted'
else:
hint_status = path.basename(directory)
assert hint_status in ['hinted', 'unhinted']
key = family.replace(' ', '-')
if script:
key += '-' + script
if variant not in {None, 'UI'}:
key += '-' + variant
key = key.lower()
font = Font(file_path, hint_status, key,
family, script, variant, weight, style, platform,
charset, license_type)
all_fonts.append(font)
def test_inclusion_of_legacy_pua(self):
"""Tests that legacy PUA characters remain in the fonts."""
for font in self.fonts:
charset = coverage.character_set(font)
for char in self.LEGACY_PUA:
self.assertIn(char, charset)
def setUp(self):
self.font_files, _ = self.loaded_fonts
charset = coverage.character_set(self.font_files[0])
self.marks_to_test = [char for char in charset
if unicode_data.category(char) in ['Lm', 'Sk']]
self.advance_cache = {}
def get_families(fonts):
"""Group fonts into families, separate into hinted and unhinted, select representative."""
family_id_to_fonts = collections.defaultdict(set)
families = {}
for font in fonts:
family_id = noto_font_to_family_id(font)
family_id_to_fonts[family_id].add(font)
for family_id, fonts in family_id_to_fonts.iteritems():
hinted_members = []
unhinted_members = []
rep_member = None
for font in fonts:
if font.is_hinted:
hinted_members.append(font)
else:
unhinted_members.append(font)
if not rep_member:
if font.weight == 'Regular' and font.slope is None and not font.is_mono:
# We assume here that there's no difference between a hinted or unhinted
# rep_member in terms of what we use it for. The other filters are to ensure
# the fontTools font name is a good stand-in for the family name.
rep_member = font
if not rep_member:
raise ValueError('Family %s does not have a representative font.' % family_id)
if hinted_members and not len(hinted_members) in [1, 2, 4, 7, 8, 9]: # 9 adds the two Mono variants
raise ValueError('Family %s has %d hinted_members (%s)' % (
family_id, len(hinted_members), [path.basename(font.filepath) for font in hinted_members]))
if unhinted_members and not len(unhinted_members) in [1, 2, 4, 8]:
raise ValueError('Family %s has %d unhinted_members (%s)' % (
family_id, len(unhinted_members), [
path.basename(font.filepath) for font in unhinted_members]))
if hinted_members and unhinted_members and len(hinted_members) != len(unhinted_members):
# Let's not consider this an error for now. Just drop the members with the higher number
# of fonts, assuming it's a superset of the fonts in the smaller set, so that the fonts
# we provide and display are available to all users. This means website users will not be
# able to get these fonts via the website.
#
# We'll keep the representative font and not try to change it.
print 'Family %s has %d hinted members but %d unhinted memberts' % (
family_id, len(hinted_members), len(unhinted_members))
if len(hinted_members) < len(unhinted_members):
unhinted_members = []
else:
hinted_members = []
name = get_font_family_name(rep_member.filepath)
if rep_member.fmt in {'ttf', 'otf'}:
charset = coverage.character_set(rep_member.filepath)
else:
charset = NotImplemented
families[family_id] = NotoFamily(
name, family_id, rep_member, charset, hinted_members, unhinted_members)
return families
def setUp(self):
self.font_files, _ = self.loaded_fonts
charset = coverage.character_set(self.font_files[0])
self.marks_to_test = [char for char in charset
if unicode_data.combining(char) == 230]
self.advance_cache = {}
def find_fonts():
font_name_regexp = re.compile(
"(NotoSans|NotoSerif|NotoNaskh|NotoKufi|Arimo|Cousine|Tinos)"
"(Mono)?"
"(.*?)"
"(UI|Eastern|Estrangela|Western)?"
"-"
"(|Black|Bold|DemiLight|Light|Medium|Regular|Thin)"
"(Italic)?"
"(-Windows)?"
".[ot]t[cf]")
unicode_data.load_data()
for directory in [
path.join(FONT_DIR, "hinted"),
path.join(FONT_DIR, "unhinted"),
path.join(FONT_DIR, "alpha"),
CJK_DIR,
]:
for filename in os.listdir(directory):
match = font_name_regexp.match(filename)
if match:
family, mono, script, variant, weight, style, platform = match.groups(
)
elif filename == "NotoNastaliqUrduDraft.ttf":
family = "NotoNastaliq"
script = "Aran" # Arabic Nastaliq
weight = ""
style = variant = platform = None
else:
if not (filename == "NotoSansCJK.ttc.zip"
or filename.endswith(".ttx") # All-comprehensive CJK
or filename.endswith(".git")
or filename.startswith("README.") or filename
in ["COPYING", "LICENSE", "NEWS", "HISTORY"]):
raise ValueError("unexpected filename in %s: '%s'." %
(directory, filename))
continue
if directory == CJK_DIR:
license_type = "sil"
else:
license_type = "apache"
if mono:
# we don't provide the Mono CJK on the website
continue
if script == "Historic":
# we don't provide this either
continue
if family in {"Arimo", "Cousine", "Tinos"}:
continue # Skip these three for the website
if family.startswith("Noto"):
family = family.replace("Noto", "Noto ")
if weight == "":
weight = "Regular"
assert platform is None
if script == "": # LGC
supported_scripts.update({"Latn", "Grek", "Cyrl"})
elif script == "Aran":
supported_scripts.add(script)
elif script in {"JP", "KR", "SC", "TC", "CJK"}:
continue # Skip unified or old CJK fonts
else:
script = convert_to_four_letter(script)
supported_scripts.add(script)
file_path = path.join(directory, filename)
if filename.endswith(".ttf") or filename.endswith(".otf"):
charset = coverage.character_set(file_path)
else:
charset = NotImplemented
if directory == CJK_DIR:
hint_status = "hinted"
elif directory.endswith("alpha"):
hint_status = "unhinted"
else:
hint_status = path.basename(directory)
assert hint_status in ["hinted", "unhinted"]
key = family.replace(" ", "-")
if script:
key += "-" + script
if variant not in {None, "UI"}:
key += "-" + variant
key = key.lower()
font = Font(
file_path,
hint_status,
key,
family,
script,
variant,
weight,
style,
platform,
charset,
license_type,
)
all_fonts.append(font)
def test_inclusion_of_legacy_pua(self):
"""Tests that legacy PUA characters remain in the fonts."""
for font in self.fonts:
charset = coverage.character_set(font)
for char in self.LEGACY_PUA:
self.assertIn(char, charset)
