def _swat_fonts(dst_root, dry_run): def family_key(family): return _FAMILY_KEYS.get(family, "x" + family) def script_key(script): return _SCRIPT_KEYS.get( script, None) or cldr_data.get_english_script_name(script) def compare_key(font): return ( family_key(font.family), font.style, script_key(font.script), "a" if font.is_hinted else "", font.variant if font.variant else "", "UI" if font.is_UI else "", "" if font.weight == "Regular" else font.weight, font.slope or "", font.fmt, ) fonts = noto_fonts.get_noto_fonts() for font in sorted(fonts, key=compare_key): _swat_font(font, dst_root, dry_run) if _ttc_fonts: _construct_ttc_fonts(fonts, dst_root, dry_run)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--names", help="print family names", action="store_true") parser.add_argument( "--each", help="for each code point, show supporting families", metavar="cp", nargs="+", ) parser.add_argument( "--any", help="show families that support any of the codepoints", metavar="cp", nargs="+", ) parser.add_argument( "--all", help="show families that support all of the codepoints", metavar="cp", nargs="+", ) args = parser.parse_args() fonts = noto_fonts.get_noto_fonts() families = noto_fonts.get_families(fonts) run(args, families)
def font_cmap_data(paths): """Return CmapData for (almost) all the noto font families.""" args = [("paths", paths)] if paths else None metadata = cmap_data.create_metadata("noto_font_cmaps", args) def use_in_web(font): return ( not font.subset and not font.fmt == "ttc" and not font.script in {"CJK", "HST"} and not font.family in {"Arimo", "Cousine", "Tinos"} ) if not paths: paths = noto_fonts.NOTO_FONT_PATHS fonts = filter(use_in_web, noto_fonts.get_noto_fonts(paths=paths)) families = noto_fonts.get_families(fonts) ScriptData = collections.namedtuple("ScriptData", "family_name,script,cpset") script_to_data = collections.defaultdict(list) for family in families.values(): script = family.rep_member.script family_name = family.name cpset = family.charset script_to_data[script].append(ScriptData(family_name, script, cpset)) def report_data_error(index, script_data): sys.stderr.write( " %d: %s, %d, %s\n" % ( index, script_data.family_name, script_data.script, len(script_data.cpset), lint_config.write_int_ranges(script_data.cpset), ) ) script_to_cmap = {} for script in sorted(script_to_data): data = script_to_data[script] selected_cpset = data[0].cpset if len(data) > 1: differ = False for i in range(1, len(data)): test_data = data[i] for j in range(i): if data[j].cpset != test_data.cpset: differ = True if len(test_data.cpset) > len(selected_cpset): selected_cpset = test_data.cpset if differ: sys.stderr.write("\nscript %s cmaps differ\n" % script) differences = {i.family_name: i.cpset for i in data} report_set_differences(differences) script_to_cmap[script] = selected_cpset tabledata = cmap_data.create_table_from_map(script_to_cmap) return cmap_data.CmapData(metadata, tabledata)
def font_cmap_data(paths): """Return CmapData for (almost) all the noto font families.""" args = [('paths', paths)] if paths else None metadata = cmap_data.create_metadata('noto_font_cmaps', args) def use_in_web(font): return (not font.subset and not font.fmt == 'ttc' and not font.script in {'CJK', 'HST'} and not font.family in {'Arimo', 'Cousine', 'Tinos'}) if not paths: paths = noto_fonts.NOTO_FONT_PATHS fonts = filter(use_in_web, noto_fonts.get_noto_fonts(paths=paths)) families = noto_fonts.get_families(fonts) ScriptData = collections.namedtuple('ScriptData', 'family_name,script,cpset') script_to_data = collections.defaultdict(list) for family in families.values(): script = family.rep_member.script family_name = family.name cpset = family.charset script_to_data[script].append(ScriptData(family_name, script, cpset)) def report_data_error(index, script_data): print >> sys.stderr, ' %d: %s, %d, %s' % ( index, script_data.family_name, script_data.script, len(script_data.cpset), lint_config.write_int_ranges(script_data.cpset)) script_to_cmap = {} for script in sorted(script_to_data): data = script_to_data[script] selected_cpset = data[0].cpset if len(data) > 1: differ = False for i in range(1, len(data)): test_data = data[i] for j in range(i): if data[j].cpset != test_data.cpset: differ = True if len(test_data.cpset) > len(selected_cpset): selected_cpset = test_data.cpset if differ: print >> sys.stderr, '\nscript %s cmaps differ' % script differences = {i.family_name: i.cpset for i in data} report_set_differences(differences) script_to_cmap[script] = selected_cpset tabledata = cmap_data.create_table_from_map(script_to_cmap) return cmap_data.CmapData(metadata, tabledata)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--names', help='print family names', action='store_true') parser.add_argument('--each', help='for each code point, show supporting families', metavar='cp', nargs='+') parser.add_argument('--any', help='show families that support any of the codepoints', metavar='cp', nargs='+') parser.add_argument('--all', help='show families that support all of the codepoints', metavar='cp', nargs='+') args = parser.parse_args() fonts = noto_fonts.get_noto_fonts() families = noto_fonts.get_families(fonts) run(args, families)
def font_cmap_data(): """Return CmapData for (almost) all the noto font families.""" metadata = cmap_data.create_metadata('noto_font_cmaps') def use_in_web(font): return (not font.subset and not font.is_UI and not font.fmt == 'ttc' and not font.script in {'CJK', 'HST'} and not font.family in {'Arimo', 'Cousine', 'Tinos'}) fonts = filter(use_in_web, noto_fonts.get_noto_fonts()) families = noto_fonts.get_families(fonts) ScriptData = collections.namedtuple('ScriptData', 'family_name,script,cpset') script_to_data = collections.defaultdict(list) for family in families.values(): script = family.rep_member.script family_name = family.name cpset = family.charset script_to_data[script].append(ScriptData(family_name, script, cpset)) def report_data_error(index, script_data): print >> sys.stderr, ' %d: %s, %d, %s' % ( index, script_data.family_name, script_data.script, len(script_data.cpset), lint_config.write_int_ranges(script_data.cpset)) script_to_cmap = {} for script in sorted(script_to_data): data = script_to_data[script] selected_cpset = data[0].cpset if len(data) > 1: differ = False for i in range(1, len(data)): test_data = data[i] for j in range(i): if data[j].cpset != test_data.cpset: differ = True if len(test_data.cpset) > len(selected_cpset): selected_cpset = test_data.cpset if differ: print >> sys.stderr, '\nscript %s cmaps differ' % script differences = {i.family_name: i.cpset for i in data} report_set_differences(differences) script_to_cmap[script] = selected_cpset tabledata = cmap_data.create_table_from_map(script_to_cmap) return cmap_data.CmapData(metadata, tabledata)
def _swat_fonts(dst_root, dry_run): def family_key(family): return _FAMILY_KEYS.get(family, 'x' + family) def script_key(script): return (_SCRIPT_KEYS.get(script, None) or cldr_data.get_english_script_name(script)) def compare_key(font): return (family_key(font.family), font.style, script_key(font.script), 'a' if font.is_hinted else '', font.variant if font.variant else '', 'UI' if font.is_UI else '', '' if font.weight == 'Regular' else font.weight, font.slope or '', font.fmt) fonts = noto_fonts.get_noto_fonts() for font in sorted(fonts, key=compare_key): _swat_font(font, dst_root, dry_run) if _ttc_fonts: _construct_ttc_fonts(fonts, dst_root, dry_run)
def check_spreadsheet(src_file): filenames = set() prev_script_name = None fontdata = {} filedata = {} with open(src_file) as csvfile: reader = csv.DictReader(csvfile) for index, row in enumerate(reader): font = row['Fonts'].replace('\xc2\xa0', ' ').strip() hinting = row['Hinting'].strip() status = row['Status'].strip() accepted_version = row['Accepted Version'].strip() note = row['Note'].strip() # family script style (variant UI) weight, mostly m = re.match( r'Noto (Kufi|Naskh|Color Emoji|Emoji|Sans|Serif|Nastaliq)' r'(?: (.*?))?' r'(?: (UI))?' r' (Thin|Light|DemiLight|Regular|Medium|Bold Italic' r'|Bold|Black|Italic)(?: \(merged\))?$', font) if not m: m = re.match(r'Noto (Sans) (Myanmar) (UI)(.*)', font) if not m: print 'could not parse Myanmar exception: "%s"' % font continue style, script, ui, weight = m.groups() weight = weight or 'Regular' weight = weight.replace(' ', '') ui = ui or '' script = script or '' script = re.sub('-| ', '', script) style = style.replace(' ', '') ext = 'ttf' if script == 'CJK': ext = 'ttc' elif script.startswith('TTC'): ext = 'ttc' script = '' elif script == '(LGC)': script = '' elif script == 'UI': ui = 'UI' script = '' elif script == 'Phagspa': script = 'PhagsPa' elif script == 'SumeroAkkadianCuneiform': script = 'Cuneiform' fontname = ''.join(['Noto', style, script, ui, '-', weight, '.', ext]) # print '%s:\n--> %s\n--> %s' % ( # font, str((style, script, ui, weight)), fontname) if not hinting in [ 'hinted', 'hinted (CFF)', 'unhinted']: print 'unrecognized hinting value \'%s\' on line %d (%s)' % ( hinting, index, fontname) continue hinted = 'hinted' if hinting in ['hinted', 'hinted (CFF)'] else 'unhinted' if not status in [ 'In finishing', 'Released w. lint errors', 'Approved & Released', 'Approved & Not Released', 'In design', 'Design approved', 'Design re-approved', 'Released']: print 'unrecognized status value \'%s\' on line %d (%s)' % ( status, index, fontname) continue expect_font = status in [ 'Released w. lint errors', 'Approved & Released', 'Approved & Not Released', 'Released'] data = (fontname, (index, font, style, script, ui, weight), hinted, status, accepted_version, note, expect_font) filedata[hinted + '/' + fontname] = data # ok, now let's see if we can find these files all_noto = noto_fonts.get_noto_fonts() notodata = { ('hinted' if f.is_hinted else 'unhinted') + '/' + path.basename(f.filepath) : f for f in all_noto } noto_filenames = frozenset(notodata.keys()) spreadsheet_filenames = frozenset(k for k in filedata if filedata[k][6]) spreadsheet_extra = spreadsheet_filenames - noto_filenames spreadsheet_missing = noto_filenames - spreadsheet_filenames if spreadsheet_extra: print 'spreadsheet extra:\n ' + '\n '.join( sorted(spreadsheet_extra)) if spreadsheet_missing: print 'spreadsheet missing:\n ' + '\n '.join( sorted(spreadsheet_missing)) spreadsheet_match = spreadsheet_filenames & noto_filenames for filename in sorted(spreadsheet_match): data = filedata[filename] filepath = notodata[filename].filepath ttfont = ttLib.TTFont(filepath, fontNumber=0) font_version = font_data.printable_font_revision(ttfont) approved_version = data[4] if approved_version: warn = '!!!' if approved_version != font_version else '' print '%s%s version: %s approved: %s' % ( warn, filename, font_version, approved_version) else: print '%s version: %s' % (filename, font_version)
def check_spreadsheet(src_file): filenames = set() prev_script_name = None fontdata = {} filedata = {} with open(src_file) as csvfile: reader = csv.DictReader(csvfile) for index, row in enumerate(reader): font = row["Fonts"].replace("\xc2\xa0", " ").strip() hinting = row["Hinting"].strip() status = row["Status"].strip() accepted_version = row["Accepted Version"].strip() note = row["Note"].strip() # family script style (variant UI) weight, mostly m = re.match( r"Noto (Kufi|Naskh|Color Emoji|Emoji|Sans|Serif|Nastaliq|Rashi|Traditional)" r"(?: (.*?))?" r"(?: (UI))?" r" (Thin|Light|DemiLight|Regular|Medium|Bold Italic" r"|Bold|Black|Italic)(?: \(merged\))?$", font, ) if not m: m = re.match(r"Noto (Sans) (Myanmar) (UI)(.*)", font) if not m: print('could not parse Myanmar exception: "%s"' % font) continue style, script, ui, weight = m.groups() weight = weight or "Regular" weight = weight.replace(" ", "") ui = ui or "" script = script or "" script = re.sub("-| ", "", script) style = style.replace(" ", "") ext = "ttf" if script == "CJK": ext = "ttc" elif script.startswith("TTC"): ext = "ttc" script = "" elif script == "(LGC)": script = "" elif script == "UI": ui = "UI" script = "" elif script == "Phagspa": script = "PhagsPa" elif script == "SumeroAkkadianCuneiform": script = "Cuneiform" fontname = "".join(["Noto", style, script, ui, "-", weight, ".", ext]) # print('%s:\n--> %s\n--> %s' % ( # font, str((style, script, ui, weight)), fontname)) if not hinting in ["hinted", "hinted (CFF)", "unhinted"]: print( "unrecognized hinting value '%s' on line %d (%s)" % (hinting, index, fontname) ) continue hinted = "hinted" if hinting in ["hinted", "hinted (CFF)"] else "unhinted" if not status in [ "In finishing", "Released w. lint errors", "Approved & Released", "Approved & Not Released", "In design", "Design approved", "Design re-approved", "Released", ]: print( "unrecognized status value '%s' on line %d (%s)" % (status, index, fontname) ) continue expect_font = status in [ "Released w. lint errors", "Approved & Released", "Approved & Not Released", "Released", ] data = ( fontname, (index, font, style, script, ui, weight), hinted, status, accepted_version, note, expect_font, ) filedata[hinted + "/" + fontname] = data # ok, now let's see if we can find these files all_noto = noto_fonts.get_noto_fonts() notodata = { ("hinted" if f.is_hinted else "unhinted") + "/" + path.basename(f.filepath): f for f in all_noto } noto_filenames = frozenset(notodata.keys()) spreadsheet_filenames = frozenset(k for k in filedata if filedata[k][6]) spreadsheet_extra = spreadsheet_filenames - noto_filenames spreadsheet_missing = noto_filenames - spreadsheet_filenames if spreadsheet_extra: print("spreadsheet extra:\n " + "\n ".join(sorted(spreadsheet_extra))) if spreadsheet_missing: print("spreadsheet missing:\n " + "\n ".join(sorted(spreadsheet_missing))) spreadsheet_match = spreadsheet_filenames & noto_filenames for filename in sorted(spreadsheet_match): data = filedata[filename] filepath = notodata[filename].filepath ttfont = ttLib.TTFont(filepath, fontNumber=0) font_version = font_data.printable_font_revision(ttfont) approved_version = data[4] if approved_version: warn = "!!!" if approved_version != font_version else "" print( "%s%s version: %s approved: %s" % (warn, filename, font_version, approved_version) ) else: print("%s version: %s" % (filename, font_version))
def generate(self): if self.clean: self.clean_target_dir() if not self.no_build: self.ensure_target_dirs_exist() # debug/print # ['families', 'script_to_family_ids', 'used_lang_data', # 'family_id_to_lang_scrs', 'family_id_to_default_lang_scr'] debug = frozenset([]) def use_in_web(font): return (not font.subset and not font.is_UI and not font.fmt == 'ttc' and not font.script in {'CJK', 'HST'} and not font.family in {'Arimo', 'Cousine', 'Tinos'}) fonts = filter(use_in_web, noto_fonts.get_noto_fonts()) families = noto_fonts.get_families(fonts) if 'families' in debug: print '\nfamilies' for family_id, family in sorted(families.iteritems()): print family_id, family.rep_member.script script_to_family_ids = get_script_to_family_ids(families) if 'script_to_family_ids' in debug: print '\nscript to family ids' for script, family_ids in sorted(script_to_family_ids.iteritems()): print script, family_ids all_lang_scrs = set(['und-' + script for script in script_to_family_ids]) all_lang_scrs.update(lang_data.lang_scripts()) lang_scr_to_sample_infos = {} for lang_scr in all_lang_scrs: lang, script = lang_scr.split('-') if not script in script_to_family_ids: print 'no family supports script in %s' % lang_scr continue sample_infos = get_sample_infos(lang_scr) if not sample_infos: continue lang_scr_to_sample_infos[lang_scr] = sample_infos if 'lang_scr_to_sample_infos' in debug: print '\nlang+script to sample infos' for lang_scr, info_list in sorted(lang_scr_to_sample_infos.iteritems()): for info in info_list: print '%s key: %s rtl: %s attrib: %s len: %d' % ( lang_scr, info[3], info[0], info[2], len(info[1])) family_id_to_lang_scrs = get_family_id_to_lang_scrs( lang_scr_to_sample_infos.keys(), script_to_family_ids) family_id_to_lang_scr_to_sample_key, sample_key_to_info = get_family_id_to_lang_scr_to_sample_key( family_id_to_lang_scrs, families, lang_scr_to_sample_infos) family_id_to_regions = get_family_id_to_regions(family_id_to_lang_scr_to_sample_key) region_to_family_ids = get_region_to_family_ids(family_id_to_regions) family_id_to_default_lang_scr = get_family_id_to_default_lang_scr( family_id_to_lang_scrs, families) if 'family_id_to_default_lang_scr' in debug: print '\nfamily id to default lang scr' for family_id, lang_scr in family_id_to_default_lang_scr.iteritems(): print family_id, lang_scr region_data = get_region_lat_lng_data(region_to_family_ids.keys()) lang_scrs = get_named_lang_scrs(family_id_to_lang_scr_to_sample_key) lang_scr_sort_order = get_lang_scr_sort_order(lang_scrs) # sanity checks # all families have languages, and all those have samples. # all families have a default language, and that is in the sample list error_list = [] for family in families.values(): family_id = family.family_id if not family_id in family_id_to_lang_scr_to_sample_key: error_list.append('no entry for family %s' % family_id) continue lang_scr_to_sample_key = family_id_to_lang_scr_to_sample_key[family_id] if not lang_scr_to_sample_key: error_list.append('no langs for family %s' % family_id) continue for lang_scr in lang_scr_to_sample_key: sample_key = lang_scr_to_sample_key[lang_scr] if not sample_key: error_list.append('no sample key for lang %s in family %s' % (lang_scr, sample_key)) continue if not sample_key in sample_key_to_info: error_list.append('no sample for sample key: %s' % sample_key) if not family_id in family_id_to_default_lang_scr: error_list.append('no default lang for family %s' % family_id) continue default_lang_scr = family_id_to_default_lang_scr[family_id] if not default_lang_scr in lang_scr_to_sample_key: error_list.append('default lang %s not in samples for family %s' % (default_lang_scr, family_id)) if error_list: print 'Errors:\n' + '\n '.join(error_list) if error_list or self.no_build: print 'skipping build output' return # build outputs if self.no_zips: print 'skipping zip output' else: family_zip_info = self.build_zips(families) universal_zip_info = self.build_universal_zips(families) # build outputs not used by the json but linked to from the web page self.build_ttc_zips() if self.no_css: print 'skipping css output' else: family_css_info = self.build_css(families) if self.no_data: print 'skipping data output%s' % reason else: self.build_data_json(family_id_to_lang_scr_to_sample_key, families, family_zip_info, universal_zip_info, family_id_to_regions, region_to_family_ids) self.build_families_json(family_id_to_lang_scr_to_sample_key, families, family_id_to_default_lang_scr, family_id_to_regions, family_css_info, lang_scr_sort_order) self.build_misc_json(sample_key_to_info, region_data) if self.no_images: print 'skipping image output' else: self.build_images(family_id_to_lang_scr_to_sample_key, families, family_id_to_default_lang_scr, sample_key_to_info)
'fil,fr,gl,gu,hi,hr,hu,hy,id,is,it,iw,ja,ka,kk,km,kn,ko,ky,lo,lt,lv,' 'mk,ml,mn,mr,ms,my,ne,nl,no,pa,pl,pt-BR,pt-PT,ro,ru,si,sk,sl,sq,sr,' 'sv,sw,ta,te,th,tl,tr,uk,ur,uz,vi,zh-CN,zh-TW,zu').split(',') def accept_font(f): return ( f.family == 'Noto' and # exclude Arimo, Tinos, Cousine f.style != 'Nastaliq' and # exclude Nastaliq, not suitable for maps f.script != 'HST' and # exclude Historic, tool limitation f.weight == 'Regular' and # to limit members of fonts f.width == 'Regular' and # to limit members of fonts, we don't not f.slope and # care about weights f.fmt in ['ttf', 'otf'] and # only support these formats (not f.is_cjk or f.subset)) # 'small' language-specific CJK subsets fonts = filter(accept_font, noto_fonts.get_noto_fonts()) families = noto_fonts.get_families(fonts).values() def write_csv_header(outfile): print >> outfile, 'Code,Script,Style,UI,Font Name' def write_csv(outfile, lang, script, style, ui, members): if members: print >> outfile, ','.join( [lang, script, style, ui, noto_fonts.get_font_family_name(members[0].filepath)]) with open('lang_to_font_table.csv', 'w') as outfile: write_csv_header(outfile)
def check_spreadsheet(src_file): filenames = set() prev_script_name = None fontdata = {} filedata = {} with open(src_file) as csvfile: reader = csv.DictReader(csvfile) for index, row in enumerate(reader): font = row['Fonts'].replace('\xc2\xa0', ' ').strip() hinting = row['Hinting'].strip() status = row['Status'].strip() accepted_version = row['Accepted Version'].strip() note = row['Note'].strip() # family script style (variant UI) weight, mostly m = re.match( r'Noto (Kufi|Naskh|Color Emoji|Emoji|Sans|Serif|Nastaliq)' r'(?: (.*?))?' r'(?: (UI))?' r' (Thin|Light|DemiLight|Regular|Medium|Bold Italic' r'|Bold|Black|Italic)(?: \(merged\))?$', font) if not m: m = re.match(r'Noto (Sans) (Myanmar) (UI)(.*)', font) if not m: print 'could not parse Myanmar exception: "%s"' % font continue style, script, ui, weight = m.groups() weight = weight or 'Regular' weight = weight.replace(' ', '') ui = ui or '' script = script or '' script = re.sub('-| ', '', script) style = style.replace(' ', '') ext = 'ttf' if script == 'CJK': ext = 'ttc' elif script.startswith('TTC'): ext = 'ttc' script = '' elif script == '(LGC)': script = '' elif script == 'UI': ui = 'UI' script = '' elif script == 'Phagspa': script = 'PhagsPa' elif script == 'SumeroAkkadianCuneiform': script = 'Cuneiform' fontname = ''.join( ['Noto', style, script, ui, '-', weight, '.', ext]) # print '%s:\n--> %s\n--> %s' % ( # font, str((style, script, ui, weight)), fontname) if not hinting in ['hinted', 'hinted (CFF)', 'unhinted']: print 'unrecognized hinting value \'%s\' on line %d (%s)' % ( hinting, index, fontname) continue hinted = 'hinted' if hinting in ['hinted', 'hinted (CFF)' ] else 'unhinted' if not status in [ 'In finishing', 'Released w. lint errors', 'Approved & Released', 'Approved & Not Released', 'In design', 'Design approved', 'Design re-approved', 'Released' ]: print 'unrecognized status value \'%s\' on line %d (%s)' % ( status, index, fontname) continue expect_font = status in [ 'Released w. lint errors', 'Approved & Released', 'Approved & Not Released', 'Released' ] data = (fontname, (index, font, style, script, ui, weight), hinted, status, accepted_version, note, expect_font) filedata[hinted + '/' + fontname] = data # ok, now let's see if we can find these files all_noto = noto_fonts.get_noto_fonts() notodata = {('hinted' if f.is_hinted else 'unhinted') + '/' + path.basename(f.filepath): f for f in all_noto} noto_filenames = frozenset(notodata.keys()) spreadsheet_filenames = frozenset(k for k in filedata if filedata[k][6]) spreadsheet_extra = spreadsheet_filenames - noto_filenames spreadsheet_missing = noto_filenames - spreadsheet_filenames if spreadsheet_extra: print 'spreadsheet extra:\n ' + '\n '.join( sorted(spreadsheet_extra)) if spreadsheet_missing: print 'spreadsheet missing:\n ' + '\n '.join( sorted(spreadsheet_missing)) spreadsheet_match = spreadsheet_filenames & noto_filenames for filename in sorted(spreadsheet_match): data = filedata[filename] filepath = notodata[filename].filepath ttfont = ttLib.TTFont(filepath, fontNumber=0) font_version = font_data.printable_font_revision(ttfont) approved_version = data[4] if approved_version: warn = '!!!' if approved_version != font_version else '' print '%s%s version: %s approved: %s' % ( warn, filename, font_version, approved_version) else: print '%s version: %s' % (filename, font_version)
def generate(self): if self.clean: self.clean_target_dir() if not self.no_build: self.ensure_target_dirs_exist() def use_in_web(font): return (not font.subset and not font.fmt == 'ttc' and not font.script in {'CJK', 'HST'} and not font.family in {'Arimo', 'Cousine', 'Tinos'}) fonts = filter(use_in_web, noto_fonts.get_noto_fonts()) families = noto_fonts.get_families(fonts) check_families(families) if 'families' in self.debug: print '\n#debug families' for family_id, family in sorted(families.iteritems()): print '%s (%s, %s)' % ( family_id, family.name, noto_fonts.get_family_filename(family)) if family.hinted_members: print ' hinted: %s' % ', '.join(sorted( [path.basename(m.filepath) for m in family.hinted_members])) if family.unhinted_members: print ' unhinted: %s' % ', '.join(sorted( [path.basename(m.filepath) for m in family.unhinted_members])) script_to_family_ids = get_script_to_family_ids(families) if 'script_to_family_ids' in self.debug: print '\n#debug script to family ids' for script, family_ids in sorted(script_to_family_ids.iteritems()): print '%s: %s' % (script, ', '.join(sorted(family_ids))) all_lang_scrs = set(['und-' + script for script in script_to_family_ids]) all_lang_scrs.update(lang_data.lang_scripts()) lang_scr_to_sample_infos = {} for lang_scr in all_lang_scrs: lang, script = lang_scr.split('-') if not script in script_to_family_ids: print 'no family supports script in %s' % lang_scr continue sample_infos = get_sample_infos(lang_scr) if not sample_infos: continue lang_scr_to_sample_infos[lang_scr] = sample_infos if 'lang_scr_to_sample_infos' in self.debug: print '\n#debug lang+script to sample infos' for lang_scr, info_list in sorted(lang_scr_to_sample_infos.iteritems()): for info in info_list: print '%s: %s, %s, len %d' % ( lang_scr, info[2], info[1], len(info[0])) family_id_to_lang_scrs = get_family_id_to_lang_scrs( lang_scr_to_sample_infos.keys(), script_to_family_ids) if 'family_id_to_lang_scrs' in self.debug: print '\n#debug family id to list of lang+script' for family_id, lang_scrs in sorted(family_id_to_lang_scrs.iteritems()): print '%s: (%d) %s' % ( family_id, len(lang_scrs), ' '.join(sorted(lang_scrs))) family_id_to_lang_scr_to_sample_key, sample_key_to_info = ( get_family_id_to_lang_scr_to_sample_key( family_id_to_lang_scrs, families, lang_scr_to_sample_infos)) if 'family_id_to_lang_scr_to_sample_key' in self.debug: print '\n#debug family id to map from lang+script to sample key' for family_id, lang_scr_to_sample_key in sorted( family_id_to_lang_scr_to_sample_key.iteritems()): print '%s (%d):' % (family_id, len(lang_scr_to_sample_key)) for lang_scr, sample_key in sorted(lang_scr_to_sample_key.iteritems()): print ' %s: %s' % (lang_scr, sample_key) if 'sample_key_to_info' in self.debug: print '\n#debug sample key to sample info' for sample_key, info in sorted(sample_key_to_info.iteritems()): print '%s: %s, len %d' % ( sample_key, info[1], len(info[0])) family_id_to_regions = get_family_id_to_regions( family_id_to_lang_scr_to_sample_key) if 'family_id_to_regions' in self.debug: print '\n#debug family id to regions' for family_id, regions in sorted(family_id_to_regions.iteritems()): print '%s: (%d) %s' % ( family_id, len(regions), ', '.join(sorted(regions))) region_to_family_ids = get_region_to_family_ids(family_id_to_regions) if 'region_to_family_ids' in self.debug: print '\n#debug region to family ids' for region, family_ids in sorted(region_to_family_ids.iteritems()): print '%s: (%d) %s' % ( region, len(family_ids), ', '.join(sorted(family_ids))) family_id_to_default_lang_scr = get_family_id_to_default_lang_scr( family_id_to_lang_scrs, families) if 'family_id_to_default_lang_scr' in self.debug: print '\n#debug family id to default lang scr' for family_id, lang_scr in sorted( family_id_to_default_lang_scr.iteritems()): print '%s: %s' % (family_id, lang_scr) region_data = get_region_lat_lng_data(region_to_family_ids.keys()) lang_scrs = get_named_lang_scrs(family_id_to_lang_scr_to_sample_key) lang_scr_sort_order = get_lang_scr_sort_order(lang_scrs) # sanity checks # all families have languages, and all those have samples. # all families have a default language, and that is in the sample list error_list = [] for family in families.values(): family_id = family.family_id if not family_id in family_id_to_lang_scr_to_sample_key: error_list.append('no entry for family %s' % family_id) continue lang_scr_to_sample_key = family_id_to_lang_scr_to_sample_key[family_id] if not lang_scr_to_sample_key: error_list.append('no langs for family %s' % family_id) continue for lang_scr in lang_scr_to_sample_key: sample_key = lang_scr_to_sample_key[lang_scr] if not sample_key: error_list.append( 'no sample key for lang %s in family %s' % (lang_scr, sample_key)) continue if not sample_key in sample_key_to_info: error_list.append('no sample for sample key: %s' % sample_key) if not family_id in family_id_to_default_lang_scr: error_list.append('no default lang for family %s' % family_id) continue default_lang_scr = family_id_to_default_lang_scr[family_id] if not default_lang_scr in lang_scr_to_sample_key: error_list.append('default lang %s not in samples for family %s' % (default_lang_scr, family_id)) if error_list: print 'Errors:\n' + '\n '.join(error_list) if error_list or self.no_build: print 'skipping build output' return # build outputs # zips are required for data if self.no_zips and self.no_data: print 'skipping zip output' else: self.build_readmes() family_zip_info = self.build_zips(families) universal_zip_info = self.build_universal_zips(families) # build outputs not used by the json but linked to from the web page if not self.no_zips: self.build_ttc_zips() if self.no_css: print 'skipping css output' else: family_css_info = self.build_css(families) if self.no_data: print 'skipping data output' else: self.build_data_json(family_id_to_lang_scr_to_sample_key, families, family_zip_info, universal_zip_info, family_id_to_regions, region_to_family_ids) self.build_families_json(family_id_to_lang_scr_to_sample_key, families, family_id_to_default_lang_scr, family_id_to_regions, family_css_info, lang_scr_sort_order) self.build_misc_json(sample_key_to_info, region_data) if self.no_images: print 'skipping image output' else: self.build_images(family_id_to_lang_scr_to_sample_key, families, family_id_to_default_lang_scr, sample_key_to_info)
def generate(self): if self.clean: self.clean_target_dir() if not self.no_build: self.ensure_target_dirs_exist() # debug/print # ['families', 'script_to_family_ids', 'used_lang_data', # 'family_id_to_lang_tags', 'family_id_to_default_lang_tag'] debug = frozenset([]) fonts = noto_fonts.get_noto_fonts() families = noto_fonts.get_families(fonts) if 'families' in debug: print '\nfamilies' for family_id, family in sorted(families.iteritems()): print family_id, family.rep_member.script script_to_family_ids = get_script_to_family_ids(families) if 'script_to_family_ids' in debug: print '\nscript to family ids' for script, family_ids in sorted(script_to_family_ids.iteritems()): print script, family_ids supported_scripts = set(script_to_family_ids.keys()) used_lang_data = get_used_lang_data(supported_scripts) if 'used_lang_data' in debug: print '\nused lang data' for lang, data in sorted(used_lang_data.iteritems()): used = ', '.join(data[0]) unused = ', '.join(data[1]) if unused: unused = '(' + unused + ')' if used: unused = ' ' + unused print '%s: %s%s' % (lang, used, unused) langs_to_delete = [] for lang in used_lang_data.keys(): if not cldr_data.get_english_language_name(lang): langs_to_delete.append(lang) if langs_to_delete: print 'deleting languages with no english name: %s' % langs_to_delete for lang in langs_to_delete: del used_lang_data[lang] lang_tag_to_family_ids = get_lang_tag_to_family_ids( used_lang_data, script_to_family_ids) region_to_family_ids = get_region_to_family_ids(script_to_family_ids) family_id_to_lang_tags = get_family_id_to_lang_tags( lang_tag_to_family_ids, families) if 'family_id_to_lang_tags' in debug: print '\nfamily id to lang tags' for family_id, lang_tags in sorted( family_id_to_lang_tags.iteritems()): print '%s: %s' % (family_id, ','.join(sorted(lang_tags))) family_id_to_regions = get_family_id_to_regions( region_to_family_ids, families) family_id_to_default_lang_tag = get_family_id_to_default_lang_tag( family_id_to_lang_tags) if 'family_id_to_default_lang_tag' in debug: print '\nfamily id to default lang tag' for family_id, lang_tag in family_id_to_default_lang_tag.iteritems( ): print family_id, lang_tag used_lang_tags = get_used_lang_tags( lang_tag_to_family_ids.keys(), family_id_to_default_lang_tag.values()) lang_tag_to_sample_data = get_lang_tag_to_sample_data(used_lang_tags) # find the samples that can't be displayed. tested_keys = set() failed_keys = set() family_langs_to_remove = {} for lang_tag in sorted(lang_tag_to_sample_data): sample_info = lang_tag_to_sample_data[lang_tag] sample = sample_info[1] sample_key = sample_info[3] for family_id in sorted(lang_tag_to_family_ids[lang_tag]): full_key = sample_key + '-' + family_id if full_key in tested_keys: if full_key in failed_keys: print 'failed sample %s lang %s' % (full_key, lang_tag) if family_id not in family_langs_to_remove: family_langs_to_remove[family_id] = set() family_langs_to_remove[family_id].add(lang_tag) continue failed_cps = set() tested_keys.add(full_key) charset = families[family_id].charset for cp in sample: if ord(cp) in [ 0xa, 0x28, 0x29, 0x2c, 0x2d, 0x2e, 0x3b, 0x5b, 0x5d, 0x2010 ]: continue if ord(cp) not in charset: failed_cps.add(ord(cp)) if failed_cps: print 'sample %s cannot be displayed in %s (lang %s):\n %s' % ( sample_key, family_id, lang_tag, '\n '.join( '%04x (%s)' % (cp, unichr(cp)) for cp in sorted(failed_cps))) failed_keys.add(full_key) if family_id not in family_langs_to_remove: family_langs_to_remove[family_id] = set() family_langs_to_remove[family_id].add(lang_tag) for family_id in sorted(family_langs_to_remove): langs_to_remove = family_langs_to_remove[family_id] print 'remove from %s: %s' % (family_id, ','.join( sorted(langs_to_remove))) family_id_to_lang_tags[family_id] -= langs_to_remove default_lang_tag = family_id_to_default_lang_tag[family_id] if default_lang_tag in langs_to_remove: print '!removing default lang tag %s for family %s' % ( default_lang_tag, family_id) for lang in langs_to_remove: lang_tag_to_family_ids[lang] -= set([family_id]) region_data = get_region_lat_lng_data(region_to_family_ids.keys()) lang_tag_sort_order = get_lang_tag_sort_order( lang_tag_to_family_ids.keys()) if self.no_build: print 'skipping build output' return # build outputs if self.no_zips: print 'skipping zip output' else: family_zip_info = self.build_zips(families) universal_zip_info = self.build_universal_zips(families) # build outputs not used by the json but linked to from the web page self.build_ttc_zips() if self.no_css: print 'skipping css output' else: family_css_info = self.build_css(families) if self.no_data or self.no_zips or self.no_css: reason = '' if self.no_data else 'no zips' if self.no_zips else 'no css' print 'skipping data output%s' % reason else: self.build_data_json(families, family_zip_info, universal_zip_info, family_id_to_lang_tags, family_id_to_regions, lang_tag_to_family_ids, region_to_family_ids) self.build_families_json(families, family_id_to_lang_tags, family_id_to_default_lang_tag, family_id_to_regions, family_css_info, lang_tag_sort_order) self.build_misc_json(lang_tag_to_sample_data, region_data) if self.no_images: print 'skipping image output' else: self.build_images(families, family_id_to_lang_tags, family_id_to_default_lang_tag, lang_tag_to_sample_data)
def generate(self): if self.clean: self.clean_target_dir() if not self.no_build: self.ensure_target_dirs_exist() # debug/print # ['families', 'script_to_family_ids', 'used_lang_data', # 'family_id_to_lang_scrs', 'family_id_to_default_lang_scr'] debug = frozenset([]) def use_in_web(font): return (not font.subset and not font.is_UI and not font.fmt == 'ttc' and not font.script in {'CJK', 'HST', 'Qaae'} and not font.family in {'Arimo', 'Cousine', 'Tinos'}) fonts = [ font for font in noto_fonts.get_noto_fonts() if use_in_web(font) ] families = noto_fonts.get_families(fonts) if 'families' in debug: print '\nfamilies' for family_id, family in sorted(families.iteritems()): print family_id, family.rep_member.script script_to_family_ids = get_script_to_family_ids(families) if 'script_to_family_ids' in debug: print '\nscript to family ids' for script, family_ids in sorted(script_to_family_ids.iteritems()): print script, family_ids all_lang_scrs = set( ['und-' + script for script in script_to_family_ids]) all_lang_scrs.update(lang_data.lang_scripts()) lang_scr_to_sample_infos = {} for lang_scr in all_lang_scrs: lang, script = lang_scr.split('-') if not script in script_to_family_ids: print 'no family supports script in %s' % lang_scr continue rtl = cldr_data.is_rtl(lang_scr) sample_infos = get_sample_infos(lang_scr, rtl) if not sample_infos: continue lang_scr_to_sample_infos[lang_scr] = sample_infos if 'lang_scr_to_sample_infos' in debug: print '\nlang+script to sample infos' for lang_scr, info_list in sorted( lang_scr_to_sample_infos.iteritems()): for info in info_list: print '%s key: %s rtl: %s attrib: %s len: %d' % ( lang_scr, info[3], info[0], info[2], len(info[1])) family_id_to_lang_scrs = get_family_id_to_lang_scrs( lang_scr_to_sample_infos.keys(), script_to_family_ids) family_id_to_lang_scr_to_sample_key, sample_key_to_info = get_family_id_to_lang_scr_to_sample_key( family_id_to_lang_scrs, families, lang_scr_to_sample_infos) family_id_to_regions = get_family_id_to_regions( family_id_to_lang_scr_to_sample_key) region_to_family_ids = get_region_to_family_ids(family_id_to_regions) family_id_to_default_lang_scr = get_family_id_to_default_lang_scr( family_id_to_lang_scrs, families) if 'family_id_to_default_lang_scr' in debug: print '\nfamily id to default lang scr' for family_id, lang_scr in family_id_to_default_lang_scr.iteritems( ): print family_id, lang_scr region_data = get_region_lat_lng_data(region_to_family_ids.keys()) lang_scrs = get_named_lang_scrs(family_id_to_lang_scr_to_sample_key) lang_scr_sort_order = get_lang_scr_sort_order(lang_scrs) # sanity checks # all families have languages, and all those have samples. # all families have a default language, and that is in the sample list error_list = [] for family in families.values(): family_id = family.family_id if not family_id in family_id_to_lang_scr_to_sample_key: error_list.append('no entry for family %s' % family_id) continue lang_scr_to_sample_key = family_id_to_lang_scr_to_sample_key[ family_id] if not lang_scr_to_sample_key: error_list.append('no langs for family %s' % family_id) continue for lang_scr in lang_scr_to_sample_key: sample_key = lang_scr_to_sample_key[lang_scr] if not sample_key: error_list.append( 'no sample key for lang %s in family %s' % (lang_scr, sample_key)) continue if not sample_key in sample_key_to_info: error_list.append('no sample for sample key: %s' % sample_key) if not family_id in family_id_to_default_lang_scr: error_list.append('no default lang for family %s' % family_id) continue default_lang_scr = family_id_to_default_lang_scr[family_id] if not default_lang_scr in lang_scr_to_sample_key: error_list.append( 'default lang %s not in samples for family %s' % (default_lang_scr, family_id)) if error_list: print 'Errors:\n' + '\n '.join(error_list) if error_list or self.no_build: print 'skipping build output' return # build outputs if self.no_zips: print 'skipping zip output' else: family_zip_info = self.build_zips(families) universal_zip_info = self.build_universal_zips(families) # build outputs not used by the json but linked to from the web page self.build_ttc_zips() if self.no_css: print 'skipping css output' else: family_css_info = self.build_css(families) if self.no_data: print 'skipping data output%s' % reason else: self.build_data_json(family_id_to_lang_scr_to_sample_key, families, family_zip_info, universal_zip_info, family_id_to_regions, region_to_family_ids) self.build_families_json(family_id_to_lang_scr_to_sample_key, families, family_id_to_default_lang_scr, family_id_to_regions, family_css_info, lang_scr_sort_order) self.build_misc_json(sample_key_to_info, region_data) if self.no_images: print 'skipping image output' else: self.build_images(family_id_to_lang_scr_to_sample_key, families, family_id_to_default_lang_scr, sample_key_to_info)
def generate(self): if self.clean: self.clean_target_dir() if not self.no_build: self.ensure_target_dirs_exist() # debug/print # ['families', 'script_to_family_ids', 'used_lang_data', # 'family_id_to_lang_tags', 'family_id_to_default_lang_tag'] debug = frozenset([]) fonts = noto_fonts.get_noto_fonts() families = noto_fonts.get_families(fonts) if 'families' in debug: print '\nfamilies' for family_id, family in sorted(families.iteritems()): print family_id, family.rep_member.script script_to_family_ids = get_script_to_family_ids(families) if 'script_to_family_ids' in debug: print '\nscript to family ids' for script, family_ids in sorted(script_to_family_ids.iteritems()): print script, family_ids supported_scripts = set(script_to_family_ids.keys()) used_lang_data = get_used_lang_data(supported_scripts) if 'used_lang_data' in debug: print '\nused lang data' for lang, data in sorted(used_lang_data.iteritems()): used = ', '.join(data[0]) unused = ', '.join(data[1]) if unused: unused = '(' + unused + ')' if used: unused = ' ' + unused print '%s: %s%s' % (lang, used, unused) langs_to_delete = [] for lang in used_lang_data.keys(): if not cldr_data.get_english_language_name(lang): langs_to_delete.append(lang) if langs_to_delete: print 'deleting languages with no english name: %s' % langs_to_delete for lang in langs_to_delete: del used_lang_data[lang] lang_tag_to_family_ids = get_lang_tag_to_family_ids(used_lang_data, script_to_family_ids) region_to_family_ids = get_region_to_family_ids(script_to_family_ids) family_id_to_lang_tags = get_family_id_to_lang_tags(lang_tag_to_family_ids, families) if 'family_id_to_lang_tags' in debug: print '\nfamily id to lang tags' for family_id, lang_tags in sorted(family_id_to_lang_tags.iteritems()): print '%s: %s' % (family_id, ','.join(sorted(lang_tags))) family_id_to_regions = get_family_id_to_regions(region_to_family_ids, families) family_id_to_default_lang_tag = get_family_id_to_default_lang_tag( family_id_to_lang_tags) if 'family_id_to_default_lang_tag' in debug: print '\nfamily id to default lang tag' for family_id, lang_tag in family_id_to_default_lang_tag.iteritems(): print family_id, lang_tag used_lang_tags = get_used_lang_tags( lang_tag_to_family_ids.keys(), family_id_to_default_lang_tag.values()) lang_tag_to_sample_data = get_lang_tag_to_sample_data(used_lang_tags) # find the samples that can't be displayed. tested_keys = set() failed_keys = set() family_langs_to_remove = {} for lang_tag in sorted(lang_tag_to_sample_data): sample_info = lang_tag_to_sample_data[lang_tag] sample = sample_info[1] sample_key = sample_info[3] for family_id in sorted(lang_tag_to_family_ids[lang_tag]): full_key = sample_key + '-' + family_id if full_key in tested_keys: if full_key in failed_keys: print 'failed sample %s lang %s' % (full_key, lang_tag) if family_id not in family_langs_to_remove: family_langs_to_remove[family_id] = set() family_langs_to_remove[family_id].add(lang_tag) continue failed_cps = set() tested_keys.add(full_key) charset = families[family_id].charset for cp in sample: if ord(cp) in [0xa, 0x28, 0x29, 0x2c, 0x2d, 0x2e, 0x3b, 0x5b, 0x5d, 0x2010]: continue if ord(cp) not in charset: failed_cps.add(ord(cp)) if failed_cps: print 'sample %s cannot be displayed in %s (lang %s):\n %s' % ( sample_key, family_id, lang_tag, '\n '.join('%04x (%s)' % (cp, unichr(cp)) for cp in sorted(failed_cps))) failed_keys.add(full_key) if family_id not in family_langs_to_remove: family_langs_to_remove[family_id] = set() family_langs_to_remove[family_id].add(lang_tag) for family_id in sorted(family_langs_to_remove): langs_to_remove = family_langs_to_remove[family_id] print 'remove from %s: %s' % (family_id, ','.join(sorted(langs_to_remove))) family_id_to_lang_tags[family_id] -= langs_to_remove default_lang_tag = family_id_to_default_lang_tag[family_id] if default_lang_tag in langs_to_remove: print '!removing default lang tag %s for family %s' % ( default_lang_tag, family_id) for lang in langs_to_remove: lang_tag_to_family_ids[lang] -= set([family_id]) region_data = get_region_lat_lng_data(region_to_family_ids.keys()) lang_tag_sort_order = get_lang_tag_sort_order(lang_tag_to_family_ids.keys()) if self.no_build: print 'skipping build output' return # build outputs if self.no_zips: print 'skipping zip output' else: family_zip_info = self.build_zips(families) universal_zip_info = self.build_universal_zips(families) # build outputs not used by the json but linked to from the web page self.build_ttc_zips() if self.no_css: print 'skipping css output' else: family_css_info = self.build_css(families) if self.no_data or self.no_zips or self.no_css: reason = '' if self.no_data else 'no zips' if self.no_zips else 'no css' print 'skipping data output%s' % reason else: self.build_data_json(families, family_zip_info, universal_zip_info, family_id_to_lang_tags, family_id_to_regions, lang_tag_to_family_ids, region_to_family_ids) self.build_families_json(families, family_id_to_lang_tags, family_id_to_default_lang_tag, family_id_to_regions, family_css_info, lang_tag_sort_order) self.build_misc_json(lang_tag_to_sample_data, region_data) if self.no_images: print 'skipping image output' else: self.build_images(families, family_id_to_lang_tags, family_id_to_default_lang_tag, lang_tag_to_sample_data)
def generate(self): if self.clean: self.clean_target_dir() if not self.no_build: self.ensure_target_dirs_exist() def use_in_web(font): return (not font.subset and not font.fmt == 'ttc' and not font.script in {'CJK', 'HST'} and not font.family in {'Arimo', 'Cousine', 'Tinos'}) fonts = filter(use_in_web, noto_fonts.get_noto_fonts()) families = noto_fonts.get_families(fonts) check_families(families) if 'families' in self.debug: print '\n#debug families' for family_id, family in sorted(families.iteritems()): print '%s (%s, %s)' % ( family_id, family.name, noto_fonts.get_family_filename(family)) if family.hinted_members: print ' hinted: %s' % ', '.join(sorted( [path.basename(m.filepath) for m in family.hinted_members])) if family.unhinted_members: print ' unhinted: %s' % ', '.join(sorted( [path.basename(m.filepath) for m in family.unhinted_members])) script_to_family_ids = get_script_to_family_ids(families) if 'script_to_family_ids' in self.debug: print '\n#debug script to family ids' for script, family_ids in sorted(script_to_family_ids.iteritems()): print '%s: %s' % (script, ', '.join(sorted(family_ids))) all_lang_scrs = set(['und-' + script for script in script_to_family_ids]) all_lang_scrs.update(lang_data.lang_scripts()) lang_scr_to_sample_infos = {} for lang_scr in all_lang_scrs: lang, script = lang_scr.split('-') if not script in script_to_family_ids: print 'no family supports script in %s' % lang_scr continue sample_infos = get_sample_infos(lang_scr) if not sample_infos: continue lang_scr_to_sample_infos[lang_scr] = sample_infos if 'lang_scr_to_sample_infos' in self.debug: print '\n#debug lang+script to sample infos' for lang_scr, info_list in sorted(lang_scr_to_sample_infos.iteritems()): for info in info_list: print '%s: %s, %s, len %d' % ( lang_scr, info[2], info[1], len(info[0])) family_id_to_lang_scrs = get_family_id_to_lang_scrs( lang_scr_to_sample_infos.keys(), script_to_family_ids) if 'family_id_to_lang_scrs' in self.debug: print '\n#debug family id to list of lang+script' for family_id, lang_scrs in sorted(family_id_to_lang_scrs.iteritems()): print '%s: (%d) %s' % ( family_id, len(lang_scrs), ' '.join(sorted(lang_scrs))) family_id_to_lang_scr_to_sample_key, sample_key_to_info = ( get_family_id_to_lang_scr_to_sample_key( family_id_to_lang_scrs, families, lang_scr_to_sample_infos)) if 'family_id_to_lang_scr_to_sample_key' in self.debug: print '\n#debug family id to map from lang+script to sample key' for family_id, lang_scr_to_sample_key in sorted( family_id_to_lang_scr_to_sample_key.iteritems()): print '%s (%d):' % (family_id, len(lang_scr_to_sample_key)) for lang_scr, sample_key in sorted(lang_scr_to_sample_key.iteritems()): print ' %s: %s' % (lang_scr, sample_key) if 'sample_key_to_info' in self.debug: print '\n#debug sample key to sample info' for sample_key, info in sorted(sample_key_to_info.iteritems()): print '%s: %s, len %d' % ( sample_key, info[1], len(info[0])) family_id_to_regions = get_family_id_to_regions( family_id_to_lang_scr_to_sample_key) if 'family_id_to_regions' in self.debug: print '\n#debug family id to regions' for family_id, regions in sorted(family_id_to_regions.iteritems()): print '%s: (%d) %s' % ( family_id, len(regions), ', '.join(sorted(regions))) region_to_family_ids = get_region_to_family_ids(family_id_to_regions) if 'region_to_family_ids' in self.debug: print '\n#debug region to family ids' for region, family_ids in sorted(region_to_family_ids.iteritems()): print '%s: (%d) %s' % ( region, len(family_ids), ', '.join(sorted(family_ids))) family_id_to_default_lang_scr = get_family_id_to_default_lang_scr( family_id_to_lang_scrs, families) if 'family_id_to_default_lang_scr' in self.debug: print '\n#debug family id to default lang scr' for family_id, lang_scr in sorted( family_id_to_default_lang_scr.iteritems()): print '%s: %s' % (family_id, lang_scr) region_data = get_region_lat_lng_data(region_to_family_ids.keys()) lang_scrs = get_named_lang_scrs(family_id_to_lang_scr_to_sample_key) lang_scr_sort_order = get_lang_scr_sort_order(lang_scrs) # sanity checks # all families have languages, and all those have samples. # all families have a default language, and that is in the sample list error_list = [] for family in families.values(): family_id = family.family_id if not family_id in family_id_to_lang_scr_to_sample_key: error_list.append('no entry for family %s' % family_id) continue lang_scr_to_sample_key = family_id_to_lang_scr_to_sample_key[family_id] if not lang_scr_to_sample_key: error_list.append('no langs for family %s' % family_id) continue for lang_scr in lang_scr_to_sample_key: sample_key = lang_scr_to_sample_key[lang_scr] if not sample_key: error_list.append( 'no sample key for lang %s in family %s' % (lang_scr, sample_key)) continue if not sample_key in sample_key_to_info: error_list.append('no sample for sample key: %s' % sample_key) if not family_id in family_id_to_default_lang_scr: error_list.append('no default lang for family %s' % family_id) continue default_lang_scr = family_id_to_default_lang_scr[family_id] if not default_lang_scr in lang_scr_to_sample_key: error_list.append('default lang %s not in samples for family %s' % (default_lang_scr, family_id)) if error_list: print 'Errors:\n' + '\n '.join(error_list) if error_list or self.no_build: print 'skipping build output' return # build outputs # zips are required for data if self.no_zips and self.no_data: print 'skipping zip output' else: family_zip_info = self.build_zips(families) universal_zip_info = self.build_universal_zips(families) # build outputs not used by the json but linked to from the web page if not self.no_zips: self.build_ttc_zips() if self.no_css: print 'skipping css output' else: family_css_info = self.build_css(families) if self.no_data: print 'skipping data output' else: self.build_data_json(family_id_to_lang_scr_to_sample_key, families, family_zip_info, universal_zip_info, family_id_to_regions, region_to_family_ids) self.build_families_json(family_id_to_lang_scr_to_sample_key, families, family_id_to_default_lang_scr, family_id_to_regions, family_css_info, lang_scr_sort_order) self.build_misc_json(sample_key_to_info, region_data) if self.no_images: print 'skipping image output' else: self.build_images(family_id_to_lang_scr_to_sample_key, families, family_id_to_default_lang_scr, sample_key_to_info)