def get_raw_usages(td, fds): gusages = [0 for _ in td.GlobalSubrs] lusages = [[0 for _ in fd.Private.Subrs] for fd in fds] gbias = psCharStrings.calcSubrBias(td.GlobalSubrs) lbias = map(lambda fd: psCharStrings.calcSubrBias(fd.Private.Subrs) if hasattr(fd.Private, 'Subrs') else 0, fds) gsels = [None for _ in td.GlobalSubrs] for g in td.charset: cs, sel = td.CharStrings.getItemAndSelector(g) for before, tok in zip(cs.program, cs.program[1:]): if tok == 'callgsubr': gusages[before + gbias] += 1 gsels[before + gbias] = sel elif tok == 'callsubr': lusages[sel][before + lbias[sel]] += 1 for cs, sel in zip(td.GlobalSubrs, gsels): for before, tok in zip(cs.program, cs.program[1:]): if tok == 'callgsubr': gusages[before + gbias] += 1 elif tok == 'callsubr': lusages[sel][before + lbias[sel]] += 1 for sel, fd in enumerate(fds): if hasattr(fd.Private, 'Subrs'): for cs in fd.Private.Subrs: for before, tok in zip(cs.program, cs.program[1:]): if tok == 'callgsubr': gusages[before + gbias] += 1 elif tok == 'callsubr': lusages[sel][before + lbias[sel]] += 1 return (gusages, lusages)
def inlineProgram(localSubrs, globalSubrs, program): if len(program) < 2: return program inlinedProgram = [] context = program[0] for i in range(1, len(program)): if program[i] == "callsubr": index = context + psCharStrings.calcSubrBias(localSubrs) inlinedProgram.extend( inlineProgram(localSubrs, globalSubrs, localSubrs[index].program)) context = None elif program[i] == "callgsubr": index = context + psCharStrings.calcSubrBias(globalSubrs) inlinedProgram.extend( inlineProgram(localSubrs, globalSubrs, globalSubrs[index].program)) context = None elif program[i] == "return": if context is not None: inlinedProgram.append(context) return inlinedProgram else: if context is not None: inlinedProgram.append(context) context = program[i] if context is not None: inlinedProgram.append(context) return inlinedProgram
def get_raw_usages(td, fds): gusages = [0 for _ in td.GlobalSubrs] lusages = [[0 for _ in fd.Private.Subrs] for fd in fds] gbias = psCharStrings.calcSubrBias(td.GlobalSubrs) lbias = map( lambda fd: psCharStrings.calcSubrBias(fd.Private.Subrs) if hasattr(fd.Private, 'Subrs') else 0, fds) gsels = [None for _ in td.GlobalSubrs] for g in td.charset: cs, sel = td.CharStrings.getItemAndSelector(g) for before, tok in zip(cs.program, cs.program[1:]): if tok == 'callgsubr': gusages[before + gbias] += 1 gsels[before + gbias] = sel elif tok == 'callsubr': lusages[sel][before + lbias[sel]] += 1 for cs, sel in zip(td.GlobalSubrs, gsels): for before, tok in zip(cs.program, cs.program[1:]): if tok == 'callgsubr': gusages[before + gbias] += 1 elif tok == 'callsubr': lusages[sel][before + lbias[sel]] += 1 for sel, fd in enumerate(fds): if hasattr(fd.Private, 'Subrs'): for cs in fd.Private.Subrs: for before, tok in zip(cs.program, cs.program[1:]): if tok == 'callgsubr': gusages[before + gbias] += 1 elif tok == 'callsubr': lusages[sel][before + lbias[sel]] += 1 return (gusages, lusages)
def apply_subrs(top_dict, encoding, gsubrs, lsubrs): multi_font = hasattr(top_dict, "FDArray") gbias = psCharStrings.calcSubrBias(gsubrs) lbias = [psCharStrings.calcSubrBias(subrs) for subrs in lsubrs] if multi_font: for g in top_dict.charset: charstring, sel = top_dict.CharStrings.getItemAndSelector(g) enc = encoding[g] Compreffor.collapse_hintmask(charstring.program) Compreffor.update_program(charstring.program, enc, gbias, lbias, sel) Compreffor.expand_hintmask(charstring.program) for fd in top_dict.FDArray: if not hasattr(fd.Private, "Subrs"): fd.Private.Subrs = cffLib.SubrsIndex() for subrs, subrs_index in zip( itertools.chain([gsubrs], lsubrs), itertools.chain( [top_dict.GlobalSubrs], [fd.Private.Subrs for fd in top_dict.FDArray])): for subr in subrs: item = psCharStrings.T2CharString(program=subr._program) subrs_index.append(item) for fd in top_dict.FDArray: if not fd.Private.Subrs: del fd.Private.Subrs else: for glyph, enc in encoding.items(): charstring = top_dict.CharStrings[glyph] Compreffor.collapse_hintmask(charstring.program) Compreffor.update_program(charstring.program, enc, gbias, lbias, 0) Compreffor.expand_hintmask(charstring.program) assert len(lsubrs) == 1 if not hasattr(top_dict.Private, "Subrs"): top_dict.Private.Subrs = cffLib.SubrsIndex() for subr in lsubrs[0]: item = psCharStrings.T2CharString(program=subr._program) top_dict.Private.Subrs.append(item) if not top_dict.Private.Subrs: del top_dict.Private.Subrs for subr in gsubrs: item = psCharStrings.T2CharString(program=subr._program) top_dict.GlobalSubrs.append(item)
def apply_subrs(top_dict, encoding, gsubrs, lsubrs): multi_font = hasattr(top_dict, "FDArray") gbias = psCharStrings.calcSubrBias(gsubrs) lbias = [psCharStrings.calcSubrBias(subrs) for subrs in lsubrs] if multi_font: for g in top_dict.charset: charstring, sel = top_dict.CharStrings.getItemAndSelector(g) enc = encoding[g] Compreffor.collapse_hintmask(charstring.program) Compreffor.update_program(charstring.program, enc, gbias, lbias, sel) Compreffor.expand_hintmask(charstring.program) for fd in top_dict.FDArray: if not hasattr(fd.Private, "Subrs"): fd.Private.Subrs = cffLib.SubrsIndex() for subrs, subrs_index in zip(itertools.chain([gsubrs], lsubrs), itertools.chain([top_dict.GlobalSubrs], [fd.Private.Subrs for fd in top_dict.FDArray])): for subr in subrs: item = psCharStrings.T2CharString(program=subr._program) subrs_index.append(item) for fd in top_dict.FDArray: if not fd.Private.Subrs: del fd.Private.Subrs else: for glyph, enc in encoding.items(): charstring = top_dict.CharStrings[glyph] Compreffor.collapse_hintmask(charstring.program) Compreffor.update_program(charstring.program, enc, gbias, lbias, 0) Compreffor.expand_hintmask(charstring.program) assert len(lsubrs) == 1 if not hasattr(top_dict.Private, "Subrs"): top_dict.Private.Subrs = cffLib.SubrsIndex() for subr in lsubrs[0]: item = psCharStrings.T2CharString(program=subr._program) top_dict.Private.Subrs.append(item) if not top_dict.Private.Subrs: del top_dict.Private.Subrs for subr in gsubrs: item = psCharStrings.T2CharString(program=subr._program) top_dict.GlobalSubrs.append(item)
def get_savings(td, fds): gsavings = [-(s.subr_cost + 2) if s.program else 0 for s in td.GlobalSubrs] lsavings = [[ -(s.subr_cost + 2) if s.program else 0 for s in fd.Private.Subrs ] for fd in fds] gusages = [0 for _ in td.GlobalSubrs] lusages = [[0 for _ in fd.Private.Subrs] for fd in fds] gbias = psCharStrings.calcSubrBias(td.GlobalSubrs) lbias = map( lambda fd: psCharStrings.calcSubrBias(fd.Private.Subrs) if hasattr(fd.Private, 'Subrs') else 0, fds) def count_subr(idx, is_global, fdidx=-1): if is_global: gsavings[idx + gbias] += (td.GlobalSubrs[idx + gbias].subr_saving - tokenCost(idx) - 1) gusages[idx + gbias] += 1 subr = td.GlobalSubrs[idx + gbias] else: assert fdidx >= 0 lsavings[fdidx][idx + lbias[fdidx]] += ( fds[fdidx].Private.Subrs[idx + lbias[fdidx]].subr_saving - tokenCost(idx) - 1) lusages[fdidx][idx + lbias[fdidx]] += 1 subr = fds[fdidx].Private.Subrs[idx + lbias[fdidx]] # follow called subrs: for before, tok in zip(subr.program, subr.program[1:]): if tok == 'callgsubr': count_subr(before, True, fdidx) elif tok == 'callsubr': count_subr(before, False, fdidx) for g in td.charset: cs, sel = td.CharStrings.getItemAndSelector(g) for before, tok in zip(cs.program, cs.program[1:]): if tok == 'callgsubr': count_subr(before, True, sel) elif tok == 'callsubr': count_subr(before, False, sel) return ((gsavings, lsavings), (gusages, lusages))
def get_savings(td, fds): gsavings = [-(s.subr_cost + 2) if s.program else 0 for s in td.GlobalSubrs] lsavings = [[-(s.subr_cost + 2) if s.program else 0 for s in fd.Private.Subrs] for fd in fds] gusages = [0 for _ in td.GlobalSubrs] lusages = [[0 for _ in fd.Private.Subrs] for fd in fds] gbias = psCharStrings.calcSubrBias(td.GlobalSubrs) lbias = map(lambda fd: psCharStrings.calcSubrBias(fd.Private.Subrs) if hasattr(fd.Private, 'Subrs') else 0, fds) def count_subr(idx, is_global, fdidx=-1): if is_global: gsavings[idx + gbias] += (td.GlobalSubrs[idx + gbias].subr_saving - tokenCost(idx) - 1) gusages[idx + gbias] += 1 subr = td.GlobalSubrs[idx + gbias] else: assert fdidx >= 0 lsavings[fdidx][idx + lbias[fdidx]] += (fds[fdidx].Private.Subrs[idx + lbias[fdidx]].subr_saving - tokenCost(idx) - 1) lusages[fdidx][idx + lbias[fdidx]] += 1 subr = fds[fdidx].Private.Subrs[idx + lbias[fdidx]] # follow called subrs: for before, tok in zip(subr.program, subr.program[1:]): if tok == 'callgsubr': count_subr(before, True, fdidx) elif tok == 'callsubr': count_subr(before, False, fdidx) for g in td.charset: cs, sel = td.CharStrings.getItemAndSelector(g) for before, tok in zip(cs.program, cs.program[1:]): if tok == 'callgsubr': count_subr(before, True, sel) elif tok == 'callsubr': count_subr(before, False, sel) return ((gsavings, lsavings), (gusages, lusages))
def follow_program(program, depth, subrs): bias = psCharStrings.calcSubrBias(subrs) if len(program) > 0: last = program[0] for tok in program[1:]: if tok == "callsubr": assert type(last) == int next_subr = subrs[last + bias] if (not hasattr(next_subr, "_max_call_depth") or next_subr._max_call_depth < depth + 1): increment_subr_depth(next_subr, depth + 1, subrs) elif tok == "callgsubr": assert type(last) == int next_subr = gsubrs[last + gbias] if (not hasattr(next_subr, "_max_call_depth") or next_subr._max_call_depth < depth + 1): increment_subr_depth(next_subr, depth + 1, subrs) last = tok else: log.warning("Compiled subr encountered")
def main(filenames, show_graphs): names = map(os.path.basename, filenames) cffs = map(get_cff, filenames) tds = map(lambda f: f.topDictIndex[0], cffs) fds = map(lambda td: td.FDArray if hasattr(td, 'FDArray') else [], tds) n_bytes = map(get_cs_bytes, tds, fds) for name, b in zip(names, n_bytes): print("%s:\n\t%d bytes" % (name, b)) map(decompile_charstrings, tds, fds) map(print_n_subroutines, names, tds, fds) sav_usag = map(get_savings, tds, fds) for name, (savings, usages) in zip(names, sav_usag): tot_savings = savings[0] + list(itertools.chain.from_iterable(savings[1])) tot_usages = usages[0] + list(itertools.chain.from_iterable(usages[1])) avg = float(sum(tot_savings)) / len(tot_savings) print("%s:\n\tAverage savings per subr: %f\n\tMax saving subr: %d\n\tMax usage subr: %d" % (name, avg, max(tot_savings), max(tot_usages))) if show_graphs: # plot subrs SHOW_START = 0 SHOW_LEN = 200 mins = [] maxes = [] plt.figure(0) for savings, usages in sav_usag: tot_savings = savings[0] + list(itertools.chain.from_iterable(savings[1])) plot_savings = sorted(tot_savings, reverse=True)[SHOW_START:SHOW_START+SHOW_LEN] plt.plot(range(len(plot_savings)), plot_savings) mins.append(min(plot_savings)) maxes.append(max(plot_savings)) plt.ylim([min(mins) - 1, max(maxes) + 1]) plt.title("Subroutine Savings") plt.xlabel("Subroutine") plt.ylabel("Savings (bytes)") raw_usages = map(get_raw_usages, tds, fds) fig = 1 for gusages, lusages in raw_usages: for idx, usages in zip(['Global'] + range(len(lusages)), [gusages] + lusages): if usages: bias = psCharStrings.calcSubrBias(usages) if bias == 1131: orig_order_usages = usages[1024:1240] + usages[0:1024] + usages[1240:] elif bias == 32768: orig_order_usages = (usages[32661:32877] + usages[31637:32661] + usages[32877:33901] + usages[0:31637] + usages[33901:]) else: orig_order_usages = usages plt.figure(fig) plt.plot(range(len(orig_order_usages)), orig_order_usages, color='b') plt.title("Subroutine usages for FD %s" % idx) plt.axvline(215, 0, max(orig_order_usages), color='r') plt.axvline(2263, 0, max(orig_order_usages), color='r') plt.ylim([0, max(orig_order_usages)]) plt.xlim([0, len(orig_order_usages)]) fig += 1 plt.show()
def remove_unused_subroutines(self): cff = self.cff for fontname in cff.keys(): font = cff[fontname] cs = font.CharStrings # Renumber subroutines to remove unused ones # Mark all used subroutines for g in font.charset: c, _ = cs.getItemAndSelector(g) subrs = getattr(c.private, "Subrs", []) decompiler = _MarkingT2Decompiler(subrs, c.globalSubrs, c.private) decompiler.execute(c) all_subrs = [font.GlobalSubrs] if hasattr(font, 'FDArray'): all_subrs.extend(fd.Private.Subrs for fd in font.FDArray if hasattr(fd.Private, 'Subrs') and fd.Private.Subrs) elif hasattr(font.Private, 'Subrs') and font.Private.Subrs: all_subrs.append(font.Private.Subrs) subrs = set(subrs) # Remove duplicates # Prepare for subrs in all_subrs: if not hasattr(subrs, '_used'): subrs._used = set() subrs._used = _uniq_sort(subrs._used) subrs._old_bias = psCharStrings.calcSubrBias(subrs) subrs._new_bias = psCharStrings.calcSubrBias(subrs._used) # Renumber glyph charstrings for g in font.charset: c, _ = cs.getItemAndSelector(g) subrs = getattr(c.private, "Subrs", []) c.subset_subroutines (subrs, font.GlobalSubrs) # Renumber subroutines themselves for subrs in all_subrs: if subrs == font.GlobalSubrs: if not hasattr(font, 'FDArray') and hasattr(font.Private, 'Subrs'): local_subrs = font.Private.Subrs else: local_subrs = [] else: local_subrs = subrs subrs.items = [subrs.items[i] for i in subrs._used] if hasattr(subrs, 'file'): del subrs.file if hasattr(subrs, 'offsets'): del subrs.offsets for subr in subrs.items: subr.subset_subroutines (local_subrs, font.GlobalSubrs) # Delete local SubrsIndex if empty if hasattr(font, 'FDArray'): for fd in font.FDArray: _delete_empty_subrs(fd.Private) else: _delete_empty_subrs(font.Private) # Cleanup for subrs in all_subrs: del subrs._used, subrs._old_bias, subrs._new_bias
def check_cff_call_depth(cff): """Checks that the Charstrings in the provided CFFFontSet obey the rules for subroutine nesting. Return True if the subroutine nesting level does not exceed the maximum limit (10), else return False. """ SUBR_NESTING_LIMIT = 10 assert len(cff.topDictIndex) == 1 td = cff.topDictIndex[0] class track_info: pass track_info.max_for_all = 0 gsubrs = cff.GlobalSubrs gbias = psCharStrings.calcSubrBias(gsubrs) def follow_program(program, depth, subrs): bias = psCharStrings.calcSubrBias(subrs) if len(program) > 0: last = program[0] for tok in program[1:]: if tok == "callsubr": assert type(last) == int next_subr = subrs[last + bias] if (not hasattr(next_subr, "_max_call_depth") or next_subr._max_call_depth < depth + 1): increment_subr_depth(next_subr, depth + 1, subrs) elif tok == "callgsubr": assert type(last) == int next_subr = gsubrs[last + gbias] if (not hasattr(next_subr, "_max_call_depth") or next_subr._max_call_depth < depth + 1): increment_subr_depth(next_subr, depth + 1, subrs) last = tok else: log.warning("Compiled subr encountered") def increment_subr_depth(subr, depth, subrs=None): if not hasattr(subr, "_max_call_depth") or subr._max_call_depth < depth: subr._max_call_depth = depth if subr._max_call_depth > track_info.max_for_all: track_info.max_for_all = subr._max_call_depth program = subr.program follow_program(program, depth, subrs) for cs in td.CharStrings.values(): cs.decompile() follow_program(cs.program, 0, getattr(cs.private, "Subrs", [])) if track_info.max_for_all <= SUBR_NESTING_LIMIT: log.info("Subroutine nesting depth ok! [max nesting depth of %d]", track_info.max_for_all) return True else: log.warning( "Subroutine nesting depth too deep :( [max nesting depth " "of %d]", track_info.max_for_all) return False
def main(filenames, show_graphs): names = map(os.path.basename, filenames) cffs = map(get_cff, filenames) tds = map(lambda f: f.topDictIndex[0], cffs) fds = map(lambda td: td.FDArray if hasattr(td, 'FDArray') else [], tds) n_bytes = map(get_cs_bytes, tds, fds) for name, b in zip(names, n_bytes): print("%s:\n\t%d bytes" % (name, b)) map(decompile_charstrings, tds, fds) map(print_n_subroutines, names, tds, fds) sav_usag = map(get_savings, tds, fds) for name, (savings, usages) in zip(names, sav_usag): tot_savings = savings[0] + list( itertools.chain.from_iterable(savings[1])) tot_usages = usages[0] + list(itertools.chain.from_iterable(usages[1])) avg = float(sum(tot_savings)) / len(tot_savings) print( "%s:\n\tAverage savings per subr: %f\n\tMax saving subr: %d\n\tMax usage subr: %d" % (name, avg, max(tot_savings), max(tot_usages))) if show_graphs: # plot subrs SHOW_START = 0 SHOW_LEN = 200 mins = [] maxes = [] plt.figure(0) for savings, usages in sav_usag: tot_savings = savings[0] + list( itertools.chain.from_iterable(savings[1])) plot_savings = sorted( tot_savings, reverse=True)[SHOW_START:SHOW_START + SHOW_LEN] plt.plot(range(len(plot_savings)), plot_savings) mins.append(min(plot_savings)) maxes.append(max(plot_savings)) plt.ylim([min(mins) - 1, max(maxes) + 1]) plt.title("Subroutine Savings") plt.xlabel("Subroutine") plt.ylabel("Savings (bytes)") raw_usages = map(get_raw_usages, tds, fds) fig = 1 for gusages, lusages in raw_usages: for idx, usages in zip(['Global'] + range(len(lusages)), [gusages] + lusages): if usages: bias = psCharStrings.calcSubrBias(usages) if bias == 1131: orig_order_usages = usages[1024:1240] + usages[ 0:1024] + usages[1240:] elif bias == 32768: orig_order_usages = (usages[32661:32877] + usages[31637:32661] + usages[32877:33901] + usages[0:31637] + usages[33901:]) else: orig_order_usages = usages plt.figure(fig) plt.plot(range(len(orig_order_usages)), orig_order_usages, color='b') plt.title("Subroutine usages for FD %s" % idx) plt.axvline(215, 0, max(orig_order_usages), color='r') plt.axvline(2263, 0, max(orig_order_usages), color='r') plt.ylim([0, max(orig_order_usages)]) plt.xlim([0, len(orig_order_usages)]) fig += 1 plt.show()
def process_subrs(glyph_set_keys, encodings, fdlen, fdselect, substrings, rev_keymap, subr_limit, nest_limit): def mark_reachable(cand_subr, fdidx): try: if fdidx not in cand_subr._fdidx: cand_subr._fdidx.append(fdidx) except AttributeError: cand_subr._fdidx = [fdidx] for it in cand_subr._encoding: mark_reachable(it[1], fdidx) if fdselect is not None: for g, enc in zip(glyph_set_keys, encodings): sel = fdselect(g) for it in enc: mark_reachable(it[1], sel) else: for encoding in encodings: for it in encoding: mark_reachable(it[1], 0) subrs = [ s for s in substrings if s.usages() > 0 and hasattr(s, '_fdidx') and bool(s._fdidx) and s.subr_saving(use_usages=True, true_cost=True) > 0 ] bad_substrings = [ s for s in substrings if s.usages() == 0 or not hasattr(s, '_fdidx') or not bool(s._fdidx) or s.subr_saving(use_usages=True, true_cost=True) <= 0 ] log.debug("%d substrings unused or negative saving subrs", len(bad_substrings)) for s in bad_substrings: s._flatten = True gsubrs = [] lsubrs = [[] for _ in range(fdlen)] subrs.sort( key=lambda s: s.subr_saving(use_usages=True, true_cost=True)) while subrs and (any(len(s) < subr_limit for s in lsubrs) or len(gsubrs) < subr_limit): subr = subrs[-1] del subrs[-1] if len(subr._fdidx) == 1: lsub_index = lsubrs[subr._fdidx[0]] if len(gsubrs) < subr_limit: if len(lsub_index) < subr_limit: # both have space gcost = Compreffor.test_call_cost(subr, gsubrs) lcost = Compreffor.test_call_cost(subr, lsub_index) if gcost < lcost: Compreffor.insert_by_usage(subr, gsubrs) subr._global = True else: Compreffor.insert_by_usage(subr, lsub_index) else: # just gsubrs has space Compreffor.insert_by_usage(subr, gsubrs) subr._global = True elif len(lsub_index) < subr_limit: # just lsubrs has space Compreffor.insert_by_usage(subr, lsub_index) else: # we must skip :( bad_substrings.append(subr) else: if len(gsubrs) < subr_limit: # we can put it in globals Compreffor.insert_by_usage(subr, gsubrs) subr._global = True else: # no room for this one bad_substrings.append(subr) bad_substrings.extend([s[1] for s in subrs ]) # add any leftover subrs to bad_substrings if fdselect is not None: # CID-keyed: Avoid `callsubr` usage in global subroutines bad_lsubrs = Compreffor.collect_lsubrs_called_from(gsubrs) bad_substrings.extend(bad_lsubrs) lsubrs = [[s for s in lsubrarr if s not in bad_lsubrs] for lsubrarr in lsubrs] for s in bad_substrings: s._flatten = True # fix any nesting issues Compreffor.calc_nesting(gsubrs) for subrs in lsubrs: Compreffor.calc_nesting(subrs) too_nested = [ s for s in itertools.chain(*lsubrs) if s._max_call_depth > nest_limit ] too_nested.extend( [s for s in gsubrs if s._max_call_depth > nest_limit]) for s in too_nested: s._flatten = True bad_substrings.extend(too_nested) lsubrs = [[s for s in lsubrarr if s._max_call_depth <= nest_limit] for lsubrarr in lsubrs] gsubrs = [s for s in gsubrs if s._max_call_depth <= nest_limit] too_nested = len(too_nested) log.debug("%d substrings nested too deep", too_nested) log.debug("%d substrings being flattened", len(bad_substrings)) # reorganize to minimize call cost of most frequent subrs gbias = psCharStrings.calcSubrBias(gsubrs) lbias = [psCharStrings.calcSubrBias(s) for s in lsubrs] for subr_arr, bias in zip(itertools.chain([gsubrs], lsubrs), itertools.chain([gbias], lbias)): subr_arr.sort(key=lambda s: s.usages(), reverse=True) if bias == 1131: subr_arr[:] = subr_arr[216:1240] + subr_arr[0:216] + subr_arr[ 1240:] elif bias == 32768: subr_arr[:] = (subr_arr[2264:33901] + subr_arr[216:1240] + subr_arr[0:216] + subr_arr[1240:2264] + subr_arr[33901:]) for idx, subr in enumerate(subr_arr): subr._position = idx for subr in sorted(bad_substrings, key=lambda s: len(s)): # NOTE: it is important this is run in order so shorter # substrings are run before longer ones if hasattr(subr, '_fdidx') and len(subr._fdidx) > 0: program = [rev_keymap[tok] for tok in subr.value()] Compreffor.update_program(program, subr.encoding(), gbias, lbias, None) Compreffor.expand_hintmask(program) subr._program = program for subr_arr, sel in zip(itertools.chain([gsubrs], lsubrs), itertools.chain([None], range(fdlen))): for subr in subr_arr: program = [rev_keymap[tok] for tok in subr.value()] if program[-1] not in ("endchar", "return"): program.append("return") Compreffor.update_program(program, subr.encoding(), gbias, lbias, sel) Compreffor.expand_hintmask(program) subr._program = program return (gsubrs, lsubrs)
def check_cff_call_depth(cff): """Checks that the Charstrings in the provided CFFFontSet obey the rules for subroutine nesting. Return True if the subroutine nesting level does not exceed the maximum limit (10), else return False. """ SUBR_NESTING_LIMIT = 10 assert len(cff.topDictIndex) == 1 td = cff.topDictIndex[0] class track_info: pass track_info.max_for_all = 0 gsubrs = cff.GlobalSubrs gbias = psCharStrings.calcSubrBias(gsubrs) def follow_program(program, depth, subrs): bias = psCharStrings.calcSubrBias(subrs) if len(program) > 0: last = program[0] for tok in program[1:]: if tok == "callsubr": assert type(last) == int next_subr = subrs[last + bias] if (not hasattr(next_subr, "_max_call_depth") or next_subr._max_call_depth < depth + 1): increment_subr_depth(next_subr, depth + 1, subrs) elif tok == "callgsubr": assert type(last) == int next_subr = gsubrs[last + gbias] if (not hasattr(next_subr, "_max_call_depth") or next_subr._max_call_depth < depth + 1): increment_subr_depth(next_subr, depth + 1, subrs) last = tok else: log.warning("Compiled subr encountered") def increment_subr_depth(subr, depth, subrs=None): if not hasattr(subr, "_max_call_depth") or subr._max_call_depth < depth: subr._max_call_depth = depth if subr._max_call_depth > track_info.max_for_all: track_info.max_for_all = subr._max_call_depth program = subr.program follow_program(program, depth, subrs) for cs in td.CharStrings.values(): cs.decompile() follow_program(cs.program, 0, cs.private.Subrs) if track_info.max_for_all <= SUBR_NESTING_LIMIT: log.info("Subroutine nesting depth ok! [max nesting depth of %d]", track_info.max_for_all) return True else: log.warning("Subroutine nesting depth too deep :( [max nesting depth " "of %d]", track_info.max_for_all) return False
def process_subrs(glyph_set_keys, encodings, fdlen, fdselect, substrings, rev_keymap, subr_limit, nest_limit, verbose=False): post_time = time.time() def mark_reachable(cand_subr, fdidx): try: if fdidx not in cand_subr._fdidx: cand_subr._fdidx.append(fdidx) except AttributeError: cand_subr._fdidx = [fdidx] for it in cand_subr._encoding: mark_reachable(it[1], fdidx) if fdselect != None: for g, enc in zip(glyph_set_keys, encodings): sel = fdselect(g) for it in enc: mark_reachable(it[1], sel) else: for encoding in encodings: for it in encoding: mark_reachable(it[1], 0) subrs = [s for s in substrings if s.usages() > 0 and hasattr(s, '_fdidx') and bool(s._fdidx) and s.subr_saving(use_usages=True, true_cost=True) > 0] bad_substrings = [s for s in substrings if s.usages() == 0 or not hasattr(s, '_fdidx') or not bool(s._fdidx) or s.subr_saving(use_usages=True, true_cost=True) <= 0] if verbose: print("%d substrings unused or negative saving subrs" % len(bad_substrings)) def set_flatten(s): s._flatten = True map(set_flatten, bad_substrings) gsubrs = [] lsubrs = [[] for _ in xrange(fdlen)] subrs.sort(key=lambda s: s.subr_saving(use_usages=True, true_cost=True)) while subrs and (any(len(s) < subr_limit for s in lsubrs) or len(gsubrs) < subr_limit): subr = subrs[-1] del subrs[-1] if len(subr._fdidx) == 1: lsub_index = lsubrs[subr._fdidx[0]] if len(gsubrs) < subr_limit: if len(lsub_index) < subr_limit: # both have space gcost = Compreffor.test_call_cost(subr, gsubrs) lcost = Compreffor.test_call_cost(subr, lsub_index) if gcost < lcost: Compreffor.insert_by_usage(subr, gsubrs) subr._global = True else: Compreffor.insert_by_usage(subr, lsub_index) else: # just gsubrs has space Compreffor.insert_by_usage(subr, gsubrs) subr._global = True elif len(lsub_index) < subr_limit: # just lsubrs has space Compreffor.insert_by_usage(subr, lsub_index) else: # we must skip :( bad_substrings.append(subr) else: if len(gsubrs) < subr_limit: # we can put it in globals Compreffor.insert_by_usage(subr, gsubrs) subr._global = True else: # no room for this one bad_substrings.append(subr) bad_substrings.extend([s[1] for s in subrs]) # add any leftover subrs to bad_substrings map(set_flatten, bad_substrings) # fix any nesting issues Compreffor.calc_nesting(gsubrs) map(Compreffor.calc_nesting, lsubrs) too_nested = [s for s in itertools.chain(*lsubrs) if s._max_call_depth > nest_limit] too_nested.extend([s for s in gsubrs if s._max_call_depth > nest_limit]) map(set_flatten, too_nested) bad_substrings.extend(too_nested) lsubrs = [[s for s in lsubrarr if s._max_call_depth <= nest_limit] for lsubrarr in lsubrs] gsubrs = [s for s in gsubrs if s._max_call_depth <= nest_limit] too_nested = len(too_nested) if verbose: print("%d substrings nested too deep" % too_nested) print("%d substrings being flattened" % len(bad_substrings)) # reorganize to minimize call cost of most frequent subrs def update_position(idx, subr): subr._position = idx gbias = psCharStrings.calcSubrBias(gsubrs) lbias = [psCharStrings.calcSubrBias(s) for s in lsubrs] for subr_arr, bias in zip(itertools.chain([gsubrs], lsubrs), itertools.chain([gbias], lbias)): subr_arr.sort(key=lambda s: s.usages(), reverse=True) if bias == 1131: subr_arr[:] = subr_arr[216:1240] + subr_arr[0:216] + subr_arr[1240:] elif bias == 32768: subr_arr[:] = (subr_arr[2264:33901] + subr_arr[216:1240] + subr_arr[0:216] + subr_arr[1240:2264] + subr_arr[33901:]) map(update_position, range(len(subr_arr)), subr_arr) for subr in sorted(bad_substrings, key=lambda s: len(s)): # NOTE: it is important this is run in order so shorter # substrings are run before longer ones if hasattr(subr, '_fdidx') and len(subr._fdidx) > 0: program = [rev_keymap[tok] for tok in subr.value()] Compreffor.update_program(program, subr.encoding(), gbias, lbias, None) Compreffor.expand_hintmask(program) subr._program = program for subr_arr, sel in zip(itertools.chain([gsubrs], lsubrs), itertools.chain([None], xrange(fdlen))): for subr in subr_arr: program = [rev_keymap[tok] for tok in subr.value()] if program[-1] not in ("endchar", "return"): program.append("return") Compreffor.update_program(program, subr.encoding(), gbias, lbias, sel) Compreffor.expand_hintmask(program) subr._program = program if verbose: print("POST-TIME: %gs" % (time.time() - post_time)) return (gsubrs, lsubrs)
def prune_post_subset(self, font, options): cff = self.cff for fontname in cff.keys(): font = cff[fontname] cs = font.CharStrings # Drop unused FontDictionaries if hasattr(font, "FDSelect"): sel = font.FDSelect indices = _uniq_sort(sel.gidArray) sel.gidArray = [indices.index (ss) for ss in sel.gidArray] arr = font.FDArray arr.items = [arr[i] for i in indices] del arr.file, arr.offsets # Desubroutinize if asked for if options.desubroutinize: for g in font.charset: c, _ = cs.getItemAndSelector(g) c.decompile() subrs = getattr(c.private, "Subrs", []) decompiler = _DesubroutinizingT2Decompiler(subrs, c.globalSubrs) decompiler.execute(c) c.program = c._desubroutinized del c._desubroutinized # Drop hints if not needed if not options.hinting: # This can be tricky, but doesn't have to. What we do is: # # - Run all used glyph charstrings and recurse into subroutines, # - For each charstring (including subroutines), if it has any # of the hint stem operators, we mark it as such. # Upon returning, for each charstring we note all the # subroutine calls it makes that (recursively) contain a stem, # - Dropping hinting then consists of the following two ops: # * Drop the piece of the program in each charstring before the # last call to a stem op or a stem-calling subroutine, # * Drop all hintmask operations. # - It's trickier... A hintmask right after hints and a few numbers # will act as an implicit vstemhm. As such, we track whether # we have seen any non-hint operators so far and do the right # thing, recursively... Good luck understanding that :( css = set() for g in font.charset: c, _ = cs.getItemAndSelector(g) c.decompile() subrs = getattr(c.private, "Subrs", []) decompiler = _DehintingT2Decompiler(css, subrs, c.globalSubrs, c.private.nominalWidthX, c.private.defaultWidthX) decompiler.execute(c) c.width = decompiler.width for charstring in css: charstring.drop_hints() del css # Drop font-wide hinting values all_privs = [] if hasattr(font, 'FDSelect'): all_privs.extend(fd.Private for fd in font.FDArray) else: all_privs.append(font.Private) for priv in all_privs: for k in ['BlueValues', 'OtherBlues', 'FamilyBlues', 'FamilyOtherBlues', 'BlueScale', 'BlueShift', 'BlueFuzz', 'StemSnapH', 'StemSnapV', 'StdHW', 'StdVW', 'ForceBold', 'LanguageGroup', 'ExpansionFactor']: if hasattr(priv, k): setattr(priv, k, None) # Renumber subroutines to remove unused ones # Mark all used subroutines for g in font.charset: c, _ = cs.getItemAndSelector(g) subrs = getattr(c.private, "Subrs", []) decompiler = _MarkingT2Decompiler(subrs, c.globalSubrs) decompiler.execute(c) all_subrs = [font.GlobalSubrs] if hasattr(font, 'FDSelect'): all_subrs.extend(fd.Private.Subrs for fd in font.FDArray if hasattr(fd.Private, 'Subrs') and fd.Private.Subrs) elif hasattr(font.Private, 'Subrs') and font.Private.Subrs: all_subrs.append(font.Private.Subrs) subrs = set(subrs) # Remove duplicates # Prepare for subrs in all_subrs: if not hasattr(subrs, '_used'): subrs._used = set() subrs._used = _uniq_sort(subrs._used) subrs._old_bias = psCharStrings.calcSubrBias(subrs) subrs._new_bias = psCharStrings.calcSubrBias(subrs._used) # Renumber glyph charstrings for g in font.charset: c, _ = cs.getItemAndSelector(g) subrs = getattr(c.private, "Subrs", []) c.subset_subroutines (subrs, font.GlobalSubrs) # Renumber subroutines themselves for subrs in all_subrs: if subrs == font.GlobalSubrs: if not hasattr(font, 'FDSelect') and hasattr(font.Private, 'Subrs'): local_subrs = font.Private.Subrs else: local_subrs = [] else: local_subrs = subrs subrs.items = [subrs.items[i] for i in subrs._used] if hasattr(subrs, 'file'): del subrs.file if hasattr(subrs, 'offsets'): del subrs.offsets for subr in subrs.items: subr.subset_subroutines (local_subrs, font.GlobalSubrs) # Delete local SubrsIndex if empty if hasattr(font, 'FDSelect'): for fd in font.FDArray: _delete_empty_subrs(fd.Private) else: _delete_empty_subrs(font.Private) # Cleanup for subrs in all_subrs: del subrs._used, subrs._old_bias, subrs._new_bias return True