def doit(args):
logger = args.logger
# Read input csv
builder = FB.FTMLBuilder(logger, incsv = args.input, font = args.ifont)
# Initialize FTML document:
test = "ftml_builder test"
ftml = FB.FTML(test, logger)
# all chars that should be in the font:
ftml.startTestGroup('Encoded characters')
for uid in sorted(builder.uids()):
if uid < 32: continue
c = builder.char(uid)
# iterate over all permutations of feature settings that might affect this character:
for featlist in builder.permuteFeatures(uids = (uid,)):
ftml.setFeatures(featlist)
builder.render((uid,), ftml)
# Test one character with RTL enabled:
if uid == 67:
builder.render((uid,), ftml, rtl = True)
# Don't close test -- collect consecutive encoded chars in a single row
ftml.clearFeatures()
for langID in sorted(c.langs):
ftml.setLang(langID)
builder.render((uid,), ftml)
ftml.clearLang()
# Write the output ftml file
ftml.writeFile(args.output)
def doit(args):
logger = args.logger
# Read input csv
builder = FB.FTMLBuilder(logger,
incsv=args.input,
fontcode=args.fontcode,
font=args.ifont,
ap=args.ap,
rtlenable=True)
# Override default base (25CC) for displaying combining marks
builder.diacBase = 0x0628 # beh
# Initialize FTML document:
test = args.test or "AllChars (NG)" # Default to AllChars
ftml = FB.FTML(test,
logger,
rendercheck=True,
fontscale=args.scale,
xslfn=args.xsl,
fontsrc=args.fontsrc,
defaultrtl=args.rtl)
if test.lower().startswith("allchars"):
# all chars that should be in the font:
ftml.startTestGroup('Encoded characters')
for uid in sorted(builder.uids()):
if uid < 32: continue
c = builder.char(uid)
for featlist in builder.permuteFeatures(uids=(uid, )):
ftml.setFeatures(featlist)
builder.render((uid, ), ftml)
ftml.clearFeatures()
for langID in sorted(c.langs):
ftml.setLang(langID)
builder.render((uid, ), ftml)
ftml.clearLang()
# Add specials and ligatures that were in the glyph_data:
ftml.startTestGroup('Specials & ligatures from glyph_data')
for basename in sorted(builder.specials()):
special = builder.special(basename)
for featlist in builder.permuteFeatures(uids=special.uids):
ftml.setFeatures(featlist)
builder.render(special.uids, ftml)
ftml.closeTest()
ftml.clearFeatures()
if len(special.langs):
for langID in sorted(special.langs):
ftml.setLang(langID)
builder.render(special.uids, ftml)
ftml.closeTest()
ftml.clearLang()
# Add Lam-Alef data manually
ftml.startTestGroup('Lam-Alef')
lamlist = filter(
lambda x: x in builder.uids(),
(0x0644, 0x06B5, 0x06B6, 0x06B7, 0x06B8, 0x076A, 0x08A6))
aleflist = filter(lambda x: x in builder.uids(),
(0x0627, 0x0622, 0x0623, 0x0625, 0x0671, 0x0672,
0x0673, 0x0675, 0x0773, 0x0774))
for lam in lamlist:
for alef in aleflist:
for featlist in builder.permuteFeatures(uids=(lam, alef)):
ftml.setFeatures(featlist)
builder.render((lam, alef), ftml)
ftml.closeTest()
ftml.clearFeatures()
if test.lower().startswith("diac"):
# Diac attachment:
# Representative base and diac chars:
repDiac = filter(
lambda x: x in builder.uids(),
(0x064E, 0x0650, 0x065E, 0x0670, 0x0616, 0x06E3, 0x08F0, 0x08F2))
repBase = filter(
lambda x: x in builder.uids(),
(0x0627, 0x0628, 0x062B, 0x0647, 0x064A, 0x77F, 0x08AC))
lamlist = filter(
lambda x: x in builder.uids(),
(0x0644, 0x06B5, 0x06B6, 0x06B7, 0x06B8, 0x076A, 0x08A6))
aleflist = filter(lambda x: x in builder.uids(),
(0x0627, 0x0622, 0x0623, 0x0625, 0x0671, 0x0672,
0x0673, 0x0675, 0x0773, 0x0774))
ftml.startTestGroup(
'Representative diacritics on all bases that take diacritics')
for uid in sorted(builder.uids()):
if uid < 32 or uid in (0xAA, 0xBA): continue
c = builder.char(uid)
# Always process Lo, but others only if that take marks:
if c.general == GC.OTHER_LETTER or c.isBase:
for diac in repDiac:
for featlist in builder.permuteFeatures(uids=(uid, diac)):
ftml.setFeatures(featlist)
builder.render((uid, diac), ftml, addBreaks=False)
if diac != 0x0651: # If not shadda
# include shadda, in either order:
builder.render((uid, diac, 0x0651),
ftml,
addBreaks=False)
builder.render((uid, 0x0651, diac),
ftml,
addBreaks=False)
if diac != 0x0654: # If not hamza above
# include hamza above, in either order:
builder.render((uid, diac, 0x0654),
ftml,
addBreaks=False)
builder.render((uid, 0x0654, diac),
ftml,
addBreaks=False)
ftml.clearFeatures()
ftml.closeTest()
ftml.startTestGroup('All diacritics on representative bases')
for uid in sorted(builder.uids()):
# ignore non-ABS marks
if uid < 0x600 or uid in range(0xFE00, 0xFE10): continue
c = builder.char(uid)
if c.general == GC.NON_SPACING_MARK:
for base in repBase:
for featlist in builder.permuteFeatures(uids=(uid, base)):
ftml.setFeatures(featlist)
builder.render((base, uid),
ftml,
keyUID=uid,
addBreaks=False)
if uid != 0x0651: # if not shadda
# include shadda, in either order:
builder.render((base, uid, 0x0651),
ftml,
keyUID=uid,
addBreaks=False)
builder.render((base, 0x0651, uid),
ftml,
keyUID=uid,
addBreaks=False)
if diac != 0x0670: # If not superscript alef
# include superscript alef, in either order:
builder.render((uid, diac, 0x0670),
ftml,
addBreaks=False)
builder.render((uid, 0x0670, diac),
ftml,
addBreaks=False)
ftml.clearFeatures()
ftml.closeTest()
ftml.startTestGroup('Special cases')
builder.render((0x064A, 0x065E), ftml) # Yeh + Fatha should keep dots
builder.render((0x064A, 0x0654), ftml) # Yeh + Hamza should loose dots
ftml.closeTest()
ftml.startTestGroup('LamAlef ligatures')
diaB = 0x064D
diaA = 0x064B
for lam in lamlist:
for alef in aleflist:
for featlist in builder.permuteFeatures(uids=(lam, alef)):
ftml.setFeatures(featlist)
builder.render((lam, alef), ftml, addBreaks=False)
builder.render((lam, diaA, alef, diaA),
ftml,
addBreaks=False)
builder.render((lam, diaB, alef), ftml, addBreaks=False)
builder.render((lam, alef, diaB), ftml, addBreaks=False)
builder.render((lam, diaB, alef, diaB),
ftml,
addBreaks=False)
ftml.clearFeatures()
ftml.closeTest()
if test.lower().startswith("subtending"):
# Generates sample data for all subtending marks. Data includes sequences of 0 to n+1
# digits, where n is the maximum expected to be supported on the mark. Latin, Arbic-Indic,
# and Extended Arabic-Indic digits are included.
for digitSample in filter(lambda x: x in builder.uids(),
(0x0032, 0x0668, 0x06F8)):
digitOne = (digitSample & 0xFFF0) + 1
for uid, lgt in filter(lambda x: x[0] in builder.uids(),
([0x600, 3], [0x0601, 4], [0x0602, 2], [
0x0603, 4
], [0x0604, 4], [0x0605, 4], [0x06DD, 3])):
c = unichr(uid)
label = "U+{0:04X}".format(uid)
comment = builder.char(uid).basename
for featlist in builder.permuteFeatures(uids=(uid, )):
ftml.setFeatures(featlist)
ftml.addToTest(uid, "\u0628" + c + "\u0645", label,
comment)
for ln in range(1, lgt + 1):
ftml.addToTest(uid, c + unichr(digitSample) * ln)
ftml.addToTest(uid,
c + unichr(digitOne) + unichr(digitOne + 1))
ftml.clearFeatures()
ftml.closeTest()
if uid == 0x06DD and digitOne == 0x06F1:
# Extra items for Eastern digits
for featlist in builder.permuteFeatures(uids=(uid,
0x06F7)):
ftml.setFeatures(featlist)
ftml.addToTest(uid, c + "\u06F4\u06F6\u06F7", label,
"4 6 7")
ftml.clearFeatures()
for langID in sorted(builder.char(0x06F7).langs):
ftml.setLang(langID)
ftml.addToTest(uid, c + "\u06F4\u06F6\u06F7", label,
"4 6 7")
ftml.clearLang()
ftml.closeTest()
ftml.writeFile(args.output)
def doit(args):
logger = args.logger
# Read input csv
builder = FB.FTMLBuilder(logger, incsv=args.input, fontcode=args.fontcode, font=args.ifont, ap=args.ap,
rtlenable=args.rtl, langs=args.langs)
# Override default base (25CC) for displaying combining marks:
builder.diacBase = 0x0B95 # ka
# Specify blocks of primary and secondary scripts
comb = range(0x0300, 0x036F+1)
taml = range(0x0B80, 0x0BFF+1)
deva = range(0x0900, 0x0097F+1)
vedic = range(0x1CD0, 0x1CFF+1)
gran = range(0x11300, 0x1137F+1)
block = list(comb) + list(taml) + list(deva) + list(vedic) + list(gran)
# Useful ranges of codepoints
uids = sorted(builder.uids())
vowels = [uid for uid in uids if get_ucd(uid, 'InSC') == 'Vowel_Independent']
consonants = [uid for uid in uids if get_ucd(uid, 'InSC') == 'Consonant']
matras = [uid for uid in uids if 'VOWEL SIGN' in get_ucd(uid, 'na')]
digits = [uid for uid in uids if builder.char(uid).general == 'Nd' and uid in block]
# Initialize FTML document:
# Default name for test: AllChars or something based on the csvdata file:
test = args.test or 'AllChars (NG)'
widths = None
if args.width:
try:
width, units = re.match(r'(\d+)(.*)$', args.width).groups()
if len(args.fontsrc):
width = int(round(int(width)/len(args.fontsrc)))
widths = {'string': f'{width}{units}'}
logger.log(f'width: {args.width} --> {widths["string"]}', 'I')
except:
logger.log(f'Unable to parse width argument "{args.width}"', 'W')
# split labels from fontsource parameter
fontsrc = []
labels = []
for sl in args.fontsrc:
try:
s, l = sl.split('=',1)
fontsrc.append(s)
labels.append(l)
except ValueError:
fontsrc.append(sl)
labels.append(None)
ftml = FB.FTML(test, logger, rendercheck=not args.norendercheck, fontscale=args.scale,
widths=widths, xslfn=args.xsl, fontsrc=fontsrc, fontlabel=labels, defaultrtl=args.rtl)
if test.lower().startswith("allchars"):
# all chars that should be in the font:
ftml.startTestGroup('Encoded characters')
for uid in uids:
if uid < 32: continue
c = builder.char(uid)
# iterate over all permutations of feature settings that might affect this character:
for featlist in builder.permuteFeatures(uids = (uid,)):
ftml.setFeatures(featlist)
builder.render((uid,), ftml)
# Don't close test -- collect consecutive encoded chars in a single row
ftml.clearFeatures()
if len(c.langs):
for langID in builder.allLangs:
ftml.setLang(langID)
builder.render((uid,), ftml)
ftml.clearLang()
# Add unencoded specials and ligatures -- i.e., things with a sequence of USVs in the glyph_data:
ftml.startTestGroup('Specials & ligatures from glyph_data')
for basename in builder.specials():
special = builder.special(basename)
# iterate over all permutations of feature settings that might affect this special
for featlist in builder.permuteFeatures(uids = special.uids):
ftml.setFeatures(featlist)
builder.render(special.uids, ftml)
# close test so each special is on its own row:
ftml.closeTest()
ftml.clearFeatures()
if len(special.langs):
for langID in builder.allLangs:
ftml.setLang(langID)
builder.render(special.uids, ftml)
ftml.closeTest()
ftml.clearLang()
# Characters used to create SILE test data
ftml.startTestGroup('Proof')
for uid in vowels:
builder.render((uid,), ftml)
ftml.closeTest()
for uid in matras:
builder.render((uid,), ftml)
ftml.closeTest()
for uid in consonants:
builder.render((uid,), ftml)
ftml.closeTest()
for uid in digits:
builder.render((uid,), ftml)
ftml.closeTest()
below_marks = (0x0323, 0x1133B, 0x1133C) # 0x1CDC, 0x1CDD, 0x1CDE, 0x1CDF
above_marks = (0x0307, 0x0B82, 0x0BCD) # 0x1CDA
marks = below_marks + above_marks
if test.lower().startswith("diac"):
# Diac attachment:
# Representative base and diac chars:
repDiac = list(filter(lambda x: x in builder.uids(), marks))
repBase = list(filter(lambda x: x in builder.uids(), (0x0B95, 0x0B85)))
ftml.startTestGroup('Representative diacritics on all bases that take diacritics')
for uid in uids:
# ignore bases outside of the primary script:
if uid not in block: continue
c = builder.char(uid)
# Always process Lo, but others only if that take marks:
if c.general == 'Lo' or c.isBase:
for diac in repDiac:
for featlist in builder.permuteFeatures(uids = (uid,diac)):
ftml.setFeatures(featlist)
# Don't automatically separate connecting or mirrored forms into separate lines:
builder.render((uid,diac), ftml, addBreaks = False)
ftml.clearFeatures()
ftml.closeTest()
ftml.startTestGroup('All diacritics on representative bases')
for uid in uids:
# ignore bases outside of the primary and Latin scripts:
if uid < 0x0300 or uid in range(0xFE00, 0xFE10): continue
c = builder.char(uid)
if c.general == 'Mn':
for base in repBase:
for featlist in builder.permuteFeatures(uids = (uid,base)):
ftml.setFeatures(featlist)
builder.render((base,uid), ftml, keyUID = uid, addBreaks = False)
ftml.clearFeatures()
ftml.closeTest()
if test.lower().startswith("matras"):
# Combinations with matras:
ftml.startTestGroup('Consonants with matras')
for c in consonants:
for m in matras:
builder.render((c,m), ftml, label=f'{c:04X}', comment=builder.char(c).basename)
ftml.closeTest()
if test.lower().startswith("nuktas"):
# Nuktas:
ftml.startTestGroup('Nuktas')
test_name = test.lower().split()[0]
with open(f'tests/{test_name}.template') as nuktas:
line_number = 0
for line in nuktas:
line = line.strip()
line_number += 1
if line == '':
continue
for n in below_marks:
for v in above_marks:
text = line.replace('N', chr(n))
text = text.replace('V', chr(v))
ftml.addToTest(None, text, label=f'line {line_number}', comment=f'n={n:04X} v={v:04X}')
ftml.closeTest()
# Write the output ftml file
ftml.writeFile(args.output)
def doit(args):
logger = args.logger
# Read input csv
builder = FB.FTMLBuilder(logger, incsv = args.input, fontcode = args.fontcode, font = args.ifont, ap = args.ap, rtlenable = True)
# Override default base (25CC) for displaying combining marks:
builder.diacBase = 0x0628 # beh
# Initialize FTML document:
test = args.test or "AllChars (NG)" # Default to "AllChars (NG)"
ftml = FB.FTML(test, logger, rendercheck = True, fontscale = args.scale, xslfn = args.xsl, fontsrc = args.fontsrc)
if test.lower().startswith("allchars"):
# all chars that should be in the font:
ftml.startTestGroup('Encoded characters')
for uid in sorted(builder.uids()):
if uid < 32: continue
c = builder.char(uid)
# iterate over all permutations of feature settings that might affect this character:
for featlist in builder.permuteFeatures(uids = (uid,)):
ftml.setFeatures(featlist)
builder.render((uid,), ftml)
# Don't close test -- collect consecutive encoded chars in a single row
ftml.clearFeatures()
for langID in sorted(c.langs):
ftml.setLang(langID)
builder.render((uid,), ftml)
ftml.clearLang()
# Add unencoded specials and ligatures -- i.e., things with a sequence of USVs in the glyph_data:
ftml.startTestGroup('Specials & ligatures from glyph_data')
for basename in sorted(builder.specials()):
special = builder.special(basename)
# iterate over all permutations of feature settings that might affect this special
for featlist in builder.permuteFeatures(uids = special.uids):
ftml.setFeatures(featlist)
builder.render(special.uids, ftml)
# close test so each special is on its own row:
ftml.closeTest()
ftml.clearFeatures()
if len(special.langs):
for langID in sorted(special.langs):
ftml.setLang(langID)
builder.render(special.uids, ftml)
ftml.closeTest()
ftml.clearLang()
# Add Lam-Alef data manually
ftml.startTestGroup('Lam-Alef')
# generate list of lam and alef characters that should be in the font:
lamlist = filter(lambda x: x in builder.uids(), (0x0644, 0x06B5, 0x06B6, 0x06B7, 0x06B8, 0x076A, 0x08A6))
aleflist = filter(lambda x: x in builder.uids(), (0x0627, 0x0622, 0x0623, 0x0625, 0x0671, 0x0672, 0x0673, 0x0675, 0x0773, 0x0774))
# iterate over all combinations:
for lam in lamlist:
for alef in aleflist:
for featlist in builder.permuteFeatures(uids = (lam, alef)):
ftml.setFeatures(featlist)
builder.render((lam,alef), ftml)
# close test so each combination is on its own row:
ftml.closeTest()
ftml.clearFeatures()
if test.lower().startswith("diac"):
# Diac attachment:
# Representative base and diac chars:
repDiac = filter(lambda x: x in builder.uids(), (0x064E, 0x0650, 0x065E, 0x0670, 0x0616, 0x06E3, 0x08F0, 0x08F2))
repBase = filter(lambda x: x in builder.uids(), (0x0627, 0x0628, 0x062B, 0x0647, 0x064A, 0x77F, 0x08AC))
ftml.startTestGroup('Representative diacritics on all bases that take diacritics')
for uid in sorted(builder.uids()):
# ignore some I don't care about:
if uid < 32 or uid in (0xAA, 0xBA): continue
c = builder.char(uid)
# Always process Lo, but others only if that take marks:
if c.general == 'Lo' or c.isBase:
for diac in repDiac:
for featlist in builder.permuteFeatures(uids = (uid,diac)):
ftml.setFeatures(featlist)
# Don't automatically separate connecting or mirrored forms into separate lines:
builder.render((uid,diac), ftml, addBreaks = False)
ftml.clearFeatures()
ftml.closeTest()
ftml.startTestGroup('All diacritics on representative bases')
for uid in sorted(builder.uids()):
# ignore non-ABS marks
if uid < 0x600 or uid in range(0xFE00, 0xFE10): continue
c = builder.char(uid)
if c.general == 'Mn':
for base in repBase:
for featlist in builder.permuteFeatures(uids = (uid,base)):
ftml.setFeatures(featlist)
builder.render((base,uid), ftml, keyUID = uid, addBreaks = False)
ftml.clearFeatures()
ftml.closeTest()
ftml.startTestGroup('Special cases')
builder.render((0x064A, 0x065E), ftml) # Yeh + Fatha should keep dots
builder.render((0x064A, 0x0654), ftml) # Yeh + Hamza should loose dots
ftml.closeTest()
# Write the output ftml file
ftml.writeFile(args.output)
def doit(args):
logger = args.logger
builder = FTMLBuilder_LCG(logger,
incsv=args.input,
fontcode=args.fontcode,
font=args.ifont,
ap=args.ap)
# Initialize FTML document:
test = args.test or "AllChars" # Default to "AllChars"
# split labels from fontsource parameter
fontsrc_lst, fontlabel_lst = [], []
for sl in args.fontsrc:
try:
s, l = sl.split('=', 1)
fontsrc_lst.append(s)
fontlabel_lst.append(l)
except ValueError:
fontsrc_lst.append(sl)
fontlabel_lst.append(None)
ftml = FB.FTML(test,
logger,
rendercheck=False,
fontscale=args.scale,
xslfn=args.xsl,
fontsrc=fontsrc_lst,
fontlabel=fontlabel_lst)
# Char to use in allframed test to surround other chars for checking spacing
frame_uid = 0x006F
# Representative base and diac chars:
# cedilla (H), vertical line below (L), ogonek (O), comma abov right (R), vertical line above (U)
repDiac = [
x for x in [0x0327, 0x0329, 0x0328, 0x0315, 0x030D]
if x in builder.uids()
]
ap_type_uid = {}
for diac_uid in repDiac:
c = builder.char(diac_uid)
for ap in c.aps:
if ap.startswith("_"):
ap_type_uid[ap[1:]] = diac_uid
if test.lower().startswith("allchars") or test.lower().startswith(
"allframed"):
# all chars that should be in the font:
framed = test.lower().startswith("allframed")
uids = special_uids = None
ftml.startTestGroup('Encoded characters')
for uid in sorted(builder.uids()):
if uid < 32: continue
c = builder.char(uid)
if not framed:
builder.render((uid, ), ftml)
else: # TODO: is there a cleaner way to create uids? (also see special_uids creation below)
uids = [frame_uid]
uids.extend(
(uid, )
) # used extend() instead of append() to be parallel to special_uids
uids.append(frame_uid)
builder.render(uids, ftml, keyUID=uid, descUIDs=(uid, ))
ftml.closeTest()
for langID in sorted(c.langs):
ftml.setLang(langID)
if not framed:
builder.render((uid, ), ftml)
else:
builder.render(uids, ftml, keyUID=uid, descUIDs=(uid, ))
ftml.clearLang()
# Add unencoded specials and ligatures -- i.e., things with a sequence of USVs in the glyph_data:
ftml.startTestGroup('Specials & ligatures from glyph_data')
for gname in sorted(builder.specials()):
special = builder.special(gname)
if not framed:
builder.render(special.uids, ftml)
else:
special_uids = [frame_uid]
special_uids.extend(special.uids)
special_uids.append(frame_uid)
builder.render(special_uids,
ftml,
keyUID=special.uids[0],
descUIDs=(special.uids))
ftml.closeTest()
if len(special.langs):
for langID in sorted(special.langs):
ftml.setLang(langID)
if not framed:
builder.render(special.uids, ftml)
else:
builder.render(special_uids,
ftml,
keyUID=special.uids[0],
descUIDs=(special.uids))
ftml.clearLang()
if test.lower().startswith("features"):
# support for adding a diac after each char that is being tested
# tests include: feature, feature_U, feature_L, feature_O, feature_H, feature_R
ap_type, c_mark = None, None
ix = test.find("_")
if ix != -1:
ap_type = test[ix + 1:]
if ap_type:
try:
ap_uid = ap_type_uid[ap_type]
except KeyError:
logger.log("Invalid AP type: %s" % ap_type, "S")
c_mark = builder.char(ap_uid)
ftml.startTestGroup('Features from glyph_data')
# build map from features (and all combinations) to the uids that are affected by a feature
feats_to_uid = {}
char_special_iter = chain(
[builder.char(uid) for uid in builder.uids()],
[builder.special(gname) for gname in builder.specials()])
for cs in char_special_iter:
feat_combs = chain.from_iterable(
combinations(cs.feats, r) for r in range(len(cs.feats) + 1))
for feats in feat_combs:
if not feats: continue
feats = sorted(feats)
feat_set = " ".join(feats)
if type(cs) is FB.FChar:
feats_to_uid.setdefault(feat_set, []).append(cs.uid)
elif type(cs) is FB.FSpecial:
feats_to_uid.setdefault(feat_set, []).append(cs.uids)
else:
pass
# create tests for all uids affected by a feature, organized by features
feats_sort = {} # sort based on number of features in combo
for feat_set in feats_to_uid.keys():
cnt = len(feat_set.split(" "))
try:
feats_sort[cnt].append(feat_set)
except:
feats_sort[cnt] = [feat_set]
for i in sorted(feats_sort.keys()):
feat_set_lst = sorted(
feats_sort[i]
) # list of feat combos where number in combo is i
for feat_set in feat_set_lst: # one feat combo
if feat_set == "smcp":
continue # skip smcp test (but not interaction of smcp with other features); see "smcp" test
ftml.startTestGroup(f'{feat_set}')
feats = feat_set.split(" ") # separate feat combo into feats
tvlist_lst = []
for feat in feats:
tvlist = builder.features[feat].tvlist[
1:] # all values of feats except default
tvlist_lst.append(
tvlist
) # build list of list of all value for each feat
p_lst = list(
product(*tvlist_lst)
) # find all combo of all values, MUST flatten the list of lists
# list of uids to test against feat combo
uid_lst = sorted(feats_to_uid[feat_set],
key=lambda x: x[0] if type(x) is list else x)
uid_char_lst = [u for u in uid_lst if type(u) is int]
uid_lig_lst = [u for u in uid_lst if type(u) is list]
# break uids into groups
uidlst_ct = 16 if not ap_type else 8
# generate ftml for all chars associated with the features
uidlst_lst = [
uid_char_lst[i:i + uidlst_ct]
for i in range(0, len(uid_char_lst), uidlst_ct)
]
for uidlst in uidlst_lst: #uidlst contains uids to test
base_diac_lst, base_lst = [], []
if not ap_type:
# features test (no '_<AP>')
base_diac_lst, base_lst = uidlst, uidlst
else:
# features_U, features_L, etc. tests
for uid in uidlst:
c_base = builder.char(uid)
if builder.matchMarkBase(c_mark, c_base):
base_lst.append(uid)
base_diac_lst.extend((uid, ap_uid))
ftml.clearFeatures()
builder.render(base_diac_lst, ftml, descUIDs=base_lst
) # render all uids without feat setting
for tv_lst in p_lst: # for one list of values out of all lists of values
ftml.setFeatures(tv_lst)
builder.render(base_diac_lst, ftml, descUIDs=base_lst)
# generate ftml for all ligatures associated with the features
# the ligature sequence is followed by a space
liglst_lst = [
uid_lig_lst[i:i + uidlst_ct]
for i in range(0, len(uid_lig_lst), uidlst_ct)
]
for liglst in liglst_lst: # liglst contains lists of uids & name to tests
lig_lst, lig_diac_lst = [], []
for lig in liglst: # lig is alist of uids & name
if not ap_type:
# 'features test (no '_<AP>')
lig_lst.extend(lig)
lig_lst.append(ord(' '))
lig_diac_lst.extend(lig)
lig_diac_lst.append(ord(' '))
else:
# features_U, features_L, etc. tests
c_base = builder.special(lig)
if builder.matchMarkBase(c_mark, c_base):
lig_lst.extend(lig)
lig_lst.append(ord(' '))
lig_diac_lst.extend(lig)
lig_diac_lst.append(
ap_uid
) # add AP based on type of features_<AP> test
lig_diac_lst.append(ord(' '))
lig_lst, lig_diac_lst = lig_lst[0:-1], lig_diac_lst[
0:-1] # trim trailing spaces
ftml.clearFeatures()
builder.render(lig_diac_lst, ftml, descUIDs=lig_lst
) # render all uids without feat setting
for tv_lst in p_lst: # for one list of values out of all lists of values
ftml.setFeatures(tv_lst)
builder.render(lig_diac_lst, ftml, descUIDs=lig_lst)
if test.lower().startswith("smcp"):
# TODO: improve test for "c2sc" ?
# Example of what report needs to show: LtnSmEgAlef LtnSmEgAlef.sc LtnCapEgAlef
# could add "LtnCapEgAlef <with 'c2sc' feature applied>" but commented out below
# support adding a diac after each char that is being tested
# tests include: smcp, smcp_U, etc
ap_type = None
ix = test.find("_")
if ix != -1:
ap_type = test[ix + 1:]
if ap_type:
try:
ap_uid = ap_type_uid[ap_type]
except KeyError:
logger.log("Invalid AP type: %s" % ap_type, "S")
c_mark = builder.char(ap_uid)
ftml.startTestGroup('Small caps from glyph_data')
char_special_iter = chain(
[builder.char(uid) for uid in builder.uids()],
[builder.special(gname) for gname in builder.specials()])
smcp_uid_lst = []
for cs in char_special_iter:
if 'smcp' in cs.feats:
if type(cs) is FB.FChar: smcp_uid_lst.append(cs.uid)
if type(cs) is FB.FSpecial: smcp_uid_lst.append(cs.uids)
smcp_uid_lst.sort(key=lambda x: x[0] if type(x) is list else x)
smcp_char_lst = [u for u in smcp_uid_lst if type(u) is int]
smcp_lig_lst = [u for u in smcp_uid_lst if type(u) is list]
uidlst_ct = 16 if not ap_type else 8
uidlst_lst = [
smcp_char_lst[i:i + uidlst_ct]
for i in range(0, len(smcp_char_lst), uidlst_ct)
]
for uidlst in uidlst_lst:
base_diac_lst, base_lst = [], []
if not ap_type:
base_diac_lst, base_lst = uidlst, uidlst
else:
for uid in uidlst:
c_base = builder.char(uid)
if builder.matchMarkBase(c_mark, c_base):
base_lst.append(uid)
base_diac_lst.extend((uid, ap_uid))
ftml.clearFeatures()
builder.render(
base_diac_lst, ftml,
descUIDs=base_lst) # render all uids without feat setting
ftml.setFeatures(builder.features['smcp'].tvlist[1:])
builder.render(base_diac_lst, ftml, descUIDs=base_lst)
ftml.clearFeatures()
# add uppercase uids to test
# TODO: may need a better way to convert lower to upper
upper_base_diac_lst, upper_base_lst = [], []
for lower_uid in base_diac_lst:
try:
upper_base_diac_lst.append(ord(chr(lower_uid).upper()))
except:
upper_base_diac_lst.append(ord('X'))
for lower_uid in base_lst:
try:
upper_base_lst.append(ord(chr(lower_uid).upper()))
except:
upper_base_lst.append(ord('X'))
builder.render(upper_base_diac_lst, ftml, descUIDs=upper_base_lst)
ftml.setFeatures([("c2sc", "1")
]) # TODO: kludgy way to add a 'c2sc' test
builder.render(upper_base_diac_lst, ftml, descUIDs=upper_base_lst)
liglst_lst = [
smcp_lig_lst[i:i + uidlst_ct]
for i in range(0, len(smcp_lig_lst), uidlst_ct)
]
for liglst in liglst_lst: # liglst contains lists of uids & name to tests
lig_lst, lig_diac_lst = [], []
for lig in liglst: # lig is alist of uids & name
if not ap_type:
# 'smcp test (no '_<AP>')
lig_lst.extend(lig)
lig_lst.append(ord(' '))
lig_diac_lst.extend(lig)
lig_diac_lst.append(ord(' '))
else:
# scmp_U, smcp_L, etc. tests
c_base = builder.special(lig)
if builder.matchMarkBase(c_mark, c_base):
lig_lst.extend(lig)
lig_lst.append(ord(' '))
lig_diac_lst.extend(lig)
lig_diac_lst.append(
ap_uid
) # add AP based on type of features_<AP> test
lig_diac_lst.append(ord(' '))
lig_lst, lig_diac_lst = lig_lst[0:-1], lig_diac_lst[
0:-1] # trim trailing spaces
ftml.clearFeatures()
builder.render(
lig_diac_lst, ftml,
descUIDs=lig_lst) # render all uids without feat setting
ftml.setFeatures(builder.features['smcp'].tvlist[1:])
builder.render(lig_diac_lst, ftml, descUIDs=lig_lst)
if test.lower().startswith("diac"):
# A E H O a e i o modifier-small-letter-o
repBase = [
x for x in [
0x0041, 0x0045, 0x0048, 0x004F, 0x0061, 0x0065, 0x0069, 0x006F,
0x1D52
] if x in builder.uids()
]
# Diac attachment:
ftml.startTestGroup(
'Representative diacritics on all bases that take diacritics')
for uid in sorted(builder.uids()):
# adjust for Latin
# ignore some I don't care about:
# if uid < 32 or uid in (0xAA, 0xBA): continue
if uid < 32: continue
c = builder.char(uid)
# Always process Lo, but others only if that take marks:
if c.general == 'Lo' or c.isBase:
for diac in repDiac:
c_mark = builder.char(diac)
if builder.matchMarkBase(c_mark, c):
builder.render((uid, diac),
ftml,
keyUID=uid,
descUIDs=[uid])
else:
# TODO: possibly output 'X' to mark place where mismatches occur
pass
# for featlist in builder.permuteFeatures(uids = (uid,diac)):
# ftml.setFeatures(featlist)
# # Don't automatically separate connecting or mirrored forms into separate lines:
# builder.render((uid,diac), ftml, addBreaks = False)
# ftml.clearFeatures()
ftml.closeTest()
ftml.startTestGroup('All diacritics on representative bases')
for uid in sorted(builder.uids()):
c = builder.char(uid)
if c.general == 'Mn':
for base in repBase:
c_base = builder.char(base)
if builder.matchMarkBase(c, c_base):
builder.render((base, uid),
ftml,
keyUID=uid,
descUIDs=[uid])
else: # TODO: possibly output 'X' to mark place where mismatches occur
pass
# for featlist in builder.permuteFeatures(uids = (uid,base)):
# ftml.setFeatures(featlist)
# builder.render((base,uid), ftml, keyUID = uid, addBreaks = False)
# ftml.clearFeatures()
ftml.closeTest()
ftml.startTestGroup('Special case - cv79 (NFD)')
# cv79 - Kayan grave_acute
kayan_diac_lst = [0x0300, 0x0301] # comb_grave, comb_acute
kayan_base_char_lst = [
'a', 'e', 'i', 'o', 'n', 'u', 'w', 'y', 'A', 'E', 'I', 'O', 'N',
'U', 'W', 'Y'
]
kayan_base_lst = [ord(x) for x in kayan_base_char_lst]
builder.render_lists(kayan_base_lst,
kayan_diac_lst,
ftml, [('cv79', '1')],
keyUID=kayan_diac_lst[0])
ftml.closeTestGroup()
ftml.startTestGroup('Special case - cv79 (NFC)')
# cv79 - Kayan grave_acute
kayan_diac_lst = [0x0301] # comb_acute
kayan_base_name_lst = [
'LtnSmAGrave', 'LtnSmAGrave.SngStory', 'LtnSmEGrave',
'LtnSmIGrave', 'LtnSmOGrave', 'LtnSmNGrave', 'LtnSmUGrave',
'LtnSmWGrave', 'LtnSmYGrave', 'LtnCapAGrave', 'LtnCapEGrave',
'LtnCapIGrave', 'LtnCapOGrave', 'LtnCapNGrave', 'LtnCapUGrave',
'LtnCapWGrave', 'LtnCapYGrave'
]
# try: kayan_base_lst = [builder.char(x).uid for x in kayan_base_name_lst]
# except KeyError as key_exc: logger.log('glyph missing: {}'.format(key_exc), 'W')
kayan_base_lst = []
for base in kayan_base_name_lst:
# TODO: fix this kludgy way of handling Andika's encoding of literacy glyphs
# LtnSmAGrave.SngStory will raise an exception in non-Andika fonts
# LtnSmAGrave will do the same in Andika
try:
kayan_base_lst.append(builder.char(base).uid)
except KeyError as key_exc:
logger.log('glyph missing: {}'.format(key_exc), 'W')
builder.render_lists(kayan_base_lst,
kayan_diac_lst,
ftml, [('cv79', '1')],
keyUID=kayan_diac_lst[0])
ftml.closeTestGroup()
ftml.startTestGroup('Dot removal')
diac_lst = [0x301] #comb_acute
# the below glyph list was copied from classes.xml and massaged into Python format
base_name_lst = [
'CySmByelorusUkrainI', 'CySmJe', 'LtnSmI', 'LtnSmI.SItal',
'LtnSmI.sc', 'LtnSmIOgonek', 'LtnSmIRetrHook', 'LtnSmIStrk',
'LtnSmJ', 'LtnSmJCrossedTail', 'LtnSmJStrk', 'LtnSubSmI',
'LtnSubSmJ', 'LtnSupSmI', 'LtnSupSmIStrk', 'ModSmJ',
'ModSmJCrossedTail'
]
base_lst = []
for x in base_name_lst:
try:
base_lst.append(builder.char(x).uid)
except:
pass
builder.render_lists(base_lst, diac_lst, ftml, keyUID=diac_lst[0])
builder.render_lists(base_lst,
diac_lst,
ftml,
feature_lst=builder.features['smcp'].tvlist[1:],
keyUID=diac_lst[0])
builder.render_lists(base_lst,
diac_lst,
ftml,
feature_lst=builder.features['ss05'].tvlist[1:],
keyUID=diac_lst[0])
ftml.closeTestGroup()
ftml.startTestGroup('Superscript diacritics')
diac_lst = [0x308] #CombDiaer.Sup
# the below glyph list was copied from classes.xml and massaged into Python format
base_name_lst = [
'GrSubSmBeta', 'GrSubSmChi', 'GrSubSmGamma', 'GrSubSmPhi',
'GrSubSmRho', 'LtnSubSmA', 'LtnSubSmA.SngStory', 'LtnSubSmE',
'LtnSubSmH', 'LtnSubSmI', 'LtnSubSmI.Dotless', 'LtnSubSmJ',
'LtnSubSmJ.Dotless', 'LtnSubSmK', 'LtnSubSmL', 'LtnSubSmL.SItal',
'LtnSubSmM', 'LtnSubSmN', 'LtnSubSmO', 'LtnSubSmP', 'LtnSubSmR',
'LtnSubSmS', 'LtnSubSmSchwa', 'LtnSubSmT', 'LtnSubSmU',
'LtnSubSmV', 'LtnSubSmX', 'LtnSupSmA', 'LtnSupSmA.SngStory',
'LtnSupSmAe', 'LtnSupSmAlpha', 'LtnSupSmB', 'LtnSupSmBarredO',
'LtnSupSmBarredODep', 'LtnSupSmCCurl', 'LtnSupSmCCurlDep',
'LtnSupSmCapI', 'LtnSupSmCapIDep', 'LtnSupSmCapOe', 'LtnSupSmCapY',
'LtnSupSmClosedRevOpnE', 'LtnSupSmD', 'LtnSupSmDotlessJStrk',
'LtnSupSmDotlessJStrkDep', 'LtnSupSmE', 'LtnSupSmEng',
'LtnSupSmEngDep', 'LtnSupSmEsh', 'LtnSupSmEshDep', 'LtnSupSmEzh',
'LtnSupSmEzhDep', 'LtnSupSmF', 'LtnSupSmFDep', 'LtnSupSmG',
'LtnSupSmG.SngBowl', 'LtnSupSmI', 'LtnSupSmI.Dotless',
'LtnSupSmIStrk', 'LtnSupSmIStrk.Dotless', 'LtnSupSmIStrkDep',
'LtnSupSmK', 'LtnSupSmLRetrHook', 'LtnSupSmLRetrHookDep',
'LtnSupSmM', 'LtnSupSmMDep', 'LtnSupSmN', 'LtnSupSmNLftHook',
'LtnSupSmNLftHookDep', 'LtnSupSmO', 'LtnSupSmOStrk', 'LtnSupSmOe',
'LtnSupSmOeDep', 'LtnSupSmOpnE', 'LtnSupSmOpnO',
'LtnSupSmOpnO.TopSerif', 'LtnSupSmP', 'LtnSupSmRamsHorn',
'LtnSupSmRevE', 'LtnSupSmRevOpnE', 'LtnSupSmRevOpnEDep',
'LtnSupSmSchwa', 'LtnSupSmScriptG', 'LtnSupSmScriptGDep',
'LtnSupSmT', 'LtnSupSmTurnedA', 'LtnSupSmTurnedAlpha',
'LtnSupSmTurnedAlphaDep', 'LtnSupSmTurnedM',
'LtnSupSmTurnedMLngLeg', 'LtnSupSmTurnedMLngLegDep',
'LtnSupSmTurnedV', 'LtnSupSmTurnedVDep', 'LtnSupSmU',
'LtnSupSmUBar', 'LtnSupSmUBarDep', 'LtnSupSmUpsilon',
'LtnSupSmUpsilonDep', 'LtnSupSmV', 'LtnSupSmZ', 'LtnSupSmZCurl',
'LtnSupSmZCurlDep', 'LtnSupSmZDep', 'ModCapA', 'ModCapAe',
'ModCapB', 'ModCapBarredB', 'ModCapD', 'ModCapE', 'ModCapG',
'ModCapH', 'ModCapHStrk', 'ModCapI', 'ModCapJ', 'ModCapK',
'ModCapL', 'ModCapM', 'ModCapN', 'ModCapO', 'ModCapOu',
'ModCapOu.OpenTop', 'ModCapP', 'ModCapR', 'ModCapRevE',
'ModCapRevN', 'ModCapT', 'ModCapU', 'ModCapV', 'ModCapW',
'ModSmAin', 'ModSmBeta', 'ModSmBottomHalfO', 'ModSmC', 'ModSmCDep',
'ModSmCapIStrk', 'ModSmCapIStrkDep', 'ModSmCapInvR', 'ModSmCapL',
'ModSmCapLDep', 'ModSmCapN', 'ModSmCapNDep', 'ModSmCapU',
'ModSmCapUBar', 'ModSmCapUDep', 'ModSmChi', 'ModSmDelta',
'ModSmEth', 'ModSmEthDep', 'ModSmGamma', 'ModSmGrGamma',
'ModSmGrPhi', 'ModSmH', 'ModSmHHook', 'ModSmHStrk', 'ModSmHeng',
'ModSmIota', 'ModSmIotaDep', 'ModSmJ', 'ModSmJ.Dotless',
'ModSmJCrossedTail', 'ModSmJCrossedTail.Dotless',
'ModSmJCrossedTailDep', 'ModSmL', 'ModSmLMiddleTilde',
'ModSmLPalHook', 'ModSmLPalHookDep', 'ModSmMHook', 'ModSmMHookDep',
'ModSmNRetrHook', 'ModSmNRetrHookDep', 'ModSmPhi', 'ModSmPhiDep',
'ModSmR', 'ModSmRevGlottalStop', 'ModSmS', 'ModSmSHook',
'ModSmSHookDep', 'ModSmSdwysU', 'ModSmTPalHook',
'ModSmTPalHookDep', 'ModSmTheta', 'ModSmThetaDep', 'ModSmTopHalfO',
'ModSmTrndAe', 'ModSmTrndI', 'ModSmTrndOpnE', 'ModSmTrndR',
'ModSmTrndRHook', 'ModSmTurnedH', 'ModSmTurnedHDep',
'ModSmTurnedY', 'ModSmVHook', 'ModSmVHook.StraightLft',
'ModSmVHook.StraightLftHighHook', 'ModSmVHookDep', 'ModSmW',
'ModSmX', 'ModSmY', 'ModSmZRetrHook', 'ModSmZRetrHookDep',
'ModGlottalStop', 'ModRevGlottalStop'
]
base_lst = []
for x in base_name_lst:
try:
base_lst.append(builder.char(x).uid)
except:
pass
builder.render_lists(base_lst, diac_lst, ftml, keyUID=diac_lst[0])
ftml.closeTestGroup()
# ftml.startTestGroup('Special case - cv75')
# ftml.clearFeatures()
# # comb_circumflex, comb_acute, space, a, comb_circumflex, comb_acute
# builder.render((0x0302, 0x0301, 0x0020, 0x0061, 0x0302, 0x0301), ftml, descUIDs=(0x0302, 0x0301))
# # o, comb_circumflex, comb_acute
# builder.render((0x006F, 0x0302, 0x0301), ftml, keyUID=0x0302)
# ftml.setFeatures([["cv75", "1"]])
# builder.render((0x0302, 0x0301, 0x0020, 0x0061, 0x0302, 0x0301), ftml, descUIDs=(0x0302, 0x0301))
# builder.render((0x006F, 0x0302, 0x0301), ftml, keyUID=0x0302)
# ftml.closeTestGroup()
# Write the output ftml file
ftml.writeFile(args.output)
def doit(args):
logger = args.logger
# Read input csv
builder = FB.FTMLBuilder(logger,
incsv=args.input,
fontcode=args.fontcode,
font=args.ifont,
ap=args.ap,
rtlenable=True,
langs=args.langs)
# Override default base (25CC) for displaying combining marks
builder.diacBase = 0x0628 # beh
def basenameSortKey(uid: int):
return builder.char(uid).basename.lower()
# Initialize FTML document:
test = args.test or "AllChars (NG)" # Default to AllChars
ftml = FB.FTML(test,
logger,
rendercheck=not args.norendercheck,
fontscale=args.scale,
xslfn=args.xsl,
fontsrc=args.fontsrc,
defaultrtl=args.rtl)
if test.lower().startswith("allchars"):
# all chars that should be in the font:
ftml.startTestGroup('Encoded characters')
for uid in sorted(builder.uids()):
if uid < 32: continue
c = builder.char(uid)
for featlist in builder.permuteFeatures(uids=(uid, )):
ftml.setFeatures(featlist)
builder.render((uid, ), ftml)
ftml.clearFeatures()
if len(c.langs):
for langID in builder.allLangs:
ftml.setLang(langID)
builder.render((uid, ), ftml)
ftml.clearLang()
# Add specials and ligatures that were in the glyph_data:
ftml.startTestGroup('Specials & ligatures from glyph_data')
for basename in sorted(builder.specials()):
special = builder.special(basename)
for featlist in builder.permuteFeatures(uids=special.uids,
feats=special.feats):
ftml.setFeatures(featlist)
builder.render(special.uids, ftml)
ftml.closeTest()
ftml.clearFeatures()
if len(special.langs):
for langID in builder.allLangs:
ftml.setLang(langID)
builder.render(special.uids, ftml)
ftml.closeTest()
ftml.clearLang()
# Add Lam-Alef data manually
ftml.startTestGroup('Lam-Alef')
lamlist = list(
filter(lambda x: x in builder.uids(),
(0x0644, 0x06B5, 0x06B6, 0x06B7, 0x06B8, 0x076A, 0x08A6)))
aleflist = list(
filter(lambda x: x in builder.uids(),
(0x0627, 0x0622, 0x0623, 0x0625, 0x0671, 0x0672, 0x0673,
0x0675, 0x0773, 0x0774)))
for lam in lamlist:
for alef in aleflist:
for featlist in builder.permuteFeatures(uids=(lam, alef)):
ftml.setFeatures(featlist)
builder.render((lam, alef), ftml)
ftml.closeTest()
ftml.clearFeatures()
if lam == 0x0644 and 'cv02' in builder.features:
# Also test lam with hamza above for warsh variants
for featlist in builder.permuteFeatures(uids=(lam, 0x0654,
alef),
feats=('cv02', )):
ftml.setFeatures(featlist)
builder.render((lam, 0x0654, alef), ftml)
ftml.closeTest()
ftml.clearFeatures()
# Add Allah data manually
ftml.startTestGroup('Allah ligatures')
ftml.addToTest(0xFDF2, r"\uFDF2", comment="Rule 1")
ftml.closeTest()
ftml.addToTest(None,
r"\u0641\u0644\u0644\u0647",
label="f-l-l-h",
comment="shouldn't match")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u0644\u0644\u0651\u0670\u0647",
label="a-l-l-s-da-hf",
comment="Rule 2 (daggeralef)")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u0644\u0644\u0670\u0651\u0647",
label="a-l-l-da-s-hf")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u0644\u0644\u0651\u0670\u06C1",
label="a-l-l-s-da-hgf")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u0644\u0644\u0670\u0651\u06C1",
label="a-l-l-da-s-hgf")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u0644\u0644\u0651\u064E\u0647",
label="a-l-l-s-f-hf",
comment="Rule 2 (fatha)")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u0644\u0644\u064E\u0651\u0647",
label="a-l-l-f-s-hf")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u0644\u0644\u0651\u064E\u06C1",
label="a-l-l-s-f-hgf")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u0644\u0644\u064E\u0651\u06C1",
label="a-l-l-f-s-hgf")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u0644\u06EB\u0644\u064E\u0651\u06C1",
label="a-l-M-l-s-da-hgf",
comment="Rule 2c: shouldn't match")
ftml.closeTest()
ftml.addToTest(None,
r"\u0641\u0644\u0644\u064E\u0651\u06C1",
label="f-l-l-s-da-hgf",
comment="Rule 2d: non-alef")
ftml.closeTest()
ftml.addToTest(None,
r"\u0641\u0627\u0644\u0644\u064E\u0651\u06C1",
label="f-a-l-l-s-da-hgf",
comment="Rule 2d: not isolate alef")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u06EB\u0644\u0644\u064E\u0651\u06C1",
label="a-M-l-l-s-da-hgf",
comment="Rule 2d: Mark")
ftml.closeTest()
ftml.addToTest(None,
r" \u0644\u0644\u0651\u064E\u0647",
label="space-l-l-s-da-hf",
comment="Rule 2d: shouldn't match")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u0644\u0644\u0647",
label="a-l-l-h",
comment="Rule 3")
ftml.closeTest()
ftml.addToTest(None, r"\u0622\u0644\u0644\u0647", label="aM-l-l-h")
ftml.closeTest()
ftml.addToTest(None, r"\u0623\u0644\u0644\u0647", label="aH-l-l-h")
ftml.closeTest()
ftml.addToTest(None,
r"\u0671\u0644\u0644\u0647",
label="aW-l-l-h",
comment="won't work")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u06EB\u0644\u0644\u0647",
label="a-M-l-l-h")
ftml.closeTest()
ftml.addToTest(None,
r"\u0641\u0627\u0644\u0644\u0647",
label="f-a-l-l-h",
comment="Rule 3a: shouldn't match")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u0644\u06EB\u0644\u0647",
label="a-l-M-l-h",
comment="Rule 3d: shouldn't match")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u0644\u200D\u0644\u0647",
label="a-l-zwj-l-h",
comment="Rule 4a: shouldn't match")
ftml.closeTest()
ftml.addToTest(None,
r"\u0627\u0644\u200D\u0644\u0651\u0670\u0647",
label="a-l-zwj-l-s-da-h",
comment="Rule 4a: shouldn't match")
ftml.closeTest()
if test.lower().startswith("al sorted"):
# all AL chars, sorted by shape:
ftml.startTestGroup('Arabic Letters')
for uid in sorted(filter(lambda u: get_ucd(u, 'bc') == 'AL',
builder.uids()),
key=joinGoupSortKey):
c = builder.char(uid)
for featlist in builder.permuteFeatures(uids=(uid, )):
ftml.setFeatures(featlist)
builder.render((uid, ), ftml)
ftml.clearFeatures()
if len(c.langs):
for langID in builder.allLangs:
ftml.setLang(langID)
builder.render((uid, ), ftml)
ftml.clearLang()
if test.lower().startswith("diac"):
# Diac attachment:
doLongTest = 'short' not in test.lower()
# Representative base and diac chars:
if doLongTest:
repDiac = list(
filter(lambda x: x in builder.uids(),
(0x064E, 0x0650, 0x065E, 0x0670, 0x0616, 0x06E3, 0x08F0,
0x08F2)))
repBase = list(
filter(
lambda x: x in builder.uids(),
(0x0627, 0x0628, 0x062B, 0x0647, 0x064A, 0x77F, 0x08AC)))
lamlist = list(
filter(
lambda x: x in builder.uids(),
(0x0644, 0x06B5, 0x06B6, 0x06B7, 0x06B8, 0x076A, 0x08A6)))
aleflist = list(
filter(lambda x: x in builder.uids(),
(0x0627, 0x0622, 0x0623, 0x0625, 0x0671, 0x0672, 0x0673,
0x0675, 0x0773, 0x0774)))
else:
repDiac = list(
filter(lambda x: x in builder.uids(),
(0x064E, 0x0650, 0x0670)))
repBase = list(
filter(lambda x: x in builder.uids(), (0x0627, 0x0628)))
lamlist = list(
filter(
lambda x: x in builder.uids(),
(0x0644, 0x06B5, 0x06B6, 0x06B7, 0x06B8, 0x076A, 0x08A6)))
aleflist = list(
filter(lambda x: x in builder.uids(),
(0x0627, 0x0622, 0x0623, 0x0625, 0x0671, 0x0672, 0x0673,
0x0675, 0x0773, 0x0774)))
ftml.startTestGroup(
'Representative diacritics on all bases that take diacritics')
for uid in sorted(builder.uids()):
if uid < 32 or uid in (0xAA, 0xBA): continue
c = builder.char(uid)
# Always process Lo, but others only if that take marks:
if c.general == 'Lo' or c.isBase:
for diac in repDiac:
for featlist in builder.permuteFeatures(uids=(uid, diac)):
ftml.setFeatures(featlist)
builder.render((uid, diac),
ftml,
addBreaks=False,
dualJoinMode=2)
if doLongTest:
if diac != 0x0651: # If not shadda
# include shadda, in either order:
builder.render((uid, diac, 0x0651),
ftml,
addBreaks=False,
dualJoinMode=2)
builder.render((uid, 0x0651, diac),
ftml,
addBreaks=False,
dualJoinMode=2)
if diac != 0x0654: # If not hamza above
# include hamza above, in either order:
builder.render((uid, diac, 0x0654),
ftml,
addBreaks=False,
dualJoinMode=2)
builder.render((uid, 0x0654, diac),
ftml,
addBreaks=False,
dualJoinMode=2)
ftml.clearFeatures()
ftml.closeTest()
ftml.startTestGroup('All Arabic diacritics on representative bases')
for uid in sorted(builder.uids()):
# ignore non-ABS marks
if uid < 0x600 or uid in range(0xFE00, 0xFE10): continue
c = builder.char(uid)
if c.general == 'Mn':
for base in repBase:
for featlist in builder.permuteFeatures(uids=(uid, base)):
ftml.setFeatures(featlist)
builder.render((base, uid),
ftml,
keyUID=uid,
addBreaks=False,
dualJoinMode=2)
if doLongTest:
if uid != 0x0651: # if not shadda
# include shadda, in either order:
builder.render((base, uid, 0x0651),
ftml,
keyUID=uid,
addBreaks=False,
dualJoinMode=2)
builder.render((base, 0x0651, uid),
ftml,
keyUID=uid,
addBreaks=False,
dualJoinMode=2)
if diac != 0x0670: # If not superscript alef
# include superscript alef, in either order:
builder.render((uid, diac, 0x0670),
ftml,
addBreaks=False,
dualJoinMode=2)
builder.render((uid, 0x0670, diac),
ftml,
addBreaks=False,
dualJoinMode=2)
ftml.clearFeatures()
ftml.closeTest()
ftml.startTestGroup('Special cases')
builder.render((0x064A, 0x064E), ftml) # Yeh + Fatha should keep dots
builder.render((0x064A, 0x0654), ftml) # Yeh + Hamza should loose dots
ftml.closeTest()
ftml.startTestGroup('LamAlef ligatures')
diaB = 0x064D
diaA = 0x064B
for lam in lamlist:
for alef in aleflist:
for featlist in builder.permuteFeatures(uids=(lam, alef)):
ftml.setFeatures(featlist)
builder.render((lam, alef), ftml, addBreaks=False)
builder.render((lam, diaA, alef, diaA),
ftml,
addBreaks=False)
builder.render((lam, diaB, alef), ftml, addBreaks=False)
builder.render((lam, alef, diaB), ftml, addBreaks=False)
builder.render((lam, diaB, alef, diaB),
ftml,
addBreaks=False)
ftml.clearFeatures()
ftml.closeTest()
if test.lower().startswith("subtending"):
# Generates sample data for all subtending marks. Data includes sequences of 0 to n+1
# digits, where n is the maximum expected to be supported on the mark. Latin, Arbic-Indic,
# and Extended Arabic-Indic digits are included.
for digitSample in filter(lambda x: x in builder.uids(),
(0x0032, 0x0668, 0x06F8)):
digitOne = (digitSample & 0xFFF0) + 1
for uid, lgt in filter(lambda x: x[0] in builder.uids(),
([0x600, 3], [0x0601, 4], [0x0602, 2], [
0x0603, 4
], [0x0604, 4], [0x0605, 4], [0x06DD, 3])):
c = chr(uid)
label = "U+{0:04X} {1}".format(
uid, 'latn' if digitOne == 0x0031 else
'arab' if digitOne == 0x0661 else 'urdu')
comment = builder.char(uid).basename
for featlist in builder.permuteFeatures(uids=(uid, )):
ftml.setFeatures(featlist)
ftml.addToTest(uid, "\u0628" + c + "\u0645", label,
comment)
for ln in range(1, lgt + 1):
ftml.addToTest(uid, c + chr(digitSample) * ln)
ftml.addToTest(uid, c + chr(digitOne) + chr(digitOne + 1))
ftml.clearFeatures()
ftml.closeTest()
if uid == 0x06DD and digitOne == 0x06F1:
# Extra items for Eastern digits
for featlist in builder.permuteFeatures(uids=(uid,
0x06F7)):
ftml.setFeatures(featlist)
ftml.addToTest(uid, c + "\u06F4\u06F6\u06F7", label,
"4 6 7")
ftml.clearFeatures()
for langID in builder.allLangs:
ftml.setLang(langID)
for featlist in ((None, ), (['cv80',
'1'], ), (['cv80',
'2'], )):
ftml.setFeatures(featlist)
ftml.addToTest(uid, c + "\u06F4\u06F6\u06F7",
label, "4 6 7")
ftml.clearFeatures()
ftml.clearLang()
ftml.closeTest()
if test.lower().startswith("showinv"):
# Sample data for chars that have a "show invisible" feature
# The 'r', 'a', 'ra' indicates whether this is standard in Roman fonts, Arabic fonts, or both.
invlist = [(0x034F, 'r'), (0x061C, 'a'), (0x200B, 'r'), (0x200C, 'ra'),
(0x200D, 'ra'), (0x200E, 'ra'), (0x200F, 'ra'),
(0x202A, 'ra'), (0x202B, 'ra'), (0x202C, 'ra'),
(0x202D, 'ra'), (0x202E, 'ra'), (0x202E, 'r'),
(0x2060, 'r'), (0x2061, 'r'), (0x2062, 'r'), (0x2063, 'r'),
(0x2066, 'a'), (0x2067, 'a'), (0x2068, 'a'), (0x2069, 'a'),
(0xFE00, 'ra'), (0xFE01, 'ra'), (0xFE02, 'ra'),
(0xFE03, 'ra'), (0xFE04, 'ra'), (0xFE05, 'ra'),
(0xFE06, 'ra'), (0xFE07, 'ra'), (0xFE08, 'ra'),
(0xFE09, 'ra'), (0xFE0A, 'ra'), (0xFE0B, 'ra'),
(0xFE0C, 'ra'), (0xFE0D, 'ra'), (0xFE0E, 'ra'),
(0xFE0F, 'ra')]
featlist = (('invs', '1'), ('ss06', '1'))
ftml.setFeatures(featlist)
for inv in invlist:
uid = inv[0]
c = chr(uid)
label = 'U+{0:04X} ({1})'.format(uid, inv[1])
comment = builder.char(
uid).basename if uid in builder.uids() else ""
ftml.addToTest(uid, " " + c + " ", label, comment)
ftml.closeTest()
ftml.clearFeatures()
if test.lower().startswith('daggeralef'):
for uid in sorted(builder.uids(), key=joinGoupSortKey):
if get_ucd(uid, 'jg') not in ('Sad', 'Seen', 'Yeh'):
# If not Yeh, Sad or seen joining group we're not interested
continue
for featlist in builder.permuteFeatures(uids=(uid, 0x0670)):
ftml.setFeatures(featlist)
builder.render((uid, 0x0670), ftml)
ftml.clearFeatures()
ftml.closeTest()
if test.lower().startswith('kern'):
rehs = sorted(
filter(lambda uid: get_ucd(uid, 'jg') == 'Reh', builder.uids()))
waws = sorted(
filter(lambda uid: get_ucd(uid, 'jg') == 'Waw', builder.uids()))
uids = sorted(filter(
lambda uid: get_ucd(uid, 'jt') in ('D', 'R') or uid == 0xFD3E,
builder.uids()),
key=joinGoupSortKey)
# NB: I wondered about including punctuation, i.e., get_ucd(uid, 'gc').startswith('P'), but the default
# spacing is pretty good and graphite collision avoidance makes it worse, so the only one we need is FDFE
dbehf = chr(0x066E) + chr(0x200D) # dotless beh final
alef = chr(0x0627) # alef
zwj = chr(0x200D) # Zero width joiner
ma = 0x064B # Mark above (fathatan)
mb = 0x064D # chr(0x064D) # Mark below (kasratan)
if "data" not in test.lower():
ftml.startTestGroup('All the rehs')
for uid in rehs:
c = chr(uid)
label = 'U+{0:04X}'.format(uid)
comment = builder.char(uid).basename
for featlist in builder.permuteFeatures(uids=(uid, )):
ftml.setFeatures(featlist)
ftml.addToTest(uid, c + dbehf + ' ' + zwj + c + dbehf,
label, comment)
ftml.clearFeatures()
ftml.closeTest()
ftml.startTestGroup('All the waws')
for uid in waws:
c = chr(uid)
label = 'U+{0:04X}'.format(uid)
comment = builder.char(uid).basename
for featlist in builder.permuteFeatures(uids=(uid, )):
ftml.setFeatures(featlist)
ftml.addToTest(uid, c + dbehf + ' ' + zwj + c + dbehf,
label, comment)
ftml.clearFeatures()
ftml.closeTest()
# reh or waw plus the others
for uid1 in (0x631, 0x648): # (reh, waw)
ftml.startTestGroup('{} + all the others'.format(
get_ucd(uid1, 'jg')))
c1 = chr(uid1)
for uid2 in uids:
c2 = chr(uid2)
comment = builder.char(uid2).basename
label = 'U+{:04X}'.format(uid2)
for featlist in builder.permuteFeatures(uids=(uid1, uid2)):
ftml.setFeatures(featlist)
if get_ucd(uid2, 'jt') == 'D':
ftml.addToTest(uid2, zwj + c1 + c2 + zwj, label,
comment)
ftml.addToTest(uid2, c1 + c2 + zwj)
ftml.addToTest(uid2, zwj + c1 + c2, label, comment)
ftml.addToTest(uid2, c1 + c2)
ftml.clearFeatures()
ftml.closeTest()
else:
# exhaustive test for kerning data extraction
ftml.defaultRTL = True
addMarks = "with marks" in test.lower()
for uid1 in rehs: # (rehs[0],)
for uid2 in uids:
for featlist in builder.permuteFeatures(uids=(uid1, uid2)):
ftml.setFeatures(featlist)
builder.render([uid1, uid2],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
if addMarks:
builder.render([uid1, uid2, mb],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
builder.render([uid1, uid2, ma],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
builder.render([uid1, uid2, mb, ma],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
builder.render([uid1, uid2, ma, mb],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
builder.render([uid1, ma, uid2],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
builder.render([uid1, ma, uid2, mb],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
builder.render([uid1, ma, uid2, ma],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
builder.render([uid1, ma, uid2, mb, ma],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
builder.render([uid1, ma, uid2, ma, mb],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
builder.render([uid1, mb, uid2],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
builder.render([uid1, mb, uid2, mb],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
builder.render([uid1, mb, uid2, ma],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
builder.render([uid1, mb, uid2, mb, ma],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
builder.render([uid1, mb, uid2, ma, mb],
ftml,
addBreaks=False,
rtl=True,
dualJoinMode=1)
ftml.clearFeatures()
ftml.closeTest()
if test.lower().startswith('chadian'):
rehs = '[' + ''.join(
map(
chr,
filter(lambda uid: get_ucd(uid, 'jg') == 'Reh',
builder.uids()))) + ']'
uids = '[' + ''.join(
map(
chr,
filter(
lambda uid: get_ucd(uid, 'jt') in
('D', 'R') or uid == 0xFD3E, builder.uids()))) + ']'
marks = '[' + ''.join(
map(
chr,
filter(lambda uid: get_ucd(uid, 'gc').startswith('M'),
builder.uids()))) + ']'
rehwordsRE = re.compile(f'({rehs}{marks}{uids}{marks}*)')
with open('/SRC/ABS Text Samples/Chad/Chadian Arabic AS word list.txt',
encoding="utf8") as f:
for line_no, line in enumerate(f):
res = ''
matches = ''
lastEnd = 0
for m in rehwordsRE.finditer(line):
if m.start() > 0:
res += line[lastEnd:m.start()]
# I wish I could output <em> around the kerned pair, something like:
# res += f'<em>{m.group()}</em>'
# but apparently ftml.py doesn't support this :-(
# So just append
res += m.group()
# Keep track af all matched strings for feature permutations
matches += m.group()
lastEnd = m.end()
if len(res) > 0:
# Add tail to result
res += line[lastEnd:]
# figure features based only on what matched
matchedUids = map(ord, list(matches))
for featlist in builder.permuteFeatures(uids=matchedUids):
ftml.setFeatures(featlist)
# Add to test:
ftml.addToTest(None, res, f'line {line_no}')
ftml.clearFeatures()
ftml.closeTest()
if test.lower().startswith('yehbar'):
# Yehbarree tail interacting with diacs below previous char
uids = sorted(filter(lambda uid: get_ucd(uid, 'jt') in ('D', ),
builder.uids()),
key=basenameSortKey)
markbelow = r'\u064D' # kasratan
markabove = r'\u06EC' # dotStopabove-ar
zwj = r'\u200D' # Zero width joiner
ftml.startTestGroup('U+06D2 yehbarree')
yehbarree = r'\u06D2'
for uid in uids:
if uid < 32: continue
c = r'\u{:04X}'.format(uid)
label = 'U+{:04X}'.format(uid)
comment = builder.char(uid).basename
for featlist in builder.permuteFeatures(uids=(uid, )):
ftml.setFeatures(featlist)
ftml.addToTest(
uid,
f"{c}{markabove}{yehbarree} {zwj}{c}{markabove}{yehbarree} {c}{markbelow}{markabove}{yehbarree} {zwj}{c}{markbelow}{markabove}{yehbarree}",
label, comment)
ftml.closeTest()
ftml.clearFeatures()
# Also test other forms of yehbarree (yehbarreeHamzaabove-ar, yehbarreeTwoabove, yehbarreeThreeabove-ar)
ftml.startTestGroup('yehbarree-like')
for yehbarree in filter(lambda x: x in builder.uids(),
(0x06D3, 0x077A, 0x077B)):
for uid in filter(lambda x: x in builder.uids(), (0x06A0, 0x08B3)):
c = r'\u{:04X}'.format(uid)
yb = r'\u{:04X}'.format(yehbarree)
label = 'U+{:04X} U+{:04X}'.format(uid, yehbarree)
comment = builder.char(uid).basename + ' ' + builder.char(
yehbarree).basename
for featlist in builder.permuteFeatures(uids=(uid, )):
ftml.setFeatures(featlist)
ftml.addToTest(
uid,
f"{c}{markabove}{yb} {zwj}{c}{markabove}{yb} {c}{markbelow}{markabove}{yb} {zwj}{c}{markbelow}{markabove}{yb}",
label, comment)
ftml.closeTest()
ftml.clearFeatures()
if test.lower().startswith('classes'):
zwj = chr(0x200D)
lsb = '' # chr(0xF130)
rsb = '' # chr(0xF131)
glyphsSeen = set()
uids = sorted(
filter(lambda uid: builder.char(uid).general == 'Lo' and uid > 255,
builder.uids()))
uids = sorted(uids, key=joinGoupSortKey)
for uid in uids:
c = chr(uid)
thischar = builder.char(uid)
label = 'U+{:04X}'.format(uid)
for featlist in builder.permuteFeatures(uids=(uid, )):
gname = thischar.basename
if len(featlist) == 1 and featlist[0] is not None:
# See if we can find an alternate glyph name:
feat = '{}={}'.format(featlist[0][0], featlist[0][1])
gname = thischar.altnames.get(feat, gname)
if gname not in glyphsSeen:
glyphsSeen.add(gname)
comment = gname
ftml.setFeatures(featlist)
ftml.addToTest(uid, lsb + c + rsb, label,
comment) #isolate
if get_ucd(uid, 'jt') == 'D':
ftml.addToTest(uid, lsb + c + zwj + rsb) # initial
ftml.addToTest(uid,
lsb + zwj + c + zwj + rsb) # medial
if get_ucd(uid, 'jt') in ('R', 'D'):
ftml.addToTest(uid, lsb + zwj + c + rsb) # final
ftml.clearFeatures()
ftml.closeTest()
ftml.writeFile(args.output)