def ldml_write(self, exemplars, sort=True):
"""Write exemplars to a string that can be written to a LDML formatted file."""
if sort:
# Exemplars mentioned in UTS #35 need to be sorted.
list_exemplars = list()
for exemplar in sorted(exemplars):
list_exemplars.append(exemplar)
else:
# Graphemes should be sorted by frequency,
# and since they already are,
# do nothing further here with the order.
list_exemplars = exemplars
list_nfc_exemplars = map(self.ucd.normalize_nfc, list_exemplars)
# Ignore exemplars not of most common script found in the data.
list_nfc_exemplars_main_script = list()
for exemplar in list_nfc_exemplars:
char = exemplar[
0] # only look at the first character in an exemplar.
script = Script.getScript(char)
script_name = Script.getShortName(script)
if script_name == self._get_script(
) or not self.ucd.is_specific_script(char):
list_nfc_exemplars_main_script.append(exemplar)
if self.unittest:
return ' '.join(list_nfc_exemplars_main_script)
else:
return palaso.sldr.UnicodeSets.list2us(
list_nfc_exemplars_main_script, self.ucd)
def ldml_write(self, exemplars, sort=True):
"""Write exemplars to a string that can be written to a LDML formatted file."""
if sort:
# Exemplars mentioned in UTS #35 need to be sorted.
list_exemplars = list()
for exemplar in sorted(exemplars):
list_exemplars.append(exemplar)
else:
# Graphemes should be sorted by frequency,
# and since they already are,
# do nothing further here with the order.
list_exemplars = exemplars
list_nfc_exemplars = map(self.ucd.normalize_nfc, list_exemplars)
# Ignore exemplars not of most common script found in the data.
list_nfc_exemplars_main_script = list()
for exemplar in list_nfc_exemplars:
char = exemplar[0] # only look at the first character in an exemplar.
script = Script.getScript(char)
script_name = Script.getShortName(script)
if script_name == self._get_script() or not self.ucd.is_specific_script(char):
list_nfc_exemplars_main_script.append(exemplar)
if self.unittest:
return ' '.join(list_nfc_exemplars_main_script)
else:
return palaso.sldr.UnicodeSets.list2us(list_nfc_exemplars_main_script, self.ucd)
def is_specific_script(char):
"""True if the character has a specific Script property,
that is, not the values Common or Inherited.
"""
script = Script.getScript(char)
script_code = Script.getScriptCode(script)
if script_code == UScriptCode.COMMON or script_code == UScriptCode.INHERITED:
return False
return True
def is_specific_script(char):
"""True if the character has a specific Script property,
that is, not the values Common or Inherited.
"""
script = Script.getScript(char)
script_code = Script.getScriptCode(script)
if script_code == UScriptCode.COMMON or script_code == UScriptCode.INHERITED:
return False
return True
def make_json(puz_uid, hint_uid, size=(30, 30), limit=1000, sample=100, numtrans=3):
script = get_script(puz_uid)
rtl = Script(Script.getCode(script)[0]).isRightToLeft()
c = translate_clues(gen_puzzle2(puz_uid, size, limit, script), puz_uid, hint_uid, numtrans)
grid = []
for row in c.best_grid:
new_row = [cell if cell != " " else "" for cell in row]
if rtl: new_row = new_row[::-1]
grid.append(new_row)
wl = []
for clue in sorted(c.best_wordlist, key=lambda x: (x[4], x[2], x[3])):
new_clue = clue[:]
if rtl:
new_clue[3] = len(grid[0]) - 1 - clue[3]
wl.append(new_clue)
allcaps = "".join(["".join(clue[0]) for clue in wl]).isupper()
json.dump({"grid": grid, "clues": wl, "rtl": rtl, "allcaps": allcaps}, open("puzzle.json", "w"))
return wl
def _get_script(self):
"""Return most frequently occurring script."""
script_code_and_count_list = self.scripts.most_common(1)
if len(script_code_and_count_list) == 0:
return ''
else:
script_code_and_count = script_code_and_count_list[0]
script_code = script_code_and_count[0]
script = self.codes_for_scripts[script_code]
script_name = Script.getShortName(script)
return script_name
def _get_script(self):
"""Return most frequently occurring script."""
script_code_and_count_list = self.scripts.most_common(1)
if len(script_code_and_count_list) == 0:
return ''
else:
script_code_and_count = script_code_and_count_list[0]
script_code = script_code_and_count[0]
script = self.codes_for_scripts[script_code]
script_name = Script.getShortName(script)
return script_name
def normalize_string(in_str, allowed_scripts):
"""
Normalizes in_str by replacing letters and digits in other scripts with
exemplar values.
Args:
in_str: String to process
allowed_scripts: List of script short names (like "Mymr") to preserve
"""
# TODO: Consider checking ScriptExtensions here as well
output = ""
for ch in in_str:
ch_script = Script.getScript(ch)
ch_type = Char.charType(ch)
ch_bucket = CHAR_TYPE_TO_BUCKET[ch_type]
ch_digit = Char.digit(ch)
if ch_script.getShortName() in allowed_scripts:
# ch is in an allowed script:
# copy directly to the output
output += ch
elif ch_bucket == 1:
# ch is a letter in a disallowed script:
# normalize to the sample char for that script
output += Script.getSampleString(ch_script)
elif ch_bucket == 3 and ch_digit != -1:
# ch is a decimal digit in a disallowed script:
# normalize to the zero digit in that numbering system
output += chr(ord(ch) - ch_digit)
elif ch_type == UCharCategory.CURRENCY_SYMBOL:
# ch is a currency symbol in a disallowed script:
# normalize to $
output += "$"
else:
# all other characters:
# copy directly to the output
output += ch
return output
def process(self, text):
"""Analyze a string."""
i = 0
text = self.ucd.normalize('NFD', text)
# Record script of each character.
for char in text:
script = Script.getScript(char)
script_code = Script.getScriptCode(script)
self.scripts[script_code] += 1
self.codes_for_scripts[script_code] = script
# Record clusters
while i < len(text):
# Look for multigraphs (from length of max_multigraph_length down to 1) character(s)
# of multigraphs already specified in a LDML file.
# Longest possible matches are looked at first.
for multigraph_length in range(self.max_multigraph_length, 0, -1):
multigraph = text[i:i + multigraph_length]
if (multigraph in self._main or
multigraph in self._auxiliary or
multigraph in self._index or
multigraph in self._punctuation):
exemplar = Exemplar(multigraph)
self.clusters[exemplar] += 1
i += multigraph_length
break
# No multigraphs were found at this position,
# so continue processing a single character
# if we have not gone beyond the end of the text.
if not i < len(text):
break
char = text[i]
# Test for punctuation.
if self.ucd.ispunct(char):
exemplar = Exemplar(char)
self.clusters[exemplar] += 1
i += 1
continue
# Find grapheme clusters.
# Ensure exemplar base has needed properties.
if not self.allowable(char):
i += 1
continue
# The current character is a base character.
base = char
# Then find the end of the cluster
# (which may consist of only base characters).
length = base_length = 1
while i + length < len(text):
trailer = text[i + length]
if Char.hasBinaryProperty(trailer, UProperty.DEFAULT_IGNORABLE_CODE_POINT):
# A Default_Ignorable_Code_Point was found, so the cluster continues.
length += 1
continue
if self.ucd.ismark(trailer):
# A Mark was found, so the cluster continues.
length += 1
# Marks such as nuktas are considered part of the base.
if self.ucd.is_always_combine(trailer):
# A Mark such as a nukta was found, so the base continues,
# as well as the cluster.
base_length += 1
base = text[i:i + base_length]
continue
else:
# No more marks, so the end of the cluster has been reached.
break
# Extract cluster
# If no nuktas have been found,
# then the base will be the single character already called base (or char).
# If no non-nukta marks have been found,
# then the trailers variable will be an empty string.
trailers = text[i + base_length:i + length]
exemplar = Exemplar(base, trailers)
self.clusters[exemplar] += 1
i += length
def process(self, text):
"""Analyze a string."""
i = 0
text = self.ucd.normalize('NFD', text)
# Record script of each character.
for char in text:
script = Script.getScript(char)
script_code = Script.getScriptCode(script)
self.scripts[script_code] += 1
self.codes_for_scripts[script_code] = script
# Record clusters
while i < len(text):
# Look for multigraphs (from length of max_multigraph_length down to 1) character(s)
# of multigraphs already specified in a LDML file.
# Longest possible matches are looked at first.
for multigraph_length in range(self.max_multigraph_length, 0, -1):
multigraph = text[i:i + multigraph_length]
if (multigraph in self._main or
multigraph in self._auxiliary or
multigraph in self._index or
multigraph in self._punctuation):
exemplar = Exemplar(multigraph)
self.clusters[exemplar] += 1
i += multigraph_length
break
# No multigraphs were found at this position,
# so continue processing a single character
# if we have not gone beyond the end of the text.
if not i < len(text):
break
char = text[i]
# Test for punctuation.
if self.ucd.ispunct(char):
exemplar = Exemplar(char)
self.clusters[exemplar] += 1
i += 1
continue
# Find grapheme clusters.
# Ensure exemplar base has needed properties.
if not self.allowable(char):
i += 1
continue
# The current character is a base character.
base = char
# Then find the end of the cluster
# (which may consist of only base characters).
length = base_length = 1
while i + length < len(text):
trailer = text[i + length]
if self.ucd.is_zwj(trailer):
# ZWJ found, so the cluster continues.
length += 1
continue
if self.ucd.is_zwnj(trailer):
# ZWNJ found, so the end of the cluster has been reached,
# but put include ZWNJ in the cluster
length += 1
break
if self.ucd.ismark(trailer):
# A Mark was found, so the cluster continues.
length += 1
# Marks such as nuktas are considered part of the base.
if self.ucd.is_always_combine(trailer):
# A Mark such as a nukta was found, so the base continues,
# as well as the cluster.
base_length += 1
base = text[i:i + base_length]
continue
else:
# No more marks, so the end of the cluster has been reached.
break
# Extract cluster
# If no nuktas have been found,
# then the base will be the single character already called base (or char).
# If no non-nukta marks have been found,
# then the trailers variable will be an empty string.
trailers = text[i + base_length:i + length]
exemplar = Exemplar(base, trailers)
self.clusters[exemplar] += 1
i += length