def generate_translit(config, io, common_vars):
input_files = [
InFile("translit/root.txt"),
InFile("translit/en.txt"),
InFile("translit/el.txt")
]
dep_files = set(InFile(filename) for filename in io.glob("translit/*.txt"))
dep_files -= set(input_files)
dep_files = list(sorted(dep_files))
input_basenames = [v.filename[9:] for v in input_files]
output_files = [
OutFile("translit/%s.res" % v[:-4]) for v in input_basenames
]
return [
RepeatedOrSingleExecutionRequest(
name="translit_res",
category="translit",
dep_targets=dep_files,
input_files=input_files,
output_files=output_files,
tool=IcuTool("genrb"),
args="-s {IN_DIR}/translit -d {OUT_DIR}/translit -i {OUT_DIR} "
"-k "
"{INPUT_BASENAME}",
format_with={},
repeat_with={
"INPUT_BASENAME": utils.SpaceSeparatedList(input_basenames)
})
]
def generate_brkitr_lstm(config, io, common_vars):
input_files = [InFile(filename) for filename in io.glob("brkitr/lstm/*.txt")]
input_basenames = [v.filename[12:] for v in input_files]
output_files = [
OutFile("brkitr/%s.res" % v[:-4])
for v in input_basenames
]
return [
RepeatedOrSingleExecutionRequest(
name = "lstm_res",
category = "brkitr_lstm",
dep_targets = [],
input_files = input_files,
output_files = output_files,
tool = IcuTool("genrb"),
args = "-s {IN_DIR}/brkitr/lstm -d {OUT_DIR}/brkitr -i {OUT_DIR} "
"-k "
"{INPUT_BASENAME}",
format_with = {
},
repeat_with = {
"INPUT_BASENAME": utils.SpaceSeparatedList(input_basenames)
}
)
]
def generate_conversion_mappings(config, io, common_vars):
# UConv Conversion Table Files
input_files = [InFile(filename) for filename in io.glob("mappings/*.ucm")]
output_files = [OutFile("%s.cnv" % v.filename[9:-4]) for v in input_files]
# TODO: handle BUILD_SPECIAL_CNV_FILES? Means to add --ignore-siso-check flag to makeconv
return [
RepeatedOrSingleExecutionRequest(
name="conversion_mappings",
category="conversion_mappings",
dep_targets=[],
input_files=input_files,
output_files=output_files,
tool=IcuTool("makeconv"),
args="-s {IN_DIR} -d {OUT_DIR} -c {INPUT_FILE_PLACEHOLDER}",
format_with={},
repeat_with={
"INPUT_FILE_PLACEHOLDER":
utils.SpaceSeparatedList(file.filename for file in input_files)
})
]
def generate_tree(config, io, common_vars, sub_dir, out_sub_dir,
use_pool_bundle, dep_targets):
requests = []
category = "%s_tree" % sub_dir
out_prefix = "%s/" % out_sub_dir if out_sub_dir else ""
# TODO: Clean this up for curr
input_files = [
InFile(filename) for filename in io.glob("%s/*.txt" % sub_dir)
]
if sub_dir == "curr":
input_files.remove(InFile("curr/supplementalData.txt"))
input_basenames = [v.filename[len(sub_dir) + 1:] for v in input_files]
output_files = [
OutFile("%s%s.res" % (out_prefix, v[:-4])) for v in input_basenames
]
# Generate Pool Bundle
if use_pool_bundle:
input_pool_files = [OutFile("%spool.res" % out_prefix)]
pool_target_name = "%s_pool_write" % sub_dir
use_pool_bundle_option = "--usePoolBundle {OUT_DIR}/{OUT_PREFIX}".format(
OUT_PREFIX=out_prefix, **common_vars)
requests += [
SingleExecutionRequest(
name=pool_target_name,
category=category,
dep_targets=dep_targets,
input_files=input_files,
output_files=input_pool_files,
tool=IcuTool("genrb"),
args=
"-s {IN_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} "
"--writePoolBundle -k "
"{INPUT_BASENAMES_SPACED}",
format_with={
"IN_SUB_DIR":
sub_dir,
"OUT_PREFIX":
out_prefix,
"INPUT_BASENAMES_SPACED":
utils.SpaceSeparatedList(input_basenames)
}),
]
dep_targets = dep_targets + [DepTarget(pool_target_name)]
else:
use_pool_bundle_option = ""
# Generate Res File Tree
requests += [
RepeatedOrSingleExecutionRequest(
name="%s_res" % sub_dir,
category=category,
dep_targets=dep_targets,
input_files=input_files,
output_files=output_files,
tool=IcuTool("genrb"),
args=
"-s {IN_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} "
"{EXTRA_OPTION} -k "
"{INPUT_BASENAME}",
format_with={
"IN_SUB_DIR": sub_dir,
"OUT_PREFIX": out_prefix,
"EXTRA_OPTION": use_pool_bundle_option
},
repeat_with={
"INPUT_BASENAME": utils.SpaceSeparatedList(input_basenames)
})
]
# Generate res_index file
# Exclude the deprecated locale variants and root; see ICU-20628. This
# could be data-driven, but we do not want to perform I/O in this script
# (for example, we do not want to read from an XML file).
excluded_locales = set([
"ja_JP_TRADITIONAL",
"th_TH_TRADITIONAL",
"de_",
"de__PHONEBOOK",
"es_",
"es__TRADITIONAL",
"root",
])
# Put alias locales in a separate structure; see ICU-20627
dependency_data = io.read_locale_deps(sub_dir)
if "aliases" in dependency_data:
alias_locales = set(dependency_data["aliases"].keys())
else:
alias_locales = set()
alias_files = []
installed_files = []
for f in input_files:
file_stem = IndexRequest.locale_file_stem(f)
if file_stem in excluded_locales:
continue
destination = alias_files if file_stem in alias_locales else installed_files
destination.append(f)
cldr_version = dependency_data[
"cldrVersion"] if sub_dir == "locales" else None
index_file_txt = TmpFile("{IN_SUB_DIR}/{INDEX_NAME}.txt".format(
IN_SUB_DIR=sub_dir, **common_vars))
index_res_file = OutFile("{OUT_PREFIX}{INDEX_NAME}.res".format(
OUT_PREFIX=out_prefix, **common_vars))
index_file_target_name = "%s_index_txt" % sub_dir
requests += [
IndexRequest(
name=index_file_target_name,
category=category,
installed_files=installed_files,
alias_files=alias_files,
txt_file=index_file_txt,
output_file=index_res_file,
cldr_version=cldr_version,
args=
"-s {TMP_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} "
"-k "
"{INDEX_NAME}.txt",
format_with={
"IN_SUB_DIR": sub_dir,
"OUT_PREFIX": out_prefix
})
]
return requests
def generate_tree(config, glob, common_vars, sub_dir, out_sub_dir,
xml_filename, use_pool_bundle, dep_targets):
requests = []
category = "%s_tree" % sub_dir
out_prefix = "%s/" % out_sub_dir if out_sub_dir else ""
# TODO: Clean this up for curr
input_files = [InFile(filename) for filename in glob("%s/*.txt" % sub_dir)]
if sub_dir == "curr":
input_files.remove(InFile("curr/supplementalData.txt"))
input_basenames = [v.filename[len(sub_dir) + 1:] for v in input_files]
output_files = [
OutFile("%s%s.res" % (out_prefix, v[:-4])) for v in input_basenames
]
# Generate Pool Bundle
if use_pool_bundle:
input_pool_files = [OutFile("%spool.res" % out_prefix)]
pool_target_name = "%s_pool_write" % sub_dir
use_pool_bundle_option = "--usePoolBundle {OUT_DIR}/{OUT_PREFIX}".format(
OUT_PREFIX=out_prefix, **common_vars)
requests += [
SingleExecutionRequest(
name=pool_target_name,
category=category,
dep_targets=dep_targets,
input_files=input_files,
output_files=input_pool_files,
tool=IcuTool("genrb"),
args=
"-s {IN_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} "
"--writePoolBundle -k "
"{INPUT_BASENAMES_SPACED}",
format_with={
"IN_SUB_DIR":
sub_dir,
"OUT_PREFIX":
out_prefix,
"INPUT_BASENAMES_SPACED":
utils.SpaceSeparatedList(input_basenames)
}),
]
dep_targets = dep_targets + [DepTarget(pool_target_name)]
else:
use_pool_bundle_option = ""
# Generate Res File Tree
requests += [
RepeatedOrSingleExecutionRequest(
name="%s_res" % sub_dir,
category=category,
dep_targets=dep_targets,
input_files=input_files,
output_files=output_files,
tool=IcuTool("genrb"),
args=
"-s {IN_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} "
"{EXTRA_OPTION} -k "
"{INPUT_BASENAME}",
format_with={
"IN_SUB_DIR": sub_dir,
"OUT_PREFIX": out_prefix,
"EXTRA_OPTION": use_pool_bundle_option
},
repeat_with={
"INPUT_BASENAME": utils.SpaceSeparatedList(input_basenames)
})
]
# Generate res_index file
synthetic_locales = set()
deprecates_xml_path = os.path.join(os.path.dirname(__file__), xml_filename)
deprecates_xml = ET.parse(deprecates_xml_path)
for child in deprecates_xml.getroot():
if child.tag == "alias":
synthetic_locales.add(child.attrib["from"])
elif child.tag == "emptyLocale":
synthetic_locales.add(child.attrib["locale"])
else:
raise ValueError("Unknown tag in deprecates XML: %s" % child.tag)
index_input_files = []
for f in input_files:
file_stem = f.filename[f.filename.rfind("/") + 1:-4]
if file_stem == "root":
continue
if file_stem in synthetic_locales:
continue
index_input_files.append(f)
cldr_version = locale_dependencies.data[
"cldrVersion"] if sub_dir == "locales" else None
index_file_txt = TmpFile("{IN_SUB_DIR}/{INDEX_NAME}.txt".format(
IN_SUB_DIR=sub_dir, **common_vars))
index_res_file = OutFile("{OUT_PREFIX}{INDEX_NAME}.res".format(
OUT_PREFIX=out_prefix, **common_vars))
index_file_target_name = "%s_index_txt" % sub_dir
requests += [
IndexRequest(
name=index_file_target_name,
category=category,
input_files=index_input_files,
txt_file=index_file_txt,
output_file=index_res_file,
cldr_version=cldr_version,
args=
"-s {TMP_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} "
"-k "
"{INDEX_NAME}.txt",
format_with={
"IN_SUB_DIR": sub_dir,
"OUT_PREFIX": out_prefix
})
]
return requests
