def generate_translit(config, io, common_vars): input_files = [ InFile("translit/root.txt"), InFile("translit/en.txt"), InFile("translit/el.txt") ] dep_files = set(InFile(filename) for filename in io.glob("translit/*.txt")) dep_files -= set(input_files) dep_files = list(sorted(dep_files)) input_basenames = [v.filename[9:] for v in input_files] output_files = [ OutFile("translit/%s.res" % v[:-4]) for v in input_basenames ] return [ RepeatedOrSingleExecutionRequest( name="translit_res", category="translit", dep_targets=dep_files, input_files=input_files, output_files=output_files, tool=IcuTool("genrb"), args="-s {IN_DIR}/translit -d {OUT_DIR}/translit -i {OUT_DIR} " "-k " "{INPUT_BASENAME}", format_with={}, repeat_with={ "INPUT_BASENAME": utils.SpaceSeparatedList(input_basenames) }) ]
def generate_brkitr_lstm(config, io, common_vars): input_files = [InFile(filename) for filename in io.glob("brkitr/lstm/*.txt")] input_basenames = [v.filename[12:] for v in input_files] output_files = [ OutFile("brkitr/%s.res" % v[:-4]) for v in input_basenames ] return [ RepeatedOrSingleExecutionRequest( name = "lstm_res", category = "brkitr_lstm", dep_targets = [], input_files = input_files, output_files = output_files, tool = IcuTool("genrb"), args = "-s {IN_DIR}/brkitr/lstm -d {OUT_DIR}/brkitr -i {OUT_DIR} " "-k " "{INPUT_BASENAME}", format_with = { }, repeat_with = { "INPUT_BASENAME": utils.SpaceSeparatedList(input_basenames) } ) ]
def generate_conversion_mappings(config, io, common_vars): # UConv Conversion Table Files input_files = [InFile(filename) for filename in io.glob("mappings/*.ucm")] output_files = [OutFile("%s.cnv" % v.filename[9:-4]) for v in input_files] # TODO: handle BUILD_SPECIAL_CNV_FILES? Means to add --ignore-siso-check flag to makeconv return [ RepeatedOrSingleExecutionRequest( name="conversion_mappings", category="conversion_mappings", dep_targets=[], input_files=input_files, output_files=output_files, tool=IcuTool("makeconv"), args="-s {IN_DIR} -d {OUT_DIR} -c {INPUT_FILE_PLACEHOLDER}", format_with={}, repeat_with={ "INPUT_FILE_PLACEHOLDER": utils.SpaceSeparatedList(file.filename for file in input_files) }) ]
def generate_tree(config, io, common_vars, sub_dir, out_sub_dir, use_pool_bundle, dep_targets): requests = [] category = "%s_tree" % sub_dir out_prefix = "%s/" % out_sub_dir if out_sub_dir else "" # TODO: Clean this up for curr input_files = [ InFile(filename) for filename in io.glob("%s/*.txt" % sub_dir) ] if sub_dir == "curr": input_files.remove(InFile("curr/supplementalData.txt")) input_basenames = [v.filename[len(sub_dir) + 1:] for v in input_files] output_files = [ OutFile("%s%s.res" % (out_prefix, v[:-4])) for v in input_basenames ] # Generate Pool Bundle if use_pool_bundle: input_pool_files = [OutFile("%spool.res" % out_prefix)] pool_target_name = "%s_pool_write" % sub_dir use_pool_bundle_option = "--usePoolBundle {OUT_DIR}/{OUT_PREFIX}".format( OUT_PREFIX=out_prefix, **common_vars) requests += [ SingleExecutionRequest( name=pool_target_name, category=category, dep_targets=dep_targets, input_files=input_files, output_files=input_pool_files, tool=IcuTool("genrb"), args= "-s {IN_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} " "--writePoolBundle -k " "{INPUT_BASENAMES_SPACED}", format_with={ "IN_SUB_DIR": sub_dir, "OUT_PREFIX": out_prefix, "INPUT_BASENAMES_SPACED": utils.SpaceSeparatedList(input_basenames) }), ] dep_targets = dep_targets + [DepTarget(pool_target_name)] else: use_pool_bundle_option = "" # Generate Res File Tree requests += [ RepeatedOrSingleExecutionRequest( name="%s_res" % sub_dir, category=category, dep_targets=dep_targets, input_files=input_files, output_files=output_files, tool=IcuTool("genrb"), args= "-s {IN_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} " "{EXTRA_OPTION} -k " "{INPUT_BASENAME}", format_with={ "IN_SUB_DIR": sub_dir, "OUT_PREFIX": out_prefix, "EXTRA_OPTION": use_pool_bundle_option }, repeat_with={ "INPUT_BASENAME": utils.SpaceSeparatedList(input_basenames) }) ] # Generate res_index file # Exclude the deprecated locale variants and root; see ICU-20628. This # could be data-driven, but we do not want to perform I/O in this script # (for example, we do not want to read from an XML file). excluded_locales = set([ "ja_JP_TRADITIONAL", "th_TH_TRADITIONAL", "de_", "de__PHONEBOOK", "es_", "es__TRADITIONAL", "root", ]) # Put alias locales in a separate structure; see ICU-20627 dependency_data = io.read_locale_deps(sub_dir) if "aliases" in dependency_data: alias_locales = set(dependency_data["aliases"].keys()) else: alias_locales = set() alias_files = [] installed_files = [] for f in input_files: file_stem = IndexRequest.locale_file_stem(f) if file_stem in excluded_locales: continue destination = alias_files if file_stem in alias_locales else installed_files destination.append(f) cldr_version = dependency_data[ "cldrVersion"] if sub_dir == "locales" else None index_file_txt = TmpFile("{IN_SUB_DIR}/{INDEX_NAME}.txt".format( IN_SUB_DIR=sub_dir, **common_vars)) index_res_file = OutFile("{OUT_PREFIX}{INDEX_NAME}.res".format( OUT_PREFIX=out_prefix, **common_vars)) index_file_target_name = "%s_index_txt" % sub_dir requests += [ IndexRequest( name=index_file_target_name, category=category, installed_files=installed_files, alias_files=alias_files, txt_file=index_file_txt, output_file=index_res_file, cldr_version=cldr_version, args= "-s {TMP_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} " "-k " "{INDEX_NAME}.txt", format_with={ "IN_SUB_DIR": sub_dir, "OUT_PREFIX": out_prefix }) ] return requests
def generate_tree(config, glob, common_vars, sub_dir, out_sub_dir, xml_filename, use_pool_bundle, dep_targets): requests = [] category = "%s_tree" % sub_dir out_prefix = "%s/" % out_sub_dir if out_sub_dir else "" # TODO: Clean this up for curr input_files = [InFile(filename) for filename in glob("%s/*.txt" % sub_dir)] if sub_dir == "curr": input_files.remove(InFile("curr/supplementalData.txt")) input_basenames = [v.filename[len(sub_dir) + 1:] for v in input_files] output_files = [ OutFile("%s%s.res" % (out_prefix, v[:-4])) for v in input_basenames ] # Generate Pool Bundle if use_pool_bundle: input_pool_files = [OutFile("%spool.res" % out_prefix)] pool_target_name = "%s_pool_write" % sub_dir use_pool_bundle_option = "--usePoolBundle {OUT_DIR}/{OUT_PREFIX}".format( OUT_PREFIX=out_prefix, **common_vars) requests += [ SingleExecutionRequest( name=pool_target_name, category=category, dep_targets=dep_targets, input_files=input_files, output_files=input_pool_files, tool=IcuTool("genrb"), args= "-s {IN_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} " "--writePoolBundle -k " "{INPUT_BASENAMES_SPACED}", format_with={ "IN_SUB_DIR": sub_dir, "OUT_PREFIX": out_prefix, "INPUT_BASENAMES_SPACED": utils.SpaceSeparatedList(input_basenames) }), ] dep_targets = dep_targets + [DepTarget(pool_target_name)] else: use_pool_bundle_option = "" # Generate Res File Tree requests += [ RepeatedOrSingleExecutionRequest( name="%s_res" % sub_dir, category=category, dep_targets=dep_targets, input_files=input_files, output_files=output_files, tool=IcuTool("genrb"), args= "-s {IN_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} " "{EXTRA_OPTION} -k " "{INPUT_BASENAME}", format_with={ "IN_SUB_DIR": sub_dir, "OUT_PREFIX": out_prefix, "EXTRA_OPTION": use_pool_bundle_option }, repeat_with={ "INPUT_BASENAME": utils.SpaceSeparatedList(input_basenames) }) ] # Generate res_index file synthetic_locales = set() deprecates_xml_path = os.path.join(os.path.dirname(__file__), xml_filename) deprecates_xml = ET.parse(deprecates_xml_path) for child in deprecates_xml.getroot(): if child.tag == "alias": synthetic_locales.add(child.attrib["from"]) elif child.tag == "emptyLocale": synthetic_locales.add(child.attrib["locale"]) else: raise ValueError("Unknown tag in deprecates XML: %s" % child.tag) index_input_files = [] for f in input_files: file_stem = f.filename[f.filename.rfind("/") + 1:-4] if file_stem == "root": continue if file_stem in synthetic_locales: continue index_input_files.append(f) cldr_version = locale_dependencies.data[ "cldrVersion"] if sub_dir == "locales" else None index_file_txt = TmpFile("{IN_SUB_DIR}/{INDEX_NAME}.txt".format( IN_SUB_DIR=sub_dir, **common_vars)) index_res_file = OutFile("{OUT_PREFIX}{INDEX_NAME}.res".format( OUT_PREFIX=out_prefix, **common_vars)) index_file_target_name = "%s_index_txt" % sub_dir requests += [ IndexRequest( name=index_file_target_name, category=category, input_files=index_input_files, txt_file=index_file_txt, output_file=index_res_file, cldr_version=cldr_version, args= "-s {TMP_DIR}/{IN_SUB_DIR} -d {OUT_DIR}/{OUT_PREFIX} -i {OUT_DIR} " "-k " "{INDEX_NAME}.txt", format_with={ "IN_SUB_DIR": sub_dir, "OUT_PREFIX": out_prefix }) ] return requests