def convert_models(self, tatoeba_ids, dry_run=False):
entries_to_convert = [x for x in self.registry if x[0] in tatoeba_ids]
converted_paths = convert_all_sentencepiece_models(entries_to_convert, dest_dir=self.model_card_dir)
for path in converted_paths:
long_pair = remove_prefix(path.name, "opus-mt-").split("-") # eg. heb-eng
assert len(long_pair) == 2
new_p_src = self.get_two_letter_code(long_pair[0])
new_p_tgt = self.get_two_letter_code(long_pair[1])
hf_model_id = f"opus-mt-{new_p_src}-{new_p_tgt}"
new_path = path.parent.joinpath(hf_model_id) # opus-mt-he-en
os.rename(str(path), str(new_path))
self.write_model_card(hf_model_id, dry_run=dry_run)
def __init__(self, save_dir="marian_converted"):
assert Path(DEFAULT_REPO).exists(
), "need git clone [email protected]:Helsinki-NLP/Tatoeba-Challenge.git"
reg = self.make_tatoeba_registry()
self.download_metadata()
self.registry = reg
reg_df = pd.DataFrame(
reg, columns=["id", "prepro", "url_model", "url_test_set"])
assert reg_df.id.value_counts().max() == 1
reg_df = reg_df.set_index("id")
reg_df["src"] = reg_df.reset_index().id.apply(
lambda x: x.split("-")[0]).values
reg_df["tgt"] = reg_df.reset_index().id.apply(
lambda x: x.split("-")[1]).values
released_cols = [
"url_base",
"pair", # (ISO639-3/ISO639-5 codes),
"short_pair", # (reduced codes),
"chrF2_score",
"bleu",
"brevity_penalty",
"ref_len",
"src_name",
"tgt_name",
]
released = pd.read_csv("Tatoeba-Challenge/models/released-models.txt",
sep="\t",
header=None).iloc[:-1]
released.columns = released_cols
released["fname"] = released["url_base"].apply(lambda x: remove_suffix(
remove_prefix(
x, "https://object.pouta.csc.fi/Tatoeba-Challenge/opus"),
".zip"))
released["2m"] = released.fname.str.startswith("2m")
released["date"] = pd.to_datetime(released["fname"].apply(
lambda x: remove_prefix(remove_prefix(x, "2m-"), "-")))
released["base_ext"] = released.url_base.apply(lambda x: Path(x).name)
reg_df["base_ext"] = reg_df.url_model.apply(lambda x: Path(x).name)
metadata_new = reg_df.reset_index().merge(
released.rename(columns={"pair": "id"}), on=["base_ext", "id"])
metadata_renamer = {
"src": "src_alpha3",
"tgt": "tgt_alpha3",
"id": "long_pair",
"date": "train_date"
}
metadata_new = metadata_new.rename(columns=metadata_renamer)
metadata_new["src_alpha2"] = metadata_new.short_pair.apply(
lambda x: x.split("-")[0])
metadata_new["tgt_alpha2"] = metadata_new.short_pair.apply(
lambda x: x.split("-")[1])
DROP_COLS_BOTH = ["url_base", "base_ext", "fname"]
metadata_new = metadata_new.drop(DROP_COLS_BOTH, 1)
metadata_new["prefer_old"] = metadata_new.long_pair.isin([])
self.metadata = metadata_new
assert self.metadata.short_pair.value_counts().max(
) == 1, "Multiple metadata entries for a short pair"
self.metadata = self.metadata.set_index("short_pair")
# wget.download(LANG_CODE_URL)
mapper = pd.read_csv(LANG_CODE_PATH)
mapper.columns = ["a3", "a2", "ref"]
self.iso_table = pd.read_csv(
ISO_PATH, sep="\t").rename(columns=lambda x: x.lower())
more_3_to_2 = self.iso_table.set_index("id").part1.dropna().to_dict()
more_3_to_2.update(mapper.set_index("a3").a2.to_dict())
self.alpha3_to_alpha2 = more_3_to_2
self.model_card_dir = Path(save_dir)
self.constituents = GROUP_MEMBERS
def write_model_card(
self,
hf_model_id: str,
repo_root=DEFAULT_REPO,
dry_run=False,
) -> str:
"""Copy the most recent model's readme section from opus, and add metadata.
upload command: aws s3 sync model_card_dir s3://models.huggingface.co/bert/Helsinki-NLP/ --dryrun
"""
short_pair = remove_prefix(hf_model_id, "opus-mt-")
extra_metadata = self.metadata.loc[short_pair].drop("2m")
extra_metadata["short_pair"] = short_pair
lang_tags, src_multilingual, tgt_multilingual = self.resolve_lang_code(
extra_metadata)
opus_name = f"{extra_metadata.src_alpha3}-{extra_metadata.tgt_alpha3}"
# opus_name: str = self.convert_hf_name_to_opus_name(hf_model_name)
assert repo_root in ("OPUS-MT-train", "Tatoeba-Challenge")
opus_readme_path = Path(repo_root).joinpath("models", opus_name,
"README.md")
assert opus_readme_path.exists(
), f"Readme file {opus_readme_path} not found"
opus_src, opus_tgt = [x.split("+") for x in opus_name.split("-")]
readme_url = f"https://github.com/Helsinki-NLP/{repo_root}/tree/master/models/{opus_name}/README.md"
s, t = ",".join(opus_src), ",".join(opus_tgt)
metadata = {
"hf_name": short_pair,
"source_languages": s,
"target_languages": t,
"opus_readme_url": readme_url,
"original_repo": repo_root,
"tags": ["translation"],
"languages": lang_tags,
}
lang_tags = l2front_matter(lang_tags)
metadata["src_constituents"] = self.constituents[s]
metadata["tgt_constituents"] = self.constituents[t]
metadata["src_multilingual"] = src_multilingual
metadata["tgt_multilingual"] = tgt_multilingual
metadata.update(extra_metadata)
metadata.update(get_system_metadata(repo_root))
# combine with Tatoeba markdown
extra_markdown = f"### {short_pair}\n\n* source group: {metadata['src_name']} \n* target group: {metadata['tgt_name']} \n* OPUS readme: [{opus_name}]({readme_url})\n"
content = opus_readme_path.open().read()
content = content.split(
"\n# "
)[-1] # Get the lowest level 1 header in the README -- the most recent model.
splat = content.split("*")[2:]
content = "*".join(splat)
# BETTER FRONT MATTER LOGIC
content = (FRONT_MATTER_TEMPLATE.format(lang_tags) + extra_markdown +
"\n* " + content.replace("download", "download original "
"weights"))
items = "\n\n".join([f"- {k}: {v}" for k, v in metadata.items()])
sec3 = "\n### System Info: \n" + items
content += sec3
if dry_run:
return content, metadata
sub_dir = self.model_card_dir / hf_model_id
sub_dir.mkdir(exist_ok=True)
dest = sub_dir / "README.md"
dest.open("w").write(content)
pd.Series(metadata).to_json(sub_dir / "metadata.json")
return content, metadata
