def update_udhr(udhr_dir, fetch_dir, in_repo):
"""Delete udhr_dir and rebuild with files extracted from udhr_xml.zip
in fetch_dir. Stage if udhr_dir is in the repo."""
zippath = os.path.join(fetch_dir, UDHR_XML_ZIP_NAME)
tool_utils.check_file_exists(zippath)
if in_repo and os.path.isdir(
udhr_dir) and not tool_utils.git_is_clean(udhr_dir):
raise ValueError('Please clean %s.' % udhr_dir)
if os.path.isdir(udhr_dir):
shutil.rmtree(udhr_dir)
os.makedirs(udhr_dir)
tool_utils.zip_extract_with_timestamp(zippath, udhr_dir)
# dos line endings, sheesh
tool_utils.dos2unix(udhr_dir, ['*.xml', '*.rnc', '*.rng'])
if in_repo:
tool_utils.git_add_all(udhr_dir)
date = datetime.datetime.now().strftime('%Y-%m-%d')
dst = 'in %s ' % udhr_dir if not in_repo else ''
print('Update UDHR files %sfrom %s as of %s.' % (dst, fetch_dir, date))
def update_samples(
sample_dir, udhr_dir, bcp_to_code_attrib_sample, in_repo, no_stage):
"""Create samples in sample_dir based on the bcp to c_a_s map. Stage
if sample_dir is in the repo. If sample_dir is in the repo, don't
overwrite samples whose most recent log entry does not start with
'Updated by tool'."""
tool_utils.check_dir_exists(udhr_dir)
if (in_repo and not no_stage and os.path.isdir(sample_dir) and
not tool_utils.git_is_clean(sample_dir)):
raise ValueError('Please clean %s.' % sample_dir)
if in_repo:
repo, subdir = os.path.split(sample_dir)
tool_samples = frozenset(tool_utils.get_tool_generated(repo, subdir))
print 'allowing overwrite of %d files:\n %s' % (
len(tool_samples), ', '.join(sorted(tool_samples)))
comments = [
'# Attributions for sample excerpts:',
'# original - in the public domain, no attribution',
'# UN - UN, OHCHR, or affiliate, attribute to UN',
'# other - not a UN translation',
'# none - not on ohchr, not a UN translation'
]
sample_attrib_list = []
sample_dir = tool_utils.ensure_dir_exists(sample_dir)
count = 0
for bcp, (code, attrib, sample) in bcp_to_code_attrib_sample.iteritems():
dst_file = '%s_udhr.txt' % bcp
dst_path = os.path.join(sample_dir, dst_file)
if in_repo and os.path.isfile(dst_path) and dst_file not in tool_samples:
print 'Not overwriting modified file %s' % dst_file
else:
with codecs.open(dst_path, 'w', 'utf8') as f:
f.write(sample)
count += 1
sample_attrib_list.append('%s: %s' % (dst_file, attrib))
print 'Created %d samples' % count
# Some existing samples that we don't overwrite are not in
# bcp_to_code_attrib_sample, so they're not listed. Readers of the
# attributions.txt file will need to default these to 'none'.
attrib_data = '\n'.join(comments + sorted(sample_attrib_list)) + '\n'
with open(os.path.join(sample_dir, 'attributions.txt'), 'w') as f:
f.write(attrib_data)
if in_repo and not no_stage:
tool_utils.git_add_all(sample_dir)
date = datetime.datetime.now().strftime('%Y-%m-%d')
dst = 'in %s ' % sample_dir if not in_repo else ''
noto_ix = udhr_dir.find('nototools')
src = udhr_dir if noto_ix == -1 else udhr_dir[noto_ix:]
# prefix of this sample commit message indicates that these were
# tool-generated
print 'Updated by tool - sample files %sfrom %s as of %s.' % (dst, src, date)
def noto_check_clean():
errors = []
for r, p in zip(_REPOS, _REPO_PATHS):
if not tool_utils.git_is_clean(p):
errors.append(r)
if errors:
print >> sys.stderr, '%s %s not clean' % (
' '.join(errors), 'is' if len(errors) == 1 else 'are')
return False
return True
def noto_check_clean():
errors = []
for r, p in zip(_REPOS, _REPO_PATHS):
if not tool_utils.git_is_clean(p):
errors.append(r)
if errors:
print >> sys.stderr, '%s %s not clean' % (' '.join(errors), 'is' if
len(errors) == 1 else 'are')
return False
return True
def noto_check_clean():
errors = []
for r, p in zip(_REPOS, _REPO_PATHS):
if not tool_utils.git_is_clean(p):
errors.append(r)
if errors:
sys.stderr.write(
"%s %s not clean\n"
% (" ".join(errors), "is" if len(errors) == 1 else "are")
)
return False
return True
def update_cldr(noto_repo, cldr_repo, update=False, cldr_tag=""):
"""Copy needed directories/files from cldr_repo to noto_repo/third_party/cldr."""
noto_repo = os.path.abspath(noto_repo)
cldr_repo = os.path.abspath(cldr_repo)
noto_cldr = os.path.join(noto_repo, "third_party/cldr")
tool_utils.check_dir_exists(noto_cldr)
tool_utils.check_dir_exists(cldr_repo)
if not tool_utils.git_is_clean(noto_repo):
print("git in %s is not clean: please fix" % noto_repo)
return
if update:
tool_utils.svn_update(cldr_repo)
# get version of cldr. Unfortunately, this doesn't know about tags.
cldr_version = tool_utils.svn_get_version(cldr_repo)
# prepare and create README.third_party
readme_text = string.Template(README_TEMPLATE).substitute(
version=cldr_version, tag=cldr_tag)
with open(os.path.join(noto_cldr, "README.third_party"), "w") as f:
f.write(readme_text)
# remove/replace directories
for subdir in CLDR_SUBDIRS:
src = os.path.join(cldr_repo, subdir)
dst = os.path.join(noto_cldr, subdir)
print("replacing directory %s..." % subdir)
print("...by copying from %s to %s" % (src, dst))
if os.path.exists(dst):
shutil.rmtree(dst)
shutil.copytree(src, dst)
# replace files
for f in CLDR_FILES:
print("replacing file %s..." % f)
src = os.path.join(cldr_repo, f)
dst = os.path.join(noto_cldr, f)
shutil.copy(src, dst)
# stage changes in cldr dir
tool_utils.git_add_all(noto_cldr)
# print commit message
tag_string = (" tag %s" % cldr_tag) if cldr_tag else ""
print("Update CLDR data to SVN r%s%s." % (cldr_version, tag_string))
def update_repo(repo_samples, new_samples):
# Verify directory is clean.
if not tool_utils.git_is_clean(new_samples):
print("Please fix.")
return
# Copy samples into git repo
for filename in os.listdir(new_samples):
filepath = os.path.join(new_samples, filename)
if not os.path.isfile(filepath):
continue
shutil.copy2(filename, repo_samples)
# Stage changes.
tool_utils.git_add_all(new_samples)
# Sample commit message.
print("Update UDHR sample data.")
def update_repo(repo_samples, new_samples):
# Verify directory is clean.
if not tool_utils.git_is_clean(new_samples):
print 'Please fix.'
return
# Copy samples into git repo
for filename in os.listdir(new_samples):
filepath = os.path.join(new_samples, filename)
if not os.path.isfile(filepath):
continue
shutil.copy2(filename, repo_samples)
# Stage changes.
tool_utils.git_add_all(new_samples)
# Sample commit message.
print 'Update UDHR sample data.'
def get_repo_info(skip_checks):
"""Looks at the three noto fonts repos (fonts, cjk, emoji) and
gets information about the current state of each. Returns
a mapping from 'fonts', 'cjk', and 'emoji' to the corresponding
info.
If skip_checks is not set, checks that the repos are in a good
state (at a known annotated tag and there are no pending commits),
otherwise an exception is raised."""
repo_info = {}
errors = []
for repo_name in 'fonts cjk emoji'.split():
msg_lines = []
repo = tool_utils.resolve_path('[%s]' % repo_name)
repo_head_commit = tool_utils.git_head_commit(repo)
repo_branch = tool_utils.git_get_branch(repo)
msg_lines.append('Repo: noto-%s' % repo_name)
if skip_checks:
msg_lines.append('Branch: %s' % repo_branch)
msg_lines.append('Commit: %s\nSubject: %s' % repo_head_commit)
else:
if not tool_utils.git_is_clean(repo):
errors.append('repo noto-%s is not clean' % repo_name)
continue
repo_tag = None
for tag in tool_utils.git_tags(repo):
if tag[0] == repo_head_commit[0]: # matching commits
repo_tag = tag
break
if not repo_tag:
errors.append('noto-%s is not at a release tag' % repo_name)
continue
tag_commit, tag_name, tag_date = tag
tag_info = tool_utils.git_tag_info(repo, tag_name)
msg_lines.append(
'Tag: %s\nDate: %s\nCommit:%s\n\n%s' % (
tag_name, tag_date, tag_commit, tag_info))
repo_info[repo_name] = '\n'.join(msg_lines)
if errors:
for _, v in sorted(repo_info.iteritems()):
print v
raise Exception('Some repos are not clean\n' + '\n'.join(errors))
return repo_info
def update_udhr(udhr_dir, fetch_dir, in_repo):
"""Delete udhr_dir and rebuild with files extracted from udhr_xml.zip
in fetch_dir. Stage if udhr_dir is in the repo."""
zippath = os.path.join(fetch_dir, UDHR_XML_ZIP_NAME)
tool_utils.check_file_exists(zippath)
if in_repo and os.path.isdir(udhr_dir) and not tool_utils.git_is_clean(udhr_dir):
raise ValueError('Please clean %s.' % udhr_dir)
if os.path.isdir(udhr_dir):
shutil.rmtree(udhr_dir)
os.makedirs(udhr_dir)
tool_utils.zip_extract_with_timestamp(zippath, udhr_dir)
# dos line endings, sheesh
tool_utils.dos2unix(udhr_dir, ['*.xml', '*.rnc', '*.rng'])
if in_repo:
tool_utils.git_add_all(udhr_dir)
date = datetime.datetime.now().strftime('%Y-%m-%d')
dst = 'in %s ' % udhr_dir if not in_repo else ''
print 'Update UDHR files %sfrom %s as of %s.' % (dst, fetch_dir, date)
def update_samples(sample_dir, udhr_dir, bcp_to_code_attrib_sample, in_repo, no_stage):
"""Create samples in sample_dir based on the bcp to c_a_s map. Stage
if sample_dir is in the repo. If sample_dir is in the repo, don't
overwrite samples whose most recent log entry does not start with
'Updated by tool'."""
tool_utils.check_dir_exists(udhr_dir)
if (
in_repo
and not no_stage
and os.path.isdir(sample_dir)
and not tool_utils.git_is_clean(sample_dir)
):
raise ValueError("Please clean %s." % sample_dir)
if in_repo:
repo, subdir = os.path.split(sample_dir)
tool_samples = frozenset(tool_utils.get_tool_generated(repo, subdir))
print(
"allowing overwrite of %d files:\n %s"
% (len(tool_samples), ", ".join(sorted(tool_samples)))
)
comments = [
"# Attributions for sample excerpts:",
"# original - in the public domain, no attribution",
"# UN - UN, OHCHR, or affiliate, attribute to UN",
"# other - not a UN translation",
"# none - not on ohchr, not a UN translation",
]
sample_attrib_list = []
sample_dir = tool_utils.ensure_dir_exists(sample_dir)
count = 0
for bcp, (code, attrib, sample) in bcp_to_code_attrib_sample.items():
dst_file = "%s_udhr.txt" % bcp
dst_path = os.path.join(sample_dir, dst_file)
if in_repo and os.path.isfile(dst_path) and dst_file not in tool_samples:
print("Not overwriting modified file %s" % dst_file)
else:
with codecs.open(dst_path, "w", "utf8") as f:
f.write(sample)
count += 1
sample_attrib_list.append("%s: %s" % (dst_file, attrib))
print("Created %d samples" % count)
# Some existing samples that we don't overwrite are not in
# bcp_to_code_attrib_sample, so they're not listed. Readers of the
# attributions.txt file will need to default these to 'none'.
attrib_data = "\n".join(comments + sorted(sample_attrib_list)) + "\n"
with open(os.path.join(sample_dir, "attributions.txt"), "w") as f:
f.write(attrib_data)
if in_repo and not no_stage:
tool_utils.git_add_all(sample_dir)
date = datetime.datetime.now().strftime("%Y-%m-%d")
dst = "in %s " % sample_dir if not in_repo else ""
noto_ix = udhr_dir.find("nototools")
src = udhr_dir if noto_ix == -1 else udhr_dir[noto_ix:]
# prefix of this sample commit message indicates that these were
# tool-generated
print("Updated by tool - sample files %sfrom %s as of %s." % (dst, src, date))
