def get_repo_symbolic_name_from_dirname(git_dir: str) -> str:
"""Return the name of the repo in `git_dir`.
E.g., "alphamatic/amp", "ParticleDev/commodity_research"
"""
dbg.dassert_exists(git_dir)
cmd = "cd %s; (git remote -v | grep fetch)" % git_dir
# TODO(gp): Make it more robust, by checking both fetch and push.
# "origin [email protected]:alphamatic/amp (fetch)"
_, output = si.system_to_string(cmd)
data: List[str] = output.split()
_LOG.debug("data=%s", data)
dbg.dassert_eq(len(data), 3, "data='%s'", str(data))
# [email protected]:alphamatic/amp
repo_name = data[1]
m = re.match(r"^.*\.com:(.*)$", repo_name)
dbg.dassert(m, "Can't parse '%s'", repo_name)
repo_name = m.group(1) # type: ignore
_LOG.debug("repo_name=%s", repo_name)
# We expect something like "alphamatic/amp".
m = re.match(r"^\S+/\S+$", repo_name)
dbg.dassert(m, "repo_name='%s'", repo_name)
# origin [email protected]:ParticleDev/ORG_Particle.git (fetch)
suffix_to_remove = ".git"
if repo_name.endswith(suffix_to_remove):
repo_name = repo_name[:-len(suffix_to_remove)]
return repo_name
def from_file(
file_name: str, use_gzip: bool = False, encoding: Optional[Any] = None
) -> str:
"""Read contents of a file as string.
Use `use_gzip` flag to load a compressed file with correct extenstion.
:param file_name: path to .txt or .gz file
:param use_gzip: whether to decompress the archived file
:param encoding: encoding to use when reading the string
:return: contents of file as string
"""
# Verify that file name is not empty.
dbg.dassert_ne(file_name, "")
# Verify that the file name exists.
dbg.dassert_exists(file_name)
if use_gzip:
# Check if user provided correct file name.
if not file_name.endswith(("gz", "gzip")):
_LOG.warning("The provided file extension is not for a gzip file.")
# Open gzipped file.
f = gzip.open(file_name, "rt", encoding=encoding)
else:
# Open regular text file.
f = open(file_name, "r", encoding=encoding)
try:
# Read data.
data: str = f.read()
except UnicodeDecodeError as e:
# Raise unicode decode error message.
_raise_file_decode_error(e, file_name)
finally:
f.close()
dbg.dassert_isinstance(data, str)
return data
def get_current_commit_hash(dir_name: str = "./") -> str:
dbg.dassert_exists(dir_name)
cmd = f"cd {dir_name} && git rev-parse HEAD"
data: Tuple[int, str] = si.system_to_one_line(cmd)
_, sha = data
# 0011776388b4c0582161eb2749b665fc45b87e7e
_LOG.debug("sha=%s", sha)
return sha
def from_json(file_name: str) -> dict:
"""Read object from JSON file.
:param file_name: name of file
:return: dict with data
"""
dbg.dassert_exists(file_name)
with open(file_name, "r") as f:
data: dict = json.loads(f.read())
return data
def get_branch_name(git_dir: str = ".") -> str:
"""Return the name of the Git branch we are in.
E.g., `master` or `PartTask672_DEV_INFRA_Add_script_to_check_and_merge_PR`
"""
dbg.dassert_exists(git_dir)
cmd = "cd %s && git rev-parse --abbrev-ref HEAD" % git_dir
data: Tuple[int, str] = si.system_to_one_line(cmd)
_, output = data
return output
def get_remote_head_hash(dir_name: str) -> str:
"""Report the hash that the remote Git repo is at."""
dbg.dassert_exists(dir_name)
sym_name = get_repo_symbolic_name_from_dirname(dir_name)
cmd = f"git ls-remote [email protected]:{sym_name} HEAD 2>/dev/null"
data: Tuple[int, str] = si.system_to_one_line(cmd)
_, output = data
# > git ls-remote [email protected]:alphamatic/amp HEAD 2>/dev/null
# 921676624f6a5f3f36ab507baed1b886227ac2e6 HEAD
return output
def assert_equal(self, actual: str, expected: str) -> None:
dbg.dassert_in(type(actual), (bytes, str))
dbg.dassert_in(type(expected), (bytes, str))
#
dir_name = self._get_current_path()
_LOG.debug("dir_name=%s", dir_name)
io_.create_dir(dir_name, incremental=True)
dbg.dassert_exists(dir_name)
#
test_name = self._get_test_name()
_assert_equal(actual, expected, test_name, dir_name)
def get_head_hash(dir_name: str) -> str:
"""Report the hash that a Git repo is synced at.
> git rev-parse HEAD
"""
dbg.dassert_exists(dir_name)
cmd = f"cd {dir_name} && git rev-parse HEAD"
data: Tuple[int, str] = si.system_to_one_line(cmd)
_, output = data
# 4759b3685f903e6c669096e960b248ec31c63b69
return output
def find_file_in_git_tree(file_in: str, super_module: bool = True) -> str:
"""Find the path of a file `file_in` in the outermost git submodule (i.e.,
in the super-module)."""
root_dir = get_client_root(super_module=super_module)
cmd = "find %s -name '%s' | grep -v .git" % (root_dir, file_in)
_, file_name = si.system_to_one_line(cmd)
_LOG.debug("file_name=%s", file_name)
dbg.dassert(file_name != "", "Can't find file '%s' in dir '%s'", file_in,
root_dir)
file_name: str = os.path.abspath(file_name)
dbg.dassert_exists(file_name)
return file_name
def get_submodule_hash(dir_name: str) -> str:
"""Report the Git hash that a submodule (e.g., amp) is at from the point of
view of a supermodule (e.g., p1).
> git ls-tree master | grep <dir_name>
"""
dbg.dassert_exists(dir_name)
cmd = "git ls-tree master | grep %s" % dir_name
data: Tuple[int, str] = si.system_to_one_line(cmd)
_, output = data
# 160000 commit 0011776388b4c0582161eb2749b665fc45b87e7e amp
_LOG.debug("output=%s", output)
data: List[str] = output.split()
_LOG.debug("data=%s", data)
git_hash = data[2]
return git_hash
def run_notebook(file_name: str, scratch_dir: str) -> None:
"""Run jupyter notebook `file_name` using `scratch_dir` as temporary dir
storing the output.
Assert if the notebook doesn't complete successfully.
"""
file_name = os.path.abspath(file_name)
dbg.dassert_exists(file_name)
dbg.dassert_exists(scratch_dir)
# Build command line.
cmd = []
cmd.append("cd %s && " % scratch_dir)
cmd.append("jupyter nbconvert %s" % file_name)
cmd.append("--execute")
cmd.append("--to html")
cmd.append("--ExecutePreprocessor.kernel_name=python")
# No time-out.
cmd.append("--ExecutePreprocessor.timeout=-1")
# Execute.
cmd_as_str = " ".join(cmd)
si.system(cmd_as_str, abort_on_error=True)
def check_string(
self,
actual: str,
fuzzy_match: bool = False,
purify_text: bool = False,
use_gzip: bool = False,
) -> None:
"""Check the actual outcome of a test against the expected outcomes
contained in the file and/or updates the golden reference file with the
actual outcome.
:param: purify_text: remove some artifacts (e.g., user names,
directories, reference to Git client)
Raises if there is an error.
"""
dbg.dassert_in(type(actual), (bytes, str))
#
dir_name = self._get_current_path()
_LOG.debug("dir_name=%s", dir_name)
io_.create_dir(dir_name, incremental=True)
dbg.dassert_exists(dir_name)
# Get the expected outcome.
file_name = self.get_output_dir() + "/test.txt"
if use_gzip:
file_name += ".gz"
_LOG.debug("file_name=%s", file_name)
# Remove reference from the current purify.
if purify_text:
actual = purify_txt_from_client(actual)
#
if get_update_tests():
# Determine whether outcome needs to be updated.
outcome_updated = False
file_exists = os.path.exists(file_name)
if file_exists:
expected = io_.from_file(file_name, use_gzip=use_gzip)
if expected != actual:
outcome_updated = True
else:
# The golden outcome doesn't exist.
outcome_updated = True
if outcome_updated:
# Update the test result.
_LOG.warning("Test outcome updated ... ")
io_.to_file(file_name, actual, use_gzip=use_gzip)
# Add to git.
cmd = "git add %s" % file_name
rc = si.system(cmd, abort_on_error=False)
if rc:
_LOG.warning(
"Can't run '%s': you need to add the file "
"manually",
cmd,
)
else:
# Just check the test result.
if os.path.exists(file_name):
# Golden outcome is available: check the actual outcome against
# the golden outcome.
expected = io_.from_file(file_name, use_gzip=use_gzip)
test_name = self._get_test_name()
_assert_equal(actual,
expected,
test_name,
dir_name,
fuzzy_match=fuzzy_match)
else:
# No golden outcome available: save the result in a tmp file.
tmp_file_name = file_name + ".tmp"
io_.to_file(tmp_file_name, actual)
msg = "Can't find golden in %s\nSaved actual outcome in %s" % (
file_name,
tmp_file_name,
)
raise RuntimeError(msg)