def diff_files(file_name1: str,
file_name2: str,
tag: Optional[str] = None) -> NoReturn:
# Diff to screen.
_, res = si.system_to_string(
"echo; sdiff -l -w 150 %s %s" % (file_name1, file_name2),
abort_on_error=False,
log_level=logging.DEBUG,
)
if tag is not None:
_LOG.error("%s", "\n" + prnt.frame(tag))
_LOG.error(res)
# Report how to diff.
vimdiff_cmd = "vimdiff %s %s" % (
os.path.abspath(file_name1),
os.path.abspath(file_name2),
)
# Save a script to diff.
diff_script = "./tmp_diff.sh"
io_.to_file(diff_script, vimdiff_cmd)
cmd = "chmod +x " + diff_script
si.system(cmd)
msg = []
msg.append("Diff with:")
msg.append("> " + vimdiff_cmd)
msg.append("or running:")
msg.append("> " + diff_script)
msg_as_str = "\n".join(msg)
_LOG.error(msg_as_str)
raise RuntimeError(msg_as_str)
def git_stash_push(prefix: str,
msg: Optional[str] = None,
log_level: int = logging.DEBUG) -> Tuple[str, bool]:
user_name = si.get_user_name()
server_name = si.get_server_name()
timestamp = hdt.get_timestamp()
tag = "%s-%s-%s" % (user_name, server_name, timestamp)
tag = prefix + "." + tag
_LOG.debug("tag='%s'", tag)
cmd = "git stash push"
_LOG.debug("msg='%s'", msg)
push_msg = tag[:]
if msg:
push_msg += ": " + msg
cmd += " -m '%s'" % push_msg
si.system(cmd, suppress_output=False, log_level=log_level)
# Check if we actually stashed anything.
cmd = r"git stash list | \grep '%s' | wc -l" % tag
_, output = si.system_to_string(cmd)
was_stashed = int(output) > 0
if not was_stashed:
msg = "Nothing was stashed"
_LOG.warning(msg)
# raise RuntimeError(msg)
return tag, was_stashed
def git_stash_apply(mode: str, log_level: int = logging.DEBUG) -> None:
_LOG.debug("# Checking stash head ...")
cmd = "git stash list | head -3"
si.system(cmd, suppress_output=False, log_level=log_level)
#
_LOG.debug("# Restoring local changes...")
if mode == "pop":
cmd = "git stash pop --quiet"
elif mode == "apply":
cmd = "git stash apply --quiet"
else:
raise ValueError("mode='%s'" % mode)
si.system(cmd, suppress_output=False, log_level=log_level)
def create_soft_link(src: str, dst: str) -> None:
"""Create a soft-link to <src> called <dst> (where <src> and <dst> are
files or directories as in a Linux ln command).
This is equivalent to a command like "cp <src> <dst>" but creating a
soft link.
"""
_LOG.debug("# CreateSoftLink")
# Create the enclosing directory, if needed.
enclosing_dir = os.path.dirname(dst)
_LOG.debug("enclosing_dir=%s", enclosing_dir)
create_dir(enclosing_dir, incremental=True)
# Create the link. Note that the link source needs to be an absolute path.
src = os.path.abspath(src)
cmd = "ln -s %s %s" % (src, dst)
si.system(cmd)
def delete_dir(
dir_: str,
change_perms: bool = False,
errnum_to_retry_on: int = 16,
num_retries: int = 1,
num_secs_retry: int = 1,
) -> None:
"""Delete a directory.
- change_perms: change permissions to -R rwx before deleting to deal with
incorrect permissions left over
- errnum_to_retry_on: specify the error to retry on
OSError: [Errno 16] Device or resource busy:
'gridTmp/.nfs0000000002c8c10b00056e57'
"""
_LOG.debug("Deleting dir '%s'", dir_)
if not os.path.isdir(dir_):
# No directory so nothing to do.
return
if change_perms and os.path.isdir(dir_):
cmd = "chmod -R +rwx " + dir_
si.system(cmd)
i = 1
while True:
try:
shutil.rmtree(dir_)
# Command succeeded: exit.
break
except OSError as e:
if errnum_to_retry_on is not None and e.errno == errnum_to_retry_on:
# TODO(saggese): Make it less verbose once we know it's working
# properly.
_LOG.warning(
"Couldn't delete %s: attempt=%s / %s", dir_, i, num_retries
)
i += 1
if i > num_retries:
dbg.dfatal(
"Couldn't delete %s after %s attempts (%s)"
% (dir_, num_retries, str(e))
)
else:
time.sleep(num_secs_retry)
else:
# Unforeseen error: just propagate it.
raise e
def is_inside_submodule(git_dir: str = ".") -> bool:
"""Return whether we are inside a Git submodule or in a Git supermodule."""
cmd = []
cmd.append("cd %s" % git_dir)
cmd.append('cd "$(git rev-parse --show-toplevel)/.."')
cmd.append("(git rev-parse --is-inside-work-tree | grep -q true)")
cmd_as_str = " && ".join(cmd)
rc = si.system(cmd_as_str, abort_on_error=False)
ret: bool = rc == 0
return ret
def run_notebook(file_name: str, scratch_dir: str) -> None:
"""Run jupyter notebook `file_name` using `scratch_dir` as temporary dir
storing the output.
Assert if the notebook doesn't complete successfully.
"""
file_name = os.path.abspath(file_name)
dbg.dassert_exists(file_name)
dbg.dassert_exists(scratch_dir)
# Build command line.
cmd = []
cmd.append("cd %s && " % scratch_dir)
cmd.append("jupyter nbconvert %s" % file_name)
cmd.append("--execute")
cmd.append("--to html")
cmd.append("--ExecutePreprocessor.kernel_name=python")
# No time-out.
cmd.append("--ExecutePreprocessor.timeout=-1")
# Execute.
cmd_as_str = " ".join(cmd)
si.system(cmd_as_str, abort_on_error=True)
def check_string(
self,
actual: str,
fuzzy_match: bool = False,
purify_text: bool = False,
use_gzip: bool = False,
) -> None:
"""Check the actual outcome of a test against the expected outcomes
contained in the file and/or updates the golden reference file with the
actual outcome.
:param: purify_text: remove some artifacts (e.g., user names,
directories, reference to Git client)
Raises if there is an error.
"""
dbg.dassert_in(type(actual), (bytes, str))
#
dir_name = self._get_current_path()
_LOG.debug("dir_name=%s", dir_name)
io_.create_dir(dir_name, incremental=True)
dbg.dassert_exists(dir_name)
# Get the expected outcome.
file_name = self.get_output_dir() + "/test.txt"
if use_gzip:
file_name += ".gz"
_LOG.debug("file_name=%s", file_name)
# Remove reference from the current purify.
if purify_text:
actual = purify_txt_from_client(actual)
#
if get_update_tests():
# Determine whether outcome needs to be updated.
outcome_updated = False
file_exists = os.path.exists(file_name)
if file_exists:
expected = io_.from_file(file_name, use_gzip=use_gzip)
if expected != actual:
outcome_updated = True
else:
# The golden outcome doesn't exist.
outcome_updated = True
if outcome_updated:
# Update the test result.
_LOG.warning("Test outcome updated ... ")
io_.to_file(file_name, actual, use_gzip=use_gzip)
# Add to git.
cmd = "git add %s" % file_name
rc = si.system(cmd, abort_on_error=False)
if rc:
_LOG.warning(
"Can't run '%s': you need to add the file "
"manually",
cmd,
)
else:
# Just check the test result.
if os.path.exists(file_name):
# Golden outcome is available: check the actual outcome against
# the golden outcome.
expected = io_.from_file(file_name, use_gzip=use_gzip)
test_name = self._get_test_name()
_assert_equal(actual,
expected,
test_name,
dir_name,
fuzzy_match=fuzzy_match)
else:
# No golden outcome available: save the result in a tmp file.
tmp_file_name = file_name + ".tmp"
io_.to_file(tmp_file_name, actual)
msg = "Can't find golden in %s\nSaved actual outcome in %s" % (
file_name,
tmp_file_name,
)
raise RuntimeError(msg)