def check_branches(self):
"""Checks if the branch can be renamed.
We require that the source branch exists, is not the current branch and
is actually a review branch. We also require that the target branch does
not exist.
If successful, sets state to RENAME, otherwise to FINISHED.
"""
rietveld_info = None
if utils.branch_exists(self.__target_branch):
print 'Target branch %r already exists.' % (self.__target_branch,)
self.state = self.FINISHED
elif not utils.branch_exists(self.__source_branch):
print 'Branch %r doesn\'t exist.' % (self.__source_branch,)
self.state = self.FINISHED
elif self.__source_branch == utils.get_current_branch():
print 'Can\'t rename branch you\'re currently in.'
self.state = self.FINISHED
else:
rietveld_info = utils.RietveldInfo.from_branch(
branch_name=self.__source_branch)
if rietveld_info is None:
print ('Branch %r has no review in progress.' %
(self.__source_branch,))
print 'Instead, use the git command:'
print '\tgit branch -m %s %s' % (self.__source_branch,
self.__target_branch)
self.state = self.FINISHED
else:
self.state = self.RENAME
self.advance(rietveld_info)
def check_branches(self):
"""Checks if the branch can be renamed.
We require that the source branch exists, is not the current branch and
is actually a review branch. We also require that the target branch does
not exist.
If successful, sets state to RENAME, otherwise to FINISHED.
"""
rietveld_info = None
if utils.branch_exists(self.__target_branch):
print 'Target branch %r already exists.' % (self.__target_branch, )
self.state = self.FINISHED
elif not utils.branch_exists(self.__source_branch):
print 'Branch %r doesn\'t exist.' % (self.__source_branch, )
self.state = self.FINISHED
elif self.__source_branch == utils.get_current_branch():
print 'Can\'t rename branch you\'re currently in.'
self.state = self.FINISHED
else:
rietveld_info = utils.RietveldInfo.from_branch(
branch_name=self.__source_branch)
if rietveld_info is None:
print('Branch %r has no review in progress.' %
(self.__source_branch, ))
print 'Instead, use the git command:'
print '\tgit branch -m %s %s' % (self.__source_branch,
self.__target_branch)
self.state = self.FINISHED
else:
self.state = self.RENAME
self.advance(rietveld_info)
def __init__(self, in_continue, export_action_args, export_action_argv):
"""Constructor for SyncAction.
Args:
in_continue: Boolean indicating whether or not this SyncAction is
continuing or starting fresh.
export_action_args: Parsed argparse.Namespace modified to be passed
in to ExportAction.callback.
export_action_argv: Command line arguments modified to be passed in
to ExportAction.callback.
"""
self.__continue = in_continue
self.__branch = utils.get_current_branch()
self.__rietveld_info = utils.RietveldInfo.from_branch(
branch_name=self.__branch)
export_action_args.server = self.__rietveld_info.server
export_action_args.private = self.__rietveld_info.private
self.__export_action_args = export_action_args
self.__export_action_argv = export_action_argv
# Make sure we have review data
if self.__rietveld_info is None:
print 'There is no review data for branch %r.' % (self.__branch,)
self.state = self.FINISHED
else:
self.state = self.STARTING
self.advance()
def __init__(self, in_continue, export_action_args, export_action_argv):
"""Constructor for SyncAction.
Args:
in_continue: Boolean indicating whether or not this SyncAction is
continuing or starting fresh.
export_action_args: Parsed argparse.Namespace modified to be passed
in to ExportAction.callback.
export_action_argv: Command line arguments modified to be passed in
to ExportAction.callback.
"""
self.__continue = in_continue
self.__branch = utils.get_current_branch()
self.__rietveld_info = utils.RietveldInfo.from_branch(
branch_name=self.__branch)
export_action_args.server = self.__rietveld_info.server
export_action_args.private = self.__rietveld_info.private
self.__export_action_args = export_action_args
self.__export_action_argv = export_action_argv
# Make sure we have review data
if self.__rietveld_info is None:
print 'There is no review data for branch %r.' % (self.__branch, )
self.state = self.FINISHED
else:
self.state = self.STARTING
self.advance()
def callback(cls, args, argv):
"""A callback to begin an ExportAction after arguments are parsed.
If the branch is not in a clean state, won't create an ExportAction,
will just print 'git diff' and proceed.
Args:
args: An argparse.Namespace object to extract parameters from.
argv: The original command line arguments that were parsed to create
args. These may be used in a call to upload.py.
Returns:
An instance of ExportAction. Just by instantiating the instance, the
state machine will begin working.
"""
current_branch = utils.get_current_branch()
if not utils.in_clean_state():
print 'Branch %r not in clean state:' % (current_branch, )
print utils.capture_command('git', 'diff', single_line=False)
return
if args.no_mail and args.send_patch:
raise GitRvException('The flags --no_mail and --send_patch are '
'mutually exclusive.')
# This is to determine whether or not --send_mail should be added to
# the upload.py call. If --send_patch is set, we don't need to
# send mail. Similarly if --no_mail is set, we should not send mail.
no_send_mail = args.no_mail or args.send_patch
# Rietveld treats an empty string the same as if the value
# was never set.
if args.message and not args.title:
raise GitRvException('A patch description can only be set if it '
'also has a title.')
commit_subject = commit_description = None
if args.title:
if len(args.title) > 100:
raise GitRvException(utils.SUBJECT_TOO_LONG_TEMPLATE %
(args.title, ))
# If args.message is '', then both the Title and Description
# will take the value of the title.
commit_subject = args.title
commit_description = args.message or ''
return cls(current_branch,
args,
commit_subject=commit_subject,
commit_description=commit_description,
no_send_mail=no_send_mail,
argv=argv)
def callback(cls, args, argv):
"""A callback to begin an ExportAction after arguments are parsed.
If the branch is not in a clean state, won't create an ExportAction,
will just print 'git diff' and proceed.
Args:
args: An argparse.Namespace object to extract parameters from.
argv: The original command line arguments that were parsed to create
args. These may be used in a call to upload.py.
Returns:
An instance of ExportAction. Just by instantiating the instance, the
state machine will begin working.
"""
current_branch = utils.get_current_branch()
if not utils.in_clean_state():
print 'Branch %r not in clean state:' % (current_branch,)
print utils.capture_command('git', 'diff', single_line=False)
return
if args.no_mail and args.send_patch:
raise GitRvException('The flags --no_mail and --send_patch are '
'mutually exclusive.')
# This is to determine whether or not --send_mail should be added to
# the upload.py call. If --send_patch is set, we don't need to
# send mail. Similarly if --no_mail is set, we should not send mail.
no_send_mail = args.no_mail or args.send_patch
# Rietveld treats an empty string the same as if the value
# was never set.
if args.message and not args.title:
raise GitRvException('A patch description can only be set if it '
'also has a title.')
commit_subject = commit_description = None
if args.title:
if len(args.title) > 100:
raise GitRvException(utils.SUBJECT_TOO_LONG_TEMPLATE %
(args.title,))
# If args.message is '', then both the Title and Description
# will take the value of the title.
commit_subject = args.title
commit_description = args.message or ''
return cls(current_branch, args, commit_subject=commit_subject,
commit_description=commit_description,
no_send_mail=no_send_mail, argv=argv)
def check_branch(self):
"""Checks if the branch can be deleted.
We require that the branch exists, is not the current branch and is
actually a review branch.
If successful, sets state to DELETE, otherwise to FINISHED.
"""
if not utils.branch_exists(self.__branch):
print 'Branch %r doesn\'t exist.' % (self.__branch,)
self.state = self.FINISHED
elif self.__branch == utils.get_current_branch():
print 'Can\'t delete current branch.' % (self.__branch,)
self.state = self.FINISHED
else:
if not utils.in_review(current_branch=self.__branch):
print 'Branch %r has no review in progress.' % (self.__branch,)
print 'Instead, use the git command:'
print '\tgit branch -D %s' % (self.__branch,)
self.state = self.FINISHED
else:
self.state = self.DELETE
self.advance()
def __init__(self, rpc_server_args, do_close=True):
"""Constructor for SubmitAction.
Args:
rpc_server_args: A list of email, host, save_cookies and
account_type from the parsed command line arguments.
do_close: Boolean; defaults to True. Represents whether the issue
should be closed after pushing the commit.
Saves some environment data on the object such as the current branch,
and the issue, server and issue description associated with the current
branch.
"""
self.__branch = utils.get_current_branch()
self.__review_branch = None
self.__rietveld_info = utils.RietveldInfo.from_branch(
branch_name=self.__branch)
# TODO(dhermes): These assume rietveld_info is not None.
# TODO(dhermes): This assumes rietveld_info.review_info is not None.
self.__issue = self.__rietveld_info.review_info.issue
self.__server = self.__rietveld_info.server
self.__rpc_server_args = rpc_server_args
# Add the server
self.__rpc_server_args['server'] = self.__server
self.__do_close = do_close
# TODO(dhermes): These assume rietveld_info.remote_info is not None.
self.__remote = self.__rietveld_info.remote_info.remote
self.__remote_branch = self.__rietveld_info.remote_info.branch
self.__last_synced = self.__rietveld_info.remote_info.last_synced
self.state = self.CHECK_ENVIRONMENT
self.advance()
def check_branch(self):
"""Checks if the branch can be deleted.
We require that the branch exists, is not the current branch and is
actually a review branch.
If successful, sets state to DELETE, otherwise to FINISHED.
"""
if not utils.branch_exists(self.__branch):
print 'Branch %r doesn\'t exist.' % (self.__branch, )
self.state = self.FINISHED
elif self.__branch == utils.get_current_branch():
print 'Can\'t delete current branch.' % (self.__branch, )
self.state = self.FINISHED
else:
if not utils.in_review(current_branch=self.__branch):
print 'Branch %r has no review in progress.' % (
self.__branch, )
print 'Instead, use the git command:'
print '\tgit branch -D %s' % (self.__branch, )
self.state = self.FINISHED
else:
self.state = self.DELETE
self.advance()
def __init__(self, rpc_server_args, do_close=True):
"""Constructor for SubmitAction.
Args:
rpc_server_args: A list of email, host, save_cookies and
account_type from the parsed command line arguments.
do_close: Boolean; defaults to True. Represents whether the issue
should be closed after pushing the commit.
Saves some environment data on the object such as the current branch,
and the issue, server and issue description associated with the current
branch.
"""
self.__branch = utils.get_current_branch()
self.__review_branch = None
self.__rietveld_info = utils.RietveldInfo.from_branch(
branch_name=self.__branch)
# TODO(dhermes): These assume rietveld_info is not None.
# TODO(dhermes): This assumes rietveld_info.review_info is not None.
self.__issue = self.__rietveld_info.review_info.issue
self.__server = self.__rietveld_info.server
self.__rpc_server_args = rpc_server_args
# Add the server
self.__rpc_server_args['server'] = self.__server
self.__do_close = do_close
# TODO(dhermes): These assume rietveld_info.remote_info is not None.
self.__remote = self.__rietveld_info.remote_info.remote
self.__remote_branch = self.__rietveld_info.remote_info.branch
self.__last_synced = self.__rietveld_info.remote_info.last_synced
self.state = self.CHECK_ENVIRONMENT
self.advance()
def __init__(self, cxx, kokkos=None, submit=False, parallel=False, fast_fail=False, baseline_ref=None,
baseline_dir=None, machine=None, no_tests=False, keep_tree=False,
custom_cmake_opts=(), custom_env_vars=(), tests=(),
integration_test="JENKINS_HOME" in os.environ, root_dir=None, dry_run=False,
make_parallel_level=0, ctest_parallel_level=0):
###########################################################################
self._cxx = cxx
self._kokkos = kokkos
self._submit = submit
self._parallel = parallel
self._fast_fail = fast_fail
self._baseline_ref = baseline_ref
self._machine = machine
self._perform_tests = not no_tests
self._keep_tree = keep_tree
self._baseline_dir = baseline_dir
self._custom_cmake_opts = custom_cmake_opts
self._custom_env_vars = custom_env_vars
self._tests = tests
self._root_dir = root_dir
self._integration_test = integration_test
self._dry_run = dry_run
self._must_generate_baselines = False
self._testing_dir = "ctest-build"
############################################
# Sanity checks and helper structs setup #
############################################
expect (not self._baseline_dir or self._testing_dir != self._baseline_dir,
"Error! For your safety, do NOT use 'ctest-build' to store baselines. Move them to a different directory.")
expect(not (self._baseline_ref and self._baseline_dir),
"Makes no sense to specify a baseline generation commit if using pre-existing baselines ")
self._tests_cmake_args = {"dbg" : [("CMAKE_BUILD_TYPE", "Debug"),
("EKAT_DEFAULT_BFB", "ON")],
"sp" : [("CMAKE_BUILD_TYPE", "Debug"),
("SCREAM_DOUBLE_PRECISION", "False"),
("EKAT_DEFAULT_BFB", "ON")],
"fpe" : [("CMAKE_BUILD_TYPE", "Debug"),
("SCREAM_PACK_SIZE", "1"),
("SCREAM_SMALL_PACK_SIZE", "1"),
("EKAT_DEFAULT_BFB", "ON")]}
self._test_full_names = { "dbg" : "full_debug",
"sp" : "full_sp_debug",
"fpe" : "debug_nopack_fpe"}
if not self._tests:
self._tests = ["dbg", "sp", "fpe"]
else:
for t in self._tests:
expect(t in self._test_full_names,
"Requested test '{}' is not supported by test-all-scream, please choose from: {}".\
format(t, ", ".join(self._test_full_names.keys())))
# Compute root dir
if not self._root_dir:
self._root_dir = pathlib.Path(__file__).resolve().parent.parent
else:
self._root_dir = pathlib.Path(self._root_dir).resolve()
expect(self._root_dir.is_dir() and self._root_dir.parts()[-2:] == ('scream', 'components'),
"Bad root-dir '{}', should be: $scream_repo/components/scream".format(self._root_dir))
os.chdir(str(self._root_dir)) # needed, or else every git command will need repo=root_dir
expect(get_current_commit(), "Root dir: {}, does not appear to be a git repo".format(self._root_dir))
self._original_branch = get_current_branch()
self._original_commit = get_current_commit()
if not self._kokkos:
expect(self._machine, "If no kokkos provided, must provide machine name for internal kokkos build")
if self._submit:
expect(self._machine, "If dashboard submit request, must provide machine name")
print_last_commit(git_ref=self._original_branch)
###################################
# Compute baseline info #
###################################
default_baselines_root_dir = pathlib.Path(self._testing_dir,"baselines")
if self._baseline_dir is None:
if self._baseline_ref is None:
# Compute baseline ref
if self._keep_tree:
self._baseline_ref = "HEAD"
elif self._integration_test:
self._baseline_ref = "origin/master"
merge_git_ref(git_ref="origin/master",verbose=True)
else:
self._baseline_ref = get_common_ancestor("origin/master")
# Prefer a symbolic ref if possible
if self._baseline_ref is None or self._baseline_ref == get_current_commit(commit="origin/master"):
self._baseline_ref = "origin/master"
self._must_generate_baselines = True
self._baseline_dir = pathlib.Path(default_baselines_root_dir).absolute()
else:
# We treat the "AUTO" string as a request for automatic baseline dir.
if self._baseline_dir == "AUTO":
self._baseline_dir = get_mach_baseline_root_dir(self._machine,default_baselines_root_dir)
self._baseline_dir = pathlib.Path(self._baseline_dir).absolute()
# Make sure the baseline root directory exists
expect(self._baseline_dir.is_dir(), "Baseline_dir {} is not a dir".format(self._baseline_dir))
if self._integration_test:
self._baseline_ref = "origin/master"
merge_git_ref(git_ref=self._baseline_ref,verbose=True)
else:
for test in self._tests:
test_baseline_dir = self.get_preexisting_baseline(test)
expect(test_baseline_dir.is_dir(), "Missing baseline {}".format(test_baseline_dir))
# Name of the file used to store/check the git sha of the repo used to generate baselines,
# and name of the file used to store/check the builds for which baselines are available
# Store it once to avoid typos-like bugs
self._baseline_sha_file = pathlib.Path(self._baseline_dir, "baseline_git_sha")
self._baseline_names_file = pathlib.Path(self._baseline_dir, "baseline_names")
if self._integration_test:
master_sha = get_current_commit(commit=self._baseline_ref)
if not self.baselines_are_present():
print ("Some baselines were not found. Rebuilding them.")
self._must_generate_baselines = True
elif self.baselines_are_expired(expected_baseline_sha=master_sha):
print ("Baselines expired. Rebuilding them.")
self._must_generate_baselines = True
else:
print ("Baselines found and not expired. Skipping baselines generation.")
if self._must_generate_baselines:
print("Using commit {} to generate baselines".format(self._baseline_ref))
##################################################
# Deduce how many testing resources per test #
##################################################
if ctest_parallel_level > 0:
ctest_max_jobs = ctest_parallel_level
print("Note: honoring requested value for ctest parallel level: {}".format(ctest_max_jobs))
elif "CTEST_PARALLEL_LEVEL" in os.environ:
ctest_max_jobs = int(os.environ["CTEST_PARALLEL_LEVEL"])
print("Note: honoring environment value for ctest parallel level: {}".format(ctest_max_jobs))
else:
ctest_max_jobs = get_mach_testing_resources(self._machine)
print("Note: no value passed for --ctest-parallel-level. Using the default for this machine: {}".format(ctest_max_jobs))
self._testing_res_count = {"dbg" : ctest_max_jobs,
"sp" : ctest_max_jobs,
"fpe" : ctest_max_jobs}
# Deduce how many compilation resources per test
if make_parallel_level > 0:
make_max_jobs = make_parallel_level
print("Note: honoring requested value for make parallel level: {}".format(make_max_jobs))
else:
make_max_jobs = get_mach_compilation_resources(self._machine)
print("Note: no value passed for --make-parallel-level. Using the default for this machine: {}".format(make_max_jobs))
self._compile_res_count = {"dbg" : make_max_jobs,
"sp" : make_max_jobs,
"fpe" : make_max_jobs}
if self._parallel:
# We need to be aware that other builds may be running too.
# (Do not oversubscribe the machine)
make_remainder = make_max_jobs % len(self._tests)
make_count = make_max_jobs // len(self._tests)
ctest_remainder = ctest_max_jobs % len(self._tests)
ctest_count = ctest_max_jobs // len(self._tests)
# In case we have more items in self._tests than cores/gpus (unlikely)
if make_count == 0:
make_count = 1
if ctest_count == 0:
ctest_count = 1
for test in self._tests:
self._compile_res_count[test] = make_count
if self._tests.index(test)<make_remainder:
self._compile_res_count[test] = make_count + 1
self._testing_res_count[test] = ctest_count
if self._tests.index(test)<ctest_remainder:
self._testing_res_count[test] = ctest_count + 1
print("test {} can use {} jobs to compile, and {} jobs for testing".format(test,self._compile_res_count[test],self._testing_res_count[test]))
if self._keep_tree:
expect(not is_repo_clean(silent=True), "Makes no sense to use --keep-tree when repo is clean")
expect(not self._integration_test, "Should not be doing keep-tree with integration testing")
print("WARNING! You have uncommitted changes in your repo.",
" The PASS/FAIL status may depend on these changes",
" so if you want to keep them, don't forget to create a commit.",sep="\n")
if self._baseline_dir is None:
# Make sure the baseline ref is HEAD
expect(self._baseline_ref == "HEAD",
"The option --keep-tree is only available when testing against pre-built baselines "
"(--baseline-dir) or HEAD (-b HEAD)")
else:
expect(is_repo_clean(),
"Repo must be clean before running. If testing against HEAD or pre-built baselines, "
"you can pass `--keep-tree` to allow non-clean repo.")
def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None,
submit=False, parallel=False, fast_fail=False,
baseline_ref=None, baseline_dir=None, machine=None, no_tests=False, keep_tree=False,
custom_cmake_opts=(), custom_env_vars=(), preserve_env=False, tests=(),
integration_test="JENKINS_HOME" in os.environ, local=False, root_dir=None, work_dir=None,
quick_rerun=False,quick_rerun_failed=False,dry_run=False,
make_parallel_level=0, ctest_parallel_level=0):
###########################################################################
self._cxx_compiler = cxx_compiler
self._f90_compiler = f90_compiler
self._c_compiler = c_compiler
self._submit = submit
self._parallel = parallel
self._fast_fail = fast_fail
self._baseline_ref = baseline_ref
self._machine = machine
self._local = local
self._perform_tests = not no_tests
self._keep_tree = keep_tree
self._baseline_dir = baseline_dir
self._custom_cmake_opts = custom_cmake_opts
self._custom_env_vars = custom_env_vars
self._preserve_env = preserve_env
self._tests = tests
self._root_dir = root_dir
self._work_dir = work_dir
self._integration_test = integration_test
self._quick_rerun = quick_rerun
self._quick_rerun_failed = quick_rerun_failed
self._dry_run = dry_run
self._must_generate_baselines = False
if self._quick_rerun_failed:
self._quick_rerun = True
############################################
# Sanity checks and helper structs setup #
############################################
# Probe machine if none was specified
if self._machine is None:
# We could potentially integrate more with CIME here to do actual
# nodename probing.
if "CIME_MACHINE" in os.environ and is_machine_supported(os.environ["CIME_MACHINE"]):
self._machine = os.environ["CIME_MACHINE"]
else:
expect(self._local,
"test-all-scream requires either the machine arg (-m $machine) or the -l flag,"
"which makes it lookf for machine specs in '~/.cime/scream_mach_specs.py'.")
self._machine = "local"
else:
expect (not self._local, "Specifying a machine while passing '-l,--local' is ambiguous.")
##################################################
# Deduce how many testing resources per test #
##################################################
if ctest_parallel_level > 0:
ctest_max_jobs = ctest_parallel_level
print("Note: honoring requested value for ctest parallel level: {}".format(ctest_max_jobs))
elif "CTEST_PARALLEL_LEVEL" in os.environ:
ctest_max_jobs = int(os.environ["CTEST_PARALLEL_LEVEL"])
print("Note: honoring environment value for ctest parallel level: {}".format(ctest_max_jobs))
else:
ctest_max_jobs = get_mach_testing_resources(self._machine)
print("Note: no value passed for --ctest-parallel-level. Using the default for this machine: {}".format(ctest_max_jobs))
# Unless the user claims to know what he/she is doing, we setup the env.
if not self._preserve_env:
# Setup the env on this machine
setup_mach_env(self._machine, ctest_j=ctest_max_jobs)
# Compute root dir
if not self._root_dir:
self._root_dir = pathlib.Path(__file__).resolve().parent.parent
else:
self._root_dir = pathlib.Path(self._root_dir).resolve()
expect(self._root_dir.is_dir() and self._root_dir.parts()[-2:] == ('scream', 'components'),
"Bad root-dir '{}', should be: $scream_repo/components/scream".format(self._root_dir))
if self._work_dir is not None:
expect(pathlib.Path(self._work_dir).absolute().is_dir(),
"Error! Work directory '{}' does not exist.".format(self._work_dir))
else:
self._work_dir = self._root_dir.absolute().joinpath("ctest-build")
expect (not self._baseline_dir or self._work_dir != self._baseline_dir,
"Error! For your safety, do NOT use '{}' to store baselines. Move them to a different directory (even a subdirectory of that works).".format(self._work_dir))
expect(not (self._baseline_ref and self._baseline_dir),
"Makes no sense to specify a baseline generation commit if using pre-existing baselines ")
self._tests_cmake_args = {
"dbg" : [("CMAKE_BUILD_TYPE", "Debug"),
("EKAT_DEFAULT_BFB", "True")],
"sp" : [("CMAKE_BUILD_TYPE", "Debug"),
("SCREAM_DOUBLE_PRECISION", "False"),
("EKAT_DEFAULT_BFB", "True")],
"fpe" : [("CMAKE_BUILD_TYPE", "Debug"),
("SCREAM_PACK_SIZE", "1"),
("SCREAM_SMALL_PACK_SIZE", "1"),
("EKAT_DEFAULT_BFB", "True")],
"opt" : [("CMAKE_BUILD_TYPE", "Release")],
"valg" : [("CMAKE_BUILD_TYPE", "Debug"),
("EKAT_ENABLE_VALGRIND", "True")],
}
self._test_full_names = OrderedDict([
("dbg" , "full_debug"),
("sp" , "full_sp_debug"),
("fpe" , "debug_nopack_fpe"),
("opt" , "release"),
("valg" , "valgrind"),
])
if not self._tests:
# default to all test types except do not do fpe on CUDA
self._tests = list(self._test_full_names.keys())
self._tests.remove("valg") # don't want this on by default
if is_cuda_machine(self._machine):
self._tests.remove("fpe")
else:
for t in self._tests:
expect(t in self._test_full_names,
"Requested test '{}' is not supported by test-all-scream, please choose from: {}".\
format(t, ", ".join(self._test_full_names.keys())))
os.chdir(str(self._root_dir)) # needed, or else every git command will need repo=root_dir
expect(get_current_commit(), "Root dir: {}, does not appear to be a git repo".format(self._root_dir))
self._original_branch = get_current_branch()
self._original_commit = get_current_commit()
print_last_commit(git_ref=self._original_branch, dry_run=self._dry_run)
############################################
# Deduce compilers if needed/possible #
############################################
if self._cxx_compiler is None:
self._cxx_compiler = get_mach_cxx_compiler(self._machine)
if self._f90_compiler is None:
self._f90_compiler = get_mach_f90_compiler(self._machine)
if self._c_compiler is None:
self._c_compiler = get_mach_c_compiler(self._machine)
if not self._dry_run:
self._f90_compiler = run_cmd_no_fail("which {}".format(self._f90_compiler))
self._cxx_compiler = run_cmd_no_fail("which {}".format(self._cxx_compiler))
self._c_compiler = run_cmd_no_fail("which {}".format(self._c_compiler))
###################################
# Compute baseline info #
###################################
default_baselines_root_dir = pathlib.Path(self._work_dir,"baselines")
if self._baseline_dir is None:
if self._baseline_ref is None:
# Compute baseline ref
if self._keep_tree:
self._baseline_ref = "HEAD"
elif self._integration_test:
# Make sure our copy of origin/master is up-to-date (at least at the time of this script's execution)
git_fetch_remote("origin")
self._baseline_ref = "origin/master"
merge_git_ref(git_ref="origin/master", verbose=True, dry_run=self._dry_run)
else:
self._baseline_ref = get_common_ancestor("origin/master")
# Prefer a symbolic ref if possible
if self._baseline_ref is None or self._baseline_ref == get_current_commit(commit="origin/master"):
self._baseline_ref = "origin/master"
self._must_generate_baselines = True
self._baseline_dir = pathlib.Path(default_baselines_root_dir).absolute()
else:
# We treat the "AUTO" string as a request for automatic baseline dir.
if self._baseline_dir == "AUTO":
self._baseline_dir = get_mach_baseline_root_dir(self._machine)
self._baseline_dir = pathlib.Path(self._baseline_dir).absolute()
# Make sure the baseline root directory exists
expect(self._baseline_dir.is_dir(), "Baseline_dir {} is not a dir".format(self._baseline_dir))
if self._integration_test:
self._baseline_ref = "origin/master"
merge_git_ref(git_ref=self._baseline_ref, verbose=True, dry_run=self._dry_run)
else:
for test in self._tests:
test_baseline_dir = self.get_preexisting_baseline(test)
expect(test_baseline_dir.is_dir(), "Missing baseline {}".format(test_baseline_dir))
# Name of the file used to store/check the git sha of the repo used to generate baselines,
# and name of the file used to store/check the builds for which baselines are available
# Store it once to avoid typos-like bugs
self._baseline_sha_file = pathlib.Path(self._baseline_dir, "baseline_git_sha")
self._baseline_names_file = pathlib.Path(self._baseline_dir, "baseline_names")
if self._integration_test:
master_sha = get_current_commit(commit=self._baseline_ref)
if not self.baselines_are_present():
print ("Some baselines were not found. Rebuilding them.")
self._must_generate_baselines = True
elif self.baselines_are_expired(expected_baseline_sha=master_sha):
print ("Baselines expired. Rebuilding them.")
self._must_generate_baselines = True
else:
print ("Baselines found and not expired. Skipping baselines generation.")
if self._must_generate_baselines:
print("Using commit {} to generate baselines".format(self._baseline_ref))
self._testing_res_count = {
"dbg" : ctest_max_jobs,
"sp" : ctest_max_jobs,
"fpe" : ctest_max_jobs,
"opt" : ctest_max_jobs,
"valg" : ctest_max_jobs,
}
# Deduce how many compilation resources per test
if make_parallel_level > 0:
make_max_jobs = make_parallel_level
print("Note: honoring requested value for make parallel level: {}".format(make_max_jobs))
else:
make_max_jobs = get_mach_compilation_resources(self._machine)
print("Note: no value passed for --make-parallel-level. Using the default for this machine: {}".format(make_max_jobs))
self._compile_res_count = {
"dbg" : make_max_jobs,
"sp" : make_max_jobs,
"fpe" : make_max_jobs,
"opt" : make_max_jobs,
"valg" : make_max_jobs,
}
if self._parallel:
# We need to be aware that other builds may be running too.
# (Do not oversubscribe the machine)
make_remainder = make_max_jobs % len(self._tests)
make_count = make_max_jobs // len(self._tests)
ctest_remainder = ctest_max_jobs % len(self._tests)
ctest_count = ctest_max_jobs // len(self._tests)
# In case we have more items in self._tests than cores/gpus (unlikely)
if make_count == 0:
make_count = 1
if ctest_count == 0:
ctest_count = 1
for test in self._tests:
self._compile_res_count[test] = make_count
if self._tests.index(test)<make_remainder:
self._compile_res_count[test] = make_count + 1
self._testing_res_count[test] = ctest_count
if self._tests.index(test)<ctest_remainder:
self._testing_res_count[test] = ctest_count + 1
print("test {} can use {} jobs to compile, and {} jobs for testing".format(test,self._compile_res_count[test],self._testing_res_count[test]))
if self._keep_tree:
expect(not self._integration_test, "Should not be doing keep-tree with integration testing")
print("WARNING! You have uncommitted changes in your repo.",
" The PASS/FAIL status may depend on these changes",
" so if you want to keep them, don't forget to create a commit.",sep="\n")
if self._baseline_dir is None:
# Make sure the baseline ref is HEAD
expect(self._baseline_ref == "HEAD",
"The option --keep-tree is only available when testing against pre-built baselines "
"(--baseline-dir) or HEAD (-b HEAD)")
else:
expect(self._dry_run or is_repo_clean(),
"Repo must be clean before running. If testing against HEAD or pre-built baselines, "
"you can pass `--keep-tree` to allow non-clean repo.")
def __init__(self, pull=False):
"""Constructor for GetInfoAction."""
self.__branch = utils.get_current_branch()
self.__pull = pull
self.state = self.GET_INFO
self.advance()
def __init__(self, pull=False):
"""Constructor for GetInfoAction."""
self.__branch = utils.get_current_branch()
self.__pull = pull
self.state = self.GET_INFO
self.advance()