def os_wc(in_file, fields_index_list=[]):
""" Run a wc (equivalent) command on a file and then extract specific
fields of the result as a string.
Args:
in_file (str): Input string to parse
fields_index_list (list of ints): A list of ints and in the order of
fields to be returned
Returns:
(str) Space separated string of extracted fields.
"""
check_param_type("in_file", in_file, str)
check_param_type("fields_index_list", fields_index_list, list)
for index, field_index in enumerate(fields_index_list):
check_param_type("field_index - {0}".format(index), field_index, int)
cmd = "wc {0}".format(in_file)
rtn = OSCommand(cmd).run()
wc_out = rtn.output()
if fields_index_list:
wc_out = os_awk_print(wc_out, fields_index_list)
return wc_out
def _get_sst_config_include_file_value(include_dict,
include_source,
define,
default=None,
data_type=str,
disable_warning=False):
""" Retrieve a define from an SST Configuration Include File (sst_config.h or sst-element_config.h)
include_dict (dict): The dictionary to search for the define
include_source (str): The name of the include file we are searching
define (str): The define to look for
default (str|int): Default Return if failure occurs
data_type (str|int): The data type to return
disable_warning (bool): Disable the warning if define not found
:return (str|int): The returned data or default if not found in the dict
This will raise a SSTTestCaseException if a default is not provided or type
is incorrect
"""
if data_type not in (int, str):
raise SSTTestCaseException("Illegal datatype {0}".format(data_type))
check_param_type("include_source", include_source, str)
check_param_type("define", define, str)
if default is not None:
check_param_type("default", default, data_type)
try:
rtn_data = include_dict[define]
except KeyError as exc_e:
errmsg = ("Reading Config include file {0}") \
+ (" - Cannot find #define {1}".format(include_source, exc_e))
if disable_warning == False:
log_warning(errmsg)
if default is None:
raise SSTTestCaseException(exc_e)
rtn_data = default
if data_type == int:
rtn_data = int(rtn_data)
return rtn_data
def run_sst(self, sdl_file, out_file, err_file=None, set_cwd=None, mpi_out_files="",
other_args="", num_ranks=None, num_threads=None, global_args=None,
timeout_sec=60):
""" Launch sst with with the command line and send output to the
output file. The SST execution will be monitored for result errors and
timeouts. On an error or timeout, a SSTTestCase.assert() will be generated
indicating the details of the failure.
Args:
sdl_file (str): The FilePath to the test SDL (python) file.
out_file (str): The FilePath to the finalized output file.
err_file (str): The FilePath to the finalized error file.
Default = same directory as the output file.
mpi_out_files (str): The FilePath to the mpi run output files.
These will be merged into the out_file at
the end of a multi-rank run.
other_args (str): Any other arguments used in the SST cmd
that the caller wishes to use.
num_ranks (int): The number of ranks to run SST with.
num_threads (int): The number of threads to run SST with.
global_args (str): Global Arguments provided from test engine args
timeout_sec (int): Allowed runtime in seconds
"""
# NOTE: We cannot set the default of param to the global variable due to
# oddities on how this class loads, so we do it here.
if num_ranks is None:
num_ranks = test_engine_globals.TESTENGINE_SSTRUN_NUMRANKS
if num_threads is None:
num_threads = test_engine_globals.TESTENGINE_SSTRUN_NUMTHREADS
if global_args is None:
global_args = test_engine_globals.TESTENGINE_SSTRUN_GLOBALARGS
# Make sure arguments are of valid types
check_param_type("sdl_file", sdl_file, str)
check_param_type("out_file", out_file, str)
if err_file is not None:
check_param_type("err_file", out_file, str)
if set_cwd is not None:
check_param_type("set_cwd", set_cwd, str)
check_param_type("mpi_out_files", mpi_out_files, str)
check_param_type("other_args", other_args, str)
if num_ranks is not None:
check_param_type("num_ranks", num_ranks, int)
if num_threads is not None:
check_param_type("num_threads", num_threads, int)
if global_args is not None:
check_param_type("global_args", global_args, str)
if not (isinstance(timeout_sec, (int, float)) and not isinstance(timeout_sec, bool)):
raise ValueError("ERROR: Timeout_sec must be a postive int or a float")
# Make sure sdl file is exists and is a file
if not os.path.exists(sdl_file) or not os.path.isfile(sdl_file):
log_error("sdl_file {0} does not exist".format(sdl_file))
# Figure out a name for the mpi_output files if the default is provided
if mpi_out_files == "":
mpiout_filename = "{0}.testfile".format(out_file)
else:
mpiout_filename = mpi_out_files
# Set the initial os launch command for sst.
# If multi-threaded, include the number of threads
if num_threads > 1:
oscmd = "sst -n {0} {1} {2} {3}".format(num_threads,
global_args,
other_args,
sdl_file)
else:
oscmd = "sst {0} {1} {2}".format(global_args,
other_args,
sdl_file)
# Update the os launch command if we are running multi-rank
num_cores = host_os_get_num_cores_on_system()
# Perform any multi-rank checks/setup
mpi_avail = False
numa_param = ""
if num_ranks > 1:
# Check to see if mpirun is available
rtn = os.system("which mpirun > /dev/null 2>&1")
if rtn == 0:
mpi_avail = True
numa_param = "-map-by numa:PE={0}".format(num_threads)
oscmd = "mpirun -np {0} {1} -output-filename {2} {3}".format(num_ranks,
numa_param,
mpiout_filename,
oscmd)
# Identify the working directory that we are launching SST from
final_wd = os.getcwd()
if set_cwd is not None:
final_wd = os.path.abspath(set_cwd)
# Log some debug info on the launch of SST
log_debug((("-- SST Launch Command (In Dir={0}; Ranks:{1}; Threads:{2};") +
(" Num Cores:{3}) = {4}")).format(final_wd, num_ranks, num_threads,
num_cores, oscmd))
# Check for OpenMPI
if num_ranks > 1 and mpi_avail is False:
log_fatal("OpenMPI IS NOT FOUND/AVAILABLE")
# Launch SST
rtn = OSCommand(oscmd, output_file_path = out_file,
error_file_path = err_file,
set_cwd = set_cwd).run(timeout_sec=timeout_sec)
if num_ranks > 1:
testing_merge_mpi_files("{0}*".format(mpiout_filename), mpiout_filename, out_file)
# Look for runtime error conditions
err_str = "SST Timed-Out ({0} secs) while running {1}".format(timeout_sec, oscmd)
self.assertFalse(rtn.timeout(), err_str)
err_str = "SST returned {0}; while running {1}".format(rtn.result(), oscmd)
self.assertEqual(rtn.result(), 0, err_str)
def os_wget(fileurl,
targetdir,
num_tries=3,
secsbetweentries=10,
wgetparams=""):
""" Perform a wget command to download a file from a url.
Args:
fileurl (str): URL to the file to be downloaded.
targetdir (str): Where to download the file to
num_tries (int): Max number of tries to download [3].
secsbetweentries (int): Seconds between tries [10]
wgetparams (str): Max number of tries to download.
Returns:
(bool) True if wget is successful.
"""
check_param_type("fileurl", fileurl, str)
check_param_type("targetdir", targetdir, str)
check_param_type("num_tries", num_tries, int)
check_param_type("secsbetweentries", secsbetweentries, int)
check_param_type("wgetparams", wgetparams, str)
wget_success = False
# Make sure target dir exists, and cd into it
if not os.path.isdir(targetdir):
log_error("Download directory {0} does not exist".format(targetdir))
return False
wgetoutfile = "{0}/wget.out".format(test_output_get_tmp_dir())
log_debug("Downloading file via wget: {0}".format(fileurl))
cmd = "wget {0} --no-check-certificate -P {1} {2} > {3} 2>&1".\
format(fileurl, targetdir, wgetparams, wgetoutfile)
attemptnum = 1
while attemptnum <= num_tries:
log_debug("wget Attempt#{0}; cmd={1}".format(attemptnum, cmd))
rtn_value = os.system(cmd)
log_debug("wget rtn: {0}".format(rtn_value))
if rtn_value == 0:
# wget was successful, force exit of the loop
log_debug("wget download successful")
wget_success = True
attemptnum = num_tries + 1
continue
if os.path.isfile(wgetoutfile):
with open(wgetoutfile, "rb") as wgetfile:
wgetoutput = "".join(wgetfile.readlines()[1:])
log_debug("wget output:\n{0}".format(wgetoutput))
else:
log_debug("wget output: NOT FOUND")
log_debug("Download Failed, waiting {0} seconds and trying again...".\
format(secsbetweentries))
attemptnum = attemptnum + 1
time.sleep(secsbetweentries)
if not wget_success:
log_error("Failed to download via wget {0}".format(fileurl))
return wget_success
