def get_file_path(suffix, extn, prefix=None, consolidated=False, dot="."):
file_prefix = env.get("SPYTEST_FILE_PREFIX", "results")
results_prefix = env.get("SPYTEST_RESULTS_PREFIX", file_prefix)
if results_prefix: results_prefix = "{}_".format(results_prefix)
if not consolidated:
filename = "{}{}{}{}".format(results_prefix, suffix, dot, extn)
elif suffix:
filename = "{}{}_all{}{}".format(results_prefix, suffix, dot, extn)
else:
filename = "{}all{}{}".format(results_prefix, dot, extn)
if prefix:
filename = os.path.join(prefix, filename)
return filename
def _load_csvs(name, default):
csv_files = env.get(name, default)
rows = []
for csv_file in csv_files.split(","):
for row in _load_csv(csv_file, "reporting"):
rows.append(row)
return rows
def reinit(self, platform=None, cli=None):
self.root = os.path.join(os.path.dirname(__file__), '..', 'templates')
self.samples = os.path.join(self.root, 'test')
index_file = env.get("SPYTEST_TEXTFSM_INDEX_FILENAME", "index")
self.cli_table = clitable.CliTable(index_file, self.root)
self.platform = platform
self.cli = cli
def __init__(self, model=None, logger=None):
"""
Construction of Prompts object
:param logger:
:type logger:
"""
self.logger = logger or logging.getLogger()
self.oyaml = None
model = "sonic" if not model else re.sub("_(ssh|terminal)$", "", model)
filename = "{}_prompts.yaml".format(model)
filename = env.get("SPYTEST_PROMPTS_FILENAME", filename)
filename = os.path.join(os.path.abspath(prompts_root), filename)
self.oyaml = OrderedYaml(filename, [])
prompts_file_data = self.oyaml.get_data() or dict()
self.patterns = prompts_file_data.patterns if "patterns" in prompts_file_data else SpyTestDict(
)
self.modes = prompts_file_data.modes if "modes" in prompts_file_data else SpyTestDict(
)
self.required_args = prompts_file_data.required_args if "required_args" in prompts_file_data else SpyTestDict(
)
self.sudo_include_prompts = prompts_file_data.sudo_include_prompts if "sudo_include_prompts" in prompts_file_data else []
self.do_exclude_prompts = prompts_file_data.do_exclude_prompts if "do_exclude_prompts" in prompts_file_data else []
self.stored_values = SpyTestDict()
def slaves_init(logs_path):
if get_impl_type() == 2:
# present auto slave init
return
count = wa.count
for index in range(0, count):
key = "SPYTEST_TESTBED_FILE_gw{}".format(index)
slave_start(env.get(key), logs_path)
def __init__(self, file_prefix=None, filename=None, name='', level=logging.INFO, tlog=False, mlog=True):
"""
Initialization of the logger object
:param filename: filename where the logs will be generated. spytest.log if not passed
:type filename: str
:param name: name of the instance from where the logs or written
:type name: str
:param level: logging level
"""
self.logdir = None
self.name = name
self.logger = logging.getLogger(name)
self.logger.setLevel(level)
self.dut_loggers = dict()
self.alert_logger = None
self.tc_log_support = tlog
self.tc_log_handler = None
self.module_log_support = mlog
self.module_log_handler = None
self.module_logger = None
self.module_only_log_support = True
self.file_prefix = file_prefix
self.use_elapsed_time_fmt = bool(env.get("SPYTEST_LOGS_TIME_FMT_ELAPSED", "0") == "1")
self.module_only_log_support = bool(env.get("SPYTEST_LOGS_MODULE_ONLY_SUPPORT", "0") == "1")
logfile = filename if filename else "spytest.log"
logfile = self._add_prefix(logfile)
self.logdir = os.path.dirname(logfile)
self.add_file_handler(self.logger, logfile, add_prefix=False)
# Handler for Console logs
if env.get("SPYTEST_NO_CONSOLE_LOG", "0") == "0":
console_handler = logging.StreamHandler(sys.stdout)
fmt = LogFormatter(self.use_elapsed_time_fmt)
console_handler.setFormatter(fmt)
self.logger.addHandler(console_handler)
# Handler for Alert logs
if not self.alert_logger:
logfile_path = "alerts.log"
self.alert_logger = logging.getLogger(logfile_path)
self.add_file_handler(self.alert_logger, logfile_path)
self.alert_logger.propagate = False
def get_config():
global config
if config: return config
data = {}
if env.get("SPYTEST_SYSLOG_ANALISYS", "0") != "0":
root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
filename = os.path.join(root, "reporting", "syslogs.yaml")
oyaml = OrderedYaml(filename,[])
data = oyaml.get_data() or dict()
for color in ["yellow", "green", "red"]:
if color not in data:
data[color] = []
return data
def __init__(self, logger=None):
self.base_url = env.get("SPYTEST_REST_TEST_URL")
self.session = None
self.logger = logger
self.timeout = 5
self.protocol = "https"
self.ip = None
self.username = None
self.password = None
self.altpassword = None
self.curr_pwd = None
self.cli_data = SpyTestDict()
self.headers = {}
def write_report_png(filepath, rows, index):
if env.get("SPYTEST_RESULTS_PNG", "1") == "0":
return
try:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
buckets = dict()
all_colors = {
"green":0, "red":0, "orange":0, "blue":0,
"cyan":0, "olive":0, "sienna":0, "peru":0,
"indigo":0, "magenta":0, "lightblue":0,
"yellow":0, "salmon":0, "palegreen":0,
"pink":0, "crimson":0, "lightpink":0
}
res_colors = {
"Pass":"******", "DUTFail":"red", "Fail":"orange",
"ScriptError":"blue", "EnvFail":"cyan", "DepFail":"olive",
"ConfigFail":"sienna", "TopoFail":"peru", "TGenFail":"indigo",
"Timeout":"magenta", "Skipped":"lightblue", "Unsupported": "yellow"
}
for row in rows:
res = row[index]
if not res:
continue
if res not in buckets:
buckets[res] = 0
buckets[res] = buckets[res] + 1
labels = []
colors = []
for label in buckets:
if label in res_colors:
color = res_colors[label]
all_colors[color]=1
else:
for c,used in all_colors.items():
if not used:
color=c
all_colors[color]=1
break
colors.append(color)
labels.append("{} [{}]".format(label, buckets[label]))
sizes = buckets.values()
plt.pie(sizes, colors=colors, labels=labels, autopct='%1.1f%%', startangle=140)
plt.axis('equal')
plt.savefig(filepath)
plt.clf()
except Exception as e:
print(e)
def open_csv(self, prefix, index):
report_csv = self.report_csv[index]
if env.get("SPYTEST_RESULTS_PREFIX"):
rows = Result.read_report_csv(report_csv)
else:
rows = []
if sys.version_info.major < 3:
csv_fd = open(report_csv, "wb")
else:
csv_fd = open(report_csv, "w", newline='')
writer = csv.DictWriter(csv_fd, fieldnames=slave_cols[index],
dialect="excel")
self.csv_fd[index] = csv_fd
if not rows:
writer.writeheader()
else:
l_rows = Result.prepend_row_index(rows)
utils.write_csv_writer(slave_cols[index], l_rows, writer)
self.count[index] = len(rows)
self.writer[index] = writer
def get_mlog_basename(nodeid):
module = nodeid.split(':')[0]
if env.get("SPYTEST_REPEAT_MODULE_SUPPORT") != "0":
module = os.path.basename(module)
return module
def _parse_args(pre_parse=False):
# pytest hack to let it wotk with absolute paths for testbed and tclist
parser = argparse.ArgumentParser(description='Process SpyTest arguments.',
add_help=False)
if pre_parse:
parser.add_argument("--args-file",
action="store",
default=None,
help="spytest arguments from file path")
parser.add_argument("--testbed-file",
action="store",
default=None,
help="testbed file path -- default: ./testbed.yaml")
parser.add_argument("--test-suite",
action="append",
default=[],
help="test suites")
parser.add_argument("--tclist-file",
action="append",
default=None,
help="test case list file path")
parser.add_argument("--logs-path",
action="store",
default=None,
help="logs folder -- default: .")
parser.add_argument("--logs-level",
action="store",
default="info",
help="logs level -- default: info")
parser.add_argument("--log-level",
action="store",
dest="logs_level",
default="info",
help="logs level -- default: info")
parser.add_argument("--results-prefix",
action="store",
default=None,
help="Prefix to be used for results.")
parser.add_argument("--file-mode",
action="store_true",
default=False,
help="Execute in file mode -- default: false")
parser.add_argument("-n",
"--numprocesses",
action="store",
default=None,
type=int,
help="number of preocessese")
parser.add_argument("--tclist-bucket",
action="append",
default=None,
help="use test cases from buckets")
for bucket in range(1, 9):
parser.add_argument(
"--bucket-{}".format(bucket),
action="store",
default=None,
nargs="*",
help="needed topology for bucket-{}.".format(bucket))
parser.add_argument("--env",
action="append",
default=[],
nargs=2,
help="environment variables")
parser.add_argument("--exclude-devices",
action="store",
default=None,
help="exclude given duts from testbed")
parser.add_argument("--include-devices",
action="store",
default=None,
help="include given duts from testbed")
parser.add_argument(
"--open-config-api",
action="store",
default='GNMI',
help="specified open-config request API type -- default: gNMI")
parser.add_argument(
"--noop",
action="store_true",
default=False,
help="No operation, to be used while using optional arguments")
parser.add_argument("--augment-modules-csv",
action="append",
default=[],
nargs="*",
help="Add additional lines to modules.csv")
args, unknown = parser.parse_known_args()
# parse the bucket options
argsdict = vars(args)
tclist_bucket = ",".join(argsdict["tclist_bucket"] or "")
bucket_list = []
for bucket in range(1, 9):
value = argsdict["bucket_{}".format(bucket)]
if value is None: continue
bucket_list.append(str(bucket))
tclist_bucket = ",".join(bucket_list)
if not value: continue
os.environ["SPYTEST_TOPO_{}".format(bucket)] = " ".join(value)
# update sys.argv with arguments from suite args
if args.test_suite:
addl_args = parse_suite_files(args.test_suite)
print("\nSuite Arguments {}\n".format(" ".join(addl_args)))
index = sys.argv.index("--test-suite")
new_argv = []
new_argv.extend(sys.argv[:index])
new_argv.extend(addl_args)
new_argv.extend(sys.argv[index + 2:])
sys.argv = new_argv
#print("\nSuite Arguments {}\n".format(" ".join(sys.argv)))
os.environ["SPYTEST_SUITE_ARGS"] = " ".join(addl_args)
return _parse_args(pre_parse=pre_parse)
if pre_parse and args.args_file:
# read arguments from file
user_root = env.get("SPYTEST_USER_ROOT")
if user_root and not os.path.isabs(args.args_file):
filepath = os.path.join(user_root, args.args_file)
else:
filepath = args.args_file
file_args = []
for line in utils.read_lines(filepath):
file_args.extend(utils.split_with_quoted_strings(line))
# update sys.argv with arguments from file
index = sys.argv.index("--args-file")
new_argv = []
new_argv.extend(sys.argv[:index])
new_argv.extend(file_args)
new_argv.extend(sys.argv[index + 2:])
sys.argv = new_argv
# update SPYTEST_CMDLINE_ARGS with arguments from file
app_cmdline = env.get("SPYTEST_CMDLINE_ARGS", "")
app_args = utils.split_with_quoted_strings(app_cmdline)
index = app_args.index("--args-file")
app_new_args = []
app_new_args.extend(app_args[:index])
for arg in file_args:
app_new_args.append("'{}'".format(arg) if " " in arg else arg)
app_new_args.extend(app_args[index + 2:])
os.environ["SPYTEST_CMDLINE_ARGS"] = " ".join(app_new_args)
return _parse_args()
sys.argv = [sys.argv[0]]
sys.argv.extend(unknown)
for name, value in args.env:
print("setting environment {} = {}".format(name, value))
os.environ[name] = value
if args.exclude_devices:
os.environ["SPYTEST_TESTBED_EXCLUDE_DEVICES"] = args.exclude_devices
sys.argv.extend(["--exclude-devices", args.exclude_devices])
if args.include_devices:
os.environ["SPYTEST_TESTBED_INCLUDE_DEVICES"] = args.include_devices
sys.argv.extend(["--include-devices", args.include_devices])
if args.testbed_file:
os.environ["SPYTEST_TESTBED_FILE"] = args.testbed_file
if args.tclist_file:
os.environ["SPYTEST_TCLIST_FILE"] = ",".join(args.tclist_file)
if args.logs_path:
os.environ["SPYTEST_LOGS_PATH"] = args.logs_path
os.environ["SPYTEST_LOGS_LEVEL"] = args.logs_level
if args.open_config_api:
os.environ["SPYTEST_OPENCONFIG_API"] = args.open_config_api
prefix = ""
prefix = "results"
if args.results_prefix:
file_prefix = args.results_prefix
elif args.file_mode and prefix:
file_prefix = prefix
elif tclist_bucket:
file_prefix = prefix
elif prefix:
file_prefix = "{}_{}".format(prefix,
time.strftime("%Y_%m_%d_%H_%M_%S"))
else:
file_prefix = "{}".format(time.strftime("%Y_%m_%d_%H_%M_%S"))
os.environ["SPYTEST_FILE_PREFIX"] = file_prefix
# filemode is needed in more places
if args.file_mode:
os.environ["SPYTEST_FILE_MODE"] = "1"
sys.argv.append("--file-mode")
addl_args = parse_batch_args(args.numprocesses, tclist_bucket,
args.augment_modules_csv)
sys.argv.extend(addl_args)
seed = utils.get_random_seed()
print("SPYTEST_RAMDOM_SEED used = {}".format(seed))
return args
def get_repeated():
if not repeat_info:
return {}
if env.get("SPYTEST_REPEAT_MODULE_SUPPORT") == "0":
return {}
return get().repeated
raise Exception('Unable to parse command "%s" - %s' %
(cmd, str(e)))
# apply the given template on given data
def apply_textfsm(self, tmpl_file, data):
tmpl_file2 = os.path.join(self.root, tmpl_file)
tmpl_fp = open(tmpl_file2, "r")
out = textfsm.TextFSM(tmpl_fp).ParseText(data)
tmpl_fp.close()
return out
if __name__ == "__main__":
template = Template()
if len(sys.argv) <= 2:
print("USAGE: template.py <template file> <data file>")
sys.exit(0)
f = open(sys.argv[2], "r")
tmpl, rv = template.apply(f.read(), sys.argv[1])
print("============ Template: {}".format(tmpl))
if env.get("SPYTEST_TEXTFSM_DUMP_INDENT_JSON", "0") == "0":
print(rv)
sys.exit(0)
try:
print(json.dumps(rv, indent=2))
except Exception as exp:
print("============ ERROR: {}".format(exp))
print(rv)
def load(do_verify=True, items=None):
tcmap.tclist = OrderedDict()
tcmap.comp = OrderedDict()
tcmap.cadence = OrderedDict()
tcmap.func = OrderedDict()
tcmap.modules = OrderedDict()
tcmap.module_info = OrderedDict()
tcmap.function_info = OrderedDict()
tcmap.repeated = OrderedDict()
tcmap.errors = []
tcmap.warnings = []
tcmap.non_mapped = []
#Module,UIType,FasterCLI,TrySSH,MaxTime
info_csv = env.get("SPYTEST_MODULE_INFO_CSV_FILENAME", "module_info.csv")
for row in _load_csv(info_csv, "reporting"):
if len(row) < 6: continue
name, uitype, fcli, tryssh, random, maxtime = [
str(i).strip() for i in row[:6]
]
if name.strip().startswith("#"): continue
repeat_name = "" if len(row) < 7 else row[6]
repeat_topo = "" if len(row) < 8 else row[7]
ent = get_module_info(name, repeat_name, repeat_topo, True)
ent.uitype = uitype
ent.fcli = utils.integer_parse(fcli, 0)
ent.tryssh = utils.integer_parse(tryssh, 0)
ent.random = utils.integer_parse(random, 0)
ent.maxtime = utils.integer_parse(maxtime, 0)
#Function,MaxTime
info_csv = env.get("SPYTEST_FUNCTION_INFO_CSV_FILENAME",
"function_info.csv")
for row in _load_csv(info_csv, "reporting"):
if len(row) < 2: continue
name, maxtime = [str(i).strip() for i in row[:2]]
if name.strip().startswith("#"): continue
ent = tcmap.get_function_info(name)
ent.maxtime = utils.integer_parse(maxtime, 0)
tcmap_csv = env.get("SPYTEST_TCMAP_CSV_FILENAME", "tcmap.csv")
for row in _load_csv(tcmap_csv, "reporting"):
if len(row) == 4:
# TODO treat the data as module
(age, cadence, comp, name0) = (row[0], row[1], row[2], row[3])
for name in utils.list_files(name0, "test_*.py"):
if name in tcmap.modules:
msg = "duplicate module {}"
tcmap.errors.append(msg.format(name))
continue
module = SpyTestDict()
module.age = age
module.cadence = cadence
module.comp = comp
module.name = name
tcmap.modules[name] = module
continue
if len(row) < 5:
if row:
print("Invalid line", row)
continue
(age, cadence, comp, tcid, func) = (row[0], row[1], row[2], row[3],
row[4])
if age.strip().startswith("#"):
continue
_add_entry(age, cadence, comp, tcid, func)
# verify the tcmap if required
if do_verify: verify(items)
return tcmap
