def test_logger_get_log_dir(self, os_patch, get_glbl_var_patch):
os_patch.environ = {'TOBY_LOG_FOLDER': 'INFO'}
self.assertEqual(get_log_dir(), 'INFO')
os_patch.environ = {}
os_patch.getcwd.return_value = 'cwd'
get_glbl_var_patch.return_value = 'test'
os_patch.path.join.return_value = 'cwd'
self.assertEqual(get_log_dir(), 'test')
get_glbl_var_patch.return_value = 'cwd'
self.assertEqual(get_log_dir(), 'cwd/toby_logs')
def __init__(self, message, host_obj=None):
try:
from jnpr.toby.logger.logger import get_log_dir
log_dir = get_log_dir()
# First test if file can be written to log directory
p = Popen(["touch", log_dir + "/test_file.txt"], stdout=PIPE)
p.wait()
# Failure to write file
if p.returncode > 0:
t.log(level='WARN',
message="Couldn't write to log directory: " + log_dir)
else:
p = Popen(["rm", log_dir + "/test_file.txt"], stdout=PIPE)
p.wait()
# Second we test if the log dir is mounted
p = Popen(["findmnt", "-T", log_dir], stdout=PIPE)
p.wait()
# Prints the output from 'findmnt' command
t.log(level='INFO',
message="Mountpoint...\n" + str(p.communicate()[0].decode()))
raise TobyException(message, host_obj=host_obj)
except Exception:
t.log(level='INFO', message=str(message) + " Exception raised")
def create_ve_debug_log(data, message):
timestamp = datetime.datetime.now()
failed_tc_file = open(get_log_dir() + '/VerifyDebug.log', 'a')
failed_tc_file.write(str(timestamp) + "\t" + str(message) + "\n")
failed_tc_file.write(
"-------------------------------------------------------\n")
pprint.pprint(data, failed_tc_file)
failed_tc_file.write(
"\n\n\n-------------------------------------------------------\n")
failed_tc_file.close()
def __init__(self):
self.decoder_path = ''
self.decoder_type = ''
self.decoder_port = ''
self.suite_protofile = ''
self.junos_yang_path = None
self.suite_file = ''
self.suite = ''
self.val_type = None
self.gnmi_server = None
self.server_ip_address = ''
self.log_path = get_log_dir()
self.gnmi_params = ''
def __core_collect(self):
'''
Run one macro
'''
self.core_check_wait = True
for resource in self.core_collect:
resource_name = t['resources'][resource]['system']['primary'][
'name']
hostname = t['resources'][resource]['system']['primary'][
'controllers']['re0']['hostname']
log_dir = get_log_dir(
) + '/macro_logs/' + self.log_prefix + '/' + resource_name
if self.core_check_wait:
#need something more sophisticated, but for now, 45 seconds for demo
time.sleep(45)
self.core_check_wait = False
#self.__log("Searching for cores on resource" + resource)
response = detect_core(self.handles[resource])
if response:
status = False
stdout_orig = sys.stdout
sys.stdout = StringIO()
for stage in t.core.keys():
for controller in t.core[stage][hostname].keys():
core = t.core[stage][hostname][controller]['core_src_path'] + \
t.core[stage][hostname][controller]['core_name']
self.__log(
"Found core at resource " + resource_name + "(" +
controller + "):" + core, resource)
try:
status = self.handles[resource].download(
remote_file=core, local_file=log_dir)
folders_file = core.split(sep='/')
os.system('chmod 664 ' + log_dir + '/' +
folders_file[-1])
except Exception:
self.__log("Unable to copy core " + core +
" from resource " + resource_name +
" to " + log_dir)
if status:
response = "Core(s) retrieved and saved to " + log_dir
else:
response = "Core retrieval failed"
sys.stdout = stdout_orig
else:
response = "No Core found"
self.__log(
"Core detection response for resource " + resource + ": " +
str(response), resource)
def get_logs_and_stats_after_test(all_controller_handles_dict):
"""
pulling logs and stats at the end of each
test case
"""
t.log("entering get logs and stats after test")
logdir = get_log_dir()
test_case = BuiltIn().get_variable_value('${TEST NAME}')
logdir = logdir + '/' + test_case
if not os.path.exists(logdir):
os.makedirs(logdir)
for key in all_controller_handles_dict:
t.log(all_controller_handles_dict)
controller_handle = all_controller_handles_dict[key]
username, passwrd = get_credentials(os='JUNOS')
scp_obj = SCP(host=controller_handle.host,
user=username,
password=passwrd)
scp_obj.get_file('/var/log/messages',
logdir + '/' + '%s_messages.messages' % (key))
if controller_handle.is_master():
op = controller_handle.shell(
command="ps | grep rtsockmon").response()
flag = 0
for line in op.splitlines():
line = line.strip()
if 'rtsockmon -rt' in line:
pid = line.split(' ')[0]
if pid:
controller_handle.shell(command="kill -9 " +
pid).response()
flag = 1
if flag == 0:
controller_handle.log("\nprocess is not running")
scp_obj = SCP(host=controller_handle.host,
user=username,
password=passwrd)
scp_obj.get_file('/var/log/%s_rtsockmon.txt' % (key),
logdir + '/' + '%s_rtsockmon.log' % (key))
controller_handle.cli(
command=
"request support information | save /var/log/%s_req_sup_info.log" %
(key)).response()
scp_obj = SCP(host=controller_handle.host,
user=username,
password=passwrd)
scp_obj.get_file('/var/log/%s_req_sup_info.log' % (key),
logdir + '/' + '%s_req_sup_info.log' % (key))
t.log("exiting get logs and stats after test")
def __init__(self):
self.decoder_path = {'grpc': None, 'udp': None, 'gnmi': None}
self.decoder_status = {'grpc': False, 'udp': False, 'gnmi': False}
self.decoder_pid = {'grpc': None, 'udp': None, 'gnmi': None}
self.decoder_port = {'grpc': None, 'udp': None, 'gnmi': None}
self.decoder_type = ''
self.json_filename = ''
self.dut_ip_address = ''
self.server_ip_address = ''
self.db_obj = None
self.db_query = ''
self.db_params = {}
self.db_measure = ''
self.log_head = ''
self.log_path = get_log_dir()
self.db_name = self.log_head
self.db_path = '' #self.log_path+'/'+self.db_name
self.type = ''
self.json_file = ''
self.grpc_port = ''
self.file_name_pem = ''
self.cid = ''
self.json_params = {}
def __init__(self, chassis=None, landslide_manager=None, system_data=None, landslide_lib_path=None, landslide_jre_path=None, landslide_tcl_bin=None):
"""
Landslide abstraction layer for HLTAPI
-- Workflow 1 --
:param system_data: *MANDATORY* Dictionary of Spirent information
system:
primary:
controllers:
if0:
domain: englab.juniper.net
hostname: Systest-Landslide
isoaddr: 47.0005.80ff.f800.0000.0108.0001.0102.5500.7000.00
loop-ip: 10.255.7.0
loop-ipv6: abcd::10:255:7:0
mgt-intf-name: eth0
mgt-ip: 10.9.4.6
mgt-ipv6: abcd::10:9:4:6
osname: SPIRENT
cube:
- COMMON-LISP::OR
- wf-626-systest
make: spirent
model: spt-c100-ts
name: Systest-Landslide
osname: spirent
landslide_manager: 10.4.4.42
-- Workflow 2 --
:param host *MANDATORY* FQDN/mgt-ip of of chassis
:param landslide_manager *MANDATORY* landslide_manager - manager of test servers
:return: Landslide object
"""
self.debug = False
self.chassis = None
self.session_info = None
self.intf_to_port_map = None
self.log_dir = get_log_dir()
atexit.register(self.cleanup)
self.ls = None
self.user_functions = dict()
if landslide_lib_path and landslide_jre_path and landslide_tcl_bin:
self.lib_path = landslide_lib_path
self.jre_path = landslide_jre_path
self.tcl = landslide_tcl_bin
else:
environment = yaml.safe_load(open(os.path.join(os.path.dirname(credentials.__file__), "environment.yaml")))
self.lib_path = environment['landslide-lib-path']
self.jre_path = environment['landslide-jre-path']
self.tcl = environment['landslide-tcl-bin']
if system_data:
controller_key = list(system_data['system']['primary']['controllers'].keys())[0]
if 'user' in system_data['system']['primary']['controllers'][controller_key]:
self.username = system_data['system']['primary']['controllers'][controller_key]['user']
if 'password' in system_data['system']['primary']['controllers'][controller_key]:
self.password = system_data['system']['primary']['controllers'][controller_key]['password']
kwargs = dict()
kwargs['host'] = self.chassis = system_data['system']['primary']['name']
kwargs['hostname'] = self.chassis = system_data['system']['primary']['name']
kwargs['os'] = system_data['system']['primary']['controllers'][controller_key]['osname']
super(Landslide, self).__init__(**kwargs)
if 'debug' in system_data['system']['primary']:
self.debug = system_data['system']['primary']['debug']
if 'mgt-ip' in system_data['system']['primary']['controllers'][controller_key]:
self.chassis = system_data['system']['primary']['controllers'][controller_key]['mgt-ip']
self.landslide_manager = system_data['system']['primary']['landslide-manager']
elif chassis:
self.chassis = chassis
self.landslide_manager = landslide_manager
else:
raise TobyException("Missing either system_data (Workflow 1) or chassis (Workflow 2) parameter")
if not self.chassis:
raise TobyException("Unable to determine chassis host information! Check for valid labserver or mgt-ip in init yaml file.")
self.connect_info = None
self.log("CHASSIS= " + str(self.chassis))
cred = credentials.get_credentials(os='Landslide')
self.username = cred['USERNAME']
self.password = cred['PASSWORD']
self.telnetuser = cred['TELNETUSERNAME']
self.telnetpwd = cred['TELNETPASSWORD']
self.wait = 1
self.telnet_handle = None
self.version = self._get_version()
sys.path.append(self.lib_path)
self._set_envs()
#resetting sys.argv[0] is only way to get spirent htlapi logs to go to the right place
#executable = sys.argv[0]
#sys.argv[0] = self.log_dir
self.ls = __import__('ls')
#sys.argv[0] = executable
self.log("Landslide Chassis Initialization Complete")
def __init__(self, system_data=None, paragon_lib_path=None):
"""
Paragon abstraction layer for HLTAPI
-- Workflow 1 --
:param system_data: *MANDATORY* Dictionary of Paragon information
rt0:
interfaces:
intf1:
name: 1/8
intf2:
name: 1/9
system:
primary:
controllers:
unknown:
domain: englab.juniper.net
hostname: wf-paragonchassis2
mgt-intf-name: mgt0
mgt-ip: 10.9.1.107
osname: IxOS
make: paragon
model: xgs12
name: paragonx-123
osname: Calnex
-- Workflow 2 --
:param host *MANDATORY* FQDN/mgt-ip of of chassis
:return: Paragon object
"""
self.instrument_ip = None
self.server_ip = None
self.interfaces = None
self.intf_to_port_map = dict()
self.port_to_handle_map = dict()
self.handle_to_port_map = None
self.paragon = None
self.log_dir = get_log_dir()
atexit.register(self.cleanup)
self.user_functions = dict()
self.is_connected = False
if paragon_lib_path:
self.paragon_lib_path = paragon_lib_path
else:
environment = yaml.safe_load(open(os.path.join(os.path.dirname(credentials.__file__), "environment.yaml")))
self.paragon_lib_path = environment['paragon-lib-path']
if not system_data:
raise TobyException("Missing system information for paragon device initialization")
controller_key = list(system_data['system']['primary']['controllers'].keys())[0]
kwargs = dict()
kwargs['host'] = self.chassis = system_data['system']['primary']['name']
kwargs['hostname'] = self.chassis = system_data['system']['primary']['name']
kwargs['os'] = system_data['system']['primary']['controllers'][controller_key]['osname']
super(Paragon, self).__init__(**kwargs)
if 'mgt-ip' in system_data['system']['primary']['controllers'][controller_key]:
self.instrument_ip = system_data['system']['primary']['controllers'][controller_key]['mgt-ip']
if 'server-ip' in system_data['system']['primary']:
self.server_ip = system_data['system']['primary']['server-ip']
else:
raise TobyException('Missing server-ip', self)
sys.path.append(self.paragon_lib_path)
self.paragon = __import__('paragon')
# Import extended paragon modules with functions from jnpr/toby/trafficgen/calnex/paragon
current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
user_module_dir = re.sub(r"jnpr\/toby.*", "jnpr/toby/trafficgen/calnex/paragon", current_dir)
sys.path.append(user_module_dir)
file_list = list(filter(lambda x: os.path.isfile(os.path.join(user_module_dir, x)), os.listdir(user_module_dir)))
for file_name in file_list:
if file_name.endswith('.py') and not file_name.startswith('__'):
module = re.sub(r"\.py$", "", file_name)
obj = __import__(module)
function_list = [o for o in inspect.getmembers(obj) if inspect.isfunction(o[1])]
for function_tuple in function_list:
function_name, function = function_tuple
if function_name in self.user_functions:
raise TobyException("Duplicate functions in user contributed modules", host_obj=self)
self.user_functions[function_name] = function
def run_commands(self, template):
'''
Run particular template in templates file
param: template
name of template
'''
if template not in self.templates:
raise TobyException("Template " + template + " not found.")
batch_set = self.templates[template]
if type(batch_set) is not list:
raise TobyException(
"Malformed template file. First level of template " +
template + " should be a list")
for instr_set in batch_set:
if 'commands' not in instr_set:
raise TobyException(
"Missing 'commands' key in one of the batch sets for template "
+ template + ".")
commands = instr_set['commands']
if type(commands) is not list:
raise TobyException("Error in template " + template +
". 'commands' should be a list.")
resource_list = []
if 'tags' in instr_set:
resource_list = t.get_resource_list(tag=instr_set['tags'])
else:
resource_list = t.get_resource_list()
for resource in resource_list:
resource_osname = t['resources'][resource]['system'][
'primary']['osname']
if 'osname' in instr_set and instr_set['osname'].upper(
) != resource_osname.upper():
t.log_console("Skipping resource " + resource + ": OS '" + resource_osname + \
"' does not match template '" + template + "' filter '" + instr_set['osname'] + "'")
continue
try:
self.current_handle = t.get_handle(resource=resource)
except Exception:
continue
for command in commands:
if not re.match(r'^\S+\(', command):
raise TobyException(
"Malformed command: " + command +
"\nShould resemble: cli(show version)")
t.log_console("Running command " + command +
" on resource " + resource + "...")
command = command.replace(')', '')
instr, cmd = command.split('(')
response = None
if instr == 'config':
response = config(self.current_handle,
command_list=[cmd])
elif instr == 'cli':
response = cli(self.current_handle, command=cmd)
elif instr == 'shell':
response = shell(self.current_handle, command=cmd)
elif instr == 'cty':
dest, cmd = cmd.split(':')
response = cty(self.current_handle,
destination=dest,
command=cmd)
elif instr == 'vty':
dest, cmd = cmd.split(':')
response = vty(self.current_handle,
destination=dest,
command=cmd)
elif instr == 'rpc':
response = rpc(self.current_handle, command=cmd)
elif instr == 'pyez':
response = pyez(self.current_handle, command=cmd)
elif instr == 'get_file':
stdout_orig = sys.stdout
sys.stdout = StringIO()
status = self.current_handle.download(
remote_file=cmd, local_file=get_log_dir())
sys.stdout = stdout_orig
if status:
response = "File " + cmd + " retrieved and saved to " + get_log_dir(
) + "\n"
else:
response = "File " + cmd + " retrieval failed\n"
else:
t.log_console('Instruction ' + instr +
' not supported (skipping)')
if response:
t.log_console(
"Command " + command +
" executed successfully and results logged")
def __init__(self,
chassis=None,
system_data=None,
avalanche_tcl_bin=None,
avalanche_lib_path=None,
avalanche_threats=None,
avalanche_jre_path=None):
"""
Avalanche abstraction layer for HLTAPI
-- Workflow 1 --
:param system_data: *MANDATORY* Dictionary of Avalanche information
rt0:
interfaces:
intf1:
name: 1/8
intf2:
name: 1/9
system:
primary:
appserver: wf-appserver2.englab.juniper.net
controllers:
unknown:
domain: englab.juniper.net
hostname: wf-avchassis2
mgt-intf-name: mgt0
mgt-ip: 10.9.1.107
osname: IxOS
make: avalanche
model: xgs12
name: wf-ixchassis2
osname: IxOS
-- Workflow 2 --
:param host *MANDATORY* FQDN/mgt-ip of of chassis
:return: Avalanche object
"""
self.chassis = None
self.interfaces = None
self.model = 'Unknown'
self.platform_type = 'STC'
self.intf_to_port_map = dict()
self.port_to_handle_map = dict()
self.session_info = None
self.handle_to_port_map = None
self.login_complete = False
#self.tcl = '/volume/systest-proj/apptest/local/ActiveTcl-8.4/bin/tclsh'
#self.tcl = '/volume/systest-proj/apptest/local/bin/tclsh8.6'
self.log_dir = get_log_dir()
self.clientport = None
self.serverport = None
atexit.register(self.cleanup)
self.username, self.password = credentials.get_credentials(
os='Spirent')
if avalanche_lib_path and avalanche_tcl_bin and avalanche_jre_path and avalanche_threats:
self.lib_path = avalanche_lib_path
self.tcl_bin = avalanche_tcl_bin
self.jre_path = avalanche_jre_path
self.threat_path = avalanche_threats
else:
environment = yaml.safe_load(
open(
os.path.join(os.path.dirname(credentials.__file__),
"environment.yaml")))
self.lib_path = environment['avalanche-lib-path']
self.tcl_bin = environment['avalanche-tcl-bin']
self.jre_path = environment['avalanche-jre-path']
self.threat_path = environment['avalanche-threats']
self.api_path = self.lib_path + '/Avalanche/PythonAPI/1.0.0'
if system_data:
controller_key = list(
system_data['system']['primary']['controllers'].keys())[0]
if system_data['system']['primary']['controllers'][controller_key][
'user']:
self.username = system_data['system']['primary'][
'controllers'][controller_key]['user']
if system_data['system']['primary']['controllers'][controller_key][
'password']:
self.password = system_data['system']['primary'][
'controllers'][controller_key]['password']
kwargs = dict()
kwargs['host'] = self.chassis = system_data['system']['primary'][
'name']
kwargs['hostname'] = self.chassis = system_data['system'][
'primary']['name']
kwargs['os'] = system_data['system']['primary']['controllers'][
controller_key]['osname']
kwargs['user'] = self.username
kwargs['password'] = self.password
super(Avalanche, self).__init__(**kwargs)
if 'api-path' in system_data['system']['primary']:
self.log("Overriding existing av.py path with: " +
system_data['system']['primary']['api-path'])
self.api_path = system_data['system']['primary']['api-path']
if 'mgt-ip' in system_data['system']['primary']['controllers'][
controller_key]:
self.chassis = system_data['system']['primary']['controllers'][
controller_key]['mgt-ip']
self.model = system_data['system']['primary']['model'].upper()
match = re.search(r'(^C100$|3100|^C100U$)', self.model, re.I)
if match and not (system_data['system']['primary']['controllers'][controller_key]['user'] \
and system_data['system']['primary']['controllers'][controller_key]['password']):
if match.group(1) == 'C100' or match.group(1) == 'C100U':
self.log("Model %s, so Overriding username and password" %
match.group(1))
self.username = '******'
self.password = '******'
self.log("Login:"******", Password:"******"Model 3100, so Overriding username and password")
self.username = '******'
self.password = ''
self.log("Login:"******", Password:"******"ERROR: Must include fv-avalanche-license-path for models starting with C100",
host_obj=self)
self.platform_type = self.model
elif chassis:
self.chassis = chassis
else:
raise TobyException(
"Missing either system_data (Workflow 1) or chassis (Workflow 2) parameter"
)
if not self.chassis:
raise TobyException(
"Unable to determine chassis host information! Check for valid in init yaml file.",
host_obj=self)
self.connect_info = None
self.log("CHASSIS= " + str(self.chassis))
self.wait = 1
self.telnet_handle = None
self.version = self._get_version()
self._set_envs()
sys.path.append(self.api_path)
self.log("API PATH= " + self.api_path)
executable = sys.argv[0]
sys.argv[0] = self.log_dir
try:
self.av = __import__('av') # pylint: disable=invalid-name
except Exception as err:
raise TobyException("Unable to import 'av' at path " +
self.api_path + ": " + str(err),
host_obj=self)
sys.argv[0] = executable
self.av.new(CHASSIS=self.chassis, TYPE=self.platform_type)
def __run_single_macro(self, macro, macro_content):
'''
Run one macro
'''
commands = None #dict or list
constraints = {}
#iterate through outer loop (content nested within macro)
for instr_type in macro_content:
if instr_type not in [
'comment', 'constraints', 'commands', 'macros', 'system',
'target'
]:
message = "Unsupported instr_type '" + instr_type + "'. Only 'commands', 'macros' & 'system' are supported."
t.log_console(message)
self.__log(message)
return
if instr_type == 'macros':
macros = macro_content[instr_type]
if type(macros) is not list:
raise TobyException("Error in macro " + macro +
". 'macros' should be a list.")
for nested_macro in reversed(macros):
if nested_macro in self.target_macros or nested_macro in self.completed_macros:
self.__log(
"Loop in macro definitions. Skipping macro " +
nested_macro + " since it was already run")
continue
self.target_macros.insert(0, nested_macro)
if 'commands' not in macro_content:
self.__log("Completed macro " + macro +
" [additionally queued macros: " + str(macros) +
"]")
# this was just a macro of macros and nothing else, so return
return
# #remove for now - may be a bit too dangerous
# elif instr_type == 'system':
# system_calls = macro_content[instr_type]
# if type(system_calls) is not list:
# raise TobyException("Error in system " + system_calls + ". 'system' should be a list.")
# for system_call in system_calls:
# system_response = self.__system(system_call)
# self.__log("Local System Call:\n>>> " + system_call + "\n" + system_response + "\n")
# self.__log('Completed macro:' + macro)
# # system call is local, so no need to look at resource information or process commands
# return
elif instr_type == 'commands':
commands = macro_content[instr_type]
if type(commands) is not list and type(commands) is not dict:
raise TobyException("Malformated macro " + macro +
". 'commands' syntax incorrect")
elif instr_type == 'constraints':
constraints = macro_content['constraints']
if type(constraints) is not dict:
raise TobyException(
"Error in macro " + macro +
". 'constraints' should be a dictionary (no dashes).")
elif instr_type == 'comment':
self.comments[macro] = str(macro_content['comment'])
else:
# if no commands, and nested macros already processed, then nothing to do
continue
resources = self.current_resources
resources_tagged = []
#Get intersection of resources if 'tag' provided by user
if 'params' in constraints and 'tags' in constraints['params']:
resources_tagged = t.get_resource_list(
tag=constraints['params']['tags'])
resources = [
value for value in resources if value in resources_tagged
]
# fill in custom user constraints from outside imports (ex: chipset)
if constraints:
self.__fill_addon_constraints(constraints)
#Filter out resource based on 'constraints->resources' stanza
if constraints and 'resources' in constraints:
resource_constraints = constraints['resources']
new_resources = []
for resource in resources:
match = True
failed_constraint = ''
for key, constraint in resource_constraints.items():
if key in t['resources'][resource]['system'][
'primary'] and key != 'tags':
params_value = t['resources'][resource]['system'][
'primary'][key]
if type(constraint) is list:
if params_value not in constraint:
match = False
elif params_value.upper() != constraint.upper():
match = False
if match is False:
failed_constraint += key + ','
if match:
new_resources.append(resource)
else:
self.__log("Skipping resource " + resource +
". Filter constraint for macro " + macro +
"\n" + failed_constraint)
resources = new_resources
for resource in resources:
target_lst = {}
resource_name = t['resources'][resource]['system']['primary'][
'name']
self.__log(
'Processing macro:' + macro + ',resource:' + resource + '(' +
resource_name + ")\n", resource)
t.log_console('Processing macro:' + macro + ',resource:' +
resource + '(' + resource_name + ")\n")
#Filter out targets based on 'constraints->targets' stanza
if constraints and 'targets' in constraints:
target_constraints = constraints['targets']
for mode, target in target_constraints.items(
): # ex: 'part-number', (big list of part numbers)
#t.log_console("Mode: " + mode + ", target: " + str(target))
if type(target) is str:
if 'var_unknown' in target:
#original macro had a var[] that was not satisfied
target = target.replace('var_unknown(', '')
target = target.replace(')', '')
self.__log("Variable var[" + target +
"] undefined.")
target_lst[mode] = []
continue
else:
targets = target.split(',')
target_lst[mode] = targets
if type(
target
) is dict: #user is filtering - check for 'function' key
if 'function' not in target:
self.__log(
"Skipping resource macro " + macro +
". 'function' missing from constraint target "
+ mode)
#t.log_console("Skipping resource macro " + macro + ". 'function' missing from constraint target " + mode)
continue
callback = getattr(self, target['function'])
#t.log_console("Target before callback is " + str(target))
target_lst[mode] = callback(mode=mode,
resource=resource,
**target)
elif type(
target
) is list: #user is supplying simply a list of targets (ex: fpc1, fpc2, etc.)
target_lst[mode] = target
if macro in self.comments:
self.__log("\n" + self.comments[macro], resource)
#Standardize command format
final_commands = []
if type(commands) is not dict: #cli() style format
new_commands = {}
for command in commands:
command = command.replace(')', '')
command_type, cmd = command.split('(')
if not cmd:
cmd = ''
if command_type in commands:
new_commands[command_type].append(cmd)
else:
new_commands[command_type] = [cmd]
commands = new_commands
for command_type in commands.keys():
for command in commands[command_type]:
if command_type in target_lst:
for target in target_lst[command_type]:
final_commands.append({
'mode': command_type,
'target': target,
'cmd': command
})
else:
final_commands.append({
'mode': command_type,
'cmd': command
})
# and finally, run commands
for command in final_commands:
if self.verbosity in ['medium', 'high']:
t.log_console("Running Command " + str(command) +
" on resource " + resource)
# for debug, put 'continue' here as needed
# check for unassigned variables
match_obj = re.search(
r'(var_unknown|file)\[\'?[a-zA-Z0-9_]*\'?\]', str(command))
if match_obj:
match = match_obj.group(0)
self.__log('Skipping command ' + str(command) +
' due to unassigned variable ' + match)
continue
# try running a command
response = None
try:
#self.__log(resource + "(" + resource_name + ")" + " Running " + command['mode'] + ':' + str(command['cmd']))
if command['mode'] == 'cli':
response = cli(self.handles[resource],
command=command['cmd'])
elif command['mode'] == 'config':
response = config(self.handles[resource],
command_list=[command['cmd']],
commit=True)
elif command['mode'] == 'shell':
response = shell(self.handles[resource],
command=command['cmd'])
elif command['mode'] == 'cty':
response = cty(self.handles[resource],
destination=command['target'],
command=command['cmd'])
elif command['mode'] == 'vty':
response = vty(self.handles[resource],
destination=command['target'],
command=command['cmd'])
elif command['mode'] == 'cli-pfe':
if not resource in self.custom_modes:
self.custom_modes[resource] = {}
if 'cli-pfe' not in self.custom_modes[resource]:
try:
self.custom_modes[resource]['cli-pfe'] = \
add_mode(self.handles[resource], mode='cli-pfe', command='cli-pfe', pattern='>', exit_command='exit')
except Exception:
self.custom_modes[resource]['cli-pfe'] = False
if self.custom_modes[resource]['cli-pfe']:
response = execute_custom_mode(
self.handles[resource],
mode='cli-pfe',
command=command['cmd'])
else:
response = "Skipping cli-pfe command " + command[
'cmd'] + " (no cli-pfe available)"
elif command['mode'] == 'rpc':
response = rpc(self.handles[resource],
command=command['cmd'])
elif command['mode'] == 'pyez':
response = pyez(self.handles[resource],
command=command['cmd'])
# elif command['mode'] == 'cli-pfe':
# response = vty(self.handles[resource], destination=command['target'], command=command['cmd'], timeout=10)
elif command['mode'] == 'rest':
response = self._rest_call(command=command)
elif command['mode'] == 'fetch_cores':
self.core_collect[resource] = 1
elif command['mode'] == 'get_file':
stdout_orig = sys.stdout
sys.stdout = StringIO()
try:
log_dir = get_log_dir(
) + '/macro_logs/' + self.log_prefix + '/' + resource_name
status = self.handles[resource].download(
remote_file=command['cmd'], local_file=log_dir)
except Exception:
status = False
sys.stdout = stdout_orig
if status:
response = "File " + command[
'cmd'] + " retrieved and saved to " + log_dir
else:
response = "File " + command[
'cmd'] + " retrieval failed"
else:
self.__log('Instruction ' + command['mode'] +
' not supported (skipping)')
log_message = "\n" + resource_name
if 'target' in command:
log_message += '(' + command['target'] + ')'
log_message += ' ' + command['mode'] + "> " + str(
command['cmd']) + "\n" + response + "\n"
self.__log(log_message, resource)
except Exception as error:
log_message = 'Command ' + str(
command
) + ' failed on resource ' + resource + '(' + resource_name + ')'
if response:
log_message += ' with response ' + response
log_message += ' and an exception error [' + str(
error) + ']'
self.__log(log_message)
continue
def run_macros(self, macros, resources, message=None, targets='{}'):
'''
Run particular macros in macro_lib file
param: macros
name of macros (colon delimited)
'''
self.completed_macros = {}
if "=" in macros:
t.log_console("No macros specified")
self.__log("No macros specified")
return
# Get the testcase name from Robot Builtin library
testname = None
try:
testname = Vars().get_global_variable('${TEST NAME}')
testname = testname.replace(" ", "_")
testname = testname + '_'
except Exception:
testname = ""
# acquire time for timestamping
dtime = datetime.datetime.now()
self.current_time = str(dtime.year) \
+ '{:02.0f}'.format(dtime.month) \
+ '{:02.0f}'.format(dtime.day) \
+ '-' \
+ '{:02.0f}'.format(dtime.hour) \
+ '{:02.0f}'.format(dtime.minute) \
+ '{:02.0f}'.format(dtime.second)
# create general default logger
self.log_prefix = testname + self.current_time
general_log_dir = get_log_dir(
) + '/macro_logs/' + testname + self.current_time
os.makedirs(general_log_dir)
self.loggers['default'] = self.__setup_logger(
'default', general_log_dir + '/result.log')
self.log_files.append(general_log_dir + '/result.log')
BuiltIn().log(
level='INFO',
message=
"Macro Engine employed. <a href=macro_logs/ target=_blank>Macro Engine Logs</a>",
html=True)
if not self.all_macro_libs:
t.log_console("No macro libraries loaded.")
self.__log("No macro libraries loaded.")
return
#reconsistute targets dictionary into true dictionary since Robot only allows scalar variables
targets = literal_eval(targets)
#gather master resource list
resource_lst = None
if type(resources) is list:
resource_lst = resources
elif resources != 'all':
resource_lst = resources.split(':')
else: #get all resources
resource_lst = t.get_resource_list()
self.current_resources = resource_lst
# prep each resource
for resource in resource_lst:
#ensure resource handle is still connected and gracefully reconnect if not connected
try:
self.handles[resource] = t.get_handle(resource=resource)
self.handles[resource].reconnect(force=False)
except Exception:
self.__log("Skipping resource " + resource +
". Connection lost.")
if resource in self.handles:
del self.handles[resource]
continue
#set up logging
resource_name = t['resources'][resource]['system']['primary'][
'name']
log_dir = get_log_dir(
) + '/macro_logs/' + self.log_prefix + '/' + resource_name
os.makedirs(log_dir)
self.loggers[resource] = self.__setup_logger(
resource, log_dir + '/' + resource_name + "_result.log")
self.log_files.append(log_dir + '/' + resource_name +
"_result.log")
#user defined targets
if resource in targets:
self.user_targets[resource] = targets[resource]
# Gather robot variables
try:
self.robot_vars = BuiltIn().get_variables()
except Exception:
# no robot running
self.__log("Robot Variables unavailable")
all_macro_libs_yaml = yaml.dump(self.all_macro_libs)
# t.log_console(all_macro_libs_yaml)
all_macro_libs_yaml = self.__process_vars(all_macro_libs_yaml)
all_macro_libs_yaml = self.__process_file_vars(all_macro_libs_yaml)
self.all_macro_libs = yaml.safe_load(all_macro_libs_yaml)
# t.log_console(yaml.dump(self.all_macro_libs))
self.__log('Processing macros...')
if message:
self.__log('Message: ' + message)
macro_lst = macros.split(':')
for macro in macro_lst:
if macro not in self.all_macro_libs:
self.__log("Macro " + macro + " not found.")
else:
self.target_macros.append(macro)
t.log_console("Processing macro(s) " + str(self.target_macros) +
"...\n")
while self.target_macros:
macro = self.target_macros.pop(0)
if macro not in self.completed_macros:
macro_content = self.all_macro_libs[macro]
if type(macro_content) is dict:
self.__log("Processing macro " + macro + "... ")
self.__run_single_macro(macro, macro_content)
self.__log("Completed macro " + macro + "... ")
self.completed_macros[macro] = 'complete'
elif type(macro_content) is list:
for i in range(0, len(macro_content)):
self.__log("Processing macro " + macro + "[" +
str(i + 1) + "]... ")
self.__run_single_macro(macro, macro_content[i])
self.__log("Completed macro " + macro + "[" +
str(i + 1) + "]... ")
self.completed_macros[macro] = 'complete'
else:
t.log_console("Macro content is of type " +
str(type(macro_content)))
if self.core_collect:
self.__core_collect()
# clean up loggers and log files
for logger_name in self.loggers:
log = logging.getLogger(logger_name)
handlers = list(log.handlers)
for handle in handlers:
log.removeHandler(handle)
handle.flush()
handle.close()
for log_file in self.log_files:
with open(log_file, 'r') as fhandle:
new_log_content = fhandle.read().replace('\r', '')
fhandle.close()
with open(log_file, 'w') as fhandle:
fhandle.write(new_log_content)
fhandle.close()
def __init__(self,
chassis=None,
license_server=None,
system_data=None,
spirent_lib_path=None,
spirent_tcl_bin=None):
"""
Spirent abstraction layer for HLTAPI
-- Workflow 1 --
:param system_data: *MANDATORY* Dictionary of Spirent information
rt0:
interfaces:
intf1:
name: 1/8
intf2:
name: 1/9
system:
primary:
appserver: wf-appserver2.englab.juniper.net
controllers:
unknown:
domain: englab.juniper.net
hostname: wf-ixchassis2
mgt-intf-name: mgt0
mgt-ip: 10.9.1.107
osname: IxOS
license_server: sv8-pod1-ixlic1.englab.juniper.net
make: spirent
model: xgs12
name: wf-ixchassis2
osname: IxOS
-- Workflow 2 --
:param host *MANDATORY* FQDN/mgt-ip of of chassis
:param labserver *OPTIONAL* FQDN/mgt-ip of of lab server
:param license_server *OPTIONAL* FQDN/mgt-ip of license server
:return: Spirent object
"""
self.debug = False
self.virtual = False
self.port_list = None
self.port_order = None
self.license_server = None
self.chassis = None
self.labserver = None
self.labserver_session_name = 'toby' + str(
random.randrange(1000, 9999, 1))
self.labserver_preserve_session = False
self.labserver_create_new_session = 1
self.is_labserver_connected = False
self.intf_to_port_map = None
self.port_to_handle_map = None
self.session_info = None
self.config_file = None
self.handle_to_port_map = None
self.log_dir = get_log_dir()
atexit.register(self.cleanup)
self.sth = None
self.user_functions = dict()
self.username, self.password = credentials.get_credentials(
os='Spirent')
self.read_timeout = 60
self.tcl = None
self.tcl32 = None
self.tcl64 = None
if spirent_lib_path and spirent_tcl_bin:
self.lib_path = spirent_lib_path
self.tcl = spirent_tcl_bin
else:
environment = yaml.safe_load(
open(
os.path.join(os.path.dirname(credentials.__file__),
"environment.yaml")))
self.lib_path = environment['spirent-lib-path']
self.tcl32 = environment['tcl32-bin']
self.tcl64 = environment['tcl64-bin']
if system_data:
controller_key = list(
system_data['system']['primary']['controllers'].keys())[0]
if 'user' in system_data['system']['primary']['controllers'][controller_key] and \
system_data['system']['primary']['controllers'][controller_key]['user']:
self.username = system_data['system']['primary'][
'controllers'][controller_key]['user']
if 'password' in system_data['system']['primary']['controllers'][controller_key] and \
system_data['system']['primary']['controllers'][controller_key]['password']:
self.password = system_data['system']['primary'][
'controllers'][controller_key]['password']
kwargs = dict()
kwargs['host'] = self.chassis = system_data['system']['primary'][
'name']
kwargs['hostname'] = self.chassis = system_data['system'][
'primary']['name']
kwargs['os'] = system_data['system']['primary']['controllers'][
controller_key]['osname']
kwargs['user'] = self.username
kwargs['password'] = self.password
super(Spirent, self).__init__(**kwargs)
if 'labserver' in system_data['system']['primary']:
self.labserver = system_data['system']['primary']['labserver']
try:
ipaddress.ip_address(self.labserver)
except Exception:
self.labserver = socket.gethostbyname(self.labserver)
if 'labserver_session_name' in system_data['system'][
'primary']:
self.labserver_session_name = system_data['system'][
'primary']['labserver_session_name']
if 'labserver_connect_existing_session' in system_data[
'system']['primary']:
self.labserver_create_new_session = 0
if 'labserver_preserve_session' in system_data['system'][
'primary']:
self.log("Preserving labserver session")
self.labserver_preserve_session = True
if 'config-file' in system_data['system']['primary']:
self.config_file = str(
system_data['system']['primary']['config-file'])
if 'port-order' in system_data['system']['primary']:
self.port_order = system_data['system']['primary'][
'port-order']
if 'debug' in system_data['system']['primary']:
self.debug = system_data['system']['primary']['debug']
if 'mgt-ip' in system_data['system']['primary']['controllers'][
controller_key]:
self.chassis = system_data['system']['primary']['controllers'][
controller_key]['mgt-ip']
if system_data['system']['primary']['model'].upper() == 'VSPIRENT':
self.virtual = True
try:
self.license_server = system_data['system']['primary'][
'license_server']
except:
raise TobyException(
"Missing license_server 'primary' stanza: " +
str(system_data),
host_obj=self)
elif chassis:
self.chassis = chassis
if license_server:
self.license_server = license_server
else:
raise TobyException(
"Missing either system_data (Workflow 1) or chassis (Workflow 2) parameter",
host_obj=self)
if not self.chassis:
raise TobyException(
"Unable to determine chassis host information! Check for valid labserver or mgt-ip in init yaml file.",
host_obj=self)
self.connect_info = None
self.log("CHASSIS= " + str(self.chassis))
self.wait = 1
self.telnet_handle = None
self.version = self._get_version()
self.hltapi_lib_path = None
if 'hltapi-path' in system_data['system']['primary']:
self.hltapi_lib_path = system_data['system']['primary'][
'hltapi-path']
self.hltapi_lib_path.rstrip('/')
else:
hltapi_version = self._get_hltapi_version()
self.hltapi_lib_path = self.lib_path + '/HLTAPI/' + hltapi_version
sys.path.append(self.hltapi_lib_path + '/SourceCode/hltapiForPython')
self._set_envs()
#resetting sys.argv[0] is only way to get spirent htlapi logs to go to the right place
executable = sys.argv[0]
sys.argv[0] = self.log_dir
self.sth = __import__('sth')
sys.argv[0] = executable
self.log("Spirent Chassis Initialization Complete")
# Import extended spirent modules with functions from jnpr/toby/trafficgen/spirent
current_dir = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
user_module_dir = re.sub(r"jnpr\/toby.*",
"jnpr/toby/trafficgen/spirent", current_dir)
sys.path.append(user_module_dir)
file_list = list(
filter(lambda x: os.path.isfile(os.path.join(user_module_dir, x)),
os.listdir(user_module_dir)))
for file_name in file_list:
if file_name.endswith('.py') and not file_name.startswith('__'):
module = re.sub(r"\.py$", "", file_name)
obj = __import__(module)
function_list = [
o for o in inspect.getmembers(obj)
if inspect.isfunction(o[1])
]
for function_tuple in function_list:
function_name, function = function_tuple
if function_name in self.user_functions:
raise TobyException(
"Duplicate functions in user contributed modules",
host_obj=self)
self.user_functions[function_name] = function
def __init__(self, chassis=None, system_data=None):
"""
Warp17 abstraction layer for HLTAPI
-- Workflow 1 --
:param system_data: *MANDATORY* Dictionary of Warp17 information
rt0:
interfaces:
intf1:
name: 1/8
intf2:
name: 1/9
system:
primary:
appserver: wf-appserver2.englab.juniper.net
controllers:
unknown:
domain: englab.juniper.net
hostname: wf-warp17chassis2
mgt-intf-name: mgt0
mgt-ip: 10.9.1.107
osname: IxOS
make: warp17
model: xgs12
name: wf-ixchassis2
osname: IxOS
-- Workflow 2 --
:param host *MANDATORY* FQDN/mgt-ip of of chassis
:return: Warp17 object
"""
self.chassis = None
self.interfaces = None
self.model = 'Unknown'
self.intf_to_port_map = dict()
self.port_to_handle_map = dict()
self.session_info = None
self.handle_to_port_map = None
self.login_complete = False
self.log_dir = get_log_dir()
self.clientport = None
self.serverport = None
atexit.register(self.cleanup)
self.username, self.password = credentials.get_credentials(os='Unix')
self.connect_kwargs = {
'virtual_machine': 0,
'ring_if_pairs': None,
'dpdk_dev_bind_path': None,
'host_name': "localhost",
'ucb_pool_sz': 0,
'tcb_pool_sz': 0,
'lcores': None
}
if system_data:
controller_key = list(
system_data['system']['primary']['controllers'].keys())[0]
if 'user' in system_data['system']['primary']['controllers'][controller_key] and \
system_data['system']['primary']['controllers'][controller_key]['user']:
self.username = system_data['system']['primary'][
'controllers'][controller_key]['user']
if 'password' in system_data['system']['primary']['controllers'][controller_key] and \
system_data['system']['primary']['controllers'][controller_key]['password']:
self.password = system_data['system']['primary'][
'controllers'][controller_key]['password']
kwargs = dict()
kwargs['host'] = self.chassis = system_data['system']['primary'][
'name']
kwargs['hostname'] = self.chassis = system_data['system'][
'primary']['name']
kwargs['os'] = system_data['system']['primary']['controllers'][
controller_key]['osname']
super(Warp17, self).__init__(**kwargs)
if 'mgt-ip' in system_data['system']['primary']['controllers'][
controller_key]:
self.chassis = system_data['system']['primary']['controllers'][
controller_key]['mgt-ip']
self.model = system_data['system']['primary']['model'].upper()
warp17_options_str = system_data['system']['primary']['warp17']
if warp17_options_str == 'enable':
t.log(level="INFO", message="Using default warp17 options")
else:
option_list = warp17_options_str.split(':')
for option in option_list:
key, value = option.split('=')
if key and value and key in self.connect_kwargs:
self.connect_kwargs[key] = value
elif chassis:
self.chassis = chassis
else:
raise TobyException(
"Missing either system_data (Workflow 1) or chassis (Workflow 2) parameter"
)
if not self.chassis:
raise TobyException(
"Unable to determine chassis host information! Check for valid in init yaml file.",
host_obj=self)
self.connect_info = None
self.log("CHASSIS= " + str(self.chassis))
self.warp17 = Warp17api(server_name=self.chassis) # pylint: disable=import-error
def __init__(self, chassis=None, system_data=None):
"""
Spirent Elevate abstraction layer for Utils.py API
-- Workflow 1 --
:param system_data: *MANDATORY* Dictionary of Elevate information
rt0:
interfaces:
intf1:
name: 1/8
intf2:
name: 1/9
system:
primary:
appserver: wf-appserver2.englab.juniper.net
controllers:
unknown:
domain: englab.juniper.net
hostname: wf-avchassis2
mgt-intf-name: mgt0
mgt-ip: 10.9.1.107
osname: IxOS
make: spirent
model: spirent
name: elevate1
osname: Elevate
-- Workflow 2 --
:param host *MANDATORY* FQDN/mgt-ip of of chassis
:return: Elevate object
"""
self.chassis = None
self.interfaces = None
self.model = 'Unknown'
self.platform_type = 'Elevate'
self.intf_to_port_map = dict()
self.port_to_handle_map = dict()
self.session_info = None
self.handle_to_port_map = None
atexit.register(self.cleanup)
self.log_dir = get_log_dir()
atexit.register(self.cleanup)
environment = yaml.safe_load(open(os.path.join(os.path.dirname(credentials.__file__), "environment.yaml")))
self.lib_path = environment['elevate-lib-path']
self.api_path = self.lib_path + '/Elevate/PythonAPI/1.0.0'
if system_data:
controller_key = list(system_data['system']['primary']['controllers'].keys())[0]
kwargs = dict()
kwargs['host'] = self.chassis = system_data['system']['primary']['name']
kwargs['hostname'] = self.chassis = system_data['system']['primary']['name']
kwargs['os'] = system_data['system']['primary']['controllers'][controller_key]['osname']
super(Elevate, self).__init__(**kwargs)
if 'api-path' in system_data['system']['primary']:
self.log("Overriding existing utils.py path with: " + system_data['system']['primary']['api-path'])
self.api_path = system_data['system']['primary']['api-path']
if 'mgt-ip' in system_data['system']['primary']['controllers'][controller_key]:
self.chassis = system_data['system']['primary']['controllers'][controller_key]['mgt-ip']
self.model = system_data['system']['primary']['model'].upper()
elif chassis:
self.chassis = chassis
else:
raise TobyException("Missing either system_data (Workflow 1) or chassis (Workflow 2) parameter")
if not self.chassis:
raise TobyException("Unable to determine chassis host information! Check for valid in init yaml file.", host_obj=self)
self.connect_info = None
self.log("CHASSIS= " + str(self.chassis))
sys.path.append(self.api_path)
try:
self.elevate = __import__('Utils3')
except Exception as err:
raise TobyException("Unable to import Utils.py at path " + self.api_path + ": " + str(err), host_obj=self)
self.connect()
def __init__(self, rt_handle=None):
self.hdl = rt_handle
self.streamids = []
self.logdir = get_log_dir()