def test_runCapabilities_parse_and_dump_sdo_empty_dir(
self, mock_hash: mock.MagicMock, mock_load_files: mock.MagicMock):
""" Run runCapabilities.py script over empty directory - no yang files.
Test whether prepare.json file contain only empty dictionary '{}'.
Arguments:
:param mock_hash (mock.MagicMock) get_commit_hash() method is patched, to always return 'master'
:param mock_load_files (mock.MagicMock) LoadFiles is patched to load json files from test directory
"""
mock_hash.return_value = 'master'
mock_load_files.return_value = LoadFiles(self.test_private_dir,
yc_gc.logs_dir)
path = '{}/temp/standard/ietf/RFC/empty'.format(yc_gc.temp_dir)
# Load submodule and its config
module = __import__(self.module_name, fromlist=[self.script_name])
submodule = getattr(module, self.script_name)
script_conf = submodule.ScriptConfig()
# Set script arguments
script_conf.args.__setattr__('sdo', True)
script_conf.args.__setattr__('dir', path)
script_conf = self.set_script_conf_arguments(script_conf)
# Run runCapabilities.py script with corresponding configuration
submodule.main(scriptConf=script_conf)
# Load module data from dumped prepare.json file
with open('{}/prepare.json'.format(yc_gc.temp_dir), 'r') as f:
file_content = json.load(f)
self.assertEqual(file_content, {})
def test_loadJsonFiles(self):
"""
Test if 'parsed_jsons' object has the individual attributes set correctly.
Excluded file names should no be present in 'parsed_json' dict as keys.
"""
parsed_jsons = LoadFiles(self.test_private_dir, yc_gc.logs_dir)
names = []
with open('{}/json_links'.format(self.test_private_dir), 'r') as f:
for line in f:
names.append(line.replace('.json', '').replace('\n', ''))
names = [name for name in names if name not in self.excluded_names]
# Object should containt following attributes
self.assertTrue(hasattr(parsed_jsons, 'headers'))
self.assertTrue(hasattr(parsed_jsons, 'names'))
self.assertTrue(hasattr(parsed_jsons, 'status'))
self.assertEqual(names, parsed_jsons.names)
# Property should be set for each name found in json_links file
for name in names:
self.assertIn(name, parsed_jsons.headers)
self.assertIn(name, parsed_jsons.status)
# Property should NOT be set for excluded names
for name in self.excluded_names:
self.assertNotIn(name, parsed_jsons.headers)
self.assertNotIn(name, parsed_jsons.status)
def test_runCapabilities_parse_and_dump_sdo(
self, mock_hash: mock.MagicMock, mock_load_files: mock.MagicMock):
""" Run runCapabilities.py script over SDO yang files in directory.
For testing purposes there is only 1 yang file ([email protected]) in directory.
Compare content of prepare.json files.
Arguments:
:param mock_hash (mock.MagicMock) get_commit_hash() method is patched, to always return 'master'
:param mock_load_files (mock.MagicMock) LoadFiles is patched to load json files from test directory
"""
mock_hash.return_value = 'master'
mock_load_files.return_value = LoadFiles(self.test_private_dir,
yc_gc.logs_dir)
path = '{}/test/YangModels/yang/standard/ietf/RFC'.format(
yc_gc.temp_dir)
# Load submodule and its config
module = __import__(self.module_name, fromlist=[self.script_name])
submodule = getattr(module, self.script_name)
script_conf = submodule.ScriptConfig()
# Set script arguments
script_conf.args.__setattr__('sdo', True)
script_conf.args.__setattr__('dir', path)
script_conf = self.set_script_conf_arguments(script_conf)
# Run runCapabilities.py script with corresponding configuration
submodule.main(scriptConf=script_conf)
desired_module_data = self.load_desired_prepare_json_data(
'dumped_module')
dumped_module_data = self.load_dumped_prepare_json_data()
# Compare desired output with output of prepare.json
for dumped_module in dumped_module_data:
for desired_module in desired_module_data:
if desired_module.get('name') == dumped_module.get('name'):
# Compare properties/keys of desired and dumped module data objects
for key in desired_module:
if key == 'yang-tree':
# Compare only URL suffix (exclude domain)
desired_tree_suffix = '/api{}'.format(
desired_module[key].split('/api')[-1])
dumped_tree_suffix = '/api{}'.format(
dumped_module[key].split('/api')[-1])
self.assertEqual(desired_tree_suffix,
dumped_tree_suffix)
elif key == 'compilation-result':
if dumped_module[key] != '' and desired_module[
key] != '':
# Compare only URL suffix (exclude domain)
desired_compilation_result = '/results{}'.format(
desired_module[key].split('/results')[-1])
dumped_compilation_result = '/results{}'.format(
dumped_module[key].split('/results')[-1])
self.assertEqual(desired_compilation_result,
dumped_compilation_result)
else:
self.assertEqual(dumped_module[key],
desired_module[key])
def declare_sdo_module(self):
"""
Initialize Modules object for SDO (ietf) module.
:returns: Created instance of Modules object of SDO (ietf) module
:rtype: Modules
"""
parsed_jsons = LoadFiles(self.test_private_dir, yc_gc.logs_dir)
path_to_yang = os.path.join(yc_gc.temp_dir,
'test/YangModels/yang/standard/ietf/RFC',
self.sdo_module_filename)
yang = SdoModule(self.sdo_module_name, path_to_yang, parsed_jsons,
self.dir_paths, 'master', {}, self.schema_base)
return yang
def test_loadJsonFiles_json_links_not_found(self):
"""
Test if 'parsed_jsons' object has the individual attributes set correctly.
Incorrect path is passed as argument resulting no json_link file is found there.
All attributes should be empty as the file was not found.
"""
parsed_jsons = LoadFiles('path/to/random/dir', yc_gc.logs_dir)
# Object should containt following attributes
self.assertTrue(hasattr(parsed_jsons, 'headers'))
self.assertTrue(hasattr(parsed_jsons, 'names'))
self.assertTrue(hasattr(parsed_jsons, 'status'))
# Attributes should be empty as the file was not found
self.assertEqual(parsed_jsons.names, [])
self.assertEqual(parsed_jsons.headers, {})
self.assertEqual(parsed_jsons.status, {})
def declare_sdo_module(self, path_to_yang: str):
"""
Initialize Modules object for SDO (ietf) module.
:param path_to_yang (str) path to yang file
:returns: Created instance of Modules object of SDO (ietf) module
:rtype: Modules
"""
parsed_jsons = LoadFiles(self.test_private_dir, yc_gc.logs_dir)
module_name = path_to_yang.split('/')[-1].split('.yang')[0]
schema_base = os.path.join(github_raw, 'YangModels/yang/master')
if '@' in module_name:
module_name = module_name.split('@')[0]
yang = SdoModule(module_name, path_to_yang, parsed_jsons,
self.dir_paths, 'master', {}, schema_base)
return yang
def test_process_vendor(self, mock_load_files: mock.MagicMock):
mock_load_files.return_value = LoadFiles(self.private_dir, self.log_directory)
platform = self.test_data.get('capabilities-json-content')
dst = '{}/huawei/yang/network-router/8.20.0/ne5000e'.format(self.direc)
os.makedirs(dst, exist_ok=True)
shutil.copy('{}/huawei-dsa.yang'.format(self.huawei_dir), '{}/huawei-dsa.yang'.format(dst))
shutil.copy('{}/capabilities.xml'.format(self.huawei_dir), '{}/capabilities.xml'.format(dst))
with open('{}/capabilities.json'.format(dst), 'w') as f:
json.dump(platform, f)
arguments = ['POPULATE-VENDORS', '--dir', self.direc, '--api',
'--credentials', *self.credentials, 'True']
status, details = self.receiver.process(arguments)
self.assertEqual(status, StatusMessage.SUCCESS)
self.assertEqual(details, '')
def declare_vendor_module(self):
"""
Initialize Modules object for vendor (Cisco) module.
:returns: Created instance of Modules object of vendor (cisco) module
:rtype: Modules
"""
parsed_jsons = LoadFiles(self.test_private_dir, yc_gc.logs_dir)
vendor_data = 'ietf-netconf-acm&revision=2018-02-14&deviations=cisco-xr-ietf-netconf-acm-deviations'
module_name = vendor_data.split('&revision')[0]
module_path = '{}/{}.yang'.format(self.resources_path, module_name)
yang = VendorModule(module_name,
module_path,
parsed_jsons,
self.dir_paths,
'master', {},
self.schema_base,
data=vendor_data)
return yang
def test_loadJsonFiles_non_existing_json_file(
self, mock_load_names: mock.MagicMock):
"""
Test if 'parsed_jsons' object has the individual attributes set correctly.
load_names() method will return non-exisiting name of json file, so FileNotFound exceptions will be raised
while trying to load content of json/html files - headers and status properties should be empty for this name.
Arguments:
:param mock_load_names (mock.MagicMock) load_names() method is patched, to return name of non-exisiting json
"""
non_existing_json_name = 'SuperRandom'
mock_load_names.return_value = [non_existing_json_name]
parsed_jsons = LoadFiles(self.test_private_dir, yc_gc.logs_dir)
# Object should containt following attributes
self.assertTrue(hasattr(parsed_jsons, 'headers'))
self.assertTrue(hasattr(parsed_jsons, 'names'))
self.assertTrue(hasattr(parsed_jsons, 'status'))
# Status and headers should be empty for non-existing json/html files
self.assertEqual(parsed_jsons.headers[non_existing_json_name], [])
self.assertEqual(parsed_jsons.status[non_existing_json_name], {})
def test_process_sdo(self, mock_load_files: mock.MagicMock):
mock_load_files.return_value = LoadFiles(self.private_dir, self.log_directory)
data = self.test_data.get('request-data-content')
dst = '{}/YangModels/yang/standard/ietf/RFC'.format(self.direc)
os.makedirs(dst, exist_ok=True)
RFC_dir = '{}/yangmodels/yang/standard/ietf/RFC'.format(self.nonietf_dir)
shutil.copy('{}/[email protected]'.format(RFC_dir),
'{}/[email protected]'.format(dst))
with open('{}/request-data.json'.format(self.direc), 'w') as f:
json.dump(data, f)
arguments = ['POPULATE-MODULES', '--sdo', '--dir', self.direc, '--api',
'--credentials', *self.credentials]
status, details = self.receiver.process(arguments)
original_module_data = data['modules']['module'][0]
redis_module = self.modulesDB.get('ietf-yang-types@2010-09-24/ietf')
redis_data = (redis_module or b'{}').decode('utf-8')
self.assertEqual(status, StatusMessage.SUCCESS)
self.assertEqual(details, '')
self.assertNotEqual(redis_data, '{}')
def __init__(self, directory: str, dumper: Dumper, file_hasher: FileHasher,
api: bool, dir_paths: DirPaths):
"""
Arguments:
:param directory (str) the directory containing the files
:param dumper (Dumper) Dumper object
:param filehasher (FileHasher) FileHasher object
:param api (bool) whether the request came from API or not
:param dir_paths (DirPaths) paths to various needed directories according to configuration
"""
global LOGGER
LOGGER = log.get_logger(
'capability', '{}/parseAndPopulate.log'.format(dir_paths['log']))
LOGGER.debug('Running {} constructor'.format(self.__class__.__name__))
self.logger = log.get_logger(
'repoutil', '{}/parseAndPopulate.log'.format(dir_paths['log']))
self.dir_paths = dir_paths
self.dumper = dumper
self.api = api
self.file_hasher = file_hasher
self.directory = directory
self.parsed_jsons = LoadFiles(dir_paths['private'], dir_paths['log'])
def __init__(self, *args, **kwargs):
super(TestModulesClass, self).__init__(*args, **kwargs)
# Declare variables
self.schema_base = os.path.join(github_raw, 'YangModels/yang/master')
self.tmp_dir = '{}/'.format(yc_gc.temp_dir)
self.sdo_module_filename = '*****@*****.**'
self.sdo_module_name = 'ietf-yang-types'
self.hello_message_filename = 'capabilities-ncs5k.xml'
self.resources_path = os.path.join(os.environ['BACKEND'],
'tests/resources')
self.test_private_dir = os.path.join(self.resources_path,
'html/private')
self.parsed_jsons = LoadFiles(self.test_private_dir, yc_gc.logs_dir)
self.dir_paths: DirPaths = {
'cache': '',
'json': '',
'log': yc_gc.logs_dir,
'private': '',
'result': yc_gc.result_dir,
'save': yc_gc.save_file_dir,
'yang_models': yc_gc.yang_models
}
self.test_repo = os.path.join(yc_gc.temp_dir, 'test/YangModels/yang')
def test_runCapabilities_parse_and_dump_vendor_yang_lib(
self, mock_hash: mock.MagicMock, mock_load_files: mock.MagicMock):
""" Run runCapability script over yang_lib.xml. Compare content of normal.json and prepare.json files.
Arguments:
:param mock_hash (mock.MagicMock) get_commit_hash() method is patched, to always return 'master'
:param mock_load_files (mock.MagicMock) LoadFiles is patched to load json files from test directory
"""
mock_load_files.return_value = LoadFiles(self.test_private_dir,
yc_gc.logs_dir)
mock_hash.return_value = 'master'
xml_path = '{}/test/YangModels/yang/vendor/huawei/network-router/8.20.0/ne5000e'.format(
yc_gc.temp_dir)
# Load submodule and its config
module = __import__(self.module_name, fromlist=[self.script_name])
submodule = getattr(module, self.script_name)
script_conf = submodule.ScriptConfig()
# Set arguments
script_conf.args.__setattr__('sdo', False)
script_conf.args.__setattr__('dir', xml_path)
script_conf = self.set_script_conf_arguments(script_conf)
# Run runCapabilities.py script with corresponding configuration
submodule.main(scriptConf=script_conf)
key = lambda x: x.get('name')
dumped_module_data = sorted(self.load_dumped_prepare_json_data(),
key=key)
desired_module_data = sorted(
self.load_desired_prepare_json_data('yang_lib_prepare_json'),
key=key)
# Compare desired output with output of prepare.json
self.assertEqual(len(dumped_module_data), len(desired_module_data))
for dumped_module, desired_module in zip(dumped_module_data,
desired_module_data):
# Compare properties/keys of desired and dumped module data objects
for key in desired_module:
if key == 'yang-tree':
# Compare only URL suffix (exclude domain)
desired_tree_suffix = '/api{}'.format(
desired_module[key].split('/api')[-1])
dumped_tree_suffix = '/api{}'.format(
dumped_module[key].split('/api')[-1])
self.assertEqual(desired_tree_suffix, dumped_tree_suffix)
elif key == 'compilation-result':
if dumped_module[key] != '' and desired_module[key] != '':
# Compare only URL suffix (exclude domain)
desired_compilation_result = '/results{}'.format(
desired_module[key].split('/results')[-1])
dumped_compilation_result = '/results{}'.format(
dumped_module[key].split('/results')[-1])
self.assertEqual(desired_compilation_result,
dumped_compilation_result)
else:
if isinstance(desired_module[key], list):
for i in desired_module[key]:
self.assertIn(i, dumped_module[key], key)
else:
self.assertEqual(dumped_module[key],
desired_module[key])
# Load desired normal.json data from .json file
with open(
os.path.join(self.resources_path,
'parseAndPopulate_tests_data.json'), 'r') as f:
file_content = json.load(f)
desired_vendor_data = file_content['yang_lib_normal_json']['vendors'][
'vendor']
# Load vendor module data from normal.json file
with open(os.path.join(yc_gc.temp_dir, 'normal.json'), 'r') as f:
file_content = json.load(f)
self.assertIn('vendors', file_content)
self.assertIn('vendor', file_content['vendors'])
self.assertNotEqual(len(file_content['vendors']['vendor']), 0)
dumped_vendor_data = file_content['vendors']['vendor']
for dumped_vendor in dumped_vendor_data:
self.assertIn(dumped_vendor, desired_vendor_data)
def test_runCapabilities_parse_and_dump_vendor(
self, mock_commit_hash: mock.MagicMock,
mock_load_files: mock.MagicMock):
""" Run runCapabilities.py script over vendor yang files in directory which also contains capability xml file.
Compare content of normal.json and prepare.json files.
Arguments:
:param mock_hash (mock.MagicMock) get_commit_hash() method is patched, to always return 'master'
:param mock_load_files (mock.MagicMock) LoadFiles is patched to load json files from test directory
"""
mock_commit_hash.return_value = 'master'
mock_load_files.return_value = LoadFiles(self.test_private_dir,
yc_gc.logs_dir)
xml_path = '{}/test/YangModels/yang/vendor/cisco/xr/701'.format(
yc_gc.temp_dir)
# Load submodule and its config
module = __import__(self.module_name, fromlist=[self.script_name])
submodule = getattr(module, self.script_name)
script_conf = submodule.ScriptConfig()
# Set arguments
script_conf.args.__setattr__('sdo', False)
script_conf.args.__setattr__('dir', xml_path)
script_conf = self.set_script_conf_arguments(script_conf)
# Run runCapabilities.py script with corresponding configuration
submodule.main(scriptConf=script_conf)
desired_module_data = self.load_desired_prepare_json_data(
'ncs5k_prepare_json')
dumped_module_data = self.load_dumped_prepare_json_data()
# Compare desired output with output of prepare.json
for dumped_module in dumped_module_data:
for desired_module in desired_module_data:
if desired_module.get('name') == dumped_module.get('name'):
# Compare properties/keys of desired and dumped module data objects
for key in desired_module:
if key == 'yang-tree':
# Compare only URL suffix (exclude domain)
desired_tree_suffix = '/api{}'.format(
desired_module[key].split('/api')[-1])
dumped_tree_suffix = '/api{}'.format(
dumped_module[key].split('/api')[-1])
self.assertEqual(desired_tree_suffix,
dumped_tree_suffix)
elif key == 'compilation-result':
if dumped_module[key] != '' and desired_module[
key] != '':
# Compare only URL suffix (exclude domain)
desired_compilation_result = '/results{}'.format(
desired_module[key].split('/results')[-1])
dumped_compilation_result = '/results{}'.format(
dumped_module[key].split('/results')[-1])
self.assertEqual(desired_compilation_result,
dumped_compilation_result)
else:
self.assertEqual(dumped_module[key],
desired_module[key])
# Load desired normal.json data from .json file
with open(
'{}/parseAndPopulate_tests_data.json'.format(
self.resources_path), 'r') as f:
file_content = json.load(f)
desired_vendor_data = file_content.get('ncs5k_normal_json',
{}).get('vendors', {}).get(
'vendor', [])
self.assertNotEqual(len(desired_vendor_data), 0)
# Load vendor module data from normal.json file
with open('{}/normal.json'.format(yc_gc.temp_dir), 'r') as f:
file_content = json.load(f)
self.assertIn('vendors', file_content)
self.assertIn('vendor', file_content.get('vendors', []))
self.assertNotEqual(len(file_content['vendors']['vendor']), 0)
dumped_vendor_data = file_content.get('vendors',
{}).get('vendor', [])
for dumped_vendor in dumped_vendor_data:
self.assertIn(dumped_vendor, desired_vendor_data)
def __init__(self,
log_directory,
hello_message_file,
index,
prepare,
integrity_checker,
api,
sdo,
json_dir,
html_result_dir,
save_file_to_dir,
private_dir,
yang_models_dir,
run_integrity=False):
global LOGGER
LOGGER = log.get_logger('capability',
log_directory + '/parseAndPopulate.log')
LOGGER.debug('Running constructor')
self.log_directory = log_directory
self.run_integrity = run_integrity
self.to = save_file_to_dir
self.html_result_dir = html_result_dir
self.json_dir = json_dir
self.index = index
self.prepare = prepare
self.integrity_checker = integrity_checker
self.parsed_yang = None
self.api = api
self.sdo = sdo
self.yang_models_dir = yang_models_dir
# Get hello message root
if 'xml' in hello_message_file:
try:
LOGGER.debug('Checking for xml hello message file')
self.root = ET.parse(hello_message_file).getroot()
except:
#try to change & to &
hello_file = fileinput.FileInput(hello_message_file,
inplace=True)
for line in hello_file:
print(line.replace('&', '&'), end='')
hello_file.close()
LOGGER.warning(
'Hello message file has & instead of &, automatically changing to &'
)
self.root = ET.parse(hello_message_file).getroot()
# Split it so we can get vendor, os-type, os-version
self.split = hello_message_file.split('/')
self.hello_message_file = hello_message_file
if self.api and not self.sdo:
self.platform_data = []
json_file = open(hello_message_file.split('.xml')[0] + '.json')
impl = json.load(json_file)
self.initialize(impl)
json_file.close()
if not self.api and not self.sdo:
if os.path.isfile('/'.join(self.split[:-1]) +
'/platform-metadata.json'):
self.platform_data = []
json_file = open('/'.join(self.split[:-1]) +
'/platform-metadata.json')
platforms = json.load(json_file)['platforms']['platform']
for impl in platforms:
self.initialize(impl)
json_file.close()
else:
self.platform_data = []
LOGGER.debug('Setting metadata concerning whole directory')
self.owner = 'YangModels'
self.repo = 'yang'
self.path = None
self.branch = 'master'
self.feature_set = 'ALL'
self.software_version = self.split[5]
self.vendor = self.split[3]
# Solve for os-type
if 'nx' in self.split[4]:
self.os = 'NX-OS'
self.platform = self.split[6].split('-')[0]
elif 'xe' in self.split[4]:
self.os = 'IOS-XE'
self.platform = self.split[6].split('-')[0]
elif 'xr' in self.split[4]:
self.os = 'IOS-XR'
self.platform = self.split[6].split('-')[1]
else:
self.os = 'Unknown'
self.platform = 'Unknown'
self.os_version = self.split[5]
self.software_flavor = 'ALL'
self.platform_data.append({
'software-flavor': self.software_flavor,
'platform': self.platform
})
integrity_checker.add_platform('/'.join(self.split[:-2]),
self.platform)
self.parsed_jsons = None
if not run_integrity:
self.parsed_jsons = LoadFiles(private_dir, log_directory)