def test_sdo_directory_parse_and_load(self, mock_hash: mock.MagicMock):
"""
Test whether keys were created and prepare object values were set correctly
from all the .yang files which are located in 'path' directory.
Arguments:
:param mock_hash (mock.MagicMock) get_commit_hash() method is patched, to always return 'master'
"""
mock_hash.return_value = 'master'
repo = self.get_yangmodels_repository()
path = '{}/standard/ieee/published/802.3'.format(yc_gc.yang_models)
api = False
dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename,
self.yangcatalog_api_prefix)
sdo_directory = SdoDirectory(path, dumper, self.fileHasher, api,
self.dir_paths)
sdo_directory.parse_and_load(repo)
for root, _, sdos in os.walk(path):
for file_name in sdos:
if '.yang' in file_name and ('vendor' not in root
or 'odp' not in root):
path_to_yang = os.path.join(path, file_name)
yang = self.declare_sdo_module(path_to_yang)
key = '{}@{}/{}'.format(yang.name, yang.revision,
yang.organization)
self.assertIn(key, sdo_directory.dumper.yang_modules)
def test_sdo_directory_parse_and_load_api(self, mock_hash: mock.MagicMock):
"""
Test whether key was created and prepare object value was set correctly
from all modules loaded from request-data.json file.
Arguments:
:param mock_hash (mock.MagicMock) get_commit_hash() method is patched, to always return 'master'
"""
mock_hash.return_value = 'master'
repo = self.get_yangmodels_repository()
path = os.path.join(yc_gc.temp_dir, 'groupings-tests')
api = True
sdo = True
dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename,
self.yangcatalog_api_prefix)
sdo_directory = SdoDirectory(path, dumper, self.fileHasher, api,
self.dir_paths)
sdo_directory.parse_and_load(repo)
with open(os.path.join(self.dir_paths['json'], 'request-data.json'),
'r') as f:
sdos_json = json.load(f)
sdos_list = sdos_json.get('modules', {}).get('module', [])
self.assertNotEqual(len(sdos_list), 0)
for sdo in sdos_list:
key = '{}@{}/{}'.format(sdo.get('name'), sdo.get('revision'),
sdo.get('organization'))
self.assertIn(key, sdo_directory.dumper.yang_modules)
def test_vendor_capabilities_solve_xe_os_type(self):
""" Test if platform_data are set correctly when platform_metadata.json file is not present in the folder.
"""
directory = os.path.join(self.test_repo, 'vendor/cisco/xe/16101')
xml_file = os.path.join(directory, 'capability-asr1k.xml')
api = False
dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename,
self.yangcatalog_api_prefix)
vendor_capabilities = VendorCapabilities(directory, xml_file, dumper,
self.fileHasher, api,
self.dir_paths)
vendor_capabilities._parse_platform_metadata()
platform_data = vendor_capabilities.platform_data
# Load desired module data from .json file
with open(
os.path.join(self.resources_path,
'parseAndPopulate_tests_data.json'), 'r') as f:
file_content = json.load(f)
desired_platform_data = file_content.get('xe_platform_data', {})
self.assertNotEqual(len(platform_data), 0)
self.assertEqual(desired_platform_data, platform_data[0])
def test_dumper_add_module(self):
"""
Dumper object is initialized and key of one Modules object is added to 'yang_modules' dictionary.
Created key is then retreived from 'yang_modules' dictionary and compared with desired format of key.
"""
desired_key = 'ietf-yang-types@2013-07-15/ietf'
yang = self.declare_sdo_module()
dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename,
self.yangcatalog_api_prefix)
dumper.add_module(yang)
created_key = list(dumper.yang_modules.keys())[0]
self.assertEqual(created_key, desired_key)
self.assertIn(desired_key, dumper.yang_modules)
def test_sdo_directory_parse_and_load_submodule(self,
mock_hash: mock.MagicMock):
"""
Test whether keys were created and dumper object values were set correctly
from all the .yang files which are located in 'path' directory. Created 'path' is submodule of git repository.
Arguments:
:param mock_hash (mock.MagicMock) get_commit_hash() method is patched, to always return 'master'
"""
mock_hash.return_value = 'master'
path = os.path.join(self.test_repo,
'vendor/huawei/network-router/8.20.0/ne5000e')
repo = self.get_yangmodels_repository()
api = False
dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename,
self.yangcatalog_api_prefix)
sdo_directory = SdoDirectory(path, dumper, self.fileHasher, api,
self.dir_paths)
sdo_directory.parse_and_load(repo)
sdo_directory.dumper.dump_modules(yc_gc.temp_dir)
desired_module_data = self.load_desired_prepare_json_data(
'git_submodule_huawei')
dumped_module_data = self.load_dumped_prepare_json_data()
# Compare desired output with output of prepare.json
for dumped_module in dumped_module_data:
for desired_module in desired_module_data:
if desired_module.get('name') == dumped_module.get('name'):
# Compare properties/keys of desired and dumped module data objects
for key in desired_module:
if key == 'yang-tree':
# Compare only URL suffix (exclude domain)
desired_tree_suffix = '/api{}'.format(
desired_module[key].split('/api')[1])
dumped_tree_suffix = '/api{}'.format(
dumped_module[key].split('/api')[1])
self.assertEqual(desired_tree_suffix,
dumped_tree_suffix)
elif key == 'compilation-result':
if dumped_module[key] != '' and desired_module[
key] != '':
# Compare only URL suffix (exclude domain)
desired_compilation_result = '/results{}'.format(
desired_module[key].split('/results')[-1])
dumped_compilation_result = '/results{}'.format(
dumped_module[key].split('/results')[-1])
self.assertEqual(desired_compilation_result,
dumped_compilation_result)
else:
if isinstance(desired_module[key], list):
for i in desired_module[key]:
self.assertIn(i, dumped_module[key])
else:
self.assertEqual(dumped_module[key],
desired_module[key])
def test_vendor_yang_lib_parse_and_dump(self, mock_hash: mock.MagicMock):
""" Test if all the modules from ietf-yang-library xml file (with their submodules) have correctly set information
about implementaton from platform_metadata.json file.
Parsed modules are dumped to prepare.json file, then loaded and implementation information is checked.
Arguments:
:param mock_hash (mock.MagicMock) get_commit_hash() method is patched, to always return 'master'
"""
mock_hash.return_value = 'master'
directory = os.path.join(
self.test_repo, 'vendor/huawei/network-router/8.20.0/ne5000e')
xml_file = os.path.join(directory, 'ietf-yang-library.xml')
platform_json_path = os.path.join(
self.test_repo,
'vendor/huawei/network-router/8.20.0/ne5000e/platform-metadata.json'
)
platform_name = 'ne5000e'
api = False
dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename,
self.yangcatalog_api_prefix)
vendor_yang_lib = VendorYangLibrary(directory, xml_file, dumper,
self.fileHasher, api,
self.dir_paths)
vendor_yang_lib.parse_and_load()
vendor_yang_lib.dumper.dump_modules(yc_gc.temp_dir)
dumped_modules_data = self.load_dumped_prepare_json_data()
platform_data = self.get_platform_data(platform_json_path,
platform_name)
self.assertNotEqual(len(platform_data), 0)
self.assertNotEqual(len(dumped_modules_data), 0)
for yang_module in dumped_modules_data:
self.assertIn('implementations', yang_module)
implementations = yang_module.get('implementations',
{}).get('implementation', [])
self.assertNotEqual(len(implementations), 0)
for implementation in implementations:
if implementation.get('platform') == platform_name:
self.assertEqual(implementation.get('vendor'),
platform_data.get('vendor'))
self.assertEqual(implementation.get('platform'),
platform_data.get('name'))
self.assertEqual(implementation.get('software-version'),
platform_data.get('software-version'))
self.assertEqual(implementation.get('software-flavor'),
platform_data.get('software-flavor'))
self.assertEqual(implementation.get('os-version'),
platform_data.get('software-version'))
self.assertEqual(implementation.get('feature-set'), 'ALL')
self.assertEqual(implementation.get('os-type'),
platform_data.get('os-type'))
def test_dumper_get_deviations_none(self):
"""
Set value of deviations property to None, to test if __get_deviations() method
correctly set value.
If value is set to None, it should not be dumped into .json file.
"""
directory = os.path.join(yc_gc.temp_dir,
'test/YangModels/yang/vendor/cisco/xr/701')
platform_data, netconf_version, netconf_capabilities = self.get_platform_data(
directory)
yang = self.declare_vendor_module()
yang.add_vendor_information(platform_data, 'implement',
netconf_capabilities, netconf_version)
# Clear deviations property to test functionality of __get_deviations() method
for implementation in yang.implementations:
implementation.deviations = []
# Dumper object
dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename,
self.yangcatalog_api_prefix)
dumper.add_module(yang)
dumper.dump_vendors(yc_gc.temp_dir)
# Load vendor module data from normal.json file
with open(os.path.join(yc_gc.temp_dir, 'normal.json'), 'r') as f:
dumped_vendor_data = json.load(f)
# Since deviations property has value None, it should not be present in dumped module
self.assertNotIn('deviations', dumped_vendor_data)
def test_dumper_get_dependencies_none(self):
"""
Set value of dependencies property to None, to test if __get_dependencies() method
correctly set value.
If value is set to None, it should not be dumped into normal.json file.
"""
yang = self.declare_sdo_module()
# Clear dependencies property to test functionality of __get_dependencies() method
yang.dependencies = []
dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename,
self.yangcatalog_api_prefix)
dumper.add_module(yang)
dumper.dump_modules(yc_gc.temp_dir)
# Load module data from dumped .json file
with open(
'{}/{}.json'.format(yc_gc.temp_dir,
self.prepare_output_filename), 'r') as f:
file_content = json.load(f)
dumped_module_data = file_content['module'][0]
# Since dependencies property has value None, it should not be present in dumped module
self.assertNotIn('dependencies', dumped_module_data)
def test_dumper_dump_vendors(self):
"""
Dumper object is initialized and key of one Modules object is added to 'yang_modules' dictionary.
It is necessary that this module has filled information about the implementation.
This can be achieved by calling add_vendor_information() method.
Vendor data are then dumped into normal.json file using dump_vendors() method.
Content of dumped normal.json file is then compared with desired content loaded from parseAndPopulate_tests_data.json file.
"""
# Modules object
directory = os.path.join(yc_gc.temp_dir,
'test/YangModels/yang/vendor/cisco/xr/701')
platform_data, netconf_version, netconf_capabilities = self.get_platform_data(
directory)
yang = self.declare_vendor_module()
yang.add_vendor_information(platform_data, 'implement',
netconf_capabilities, netconf_version)
# Dumper object
dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename,
self.yangcatalog_api_prefix)
dumper.add_module(yang)
dumper.dump_vendors(yc_gc.temp_dir)
# Load desired module data from .json file
with open(
os.path.join(self.resources_path,
'parseAndPopulate_tests_data.json'), 'r') as f:
file_content = json.load(f)
desired_vendor_data = file_content.get('dumped_vendor_data', {})
# Load vendor module data from normal.json file
os.path.join(yc_gc.temp_dir, 'normal.json')
with open(os.path.join(yc_gc.temp_dir, 'normal.json'), 'r') as f:
dumped_vendor_data = json.load(f)
self.assertEqual(desired_vendor_data, dumped_vendor_data)
def test_vendor_capabilities_parse_and_load(self,
mock_hash: mock.MagicMock):
""" Test if all the modules from capability file (with their submodules) have correctly set information
about implementaton from platform_metadata.json file.
Parsed modules are dumped to prepare.json file, then loaded and implementation information is chcecked.
Arguments:
:param mock_hash (mock.MagicMock) get_commit_hash() method is patched, to always return 'master'
"""
mock_hash.return_value = 'master'
directory = os.path.join(self.test_repo, 'vendor/cisco/xr/701')
xml_file = os.path.join(directory, self.hello_message_filename)
platform_json_path = os.path.join(
self.test_repo, 'vendor/cisco/xr/701/platform-metadata.json')
api = False
dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename,
self.yangcatalog_api_prefix)
vendor_capabilities = VendorCapabilities(directory, xml_file, dumper,
self.fileHasher, api,
self.dir_paths)
vendor_capabilities.parse_and_load()
vendor_capabilities.dumper.dump_modules(yc_gc.temp_dir)
dumped_modules_data = self.load_dumped_prepare_json_data()
self.assertNotEqual(len(dumped_modules_data), 0)
platform_data = self.get_platform_data(platform_json_path,
self.platform_name)
for yang_module in dumped_modules_data:
self.assertIn('implementations', yang_module)
implementations = yang_module.get('implementations',
{}).get('implementation', [])
self.assertNotEqual(len(implementations), 0)
for implementation in implementations:
if implementation.get('platform') == self.platform_name:
self.assertEqual(implementation.get('vendor'),
platform_data.get('vendor'))
self.assertEqual(implementation.get('platform'),
platform_data.get('name'))
self.assertEqual(implementation.get('software-version'),
platform_data.get('software-version'))
self.assertEqual(implementation.get('software-flavor'),
platform_data.get('software-flavor'))
self.assertEqual(implementation.get('os-version'),
platform_data.get('software-version'))
self.assertEqual(implementation.get('feature-set'), 'ALL')
self.assertEqual(implementation.get('os-type'),
platform_data.get('os-type'))
def test_dumper_dump_modules(self):
"""
Dumper object is created and one SDO module is added.
Modules are then dumped into prepare.json file using dump_modules() method.
Content of prepare.json file is then checked, data from file are compared with Modules object properties.
"""
yang = self.declare_sdo_module()
dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename,
self.yangcatalog_api_prefix)
dumper.add_module(yang)
dumper.dump_modules(yc_gc.temp_dir)
# Load desired module data from .json file
with open(
'{}/parseAndPopulate_tests_data.json'.format(
self.resources_path), 'r') as f:
file_content = json.load(f)
desired_module_data = file_content['dumped_module']['module'][0]
# Load module data from dumped prepare.json file
with open(
'{}/{}.json'.format(yc_gc.temp_dir,
self.prepare_output_filename), 'r') as f:
file_content = json.load(f)
dumped_module_data = file_content['module'][0]
# Compare properties/keys of desired and dumped module data objects
for key in desired_module_data:
self.assertIn(key, dumped_module_data, desired_module_data[key])
if key == 'yang-tree':
# Compare only URL suffix (exclude domain)
desired_tree_suffix = '/api{}'.format(
desired_module_data[key].split('/api')[1])
dumped_tree_suffix = '/api{}'.format(
dumped_module_data[key].split('/api')[1])
self.assertEqual(desired_tree_suffix, dumped_tree_suffix)
elif key == 'compilation-result':
if dumped_module_data[key] != '' and desired_module_data[
key] != '':
# Compare only URL suffix (exclude domain)
desired_compilation_result = '/results{}'.format(
desired_module_data[key].split('/results')[1])
dumped_compilation_result = '/results{}'.format(
dumped_module_data[key].split('/results')[1])
self.assertEqual(desired_compilation_result,
dumped_compilation_result)
else:
self.assertEqual(dumped_module_data[key],
desired_module_data[key])
def test_vendor_capabilities_ampersand_exception(self):
""" Test if ampersand character will be replaced in .xml file if occurs.
If ampersand character occurs, exception is raised, and character is replaced.
"""
directory = os.path.join(self.test_repo, 'vendor/cisco/xr/701')
xml_file = os.path.join(directory, self.hello_message_filename)
api = False
dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename,
self.yangcatalog_api_prefix)
# Change to achieve Exception will be raised
hello_file = fileinput.FileInput(xml_file, inplace=True)
for line in hello_file:
print(line.replace('&', '&'), end='')
hello_file.close()
vendor_capabilities = VendorCapabilities(directory, xml_file, dumper,
self.fileHasher, api,
self.dir_paths)
self.assertEqual(vendor_capabilities.root.tag,
'{urn:ietf:params:xml:ns:netconf:base:1.0}hello')
def main(scriptConf=None):
if scriptConf is None:
scriptConf = ScriptConfig()
args = scriptConf.args
config_path = args.config_path
config = create_config(config_path)
dir_paths: DirPaths = {
'log': config.get('Directory-Section', 'logs', fallback='/var/yang/logs'),
'private': config.get('Web-Section', 'private-directory', fallback='tests/resources/html/private'),
'yang_models': config.get('Directory-Section', 'yang-models-dir', fallback='tests/resources/yangmodels/yang'),
'cache': config.get('Directory-Section', 'cache', fallback='tests/resources/cache'),
'json': args.json_dir,
'result': args.result_html_dir,
'save': args.save_file_dir
}
LOGGER = log.get_logger('runCapabilities', '{}/parseAndPopulate.log'.format(dir_paths['log']))
is_uwsgi = config.get('General-Section', 'uwsgi', fallback='True')
separator = ':'
suffix = args.api_port
if is_uwsgi == 'True':
separator = '/'
suffix = 'api'
yangcatalog_api_prefix = '{}://{}{}{}/'.format(args.api_protocol, args.api_ip, separator, suffix)
start = time.time()
dumper = Dumper(dir_paths['log'], 'prepare', yangcatalog_api_prefix)
fileHasher = FileHasher('backend_files_modification_hashes', dir_paths['cache'],
args.save_file_hash, dir_paths['log'])
LOGGER.info('Starting to iterate through files')
if args.sdo:
LOGGER.info('Found directory for sdo {}'.format(args.dir))
# If yang-parameters.xml exists -> parsing IANA-maintained modules
if os.path.isfile(os.path.join(args.dir, 'yang-parameters.xml')):
LOGGER.info('yang-parameters.xml file found')
grouping = IanaDirectory(args.dir, dumper, fileHasher, args.api, dir_paths)
grouping.parse_and_load()
else:
LOGGER.info('Starting to parse files in sdo directory')
grouping = SdoDirectory(args.dir, dumper, fileHasher, args.api, dir_paths)
grouping.parse_and_load()
dumper.dump_modules(dir_paths['json'])
else:
for pattern in ['*capabilit*.xml', '*ietf-yang-library*.xml']:
for root, basename in find_files(args.dir, pattern):
filename = os.path.join(root, basename)
LOGGER.info('Found xml source {}'.format(filename))
if pattern == '*capabilit*.xml':
grouping = VendorCapabilities(root, filename, dumper, fileHasher, args.api, dir_paths)
else:
grouping = VendorYangLibrary(root, filename, dumper, fileHasher, args.api, dir_paths)
try:
grouping.parse_and_load()
except Exception as e:
LOGGER.exception('Skipping {}, error while parsing'.format(filename))
dumper.dump_modules(dir_paths['json'])
dumper.dump_vendors(dir_paths['json'])
end = time.time()
LOGGER.info('Time taken to parse all the files {} seconds'.format(int(end - start)))
# Dump updated hashes into temporary directory
if len(fileHasher.updated_hashes) > 0:
fileHasher.dump_tmp_hashed_files_list(fileHasher.updated_hashes, dir_paths['json'])