def test_parse_semver_update_versions(self, mock_requests_get: mock.MagicMock):
""" Check whether the value of the 'derived-semantic-version' property was set correctly.
Module between two other revisions is parsed, which means, it will loop through all the available
module revisions and assign 'derived-semantic-version' to them.
Expected 'derived-semantic-version' order: '1.0.0', '2.0.0', '3.0.0', '4.0.0', '4.1.0', '4.1.1'
Arguments:
:param mock_requests_get (mock.MagicMock) requests.get() method is patched to return only the necessary modules
"""
expected_semver_order = ['1.0.0', '2.0.0', '3.0.0', '4.0.0', '4.1.0', '4.1.1']
modules = self.payloads['modulesComplicatedAlgorithms_prepare_json']['module']
modules = sorted(modules, key=lambda k: k['revision'])
# List of modules returned from patched /api/search/modules GET request
existing_modules = {}
existing_modules['module'] = deepcopy(modules[:4] + modules[5:])
mock_requests_get.return_value.json.return_value = existing_modules
module_to_parse = modules[4]
all_modules = {}
all_modules['module'] = [module_to_parse]
complicatedAlgorithms = ModulesComplicatedAlgorithms(yc_gc.logs_dir, self.yangcatalog_api_prefix,
yc_gc.credentials, self.save_file_dir,
yc_gc.temp_dir, all_modules, yc_gc.yang_models, yc_gc.temp_dir,
yc_gc.json_ytree)
complicatedAlgorithms.parse_semver()
self.assertIn('2020-05-01', complicatedAlgorithms.new_modules['semver-test'])
self.assertEqual(complicatedAlgorithms.new_modules['semver-test']['2020-05-01']['derived-semantic-version'], '4.1.0')
def test_parse_semver_update_major(self, mock_requests_get: mock.MagicMock):
""" Check whether the value of the 'derived-semantic-version' property was set correctly.
Module compilation status is 'passed-with-warnings' so a major version update is expected.
Expected 'derived-semantic-version': 2.0.0
Arguments:
:param mock_requests_get (mock.MagicMock) requests.get() method is patched to return only the necessary modules
"""
modules = self.payloads['modulesComplicatedAlgorithms_prepare_json']['module']
modules = sorted(modules, key=lambda k: k['revision'])
# List od modules returned from patched /api/search/modules GET request
modules[1].pop('derived-semantic-version')
existing_modules = {}
existing_modules['module'] = deepcopy(modules[:2])
mock_requests_get.return_value.json.return_value = existing_modules
module_to_parse = modules[1]
all_modules = {}
all_modules['module'] = [module_to_parse]
complicatedAlgorithms = ModulesComplicatedAlgorithms(yc_gc.logs_dir, self.yangcatalog_api_prefix,
yc_gc.credentials, self.save_file_dir,
yc_gc.temp_dir, all_modules, yc_gc.yang_models, yc_gc.temp_dir,
yc_gc.json_ytree)
complicatedAlgorithms.parse_semver()
self.assertNotEqual(len(complicatedAlgorithms.new_modules), 0)
name = module_to_parse['name']
revision = module_to_parse['revision']
new_module = complicatedAlgorithms.new_modules[name].get(revision, {})
self.assertEqual(new_module.get('derived-semantic-version'), '2.0.0')
def main(scriptConf=None):
start_time = int(time.time())
if scriptConf is None:
scriptConf = ScriptConfig()
api_protocol = scriptConf.api_protocol
ip = scriptConf.ip
api_port = scriptConf.api_port
is_uwsgi = scriptConf.is_uwsgi
temp_dir = scriptConf.temp_dir
log_directory = scriptConf.log_directory
save_file_dir = scriptConf.save_file_dir
yang_models = scriptConf.yang_models
credentials = scriptConf.credentials
json_ytree = scriptConf.json_ytree
LOGGER = log.get_logger('sandbox', '{}/sandbox.log'.format(log_directory))
separator = ':'
suffix = api_port
if is_uwsgi == 'True':
separator = '/'
suffix = 'api'
yangcatalog_api_prefix = '{}://{}{}{}/'.format(api_protocol, ip, separator,
suffix)
# yangcatalog_api_prefix = 'https://yangcatalog.org/api/'
url = '{}search/modules'.format(yangcatalog_api_prefix)
LOGGER.info('Getting all the modules from: {}'.format(url))
response = requests.get(url, headers={'Accept': 'application/json'})
all_existing_modules = response.json().get('module', [])
global path
path = '{}/semver_prepare.json'.format(temp_dir)
all_modules = get_list_of_unique_modules(all_existing_modules)
LOGGER.info('Number of unique modules: {}'.format(
len(all_modules['module'])))
# Uncomment the next line to read data from the file semver_prepare.json
# all_modules = load_from_json(path)
# Initialize ModulesComplicatedAlgorithms
direc = '/var/yang/tmp'
num_of_modules = len(all_modules['module'])
chunk_size = 100
chunks = (num_of_modules - 1) // chunk_size + 1
for i in range(chunks):
try:
LOGGER.info('Proccesing chunk {} out of {}'.format(i, chunks))
batch = all_modules['module'][i * chunk_size:(i + 1) * chunk_size]
batch_modules = {'module': batch}
recursion_limit = sys.getrecursionlimit()
sys.setrecursionlimit(50000)
complicatedAlgorithms = ModulesComplicatedAlgorithms(
log_directory, yangcatalog_api_prefix, credentials,
save_file_dir, direc, batch_modules, yang_models, temp_dir,
json_ytree)
complicatedAlgorithms.parse_semver()
sys.setrecursionlimit(recursion_limit)
complicatedAlgorithms.populate()
except:
LOGGER.exception(
'Exception occured during running ModulesComplicatedAlgorithms'
)
continue
messages = [{
'label': 'Number of modules checked',
'message': num_of_modules
}]
end = time.time()
LOGGER.info(
'Populate took {} seconds with the main and complicated algorithm'.
format(int(end - start_time)))
filename = os.path.basename(__file__).split('.py')[0]
job_log(start_time,
temp_dir,
filename,
messages=messages,
status='Success')
LOGGER.info('Job finished successfully')