def initialize(self, impl):
if impl['module-list-file']['path'] in self.hello_message_file:
LOGGER.info('Parsing a received json file')
self.feature_set = 'ALL'
self.os_version = impl['software-version']
self.software_flavor = impl['software-flavor']
self.vendor = impl['vendor']
self.platform = impl['name']
self.os = impl['os-type']
self.software_version = impl['software-version']
self.owner = impl['module-list-file']['owner']
self.repo = impl['module-list-file']['repository'].split('.')[0]
self.path = impl['module-list-file']['path']
self.branch = impl['module-list-file'].get('branch')
repo = repoutil.RepoUtil(
'{}{}/{}'.format(github_url, self.owner, self.repo),
self.logger)
repo.clone()
if not self.branch:
self.branch = 'master'
self.branch = repo.get_commit_hash(self.branch)
repo.remove()
self.platform_data.append({
'software-flavor': self.software_flavor,
'platform': self.platform
})
def _load_yangmodels_repo(self):
self.repo_owner = 'YangModels'
self.repo_name = 'yang'
repo_url = os.path.join(github_url, self.repo_owner, self.repo_name)
try:
repo = repoutil.load(self.dir_paths['yang_models'], repo_url)
except InvalidGitRepositoryError:
repo = repoutil.RepoUtil(
repo_url,
clone_options={'local_dir': self.dir_paths['yang_models']},
logger=self.logger)
self.repo = repo
yang_models_url_suffix = config.get('General-Section',
'yang-models-repo-url_suffix')
temp_dir = config.get('Directory-Section', 'temp')
LOGGER = log.get_logger('draftPullLocal',
log_directory + '/jobs/draft-pull-local.log')
LOGGER.info('Starting cron job IETF pull request local')
github_credentials = ''
if len(username) > 0:
github_credentials = username + ':' + token + '@'
# Fork and clone the repository YangModles/yang
LOGGER.info('Cloning repository')
reponse = requests.post('https://' + github_credentials +
yang_models_url_suffix)
repo = repoutil.RepoUtil('https://' + token + '@github.com/' + username +
'/yang.git')
repo.clone(config_name, config_email)
LOGGER.info('Cloning repo to local directory {}'.format(repo.localdir))
ietf_draft_json = requests.get(ietf_draft_url,
auth=(private_credentials[0],
private_credentials[1])).json()
response = requests.get(ietf_rfc_url,
auth=(private_credentials[0],
private_credentials[1]))
zfile = open(repo.localdir + '/rfc.tgz', 'wb')
zfile.write(response.content)
zfile.close()
tgz = tarfile.open(repo.localdir + '/rfc.tgz')
tgz.extractall(repo.localdir + '/standard/ietf/RFC')
def _resolve_imports(self, dependencies: t.List[Dependency],
git_commit_hash: str) -> list:
LOGGER.debug('Resolving imports')
imports = []
try:
imports = self._parsed_yang.search('import')
for chunk in imports:
dependency = Dependency()
dependency.name = chunk.arg
revisions = chunk.search('revision-date')
if revisions:
dependency.revision = revisions[0].arg
if dependency.revision:
yang_file = self._find_file(dependency.name,
dependency.revision)
else:
yang_file = self._find_file(dependency.name)
if yang_file is None:
dependency.schema = None
else:
try:
if os.path.dirname(yang_file) == os.path.dirname(
self._path):
if self.schema:
dependency.schema = '/'.join(
(*self.schema.split('/')[0:-1],
yang_file.split('/')[-1]))
else:
dependency.schema = None
else:
if '/yangmodels/yang/' in yang_file:
suffix = os.path.abspath(yang_file).split(
'/yangmodels/yang/')[1]
# First load/clone YangModels/yang repo
owner_name = 'YangModels'
repo_name = 'yang'
repo_url = os.path.join(
github_url, owner_name, repo_name)
try:
repo = repoutil.load(
self.yang_models, repo_url)
except InvalidGitRepositoryError:
repo = repoutil.RepoUtil(
repo_url,
clone_options={
'local_dir': self.yang_models
})
# Check if repository submodule
for submodule in repo.repo.submodules:
if submodule.name in suffix:
repo_url = submodule.url.lower()
repo_dir = os.path.join(
self.yang_models, submodule.name)
repo = repoutil.load(
repo_dir, repo_url)
owner_name = repo.get_repo_owner()
repo_name = repo.get_repo_dir().split(
'.git')[0]
suffix = suffix.replace(
'{}/'.format(submodule.name), '')
branch = repo.get_commit_hash(suffix, 'main')
dependency.schema = os.path.join(
github_raw, owner_name, repo_name, branch,
suffix)
elif git_commit_hash in yang_file:
if self.schema:
prefix = self.schema.split(
'/{}/'.format(git_commit_hash))[0]
suffix = os.path.abspath(yang_file).split(
'/{}/'.format(git_commit_hash))[1]
dependency.schema = '{}/master/{}'.format(
prefix, suffix)
else:
dependency.schema = None
except:
LOGGER.exception(
'Unable to resolve schema for {}@{}.yang'.format(
dependency.name, dependency.revision))
dependency.schema = None
dependencies.append(dependency)
dependencies.append(dependency)
finally:
return imports
def __init__(self,
log_directory,
hello_message_file,
index,
prepare,
integrity_checker,
api,
sdo,
json_dir,
html_result_dir,
save_file_to_dir,
private_dir,
yang_models_dir,
run_integrity=False):
global LOGGER
LOGGER = log.get_logger('capability',
log_directory + '/parseAndPopulate.log')
LOGGER.debug('Running constructor')
self.logger = log.get_logger('repoutil',
log_directory + '/parseAndPopulate.log')
self.log_directory = log_directory
self.run_integrity = run_integrity
self.to = save_file_to_dir
self.html_result_dir = html_result_dir
self.json_dir = json_dir
self.index = index
self.prepare = prepare
self.integrity_checker = integrity_checker
self.parsed_yang = None
self.api = api
self.sdo = sdo
self.yang_models_dir = yang_models_dir
# Get hello message root
if 'xml' in hello_message_file:
try:
LOGGER.debug('Checking for xml hello message file')
self.root = ET.parse(hello_message_file).getroot()
except:
#try to change & to &
hello_file = fileinput.FileInput(hello_message_file,
inplace=True)
for line in hello_file:
print(line.replace('&', '&'), end='')
hello_file.close()
LOGGER.warning(
'Hello message file has & instead of &, automatically changing to &'
)
self.root = ET.parse(hello_message_file).getroot()
# Split it so we can get vendor, os-type, os-version
self.split = hello_message_file.split('/')
self.hello_message_file = hello_message_file
if self.api and not self.sdo:
self.platform_data = []
json_file = open(hello_message_file.split('.xml')[0] + '.json')
impl = json.load(json_file)
self.initialize(impl)
json_file.close()
if not self.api and not self.sdo:
if os.path.isfile('/'.join(self.split[:-1]) +
'/platform-metadata.json'):
self.platform_data = []
json_file = open('/'.join(self.split[:-1]) +
'/platform-metadata.json')
platforms = json.load(json_file)['platforms']['platform']
for impl in platforms:
self.initialize(impl)
json_file.close()
else:
self.platform_data = []
LOGGER.debug('Setting metadata concerning whole directory')
self.owner = 'YangModels'
self.repo = 'yang'
repo = repoutil.RepoUtil(
'{}{}/{}'.format(github_url, self.owner, self.repo),
self.logger)
repo.clone()
self.path = None
self.branch = 'master'
self.branch = repo.get_commit_hash(self.branch)
repo.remove()
self.feature_set = 'ALL'
self.software_version = self.split[5]
self.vendor = self.split[3]
# Solve for os-type
if 'nx' in self.split[4]:
self.os = 'NX-OS'
self.platform = self.split[6].split('-')[0]
elif 'xe' in self.split[4]:
self.os = 'IOS-XE'
self.platform = self.split[6].split('-')[0]
elif 'xr' in self.split[4]:
self.os = 'IOS-XR'
self.platform = self.split[6].split('-')[1]
else:
self.os = 'Unknown'
self.platform = 'Unknown'
self.os_version = self.split[5]
self.software_flavor = 'ALL'
self.platform_data.append({
'software-flavor': self.software_flavor,
'platform': self.platform
})
integrity_checker.add_platform('/'.join(self.split[:-2]),
self.platform)
self.parsed_jsons = None
if not run_integrity:
self.parsed_jsons = LoadFiles(private_dir, log_directory)
def parse_and_dump_sdo(self):
repo = None
if self.api:
LOGGER.debug('Parsing sdo files sent via API')
with open(self.json_dir + '/prepare-sdo.json', 'r') as f:
sdos_json = json.load(f)
sdos_list = sdos_json['modules']['module']
for sdo in sdos_list:
file_name = unicodedata.normalize('NFKD', sdo['source-file']['path'].split('/')[-1]) \
.encode('ascii', 'ignore')
LOGGER.info(
'Parsing sdo file sent via API{}'.format(file_name))
self.owner = sdo['source-file']['owner']
repo_file_path = sdo['source-file']['path']
self.repo = sdo['source-file']['repository'].split('.')[0]
if repo is None:
repo = repoutil.RepoUtil(
'{}{}/{}'.format(github_url, self.owner, self.repo),
self.logger)
repo.clone()
self.branch = sdo['source-file'].get('branch')
if not self.branch:
self.branch = 'master'
self.branch = repo.get_commit_hash(
'/'.join(repo_file_path.split('/')[:-1]), self.branch)
root = self.owner + '/' + sdo['source-file']['repository'].split('.')[0] + '/' + self.branch + '/' \
+ '/'.join(repo_file_path.split('/')[:-1])
root = self.json_dir + '/temp/' + root
if sys.version_info < (3, 4):
root = self.json_dir + '/temp/' \
+ unicodedata.normalize('NFKD', root).encode('ascii', 'ignore')
if sys.version_info >= (3, 4):
file_name = file_name.decode('utf-8', 'strict')
if not os.path.isfile(root + '/' + file_name):
LOGGER.error(
'File {} sent via API was not downloaded'.format(
file_name))
continue
if '[1]' not in file_name:
try:
yang = Modules(self.yang_models_dir,
self.log_directory,
root + '/' + file_name,
self.html_result_dir, self.parsed_jsons,
self.json_dir)
except ParseException as e:
LOGGER.error(e.msg)
continue
name = file_name.split('.')[0].split('@')[0]
schema = github_raw + self.owner + '/' + self.repo + '/' + self.branch + '/' + repo_file_path
yang.parse_all(self.branch, name,
self.prepare.name_revision_organization,
schema, self.to, sdo)
self.prepare.add_key_sdo_module(yang)
else:
LOGGER.debug('Parsing sdo files from directory')
for root, subdirs, sdos in os.walk('/'.join(self.split)):
for file_name in sdos:
if '.yang' in file_name and ('vendor' not in root
or 'odp' not in root):
LOGGER.info(
'Parsing sdo file from directory {}'.format(
file_name))
if '[1]' in file_name:
LOGGER.warning(
'File {} contains [1] it its file name'.format(
file_name))
else:
try:
yang = Modules(self.yang_models_dir,
self.log_directory,
root + '/' + file_name,
self.html_result_dir,
self.parsed_jsons,
self.json_dir)
except ParseException as e:
LOGGER.error(e.msg)
continue
name = file_name.split('.')[0].split('@')[0]
self.owner = 'YangModels'
self.repo = 'yang'
if repo is None:
repo = repoutil.RepoUtil(
'{}{}/{}'.format(github_url, self.owner,
self.repo), self.logger)
repo.clone()
self.branch = 'master'
path = root + '/' + file_name
abs_path = os.path.abspath(path)
if '/yangmodels/yang/' in abs_path:
path = abs_path.split('/yangmodels/yang/')[1]
else:
path = re.split('tmp\/\w*\/', abs_path)[1]
self.branch = repo.get_commit_hash(
path, self.branch)
schema = (github_raw + self.owner + '/' +
self.repo + '/' + self.branch + '/' +
path)
yang.parse_all(
self.branch, name,
self.prepare.name_revision_organization,
schema, self.to)
self.prepare.add_key_sdo_module(yang)
repo.remove()
process = subprocess.Popen(['python', '../runYANGallstats/runYANGallstats.py', '--rootdir',
yang_models + '/standard/mef/src/model/standard', '--removedup', 'True'],
stdout=subprocess.PIPE)
out, err = process.communicate()
process_data(out, sdo_list, yang_models + '/standard/mef/src/model/standard', 'MEF standard')
process = subprocess.Popen(['python', '../runYANGallstats/runYANGallstats.py', '--rootdir',
yang_models + '/standard/mef/src/model/draft', '--removedup', 'True'],
stdout=subprocess.PIPE)
out, err = process.communicate()
process_data(out, sdo_list, yang_models + '/standard/mef/src/model/draft', 'MEF draft')
# Openconfig is from different repo that s why we need models in github zero
LOGGER.info('Cloning the repo')
repo = repoutil.RepoUtil('https://github.com/openconfig/public')
repo.clone(config_name, config_email)
process = subprocess.Popen(['python', '../runYANGallstats/runYANGallstats.py', '--rootdir',
repo.localdir + '/release/models', '--removedup', 'True'], stdout=subprocess.PIPE)
out, err = process.communicate()
process_data(out, sdo_list, repo.localdir + '/release/models', 'openconfig')
repo.remove()
context = {'table_sdo': sdo_list,
'table_vendor': vendor_list,
'num_yang_files_vendor': vendor_modules,
'num_yang_files_vendor_ndp': vendor_modules_ndp,
'num_yang_files_standard': standard_modules,
'num_yang_files_standard_ndp': standard_modules_ndp,
'num_parsed_files': all_modules_data,