def add_vendors(): if not request.json: abort(400) body = request.json LOGGER.info('Adding vendor with body {}'.format(body)) tree_created = False resolved_authorization = authorize_for_vendors(request, body) if 'passed' != resolved_authorization: return resolved_authorization try: http_request( protocol + '://' + confd_ip + ':' + repr(confdPort) + '/api/config/platforms/', 'DELETE', None, credentials, 'application/vnd.yang.collection+json') except: pass path = protocol + '://' + confd_ip + ':' + repr(confdPort) + '/api/config/' try: http_request(path, 'POST', json.dumps(body), credentials, 'application/vnd.yang.data+json') except urllib2.HTTPError as e: abort(e.code) repo = {} direc = 0 while True: if os.path.isdir('../parseAndPopulate/' + repr(direc)): direc += 1 else: break direc = '../parseAndPopulate/' + repr(direc) for platform in body['platforms']: capability = platform['module-list-file'] file_name = capability['path'].split('/')[-1] if request.method == 'POST': repo_split = capability['repository'].split('.')[0] if os.path.isfile('../../api/vendor/' + capability['owner'] + '/' + repo_split + '/' + capability['path']): continue repo_url = url + capability['owner'] + '/' + capability['repository'] if repo_url not in repo: repo[repo_url] = repoutil.RepoUtil(repo_url) repo[repo_url].clone() for submodule in repo[repo_url].repo.submodules: submodule.update(init=True) if capability.get('branch'): branch = capability.get('branch') else: branch = 'master' directory = '/'.join(capability['path'].split('/')[:-1]) save_to = direc + '/temp/' + capability['owner'] + '/' \ + capability['repository'].split('.')[0] + '/' + branch + '/' + directory try: shutil.copytree(repo[repo_url].localdir + '/' + directory, save_to, ignore=shutil.ignore_patterns( '*.json', '*.xml', '*.sh', '*.md', '*.txt', '*.bin')) except OSError: pass with open(save_to + '/' + file_name.split('.')[0] + '.json', "w") as plat: json.dump(platform, plat) shutil.copy(repo[repo_url].localdir + '/' + capability['path'], save_to) tree_created = True for key in repo: repo[key].remove() api_protocol = 'http' api_port = 5000 if ssl_key != '': api_protocol = 'https' api_port = 8443 arguments = [ "python", "../parseAndPopulate/populate.py", "--port", repr(confdPort), "--dir", direc + "/temp", "--api", "--ip", confd_ip, "--credentials", credentials[0], credentials[1], repr(tree_created), integrity_file_location, protocol, api_protocol, repr(api_port) ] job_id = sender.send('#'.join(arguments)) LOGGER.info('job_id {}'.format(job_id)) return jsonify({'info': 'Verification successful', 'job-id': job_id})
def add_modules(): if not request.json: abort(400) body = request.json LOGGER.info('Adding modules with body {}'.format(body)) tree_created = False with open('./prepare-sdo.json', "w") as plat: json.dump(body, plat) try: http_request( protocol + '://' + confd_ip + ':' + repr(confdPort) + '/api/config/modules/', 'DELETE', None, credentials, 'application/vnd.yang.collection+json') except: pass path = protocol + '://' + confd_ip + ':' + repr( confdPort) + '/api/config/modules' try: http_request(path, 'PUT', json.dumps(body), credentials, 'application/vnd.yang.data+json') except urllib2.HTTPError as e: abort(e.code) repo = {} warning = [] direc = 0 while True: if os.path.isdir('../parseAndPopulate/' + repr(direc)): direc += 1 else: break direc = '../parseAndPopulate/' + repr(direc) for mod in body['modules']['module']: sdo = mod['source-file'] orgz = mod['organization'] if request.method == 'POST': try: path = protocol + '://' + confd_ip + ':' + repr(confdPort) + '/api/config/catalog/modules/module/' + \ mod['name'] + ',' + mod['revision'] + ',' + mod['organization'] http_request(path, 'GET', None, credentials, 'application/vnd.yang.data+json') continue except urllib2.HTTPError as e: pass directory = '/'.join(sdo['path'].split('/')[:-1]) repo_url = url + sdo['owner'] + '/' + sdo['repository'] LOGGER.debug('Cloning repository') if repo_url not in repo: repo[repo_url] = repoutil.RepoUtil(repo_url) repo[repo_url].clone() for submodule in repo[repo_url].repo.submodules: submodule.update(init=True) if sdo.get('branch'): branch = sdo.get('branch') else: branch = 'master' save_to = direc + '/temp/' + sdo['owner'] + '/' + sdo['repository'].split('.')[0] \ + '/' + branch + '/' + directory try: os.makedirs(save_to) except OSError as e: # be happy if someone already created the path if e.errno != errno.EEXIST: raise shutil.copy(repo[repo_url].localdir + '/' + sdo['path'], save_to) if os.path.isfile('./prepare-sdo.json'): shutil.move('./prepare-sdo.json', direc) tree_created = True organization = '' try: namespace = yangParser.parse(os.path.abspath(save_to + '/' + sdo['path'].split('/')[-1])) \ .search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'urn:' in namespace: organization = namespace.split('urn:')[1].split(':')[0] except: while True: try: belongs_to = yangParser.parse(os.path.abspath(repo[repo_url].localdir + '/' + sdo['path'])) \ .search('belongs-to')[0].arg except: break try: namespace = yangParser.parse( os.path.abspath(repo[repo_url].localdir + '/' + '/'.join(sdo['path'].split('/')[:-1]) + '/' + belongs_to + '.yang')).search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'urn:' in namespace: organization = namespace.split('urn:')[1].split( ':')[0] break except: pass resolved_authorization = authorize_for_sdos(request, orgz, organization) if not resolved_authorization: shutil.rmtree(direc) for key in repo: repo[key].remove() return unauthorized() if 'organization' in repr(resolved_authorization): warning.append(sdo['path'].split('/')[-1] + ' ' + resolved_authorization) for key in repo: repo[key].remove() api_protocol = 'http' api_port = 5000 if ssl_key != '': api_protocol = 'https' api_port = 8443 LOGGER.debug('Sending a new job') arguments = [ "python", "../parseAndPopulate/populate.py", "--sdo", "--port", repr(confdPort), "--dir", direc + "/temp", "--api", "--ip", confd_ip, "--credentials", credentials[0], credentials[1], repr(tree_created), protocol, api_protocol, repr(api_port) ] job_id = sender.send('#'.join(arguments)) LOGGER.info('job_id {}'.format(job_id)) if len(warning) > 0: return jsonify({ 'info': 'Verification successful', 'job-id': job_id, 'warnings': [{ 'warning': val } for val in warning] }) else: return jsonify({'info': 'Verification successful', 'job-id': job_id})
separator = ':' suffix = api_port if is_uwsgi == 'True': separator = '/' suffix = 'api' yangcatalog_api_prefix = '{}://{}{}{}/'.format(api_protocol, api_ip, separator, suffix) github_credentials = '' if len(username) > 0: github_credentials = username + ':' + token + '@' LOGGER.info('Forking repository') reponse = requests.post('https://' + github_credentials + 'api.github.com/repos/openconfig/public/forks') repo = repoutil.RepoUtil('https://' + token + '@github.com/' + username + '/public.git') repo.clone(config_name, config_email) LOGGER.info('Repository cloned to local directory {}'.format( repo.localdir)) mods = [] for root, dirs, files in os.walk(repo.localdir + '/release/models/'): for basename in files: if '.yang' in basename: mod = {} name = basename.split('.')[0].split('@')[0] mod['generated-from'] = 'not-applicable' mod['module-classification'] = 'unknown' mod['name'] = name
api_port = config.get('DraftPullLocal-Section', 'api-port') credentials = config.get('DraftPullLocal-Section', 'credentials').split(' ') token = config.get('DraftPull-Section', 'yang-catalog-token') username = config.get('DraftPull-Section', 'username') api_protocol = config.get('DraftPullLocal-Section', 'protocol') github_credentials = '' if len(username) > 0: github_credentials = username + ':' + token + '@' LOGGER.info( 'Loading all files from http://www.claise.be/IETFYANGDraft.json') LOGGER.info('Forking repository') reponse = requests.post('https://' + github_credentials + 'api.github.com/repos/openconfig/public/forks') repo = repoutil.RepoUtil('https://github.com/yang-catalog/public.git') LOGGER.info('Cloning repo to local directory {}'.format(repo.localdir)) repo.clone() mods = [] for root, dirs, files in os.walk(repo.localdir + '/release/models/'): for basename in files: if '.yang' in basename: mod = {} name = basename.split('.')[0].split('@')[0] mod['generated-from'] = 'not-applicable' mod['module-classification'] = 'unknown' mod['name'] = name mod['revision'] = resolve_revision('{}/{}'.format(
api_port = config.get('General-Section', 'api-port') credentials = config.get('General-Section', 'credentials') config_name = config.get('General-Section', 'repo-config-name') config_email = config.get('General-Section', 'repo-config-email') move_to = config.get('Statistics-Section', 'file-location') auth = credentials.split(' ') is_uwsgi = config.get('General-Section', 'uwsgi') separator = ':' suffix = api_port if is_uwsgi == 'True': separator = '/' suffix = 'api' yangcatalog_api_prefix = '{}://{}{}{}/'.format(protocol, api_ip, separator, suffix) LOGGER.info('Cloning the repo') repo = repoutil.RepoUtil('https://github.com/YangModels/yang.git') repo.clone(config_name, config_email) repo.updateSubmodule() xr = set() nx = set() xe = set() solve_platforms(repo.localdir + '/vendor/cisco/xr', xr) solve_platforms(repo.localdir + '/vendor/cisco/xe', xe) solve_platforms(repo.localdir + '/vendor/cisco/nx', nx) xr_versions = sorted(next(os.walk(repo.localdir + '/vendor/cisco/xr'))[1]) nx_versions = sorted(next(os.walk(repo.localdir + '/vendor/cisco/nx'))[1]) xe_versions = sorted(next(os.walk(repo.localdir + '/vendor/cisco/xe'))[1]) xr_values = []